Merge branch 'vhost-scsi' of /usr/src/qemu-paolo into vhost-scsi-paolovhost-scsi-paolo

2034 files changed, 127924 insertions, 51494 deletions

diff --git a/.exrc b/.exrc
new file mode 100644
index 00000000000..37755ede83a
--- /dev/null
+++ b/.exrc
@@ -0,0 +1,7 @@
+"VIM settings to match QEMU coding style.  They are activated by adding the
+"following settings (without the " symbol) as last two lines in $HOME/.vimrc:
+"set secure
+"set exrc
+set expandtab
+set shiftwidth=4
+set smarttab
diff --git a/.gitignore b/.gitignore
index 824c0d24df9..53fe9c30787 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,19 +1,19 @@
 config-devices.*
 config-all-devices.*
+config-all-disas.*
 config-host.*
 config-target.*
-trace.h
-trace.c
-trace-dtrace.h
-trace-dtrace.dtrace
+trace/generated-tracers.h
+trace/generated-tracers.c
+trace/generated-tracers-dtrace.h
+trace/generated-tracers-dtrace.dtrace
+libcacard/trace/generated-tracers.c
 *-timestamp
 *-softmmu
 *-darwin-user
 *-linux-user
 *-bsd-user
 libdis*
-libhw32
-libhw64
 libuser
 linux-headers/asm
 qapi-generated
@@ -49,6 +49,7 @@ test-qmp-output-visitor
 test-string-input-visitor
 test-string-output-visitor
 test-visitor-serialization
+fsdev/virtfs-proxy-helper
 fsdev/virtfs-proxy-helper.1
 fsdev/virtfs-proxy-helper.pod
 .gdbinit
@@ -70,6 +71,7 @@ fsdev/virtfs-proxy-helper.pod
 *.tp
 *.vr
 *.d
+!scripts/qemu-guest-agent/fsfreeze-hook.d
 *.o
 *.lo
 *.la
diff --git a/.gitmodules b/.gitmodules
index eca876f85d4..cfa2af9f7a4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -19,3 +19,6 @@
 [submodule "roms/sgabios"]
 	path = roms/sgabios
 	url = git://git.qemu.org/sgabios.git
+[submodule "pixman"]
+	path = pixman
+	url = git://anongit.freedesktop.org/pixman
diff --git a/HACKING b/HACKING
index 471cf1d1970..6654d332493 100644
--- a/HACKING
+++ b/HACKING
@@ -32,7 +32,7 @@ mandatory for VMState fields.
 
 Don't use Linux kernel internal types like u32, __u32 or __le32.
 
-Use target_phys_addr_t for guest physical addresses except pcibus_t
+Use hwaddr for guest physical addresses except pcibus_t
 for PCI addresses.  In addition, ram_addr_t is a QEMU internal address
 space that maps guest RAM physical addresses into an intermediate
 address space that can map to host virtual address spaces.  Generally
@@ -91,10 +91,11 @@ emulators.
 
 4. String manipulation
 
-Do not use the strncpy function.  According to the man page, it does
-*not* guarantee a NULL-terminated buffer, which makes it extremely dangerous
-to use.  Instead, use functionally equivalent function:
-void pstrcpy(char *buf, int buf_size, const char *str)
+Do not use the strncpy function.  As mentioned in the man page, it does *not*
+guarantee a NULL-terminated buffer, which makes it extremely dangerous to use.
+It also zeros trailing destination bytes out to the specified length.  Instead,
+use this similar function when possible, but note its different signature:
+void pstrcpy(char *dest, int dest_buf_size, const char *src)
 
 Don't use strcat because it can't check for buffer overflows, but:
 char *pstrcat(char *buf, int buf_size, const char *s)
@@ -122,3 +123,23 @@ gcc's printf attribute directive in the prototype.
 This makes it so gcc's -Wformat and -Wformat-security options can do
 their jobs and cross-check format strings with the number and types
 of arguments.
+
+6. C standard, implementation defined and undefined behaviors
+
+C code in QEMU should be written to the C99 language specification. A copy
+of the final version of the C99 standard with corrigenda TC1, TC2, and TC3
+included, formatted as a draft, can be downloaded from:
+ http://www.open-std.org/jtc1/sc22/WG14/www/docs/n1256.pdf
+
+The C language specification defines regions of undefined behavior and
+implementation defined behavior (to give compiler authors enough leeway to
+produce better code).  In general, code in QEMU should follow the language
+specification and avoid both undefined and implementation defined
+constructs. ("It works fine on the gcc I tested it with" is not a valid
+argument...) However there are a few areas where we allow ourselves to
+assume certain behaviors because in practice all the platforms we care about
+behave in the same way and writing strictly conformant code would be
+painful. These are:
+ * you may assume that integers are 2s complement representation
+ * you may assume that right shift of a signed integer duplicates
+   the sign bit (ie it is an arithmetic shift, not a logical shift)
diff --git a/MAINTAINERS b/MAINTAINERS
index 6d864c1ceb9..9dd4c20798a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -98,6 +98,7 @@ S: Maintained
 F: target-ppc/
 
 S390
+M: Richard Henderson <rth@twiddle.net>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/
@@ -132,7 +133,7 @@ Guest CPU Cores (KVM):
 ----------------------
 
 Overall
-M: Avi Kivity <avi@redhat.com>
+M: Gleb Natapov <gleb@redhat.com>
 M: Marcelo Tosatti <mtosatti@redhat.com>
 L: kvm@vger.kernel.org
 S: Supported
@@ -150,7 +151,7 @@ S: Maintained
 F: target-s390x/kvm.c
 
 X86
-M: Avi Kivity <avi@redhat.com>
+M: Gleb Natapov <gleb@redhat.com>
 M: Marcelo Tosatti <mtosatti@redhat.com>
 L: kvm@vger.kernel.org
 S: Supported
@@ -268,6 +269,7 @@ S: Maintained
 F: hw/xilinx_zynq.c
 F: hw/zynq_slcr.c
 F: hw/cadence_*
+F: hw/xilinx_spips.c
 
 CRIS Machines
 -------------
@@ -349,14 +351,36 @@ PowerPC Machines
 405
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
-S: Maintained
+S: Odd Fixes
 F: hw/ppc405_boards.c
 
+Bamboo
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/ppc440_bamboo.c
+
+e500
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppc/e500.[hc]
+F: hw/ppc/e500plat.c
+
+mpc8544ds
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppc/mpc8544ds.c
+F: hw/mpc8544_guts.c
+
 New World
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Maintained
-F: hw/ppc_newworld.c
+F: hw/ppc/mac_newworld.c
 F: hw/unin_pci.c
 F: hw/dec_pci.[hc]
 
@@ -364,7 +388,7 @@ Old World
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Maintained
-F: hw/ppc_oldworld.c
+F: hw/ppc/mac_oldworld.c
 F: hw/grackle_pci.c
 
 PReP
@@ -373,6 +397,20 @@ L: qemu-ppc@nongnu.org
 S: Odd Fixes
 F: hw/ppc_prep.c
 F: hw/prep_pci.[hc]
+F: hw/pc87312.[hc]
+
+sPAPR
+M: David Gibson <david@gibson.dropbear.id.au>
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/spapr*
+
+virtex_ml507
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/virtex_ml507.c
 
 SH4 Machines
 ------------
@@ -398,6 +436,12 @@ M: Blue Swirl <blauwirbel@gmail.com>
 S: Maintained
 F: hw/sun4u.c
 
+Leon3
+M: Fabien Chouteau <chouteau@adacore.com>
+S: Maintained
+F: hw/leon3.c
+F: hw/grlib*
+
 S390 Machines
 -------------
 S390 Virtio
@@ -448,9 +492,23 @@ F: hw/omap*
 PCI
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
+F: hw/pci/*
 F: hw/pci*
 F: hw/piix*
 
+ppc4xx
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/ppc4xx*.[hc]
+
+ppce500
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppce500_*
+
 SCSI
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Supported
@@ -463,11 +521,22 @@ M: Paul Brook <paul@codesourcery.com>
 S: Odd Fixes
 F: hw/lsi53c895a.c
 
+SSI
+M: Peter Crosthwaite <peter.crosthwaite@petalogix.com>
+S: Maintained
+F: hw/ssi.*
+F: hw/m25p80.c
+
 USB
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: hw/usb*
 
+VFIO
+M: Alex Williamson <alex.williamson@redhat.com>
+S: Supported
+F: hw/vfio*
+
 vhost
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
@@ -487,6 +556,7 @@ T: git git://github.com/kvaneesh/QEMU.git
 
 virtio-blk
 M: Kevin Wolf <kwolf@redhat.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Supported
 F: hw/virtio-blk*
 
@@ -506,6 +576,7 @@ F: hw/xilinx_intc.c
 F: hw/xilinx_ethlite.c
 F: hw/xilinx_timer.c
 F: hw/xilinx.h
+F: hw/xilinx_spi.c
 
 Subsystems
 ----------
@@ -516,6 +587,7 @@ F: audio/
 
 Block
 M: Kevin Wolf <kwolf@redhat.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Supported
 F: block*
 F: block/
@@ -525,6 +597,13 @@ M: Anthony Liguori <aliguori@us.ibm.com>
 S: Maintained
 F: qemu-char.c
 
+CPU
+M: Andreas Färber <afaerber@suse.de>
+S: Supported
+F: qom/cpu.c
+F: include/qemu/cpu.h
+F: target-i386/cpu.c
+
 Device Tree
 M: Peter Crosthwaite <peter.crosthwaite@petalogix.com>
 M: Alexander Graf <agraf@suse.de>
@@ -568,7 +647,7 @@ F: monitor.c
 
 Network device layer
 M: Anthony Liguori <aliguori@us.ibm.com>
-M: Stefan Hajnoczi <stefanha@gmail.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Maintained
 F: net/
 T: git git://github.com/stefanha/qemu.git net
@@ -588,7 +667,7 @@ F: slirp/
 T: git git://git.kiszka.org/qemu.git queues/slirp
 
 Tracing
-M: Stefan Hajnoczi <stefanha@gmail.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Maintained
 F: trace/
 F: scripts/tracetool.py
diff --git a/Makefile b/Makefile
index 1cd5bc80d50..0d9099a4730 100644
--- a/Makefile
+++ b/Makefile
@@ -8,22 +8,38 @@ ifneq ($(wildcard config-host.mak),)
 # Put the all: rule here so that config-host.mak can contain dependencies.
 all:
 include config-host.mak
+
+# Check that we're not trying to do an out-of-tree build from
+# a tree that's been used for an in-tree build.
+ifneq ($(realpath $(SRC_PATH)),$(realpath .))
+ifneq ($(wildcard $(SRC_PATH)/config-host.mak),)
+$(error This is an out of tree build but your source tree ($(SRC_PATH)) \
+seems to have been used for an in-tree build. You can fix this by running \
+"make distclean && rm -rf *-linux-user *-softmmu" in your source tree)
+endif
+endif
+
 include $(SRC_PATH)/rules.mak
 config-host.mak: $(SRC_PATH)/configure
 	@echo $@ is out-of-date, running configure
 	@sed -n "/.*Configured with/s/[^:]*: //p" $@ | sh
 else
 config-host.mak:
+ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 	@echo "Please call configure before running make!"
 	@exit 1
 endif
+endif
 
-GENERATED_HEADERS = config-host.h trace.h qemu-options.def
+GENERATED_HEADERS = config-host.h qemu-options.def
+GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h
+GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c
+
+GENERATED_HEADERS += trace/generated-tracers.h
 ifeq ($(TRACE_BACKEND),dtrace)
-GENERATED_HEADERS += trace-dtrace.h
+GENERATED_HEADERS += trace/generated-tracers-dtrace.h
 endif
-GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h
-GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c trace.c
+GENERATED_SOURCES += trace/generated-tracers.c
 
 # Don't try to regenerate Makefile or configure
 # We don't generate any of them
@@ -52,8 +68,13 @@ SUBDIR_MAKEFLAGS=$(if $(V),,--no-print-directory) BUILD_DIR=$(BUILD_DIR)
 SUBDIR_DEVICES_MAK=$(patsubst %, %/config-devices.mak, $(TARGET_DIRS))
 SUBDIR_DEVICES_MAK_DEP=$(patsubst %, %/config-devices.mak.d, $(TARGET_DIRS))
 
+ifeq ($(SUBDIR_DEVICES_MAK),)
+config-all-devices.mak:
+	$(call quiet-command,echo '# no devices' > $@,"  GEN   $@")
+else
 config-all-devices.mak: $(SUBDIR_DEVICES_MAK)
 	$(call quiet-command,cat $(SUBDIR_DEVICES_MAK) | grep =y | sort -u > $@,"  GEN   $@")
+endif
 
 -include $(SUBDIR_DEVICES_MAK_DEP)
 
@@ -81,6 +102,18 @@ defconfig:
 	rm -f config-all-devices.mak $(SUBDIR_DEVICES_MAK)
 
 -include config-all-devices.mak
+-include config-all-disas.mak
+CONFIG_SOFTMMU := $(if $(filter %-softmmu,$(TARGET_DIRS)),y)
+CONFIG_USER_ONLY := $(if $(filter %-user,$(TARGET_DIRS)),y)
+CONFIG_ALL=y
+
+ifneq ($(wildcard config-host.mak),)
+include $(SRC_PATH)/Makefile.objs
+include $(SRC_PATH)/tests/Makefile
+endif
+ifeq ($(CONFIG_SMARTCARD_NSS),y)
+include $(SRC_PATH)/libcacard/Makefile
+endif
 
 all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all
 
@@ -94,15 +127,16 @@ SUBDIR_RULES=$(patsubst %,subdir-%, $(TARGET_DIRS))
 subdir-%:
 	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C $* V="$(V)" TARGET_DIR="$*/" all,)
 
-ifneq ($(wildcard config-host.mak),)
-include $(SRC_PATH)/Makefile.objs
-endif
+subdir-pixman: pixman/Makefile
+	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C pixman V="$(V)" all,)
 
-subdir-libcacard: $(oslib-obj-y) $(trace-obj-y) qemu-timer-common.o
+pixman/Makefile: $(SRC_PATH)/pixman/configure
+	(cd pixman; CFLAGS="$(CFLAGS) -fPIC $(extra_cflags) $(extra_ldflags)" $(SRC_PATH)/pixman/configure $(AUTOCONF_HOST) --disable-gtk --disable-shared --enable-static)
 
-$(filter %-softmmu,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) $(common-obj-y) $(extra-obj-y) subdir-libdis
+$(SRC_PATH)/pixman/configure:
+	(cd $(SRC_PATH)/pixman; autoreconf -v --install)
 
-$(filter %-user,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) subdir-libdis-user subdir-libuser
+$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y)
 
 ROMSUBDIR_RULES=$(patsubst %,romsubdir-%, $(ROMS))
 romsubdir-%:
@@ -112,61 +146,31 @@ ALL_SUBDIRS=$(TARGET_DIRS) $(patsubst %,pc-bios/%, $(ROMS))
 
 recurse-all: $(SUBDIR_RULES) $(ROMSUBDIR_RULES)
 
-audio/audio.o audio/fmodaudio.o: QEMU_CFLAGS += $(FMOD_CFLAGS)
-
-QEMU_CFLAGS+=$(CURL_CFLAGS)
-
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-
-ui/cocoa.o: ui/cocoa.m
-
-ui/sdl.o audio/sdlaudio.o ui/sdl_zoom.o hw/baum.o: QEMU_CFLAGS += $(SDL_CFLAGS)
-
-ui/vnc.o: QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
-
 bt-host.o: QEMU_CFLAGS += $(BLUEZ_CFLAGS)
 
 version.o: $(SRC_PATH)/version.rc config-host.h
 	$(call quiet-command,$(WINDRES) -I. -o $@ $<,"  RC    $(TARGET_DIR)$@")
 
 version-obj-$(CONFIG_WIN32) += version.o
-######################################################################
-# Support building shared library libcacard
-
-.PHONY: libcacard.la install-libcacard
-ifeq ($(LIBTOOL),)
-libcacard.la:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
+Makefile: $(version-obj-y)
 
-install-libcacard:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-else
-libcacard.la: $(oslib-obj-y) qemu-timer-common.o $(addsuffix .lo, $(basename $(trace-obj-y)))
-	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C libcacard V="$(V)" TARGET_DIR="$*/" libcacard.la,)
+######################################################################
+# Build libraries
 
-install-libcacard: libcacard.la
-	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C libcacard V="$(V)" TARGET_DIR="$*/" install-libcacard,)
-endif
+libqemustub.a: $(stub-obj-y)
+libqemuutil.a: $(util-obj-y)
 
 ######################################################################
 
 qemu-img.o: qemu-img-cmds.h
 
-tools-obj-y = $(oslib-obj-y) $(trace-obj-y) qemu-tool.o qemu-timer.o \
-	qemu-timer-common.o main-loop.o notify.o \
-	iohandler.o cutils.o iov.o async.o
-tools-obj-$(CONFIG_POSIX) += compatfd.o
-
-qemu-img$(EXESUF): qemu-img.o $(tools-obj-y) $(block-obj-y)
-qemu-nbd$(EXESUF): qemu-nbd.o $(tools-obj-y) $(block-obj-y)
-qemu-io$(EXESUF): qemu-io.o cmd.o $(tools-obj-y) $(block-obj-y)
+qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-io$(EXESUF): qemu-io.o cmd.o $(block-obj-y) libqemuutil.a libqemustub.a
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
 
-vscclient$(EXESUF): $(libcacard-y) $(oslib-obj-y) $(trace-obj-y) $(tools-obj-y) qemu-timer-common.o libcacard/vscclient.o
-	$(call quiet-command,$(CC) $(LDFLAGS) -o $@ $^ $(libcacard_libs) $(LIBS),"  LINK  $@")
-
-fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o oslib-posix.o $(trace-obj-y)
+fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o libqemuutil.a libqemustub.a
 fsdev/virtfs-proxy-helper$(EXESUF): LIBS += -lcap
 
 qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx
@@ -177,20 +181,16 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated
 
 gen-out-type = $(subst .,-,$(suffix $@))
 
-ifneq ($(wildcard config-host.mak),)
-include $(SRC_PATH)/tests/Makefile
-endif
-
 qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
 
 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 
 qapi-types.c qapi-types.h :\
@@ -206,27 +206,27 @@ $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
 
-qemu-ga$(EXESUF): qemu-ga.o $(qga-obj-y) $(tools-obj-y) $(qapi-obj-y) $(qobject-obj-y) $(version-obj-y)
-
-QEMULIBS=libhw32 libhw64 libuser libdis libdis-user
+qemu-ga$(EXESUF): $(qga-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
 
 clean:
 # avoid old build problems by removing potentially incorrect old files
 	rm -f config.mak op-i386.h opc-i386.h gen-op-i386.h op-arm.h opc-arm.h gen-op-arm.h
 	rm -f qemu-options.def
-	find . -name '*.[od]' -exec rm -f {} +
-	rm -f *.a *.lo $(TOOLS) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
+	find . -name '*.[oda]' -type f -exec rm -f {} +
+	find . -name '*.l[oa]' -type f -exec rm -f {} +
+	rm -f $(TOOLS) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
 	rm -Rf .libs
 	rm -f qemu-img-cmds.h
-	rm -f trace-dtrace.dtrace trace-dtrace.dtrace-timestamp
 	@# May not be present in GENERATED_HEADERS
-	rm -f trace-dtrace.h trace-dtrace.h-timestamp
+	rm -f trace/generated-tracers-dtrace.dtrace*
+	rm -f trace/generated-tracers-dtrace.h*
 	rm -f $(foreach f,$(GENERATED_HEADERS),$(f) $(f)-timestamp)
 	rm -f $(foreach f,$(GENERATED_SOURCES),$(f) $(f)-timestamp)
 	rm -rf qapi-generated
 	rm -rf qga/qapi-generated
 	$(MAKE) -C tests/tcg clean
-	for d in $(ALL_SUBDIRS) $(QEMULIBS) libcacard; do \
+	for d in $(ALL_SUBDIRS); do \
 	if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \
 	rm -f $$d/qemu-options.def; \
         done
@@ -240,7 +240,7 @@ qemu-%.tar.bz2:
 
 distclean: clean
 	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
-	rm -f config-all-devices.mak
+	rm -f config-all-devices.mak config-all-disas.mak
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
 	rm -f qemu-doc.info qemu-doc.aux qemu-doc.cp qemu-doc.cps qemu-doc.dvi
 	rm -f qemu-doc.fn qemu-doc.fns qemu-doc.info qemu-doc.ky qemu-doc.kys
@@ -249,9 +249,10 @@ distclean: clean
 	rm -f config.log
 	rm -f linux-headers/asm
 	rm -f qemu-tech.info qemu-tech.aux qemu-tech.cp qemu-tech.dvi qemu-tech.fn qemu-tech.info qemu-tech.ky qemu-tech.log qemu-tech.pdf qemu-tech.pg qemu-tech.toc qemu-tech.tp qemu-tech.vr
-	for d in $(TARGET_DIRS) $(QEMULIBS); do \
+	for d in $(TARGET_DIRS); do \
 	rm -rf $$d || exit 1 ; \
         done
+	if test -f pixman/config.log; then make -C pixman distclean; fi
 
 KEYMAPS=da     en-gb  et  fr     fr-ch  is  lt  modifiers  no  pt-br  sv \
 ar      de     en-us  fi  fr-be  hr     it  lv  nl         pl  ru     th \
@@ -261,6 +262,7 @@ bepo
 ifdef INSTALL_BLOBS
 BLOBS=bios.bin sgabios.bin vgabios.bin vgabios-cirrus.bin \
 vgabios-stdvga.bin vgabios-vmware.bin vgabios-qxl.bin \
+acpi-dsdt.aml q35-acpi-dsdt.aml \
 ppc_rom.bin openbios-sparc32 openbios-sparc64 openbios-ppc \
 pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \
 pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \
@@ -297,7 +299,6 @@ install-confdir:
 
 install-sysconfig: install-datadir install-confdir
 	$(INSTALL_DATA) $(SRC_PATH)/sysconfigs/target/target-x86_64.conf "$(DESTDIR)$(qemu_confdir)"
-	$(INSTALL_DATA) $(SRC_PATH)/sysconfigs/target/cpus-x86_64.conf "$(DESTDIR)$(qemu_datadir)"
 
 install: all $(if $(BUILD_DOCS),install-doc) install-sysconfig install-datadir
 	$(INSTALL_DIR) "$(DESTDIR)$(bindir)"
@@ -398,7 +399,9 @@ qemu-doc.dvi qemu-doc.html qemu-doc.info qemu-doc.pdf: \
 
 # Add a dependency on the generated files, so that they are always
 # rebuilt before other object files
+ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 Makefile: $(GENERATED_HEADERS)
+endif
 
 # Include automatically generated dependency files
 # Dependencies in Makefile.objs files come from our recursive subdir rules
diff --git a/Makefile.dis b/Makefile.dis
deleted file mode 100644
index 2cfec6a3588..00000000000
--- a/Makefile.dis
+++ /dev/null
@@ -1,20 +0,0 @@
-# Makefile for disassemblers.
-
-include ../config-host.mak
-include config.mak
-include $(SRC_PATH)/rules.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(libdis-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	rm -f *.o *.d *.a *~
diff --git a/Makefile.hw b/Makefile.hw
deleted file mode 100644
index 59f5b483504..00000000000
--- a/Makefile.hw
+++ /dev/null
@@ -1,23 +0,0 @@
-# Makefile for qemu target independent devices.
-
-include ../config-host.mak
-include ../config-all-devices.mak
-include config.mak
-include $(SRC_PATH)/rules.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(hw-obj-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	rm -f $(addsuffix *.o, $(sort $(dir $(hw-obj-y))))
-	rm -f $(addsuffix *.d, $(sort $(dir $(hw-obj-y))))
diff --git a/Makefile.objs b/Makefile.objs
index 4412757309c..68eb0cef1a7 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,52 +1,31 @@
 #######################################################################
-# Target-independent parts used in system and user emulation
-universal-obj-y =
-universal-obj-y += qemu-log.o
-
-#######################################################################
-# QObject
-qobject-obj-y = qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
-qobject-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
-qobject-obj-y += qerror.o error.o qemu-error.o
-
-universal-obj-y += $(qobject-obj-y)
+# Common libraries for tools and emulators
+stub-obj-y = stubs/
+util-obj-y = util/ qobject/ qapi/ trace/
 
 #######################################################################
-# QOM
-qom-obj-y = qom/
-
-universal-obj-y += $(qom-obj-y)
+# block-obj-y is code used by both qemu system emulation and qemu-img
 
-#######################################################################
-# oslib-obj-y is code depending on the OS (win32 vs posix)
-oslib-obj-y = osdep.o
-oslib-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
-oslib-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
+block-obj-y = async.o thread-pool.o
+block-obj-y += nbd.o block.o blockjob.o
+block-obj-y += main-loop.o iohandler.o qemu-timer.o
+block-obj-$(CONFIG_POSIX) += aio-posix.o
+block-obj-$(CONFIG_WIN32) += aio-win32.o
+block-obj-y += block/
+block-obj-y += qapi-types.o qapi-visit.o
 
-#######################################################################
-# coroutines
-coroutine-obj-y = qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
-coroutine-obj-y += qemu-coroutine-sleep.o
+block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+block-obj-y += qemu-coroutine-sleep.o
 ifeq ($(CONFIG_UCONTEXT_COROUTINE),y)
-coroutine-obj-$(CONFIG_POSIX) += coroutine-ucontext.o
+block-obj-$(CONFIG_POSIX) += coroutine-ucontext.o
 else
 ifeq ($(CONFIG_SIGALTSTACK_COROUTINE),y)
-coroutine-obj-$(CONFIG_POSIX) += coroutine-sigaltstack.o
+block-obj-$(CONFIG_POSIX) += coroutine-sigaltstack.o
 else
-coroutine-obj-$(CONFIG_POSIX) += coroutine-gthread.o
+block-obj-$(CONFIG_POSIX) += coroutine-gthread.o
 endif
 endif
-coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o
-
-#######################################################################
-# block-obj-y is code used by both qemu system emulation and qemu-img
-
-block-obj-y = cutils.o iov.o cache-utils.o qemu-option.o module.o async.o
-block-obj-y += nbd.o block.o aio.o aes.o qemu-config.o qemu-progress.o qemu-sockets.o
-block-obj-y += $(coroutine-obj-y) $(qobject-obj-y) $(version-obj-y)
-block-obj-$(CONFIG_POSIX) += posix-aio-compat.o
-block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
-block-obj-y += block/
+block-obj-$(CONFIG_WIN32) += coroutine-win32.o
 
 ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
 # Lots of the fsdev/9pcode is pulled in by vl.c via qemu_fsdev_add.
@@ -55,182 +34,76 @@ CONFIG_REALLY_VIRTFS=y
 endif
 
 ######################################################################
+# smartcard
+
+libcacard-y += libcacard/cac.o libcacard/event.o
+libcacard-y += libcacard/vcard.o libcacard/vreader.o
+libcacard-y += libcacard/vcard_emul_nss.o
+libcacard-y += libcacard/vcard_emul_type.o
+libcacard-y += libcacard/card_7816.o
+
+######################################################################
 # Target independent part of system emulation. The long term path is to
 # suppress *all* target specific code in case of system emulation, i.e. a
 # single QEMU executable should support all CPUs and machines.
 
-common-obj-y = $(block-obj-y) blockdev.o
-common-obj-y += net.o net/
-common-obj-y += qom/
-common-obj-y += readline.o console.o cursor.o
-common-obj-y += $(oslib-obj-y)
+ifeq ($(CONFIG_SOFTMMU),y)
+common-obj-y = $(block-obj-y) blockdev.o blockdev-nbd.o block/
+common-obj-y += net/
+common-obj-y += readline.o
 common-obj-$(CONFIG_WIN32) += os-win32.o
 common-obj-$(CONFIG_POSIX) += os-posix.o
 
 common-obj-$(CONFIG_LINUX) += fsdev/
-extra-obj-$(CONFIG_LINUX) += fsdev/
 
-common-obj-y += tcg-runtime.o host-utils.o main-loop.o
-common-obj-y += input.o
-common-obj-y += buffered_file.o migration.o migration-tcp.o
+common-obj-y += migration.o migration-tcp.o
 common-obj-y += qemu-char.o #aio.o
-common-obj-y += block-migration.o iohandler.o
-common-obj-y += pflib.o
-common-obj-y += bitmap.o bitops.o
+common-obj-y += block-migration.o
 common-obj-y += page_cache.o
 
 common-obj-$(CONFIG_POSIX) += migration-exec.o migration-unix.o migration-fd.o
-common-obj-$(CONFIG_WIN32) += version.o
 
 common-obj-$(CONFIG_SPICE) += spice-qemu-char.o
 
 common-obj-y += audio/
 common-obj-y += hw/
+
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 
-common-obj-y += iov.o acl.o
-common-obj-$(CONFIG_POSIX) += compatfd.o
-common-obj-y += notify.o event_notifier.o
-common-obj-y += qemu-timer.o qemu-timer-common.o
+common-obj-y += dma-helpers.o
+common-obj-y += qtest.o
+common-obj-y += vl.o
 
 common-obj-$(CONFIG_SLIRP) += slirp/
 
-######################################################################
-# libseccomp
-ifeq ($(CONFIG_SECCOMP),y)
-common-obj-y += qemu-seccomp.o
-endif
-
-######################################################################
-# libuser
-
-user-obj-y =
-user-obj-y += envlist.o path.o
-user-obj-y += tcg-runtime.o host-utils.o
-user-obj-y += cutils.o iov.o cache-utils.o
-user-obj-y += module.o
-user-obj-y += qemu-user.o
-user-obj-y += $(trace-obj-y)
-user-obj-y += qom/
+common-obj-y += backends/
 
-######################################################################
-# libhw
-
-hw-obj-y = vl.o dma-helpers.o qtest.o hw/
-
-######################################################################
-# libdis
-# NOTE: the disassembler code is only needed for debugging
-
-libdis-y =
-libdis-$(CONFIG_ALPHA_DIS) += alpha-dis.o
-libdis-$(CONFIG_ARM_DIS) += arm-dis.o
-libdis-$(CONFIG_CRIS_DIS) += cris-dis.o
-libdis-$(CONFIG_HPPA_DIS) += hppa-dis.o
-libdis-$(CONFIG_I386_DIS) += i386-dis.o
-libdis-$(CONFIG_IA64_DIS) += ia64-dis.o
-libdis-$(CONFIG_M68K_DIS) += m68k-dis.o
-libdis-$(CONFIG_MICROBLAZE_DIS) += microblaze-dis.o
-libdis-$(CONFIG_MIPS_DIS) += mips-dis.o
-libdis-$(CONFIG_PPC_DIS) += ppc-dis.o
-libdis-$(CONFIG_S390_DIS) += s390-dis.o
-libdis-$(CONFIG_SH4_DIS) += sh4-dis.o
-libdis-$(CONFIG_SPARC_DIS) += sparc-dis.o
-libdis-$(CONFIG_LM32_DIS) += lm32-dis.o
-
-######################################################################
-# trace
-
-ifeq ($(TRACE_BACKEND),dtrace)
-TRACE_H_EXTRA_DEPS=trace-dtrace.h
-endif
-trace.h: trace.h-timestamp $(TRACE_H_EXTRA_DEPS)
-trace.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=h \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace.h")
-	@cmp -s $@ trace.h || cp $@ trace.h
-
-trace.c: trace.c-timestamp
-trace.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=c \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace.c")
-	@cmp -s $@ trace.c || cp $@ trace.c
-
-trace.o: trace.c $(GENERATED_HEADERS)
-
-trace-dtrace.h: trace-dtrace.dtrace
-	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   trace-dtrace.h")
-
-# Normal practice is to name DTrace probe file with a '.d' extension
-# but that gets picked up by QEMU's Makefile as an external dependency
-# rule file. So we use '.dtrace' instead
-trace-dtrace.dtrace: trace-dtrace.dtrace-timestamp
-trace-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=d \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace-dtrace.dtrace")
-	@cmp -s $@ trace-dtrace.dtrace || cp $@ trace-dtrace.dtrace
-
-trace-dtrace.o: trace-dtrace.dtrace $(GENERATED_HEADERS)
-	$(call quiet-command,dtrace -o $@ -G -s $<, "  GEN   trace-dtrace.o")
-
-ifeq ($(LIBTOOL),)
-trace-dtrace.lo: trace-dtrace.dtrace
-	@echo "missing libtool. please install and rerun configure."; exit 1
-else
-trace-dtrace.lo: trace-dtrace.dtrace
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, "  lt GEN trace-dtrace.o")
-endif
-
-trace/simple.o: trace/simple.c $(GENERATED_HEADERS)
-
-trace-obj-$(CONFIG_TRACE_DTRACE) += trace-dtrace.o
-ifneq ($(TRACE_BACKEND),dtrace)
-trace-obj-y = trace.o
-endif
-
-trace-obj-$(CONFIG_TRACE_DEFAULT) += trace/default.o
-trace-obj-$(CONFIG_TRACE_SIMPLE) += trace/simple.o
-trace-obj-$(CONFIG_TRACE_SIMPLE) += qemu-timer-common.o
-trace-obj-$(CONFIG_TRACE_STDERR) += trace/stderr.o
-trace-obj-y += trace/control.o
-
-$(trace-obj-y): $(GENERATED_HEADERS)
-
-######################################################################
-# smartcard
-
-libcacard-y += libcacard/cac.o libcacard/event.o
-libcacard-y += libcacard/vcard.o libcacard/vreader.o
-libcacard-y += libcacard/vcard_emul_nss.o
-libcacard-y += libcacard/vcard_emul_type.o
-libcacard-y += libcacard/card_7816.o
+common-obj-$(CONFIG_SECCOMP) += qemu-seccomp.o
 
 common-obj-$(CONFIG_SMARTCARD_NSS) += $(libcacard-y)
 
 ######################################################################
 # qapi
 
-qapi-obj-y = qapi/
-qapi-obj-y += qapi-types.o qapi-visit.o
-
 common-obj-y += qmp-marshal.o qapi-visit.o qapi-types.o
 common-obj-y += qmp.o hmp.o
+endif
 
-universal-obj-y += $(qapi-obj-y)
+#######################################################################
+# Target-independent parts used in system and user emulation
+common-obj-y += qemu-log.o
+common-obj-y += tcg-runtime.o
+common-obj-y += hw/
+common-obj-y += qom/
+common-obj-y += disas/
 
 ######################################################################
 # guest agent
 
-qga-obj-y = qga/ qemu-ga.o module.o
-qga-obj-$(CONFIG_WIN32) += oslib-win32.o
-qga-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-sockets.o qemu-option.o
+# FIXME: a few definitions from qapi-types.o/qapi-visit.o are needed
+# by libqemuutil.a.  These should be moved to a separate .json schema.
+qga-obj-y = qga/ qapi-types.o qapi-visit.o
 
 vl.o: QEMU_CFLAGS+=$(GPROF_CFLAGS)
 
@@ -239,12 +112,9 @@ vl.o: QEMU_CFLAGS+=$(SDL_CFLAGS)
 QEMU_CFLAGS+=$(GLIB_CFLAGS)
 
 nested-vars += \
-	hw-obj-y \
+	stub-obj-y \
+	util-obj-y \
 	qga-obj-y \
 	block-obj-y \
-	qom-obj-y \
-	qapi-obj-y \
-	user-obj-y \
-	common-obj-y \
-	extra-obj-y
+	common-obj-y
 dummy := $(call unnest-vars)
diff --git a/Makefile.target b/Makefile.target
index 7892a8df634..760da1edf6d 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -4,9 +4,6 @@ include ../config-host.mak
 include config-devices.mak
 include config-target.mak
 include $(SRC_PATH)/rules.mak
-ifneq ($(HWDIR),)
-include $(HWDIR)/config.mak
-endif
 
 $(call set-vpath, $(SRC_PATH))
 ifdef CONFIG_LINUX
@@ -57,7 +54,7 @@ $(QEMU_PROG).stp: $(SRC_PATH)/trace-events
 		--binary=$(bindir)/$(QEMU_PROG) \
 		--target-arch=$(TARGET_ARCH) \
 		--target-type=$(TARGET_TYPE) \
-		< $< > $@,"  GEN   $(QEMU_PROG).stp")
+		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG).stp")
 else
 stap:
 endif
@@ -72,22 +69,12 @@ all: $(PROGS) stap
 obj-y = exec.o translate-all.o cpu-exec.o
 obj-y += tcg/tcg.o tcg/optimize.o
 obj-$(CONFIG_TCG_INTERPRETER) += tci.o
+obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
-obj-y += disas.o
-obj-$(CONFIG_TCI_DIS) += tci-dis.o
 obj-y += target-$(TARGET_BASE_ARCH)/
+obj-y += disas.o
 obj-$(CONFIG_GDBSTUB_XML) += gdbstub-xml.o
 
-tci-dis.o: QEMU_CFLAGS += -I$(SRC_PATH)/tcg -I$(SRC_PATH)/tcg/tci
-
-# HELPER_CFLAGS is used for all the legacy code compiled with static register
-# variables
-user-exec.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
-
-# Note: this is a workaround. The real fix is to avoid compiling
-# cpu_signal_handler() in user-exec.c.
-%/signal.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
-
 #########################################################
 # Linux user emulator target
 
@@ -96,7 +83,7 @@ ifdef CONFIG_LINUX_USER
 QEMU_CFLAGS+=-I$(SRC_PATH)/linux-user/$(TARGET_ABI_DIR) -I$(SRC_PATH)/linux-user
 
 obj-y += linux-user/
-obj-y += gdbstub.o thunk.o user-exec.o $(oslib-obj-y)
+obj-y += gdbstub.o thunk.o user-exec.o
 
 endif #CONFIG_LINUX_USER
 
@@ -108,7 +95,7 @@ ifdef CONFIG_BSD_USER
 QEMU_CFLAGS+=-I$(SRC_PATH)/bsd-user -I$(SRC_PATH)/bsd-user/$(TARGET_ARCH)
 
 obj-y += bsd-user/
-obj-y += gdbstub.o user-exec.o $(oslib-obj-y)
+obj-y += gdbstub.o user-exec.o
 
 endif #CONFIG_BSD_USER
 
@@ -132,11 +119,6 @@ obj-$(CONFIG_NO_GET_MEMORY_MAPPING) += memory_mapping-stub.o
 obj-$(CONFIG_NO_CORE_DUMP) += dump-stub.o
 LIBS+=-lz
 
-QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
-QEMU_CFLAGS += $(VNC_SASL_CFLAGS)
-QEMU_CFLAGS += $(VNC_JPEG_CFLAGS)
-QEMU_CFLAGS += $(VNC_PNG_CFLAGS)
-
 # xen support
 obj-$(CONFIG_XEN) += xen-all.o xen-mapcache.o
 obj-$(CONFIG_NO_XEN) += xen-stub.o
@@ -154,32 +136,25 @@ GENERATED_HEADERS += hmp-commands.h qmp-commands-old.h
 
 endif # CONFIG_SOFTMMU
 
+# Workaround for http://gcc.gnu.org/PR55489, see configure.
+%/translate.o: QEMU_CFLAGS += $(TRANSLATE_OPT_CFLAGS)
+
 nested-vars += obj-y
 
 # This resolves all nested paths, so it must come last
 include $(SRC_PATH)/Makefile.objs
 
 all-obj-y = $(obj-y)
-all-obj-y += $(addprefix ../, $(universal-obj-y))
-
-ifdef CONFIG_SOFTMMU
 all-obj-y += $(addprefix ../, $(common-obj-y))
-all-obj-y += $(addprefix ../libdis/, $(libdis-y))
-all-obj-y += $(addprefix $(HWDIR)/, $(hw-obj-y))
-all-obj-y += $(addprefix ../, $(trace-obj-y))
-else
-all-obj-y += $(addprefix ../libuser/, $(user-obj-y))
-all-obj-y += $(addprefix ../libdis-user/, $(libdis-y))
-endif #CONFIG_LINUX_USER
 
 ifdef QEMU_PROGW
 # The linker builds a windows executable. Make also a console executable.
-$(QEMU_PROGW): $(all-obj-y)
+$(QEMU_PROGW): $(all-obj-y) ../libqemuutil.a ../libqemustub.a
 	$(call LINK,$^)
 $(QEMU_PROG): $(QEMU_PROGW)
 	$(call quiet-command,$(OBJCOPY) --subsystem console $(QEMU_PROGW) $(QEMU_PROG),"  GEN   $(TARGET_DIR)$(QEMU_PROG)")
 else
-$(QEMU_PROG): $(all-obj-y)
+$(QEMU_PROG): $(all-obj-y) ../libqemuutil.a ../libqemustub.a
 	$(call LINK,$^)
 endif
 
diff --git a/Makefile.user b/Makefile.user
deleted file mode 100644
index 9302d33245a..00000000000
--- a/Makefile.user
+++ /dev/null
@@ -1,24 +0,0 @@
-# Makefile for qemu target independent user files.
-
-include ../config-host.mak
-include $(SRC_PATH)/rules.mak
--include config.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-QEMU_CFLAGS += -DCONFIG_USER_ONLY
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(user-obj-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	for d in . trace; do \
-	rm -f $$d/*.o $$d/*.d $$d/*.a $$d/*~; \
-	done
diff --git a/QMP/qemu-ga-client b/QMP/qemu-ga-client
new file mode 100755
index 00000000000..46676c37504
--- /dev/null
+++ b/QMP/qemu-ga-client
@@ -0,0 +1,299 @@
+#!/usr/bin/python
+
+# QEMU Guest Agent Client
+#
+# Copyright (C) 2012 Ryota Ozaki <ozaki.ryota@gmail.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Usage:
+#
+# Start QEMU with:
+#
+# # qemu [...] -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \
+#   -device virtio-serial -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0
+#
+# Run the script:
+#
+# $ qemu-ga-client --address=/tmp/qga.sock <command> [args...]
+#
+# or
+#
+# $ export QGA_CLIENT_ADDRESS=/tmp/qga.sock
+# $ qemu-ga-client <command> [args...]
+#
+# For example:
+#
+# $ qemu-ga-client cat /etc/resolv.conf
+# # Generated by NetworkManager
+# nameserver 10.0.2.3
+# $ qemu-ga-client fsfreeze status
+# thawed
+# $ qemu-ga-client fsfreeze freeze
+# 2 filesystems frozen
+#
+# See also: http://wiki.qemu.org/Features/QAPI/GuestAgent
+#
+
+import base64
+import random
+
+import qmp
+
+
+class QemuGuestAgent(qmp.QEMUMonitorProtocol):
+    def __getattr__(self, name):
+        def wrapper(**kwds):
+            return self.command('guest-' + name.replace('_', '-'), **kwds)
+        return wrapper
+
+
+class QemuGuestAgentClient:
+    error = QemuGuestAgent.error
+
+    def __init__(self, address):
+        self.qga = QemuGuestAgent(address)
+        self.qga.connect(negotiate=False)
+
+    def sync(self, timeout=3):
+        # Avoid being blocked forever
+        if not self.ping(timeout):
+            raise EnvironmentError('Agent seems not alive')
+        uid = random.randint(0, (1 << 32) - 1)
+        while True:
+            ret = self.qga.sync(id=uid)
+            if isinstance(ret, int) and int(ret) == uid:
+                break
+
+    def __file_read_all(self, handle):
+        eof = False
+        data = ''
+        while not eof:
+            ret = self.qga.file_read(handle=handle, count=1024)
+            _data = base64.b64decode(ret['buf-b64'])
+            data += _data
+            eof = ret['eof']
+        return data
+
+    def read(self, path):
+        handle = self.qga.file_open(path=path)
+        try:
+            data = self.__file_read_all(handle)
+        finally:
+            self.qga.file_close(handle=handle)
+        return data
+
+    def info(self):
+        info = self.qga.info()
+
+        msgs = []
+        msgs.append('version: ' + info['version'])
+        msgs.append('supported_commands:')
+        enabled = [c['name'] for c in info['supported_commands'] if c['enabled']]
+        msgs.append('\tenabled: ' + ', '.join(enabled))
+        disabled = [c['name'] for c in info['supported_commands'] if not c['enabled']]
+        msgs.append('\tdisabled: ' + ', '.join(disabled))
+
+        return '\n'.join(msgs)
+
+    def __gen_ipv4_netmask(self, prefixlen):
+        mask = int('1' * prefixlen + '0' * (32 - prefixlen), 2)
+        return '.'.join([str(mask >> 24),
+                         str((mask >> 16) & 0xff),
+                         str((mask >> 8) & 0xff),
+                         str(mask & 0xff)])
+
+    def ifconfig(self):
+        nifs = self.qga.network_get_interfaces()
+
+        msgs = []
+        for nif in nifs:
+            msgs.append(nif['name'] + ':')
+            if 'ip-addresses' in nif:
+                for ipaddr in nif['ip-addresses']:
+                    if ipaddr['ip-address-type'] == 'ipv4':
+                        addr = ipaddr['ip-address']
+                        mask = self.__gen_ipv4_netmask(int(ipaddr['prefix']))
+                        msgs.append("\tinet %s  netmask %s" % (addr, mask))
+                    elif ipaddr['ip-address-type'] == 'ipv6':
+                        addr = ipaddr['ip-address']
+                        prefix = ipaddr['prefix']
+                        msgs.append("\tinet6 %s  prefixlen %s" % (addr, prefix))
+            if nif['hardware-address'] != '00:00:00:00:00:00':
+                msgs.append("\tether " + nif['hardware-address'])
+
+        return '\n'.join(msgs)
+
+    def ping(self, timeout):
+        self.qga.settimeout(timeout)
+        try:
+            self.qga.ping()
+        except self.qga.timeout:
+            return False
+        return True
+
+    def fsfreeze(self, cmd):
+        if cmd not in ['status', 'freeze', 'thaw']:
+            raise StandardError('Invalid command: ' + cmd)
+
+        return getattr(self.qga, 'fsfreeze' + '_' + cmd)()
+
+    def fstrim(self, minimum=0):
+        return getattr(self.qga, 'fstrim')(minimum=minimum)
+
+    def suspend(self, mode):
+        if mode not in ['disk', 'ram', 'hybrid']:
+            raise StandardError('Invalid mode: ' + mode)
+
+        try:
+            getattr(self.qga, 'suspend' + '_' + mode)()
+            # On error exception will raise
+        except self.qga.timeout:
+            # On success command will timed out
+            return
+
+    def shutdown(self, mode='powerdown'):
+        if mode not in ['powerdown', 'halt', 'reboot']:
+            raise StandardError('Invalid mode: ' + mode)
+
+        try:
+            self.qga.shutdown(mode=mode)
+        except self.qga.timeout:
+            return
+
+
+def _cmd_cat(client, args):
+    if len(args) != 1:
+        print('Invalid argument')
+        print('Usage: cat <file>')
+        sys.exit(1)
+    print(client.read(args[0]))
+
+
+def _cmd_fsfreeze(client, args):
+    usage = 'Usage: fsfreeze status|freeze|thaw'
+    if len(args) != 1:
+        print('Invalid argument')
+        print(usage)
+        sys.exit(1)
+    if args[0] not in ['status', 'freeze', 'thaw']:
+        print('Invalid command: ' + args[0])
+        print(usage)
+        sys.exit(1)
+    cmd = args[0]
+    ret = client.fsfreeze(cmd)
+    if cmd == 'status':
+        print(ret)
+    elif cmd == 'freeze':
+        print("%d filesystems frozen" % ret)
+    else:
+        print("%d filesystems thawed" % ret)
+
+
+def _cmd_fstrim(client, args):
+    if len(args) == 0:
+        minimum = 0
+    else:
+        minimum = int(args[0])
+    print(client.fstrim(minimum))
+
+
+def _cmd_ifconfig(client, args):
+    print(client.ifconfig())
+
+
+def _cmd_info(client, args):
+    print(client.info())
+
+
+def _cmd_ping(client, args):
+    if len(args) == 0:
+        timeout = 3
+    else:
+        timeout = float(args[0])
+    alive = client.ping(timeout)
+    if not alive:
+        print("Not responded in %s sec" % args[0])
+        sys.exit(1)
+
+
+def _cmd_suspend(client, args):
+    usage = 'Usage: suspend disk|ram|hybrid'
+    if len(args) != 1:
+        print('Less argument')
+        print(usage)
+        sys.exit(1)
+    if args[0] not in ['disk', 'ram', 'hybrid']:
+        print('Invalid command: ' + args[0])
+        print(usage)
+        sys.exit(1)
+    client.suspend(args[0])
+
+
+def _cmd_shutdown(client, args):
+    client.shutdown()
+_cmd_powerdown = _cmd_shutdown
+
+
+def _cmd_halt(client, args):
+    client.shutdown('halt')
+
+
+def _cmd_reboot(client, args):
+    client.shutdown('reboot')
+
+
+commands = [m.replace('_cmd_', '') for m in dir() if '_cmd_' in m]
+
+
+def main(address, cmd, args):
+    if not os.path.exists(address):
+        print('%s not found' % address)
+        sys.exit(1)
+
+    if cmd not in commands:
+        print('Invalid command: ' + cmd)
+        print('Available commands: ' + ', '.join(commands))
+        sys.exit(1)
+
+    try:
+        client = QemuGuestAgentClient(address)
+    except QemuGuestAgent.error, e:
+        import errno
+
+        print(e)
+        if e.errno == errno.ECONNREFUSED:
+            print('Hint: qemu is not running?')
+        sys.exit(1)
+
+    if cmd != 'ping':
+        client.sync()
+
+    globals()['_cmd_' + cmd](client, args)
+
+
+if __name__ == '__main__':
+    import sys
+    import os
+    import optparse
+
+    address = os.environ['QGA_CLIENT_ADDRESS'] if 'QGA_CLIENT_ADDRESS' in os.environ else None
+
+    usage = "%prog [--address=<unix_path>|<ipv4_address>] <command> [args...]\n"
+    usage += '<command>: ' + ', '.join(commands)
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option('--address', action='store', type='string',
+                      default=address, help='Specify a ip:port pair or a unix socket path')
+    options, args = parser.parse_args()
+
+    address = options.address
+    if address is None:
+        parser.error('address is not specified')
+        sys.exit(1)
+
+    if len(args) == 0:
+        parser.error('Less argument')
+        sys.exit(1)
+
+    main(address, args[0], args[1:])
diff --git a/QMP/qmp-events.txt b/QMP/qmp-events.txt
index 287805825f6..b2698e41537 100644
--- a/QMP/qmp-events.txt
+++ b/QMP/qmp-events.txt
@@ -50,7 +50,8 @@ Emitted when a block job has been cancelled.
 
 Data:
 
-- "type":     Job type ("stream" for image streaming, json-string)
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
 - "device":   Device name (json-string)
 - "len":      Maximum progress value (json-int)
 - "offset":   Current progress value (json-int)
@@ -73,7 +74,8 @@ Emitted when a block job has completed.
 
 Data:
 
-- "type":     Job type ("stream" for image streaming, json-string)
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
 - "device":   Device name (json-string)
 - "len":      Maximum progress value (json-int)
 - "offset":   Current progress value (json-int)
@@ -94,6 +96,46 @@ Example:
                "speed": 0 },
      "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
 
+BLOCK_JOB_ERROR
+---------------
+
+Emitted when a block job encounters an error.
+
+Data:
+
+- "device": device name (json-string)
+- "operation": I/O operation (json-string, "read" or "write")
+- "action": action that has been taken, it's one of the following (json-string):
+    "ignore": error has been ignored, the job may fail later
+    "report": error will be reported and the job canceled
+    "stop": error caused job to be paused
+
+Example:
+
+{ "event": "BLOCK_JOB_ERROR",
+    "data": { "device": "ide0-hd1",
+              "operation": "write",
+              "action": "stop" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+BLOCK_JOB_READY
+---------------
+
+Emitted when a block job is ready to complete.
+
+Data:
+
+- "device": device name (json-string)
+
+Example:
+
+{ "event": "BLOCK_JOB_READY",
+    "data": { "device": "ide0-hd1" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+Note: The "ready to complete" status is always reset by a BLOCK_JOB_ERROR
+event.
+
 DEVICE_TRAY_MOVED
 -----------------
 
diff --git a/QMP/qmp-shell b/QMP/qmp-shell
index 42dabc8c6d6..24b665c8c02 100755
--- a/QMP/qmp-shell
+++ b/QMP/qmp-shell
@@ -33,6 +33,7 @@
 import qmp
 import readline
 import sys
+import pprint
 
 class QMPCompleter(list):
     def complete(self, text, state):
@@ -52,10 +53,11 @@ class QMPShellBadPort(QMPShellError):
 # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and
 #       _execute_cmd()). Let's design a better one.
 class QMPShell(qmp.QEMUMonitorProtocol):
-    def __init__(self, address):
+    def __init__(self, address, pp=None):
         qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address))
         self._greeting = None
         self._completer = None
+        self._pp = pp
 
     def __get_address(self, arg):
         """
@@ -114,7 +116,11 @@ class QMPShell(qmp.QEMUMonitorProtocol):
         if resp is None:
             print 'Disconnected'
             return False
-        print resp
+
+        if self._pp is not None:
+            self._pp.pprint(resp)
+        else:
+            print resp
         return True
 
     def connect(self):
@@ -222,22 +228,36 @@ def die(msg):
 def fail_cmdline(option=None):
     if option:
         sys.stderr.write('ERROR: bad command-line option \'%s\'\n' % option)
-    sys.stderr.write('qemu-shell [ -H ] < UNIX socket path> | < TCP address:port >\n')
+    sys.stderr.write('qemu-shell [ -p ] [ -H ] < UNIX socket path> | < TCP address:port >\n')
     sys.exit(1)
 
 def main():
     addr = ''
+    qemu = None
+    hmp = False
+    pp = None
+
     try:
-        if len(sys.argv) == 2:
-            qemu = QMPShell(sys.argv[1])
-            addr = sys.argv[1]
-        elif len(sys.argv) == 3:
-            if sys.argv[1] != '-H':
-                fail_cmdline(sys.argv[1])
-            qemu = HMPShell(sys.argv[2])
-            addr = sys.argv[2]
-        else:
-                fail_cmdline()
+        for arg in sys.argv[1:]:
+            if arg == "-H":
+                if qemu is not None:
+                    fail_cmdline(arg)
+                hmp = True
+            elif arg == "-p":
+                if pp is not None:
+                    fail_cmdline(arg)
+                pp = pprint.PrettyPrinter(indent=4)
+            else:
+                if qemu is not None:
+                    fail_cmdline(arg)
+                if hmp:
+                    qemu = HMPShell(arg)
+                else:
+                    qemu = QMPShell(arg, pp)
+                addr = arg
+
+        if qemu is None:
+            fail_cmdline()
     except QMPShellBadPort:
         die('bad port number in command-line')
 
diff --git a/QMP/qmp.py b/QMP/qmp.py
index 36ecc1dfaea..c551df1ed7f 100644
--- a/QMP/qmp.py
+++ b/QMP/qmp.py
@@ -49,7 +49,6 @@ class QEMUMonitorProtocol:
         return socket.socket(family, socket.SOCK_STREAM)
 
     def __negotiate_capabilities(self):
-        self.__sockfile = self.__sock.makefile()
         greeting = self.__json_read()
         if greeting is None or not greeting.has_key('QMP'):
             raise QMPConnectError
@@ -73,7 +72,7 @@ class QEMUMonitorProtocol:
 
     error = socket.error
 
-    def connect(self):
+    def connect(self, negotiate=True):
         """
         Connect to the QMP Monitor and perform capabilities negotiation.
 
@@ -83,7 +82,9 @@ class QEMUMonitorProtocol:
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
         self.__sock.connect(self.__address)
-        return self.__negotiate_capabilities()
+        self.__sockfile = self.__sock.makefile()
+        if negotiate:
+            return self.__negotiate_capabilities()
 
     def accept(self):
         """
@@ -95,6 +96,7 @@ class QEMUMonitorProtocol:
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
         self.__sock, _ = self.__sock.accept()
+        self.__sockfile = self.__sock.makefile()
         return self.__negotiate_capabilities()
 
     def cmd_obj(self, qmp_cmd):
@@ -134,6 +136,26 @@ class QEMUMonitorProtocol:
             raise Exception(ret['error']['desc'])
         return ret['return']
 
+    def pull_event(self, wait=False):
+        """
+        Get and delete the first available QMP event.
+
+        @param wait: block until an event is available (bool)
+        """
+        self.__sock.setblocking(0)
+        try:
+            self.__json_read()
+        except socket.error, err:
+            if err[0] == errno.EAGAIN:
+                # No data available
+                pass
+        self.__sock.setblocking(1)
+        if not self.__events and wait:
+            self.__json_read(only_event=True)
+        event = self.__events[0]
+        del self.__events[0]
+        return event
+
     def get_events(self, wait=False):
         """
         Get a list of available QMP events.
@@ -161,3 +183,8 @@ class QEMUMonitorProtocol:
     def close(self):
         self.__sock.close()
         self.__sockfile.close()
+
+    timeout = socket.timeout
+
+    def settimeout(self, timeout):
+        self.__sock.settimeout(timeout)
diff --git a/VERSION b/VERSION
index 99188f0c6fe..52356d37e0b 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.2.50
+1.3.50
diff --git a/a.out.h b/a.out.h
deleted file mode 100644
index 33ca7f77eae..00000000000
--- a/a.out.h
+++ /dev/null
@@ -1,430 +0,0 @@
-/* a.out.h
-
-   Copyright 1997, 1998, 1999, 2001 Red Hat, Inc.
-
-This file is part of Cygwin.
-
-This software is a copyrighted work licensed under the terms of the
-Cygwin license.  Please consult the file "CYGWIN_LICENSE" for
-details. */
-
-#ifndef _A_OUT_H_
-#define _A_OUT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#define COFF_IMAGE_WITH_PE
-#define COFF_LONG_SECTION_NAMES
-
-/*** coff information for Intel 386/486.  */
-
-
-/********************** FILE HEADER **********************/
-
-struct external_filehdr {
-  short f_magic;	/* magic number			*/
-  short f_nscns;	/* number of sections		*/
-  host_ulong f_timdat;	/* time & date stamp		*/
-  host_ulong f_symptr;	/* file pointer to symtab	*/
-  host_ulong f_nsyms;	/* number of symtab entries	*/
-  short f_opthdr;	/* sizeof(optional hdr)		*/
-  short f_flags;	/* flags			*/
-};
-
-/* Bits for f_flags:
- *	F_RELFLG	relocation info stripped from file
- *	F_EXEC		file is executable (no unresolved external references)
- *	F_LNNO		line numbers stripped from file
- *	F_LSYMS		local symbols stripped from file
- *	F_AR32WR	file has byte ordering of an AR32WR machine (e.g. vax)
- */
-
-#define F_RELFLG	(0x0001)
-#define F_EXEC		(0x0002)
-#define F_LNNO		(0x0004)
-#define F_LSYMS		(0x0008)
-
-
-
-#define	I386MAGIC	0x14c
-#define I386PTXMAGIC	0x154
-#define I386AIXMAGIC	0x175
-
-/* This is Lynx's all-platform magic number for executables. */
-
-#define LYNXCOFFMAGIC	0415
-
-#define I386BADMAG(x) (((x).f_magic != I386MAGIC) \
-		       && (x).f_magic != I386AIXMAGIC \
-		       && (x).f_magic != I386PTXMAGIC \
-		       && (x).f_magic != LYNXCOFFMAGIC)
-
-#define	FILHDR	struct external_filehdr
-#define	FILHSZ	20
-
-
-/********************** AOUT "OPTIONAL HEADER"=
- **********************/
-
-
-typedef struct
-{
-  unsigned short magic;		/* type of file				*/
-  unsigned short vstamp;	/* version stamp			*/
-  host_ulong	tsize;		/* text size in bytes, padded to FW bdry*/
-  host_ulong	dsize;		/* initialized data "  "		*/
-  host_ulong	bsize;		/* uninitialized data "   "		*/
-  host_ulong	entry;		/* entry pt.				*/
-  host_ulong text_start;	/* base of text used for this file */
-  host_ulong data_start;	/* base of data used for this file=
- */
-}
-AOUTHDR;
-
-#define AOUTSZ 28
-#define AOUTHDRSZ 28
-
-#define OMAGIC          0404    /* object files, eg as output */
-#define ZMAGIC          0413    /* demand load format, eg normal ld output */
-#define STMAGIC		0401	/* target shlib */
-#define SHMAGIC		0443	/* host   shlib */
-
-
-/* define some NT default values */
-/*  #define NT_IMAGE_BASE        0x400000 moved to internal.h */
-#define NT_SECTION_ALIGNMENT 0x1000
-#define NT_FILE_ALIGNMENT    0x200
-#define NT_DEF_RESERVE       0x100000
-#define NT_DEF_COMMIT        0x1000
-
-/********************** SECTION HEADER **********************/
-
-
-struct external_scnhdr {
-  char		s_name[8];	/* section name			*/
-  host_ulong	s_paddr;	/* physical address, offset
-				   of last addr in scn */
-  host_ulong	s_vaddr;	/* virtual address		*/
-  host_ulong	s_size;		/* section size			*/
-  host_ulong	s_scnptr;	/* file ptr to raw data for section */
-  host_ulong	s_relptr;	/* file ptr to relocation	*/
-  host_ulong	s_lnnoptr;	/* file ptr to line numbers	*/
-  unsigned short s_nreloc;	/* number of relocation entries	*/
-  unsigned short s_nlnno;	/* number of line number entries*/
-  host_ulong	s_flags;	/* flags			*/
-};
-
-#define	SCNHDR	struct external_scnhdr
-#define	SCNHSZ	40
-
-/*
- * names of "special" sections
- */
-#define _TEXT	".text"
-#define _DATA	".data"
-#define _BSS	".bss"
-#define _COMMENT ".comment"
-#define _LIB ".lib"
-
-/********************** LINE NUMBERS **********************/
-
-/* 1 line number entry for every "breakpointable" source line in a section.
- * Line numbers are grouped on a per function basis; first entry in a function
- * grouping will have l_lnno = 0 and in place of physical address will be the
- * symbol table index of the function name.
- */
-struct external_lineno {
-  union {
-    host_ulong l_symndx; /* function name symbol index, iff l_lnno 0 */
-    host_ulong l_paddr;	/* (physical) address of line number	*/
-  } l_addr;
-  unsigned short l_lnno;	/* line number		*/
-};
-
-#define	LINENO	struct external_lineno
-#define	LINESZ	6
-
-/********************** SYMBOLS **********************/
-
-#define E_SYMNMLEN	8	/* # characters in a symbol name	*/
-#define E_FILNMLEN	14	/* # characters in a file name		*/
-#define E_DIMNUM	4	/* # array dimensions in auxiliary entry */
-
-struct QEMU_PACKED external_syment
-{
-  union {
-    char e_name[E_SYMNMLEN];
-    struct {
-      host_ulong e_zeroes;
-      host_ulong e_offset;
-    } e;
-  } e;
-  host_ulong e_value;
-  unsigned short e_scnum;
-  unsigned short e_type;
-  char e_sclass[1];
-  char e_numaux[1];
-};
-
-#define N_BTMASK	(0xf)
-#define N_TMASK		(0x30)
-#define N_BTSHFT	(4)
-#define N_TSHIFT	(2)
-
-union external_auxent {
-  struct {
-    host_ulong x_tagndx;	/* str, un, or enum tag indx */
-    union {
-      struct {
-	unsigned short  x_lnno; /* declaration line number */
-	unsigned short  x_size; /* str/union/array size */
-      } x_lnsz;
-      host_ulong x_fsize;	/* size of function */
-    } x_misc;
-    union {
-      struct {			/* if ISFCN, tag, or .bb */
-	host_ulong x_lnnoptr;/* ptr to fcn line # */
-	host_ulong x_endndx;	/* entry ndx past block end */
-      } x_fcn;
-      struct {			/* if ISARY, up to 4 dimen. */
-	char x_dimen[E_DIMNUM][2];
-      } x_ary;
-    } x_fcnary;
-    unsigned short x_tvndx;	/* tv index */
-  } x_sym;
-
-  union {
-    char x_fname[E_FILNMLEN];
-    struct {
-      host_ulong x_zeroes;
-      host_ulong x_offset;
-    } x_n;
-  } x_file;
-
-  struct {
-    host_ulong x_scnlen;	/* section length */
-    unsigned short x_nreloc;	/* # relocation entries */
-    unsigned short x_nlinno;	/* # line numbers */
-    host_ulong x_checksum;	/* section COMDAT checksum */
-    unsigned short x_associated;/* COMDAT associated section index */
-    char x_comdat[1];		/* COMDAT selection number */
-  } x_scn;
-
-  struct {
-    host_ulong x_tvfill;	/* tv fill value */
-    unsigned short x_tvlen;	/* length of .tv */
-    char x_tvran[2][2];		/* tv range */
-  } x_tv;	/* info about .tv section (in auxent of symbol .tv)) */
-
-};
-
-#define	SYMENT	struct external_syment
-#define	SYMESZ	18
-#define	AUXENT	union external_auxent
-#define	AUXESZ	18
-
-#define _ETEXT	"etext"
-
-/********************** RELOCATION DIRECTIVES **********************/
-
-struct external_reloc {
-  char r_vaddr[4];
-  char r_symndx[4];
-  char r_type[2];
-};
-
-#define RELOC struct external_reloc
-#define RELSZ 10
-
-/* end of coff/i386.h */
-
-/* PE COFF header information */
-
-#ifndef _PE_H
-#define _PE_H
-
-/* NT specific file attributes */
-#define IMAGE_FILE_RELOCS_STRIPPED           0x0001
-#define IMAGE_FILE_EXECUTABLE_IMAGE          0x0002
-#define IMAGE_FILE_LINE_NUMS_STRIPPED        0x0004
-#define IMAGE_FILE_LOCAL_SYMS_STRIPPED       0x0008
-#define IMAGE_FILE_BYTES_REVERSED_LO         0x0080
-#define IMAGE_FILE_32BIT_MACHINE             0x0100
-#define IMAGE_FILE_DEBUG_STRIPPED            0x0200
-#define IMAGE_FILE_SYSTEM                    0x1000
-#define IMAGE_FILE_DLL                       0x2000
-#define IMAGE_FILE_BYTES_REVERSED_HI         0x8000
-
-/* additional flags to be set for section headers to allow the NT loader to
-   read and write to the section data (to replace the addresses of data in
-   dlls for one thing); also to execute the section in .text's case=
- */
-#define IMAGE_SCN_MEM_DISCARDABLE 0x02000000
-#define IMAGE_SCN_MEM_EXECUTE     0x20000000
-#define IMAGE_SCN_MEM_READ        0x40000000
-#define IMAGE_SCN_MEM_WRITE       0x80000000
-
-/*
- * Section characteristics added for ppc-nt
- */
-
-#define IMAGE_SCN_TYPE_NO_PAD                0x00000008  /* Reserved.  */
-
-#define IMAGE_SCN_CNT_CODE                   0x00000020  /* Section contains code. */
-#define IMAGE_SCN_CNT_INITIALIZED_DATA       0x00000040  /* Section contains initialized data. */
-#define IMAGE_SCN_CNT_UNINITIALIZED_DATA     0x00000080  /* Section contains uninitialized data. */
-
-#define IMAGE_SCN_LNK_OTHER                  0x00000100  /* Reserved.  */
-#define IMAGE_SCN_LNK_INFO                   0x00000200  /* Section contains comments or some other type of information. */
-#define IMAGE_SCN_LNK_REMOVE                 0x00000800  /* Section contents will not become part of image. */
-#define IMAGE_SCN_LNK_COMDAT                 0x00001000  /* Section contents comdat. */
-
-#define IMAGE_SCN_MEM_FARDATA                0x00008000
-
-#define IMAGE_SCN_MEM_PURGEABLE              0x00020000
-#define IMAGE_SCN_MEM_16BIT                  0x00020000
-#define IMAGE_SCN_MEM_LOCKED                 0x00040000
-#define IMAGE_SCN_MEM_PRELOAD                0x00080000
-
-#define IMAGE_SCN_ALIGN_1BYTES               0x00100000
-#define IMAGE_SCN_ALIGN_2BYTES               0x00200000
-#define IMAGE_SCN_ALIGN_4BYTES               0x00300000
-#define IMAGE_SCN_ALIGN_8BYTES               0x00400000
-#define IMAGE_SCN_ALIGN_16BYTES              0x00500000  /* Default alignment if no others are specified. */
-#define IMAGE_SCN_ALIGN_32BYTES              0x00600000
-#define IMAGE_SCN_ALIGN_64BYTES              0x00700000
-
-
-#define IMAGE_SCN_LNK_NRELOC_OVFL            0x01000000  /* Section contains extended relocations. */
-#define IMAGE_SCN_MEM_NOT_CACHED             0x04000000  /* Section is not cachable.               */
-#define IMAGE_SCN_MEM_NOT_PAGED              0x08000000  /* Section is not pageable.               */
-#define IMAGE_SCN_MEM_SHARED                 0x10000000  /* Section is shareable.                  */
-
-/* COMDAT selection codes.  */
-
-#define IMAGE_COMDAT_SELECT_NODUPLICATES     (1) /* Warn if duplicates.  */
-#define IMAGE_COMDAT_SELECT_ANY		     (2) /* No warning.  */
-#define IMAGE_COMDAT_SELECT_SAME_SIZE	     (3) /* Warn if different size.  */
-#define IMAGE_COMDAT_SELECT_EXACT_MATCH	     (4) /* Warn if different.  */
-#define IMAGE_COMDAT_SELECT_ASSOCIATIVE	     (5) /* Base on other section.  */
-
-/* Magic values that are true for all dos/nt implementations */
-#define DOSMAGIC       0x5a4d
-#define NT_SIGNATURE   0x00004550
-
-/* NT allows long filenames, we want to accommodate this.  This may break
-     some of the bfd functions */
-#undef  FILNMLEN
-#define FILNMLEN	18	/* # characters in a file name		*/
-
-
-#ifdef COFF_IMAGE_WITH_PE
-/* The filehdr is only weired in images */
-
-#undef FILHDR
-struct external_PE_filehdr
-{
-  /* DOS header fields */
-  unsigned short e_magic;	/* Magic number, 0x5a4d */
-  unsigned short e_cblp;	/* Bytes on last page of file, 0x90 */
-  unsigned short e_cp;		/* Pages in file, 0x3 */
-  unsigned short e_crlc;	/* Relocations, 0x0 */
-  unsigned short e_cparhdr;	/* Size of header in paragraphs, 0x4 */
-  unsigned short e_minalloc;	/* Minimum extra paragraphs needed, 0x0 */
-  unsigned short e_maxalloc;	/* Maximum extra paragraphs needed, 0xFFFF */
-  unsigned short e_ss;		/* Initial (relative) SS value, 0x0 */
-  unsigned short e_sp;		/* Initial SP value, 0xb8 */
-  unsigned short e_csum;	/* Checksum, 0x0 */
-  unsigned short e_ip;		/* Initial IP value, 0x0 */
-  unsigned short e_cs;		/* Initial (relative) CS value, 0x0 */
-  unsigned short e_lfarlc;	/* File address of relocation table, 0x40 */
-  unsigned short e_ovno;	/* Overlay number, 0x0 */
-  char e_res[4][2];		/* Reserved words, all 0x0 */
-  unsigned short e_oemid;	/* OEM identifier (for e_oeminfo), 0x0 */
-  unsigned short e_oeminfo;	/* OEM information; e_oemid specific, 0x0 */
-  char e_res2[10][2];		/* Reserved words, all 0x0 */
-  host_ulong e_lfanew;	/* File address of new exe header, 0x80 */
-  char dos_message[16][4];	/* other stuff, always follow DOS header */
-  unsigned int nt_signature;	/* required NT signature, 0x4550 */
-
-  /* From standard header */
-
-  unsigned short f_magic;	/* magic number			*/
-  unsigned short f_nscns;	/* number of sections		*/
-  host_ulong f_timdat;	/* time & date stamp		*/
-  host_ulong f_symptr;	/* file pointer to symtab	*/
-  host_ulong f_nsyms;	/* number of symtab entries	*/
-  unsigned short f_opthdr;	/* sizeof(optional hdr)		*/
-  unsigned short f_flags;	/* flags			*/
-};
-
-
-#define FILHDR struct external_PE_filehdr
-#undef FILHSZ
-#define FILHSZ 152
-
-#endif
-
-typedef struct
-{
-  unsigned short magic;		/* type of file				*/
-  unsigned short vstamp;	/* version stamp			*/
-  host_ulong	tsize;		/* text size in bytes, padded to FW bdry*/
-  host_ulong	dsize;		/* initialized data "  "		*/
-  host_ulong	bsize;		/* uninitialized data "   "		*/
-  host_ulong	entry;		/* entry pt.				*/
-  host_ulong text_start;	/* base of text used for this file */
-  host_ulong data_start;	/* base of all data used for this file */
-
-  /* NT extra fields; see internal.h for descriptions */
-  host_ulong  ImageBase;
-  host_ulong  SectionAlignment;
-  host_ulong  FileAlignment;
-  unsigned short  MajorOperatingSystemVersion;
-  unsigned short  MinorOperatingSystemVersion;
-  unsigned short  MajorImageVersion;
-  unsigned short  MinorImageVersion;
-  unsigned short  MajorSubsystemVersion;
-  unsigned short  MinorSubsystemVersion;
-  char  Reserved1[4];
-  host_ulong  SizeOfImage;
-  host_ulong  SizeOfHeaders;
-  host_ulong  CheckSum;
-  unsigned short Subsystem;
-  unsigned short DllCharacteristics;
-  host_ulong  SizeOfStackReserve;
-  host_ulong  SizeOfStackCommit;
-  host_ulong  SizeOfHeapReserve;
-  host_ulong  SizeOfHeapCommit;
-  host_ulong  LoaderFlags;
-  host_ulong  NumberOfRvaAndSizes;
-  /* IMAGE_DATA_DIRECTORY DataDirectory[IMAGE_NUMBEROF_DIRECTORY_ENTRIES]; */
-  char  DataDirectory[16][2][4]; /* 16 entries, 2 elements/entry, 4 chars */
-
-} PEAOUTHDR;
-
-
-#undef AOUTSZ
-#define AOUTSZ (AOUTHDRSZ + 196)
-
-#undef  E_FILNMLEN
-#define E_FILNMLEN	18	/* # characters in a file name		*/
-#endif
-
-/* end of coff/pe.h */
-
-#define DT_NON		(0)	/* no derived type */
-#define DT_PTR		(1)	/* pointer */
-#define DT_FCN		(2)	/* function */
-#define DT_ARY		(3)	/* array */
-
-#define ISPTR(x)	(((x) & N_TMASK) == (DT_PTR << N_BTSHFT))
-#define ISFCN(x)	(((x) & N_TMASK) == (DT_FCN << N_BTSHFT))
-#define ISARY(x)	(((x) & N_TMASK) == (DT_ARY << N_BTSHFT))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _A_OUT_H_ */
diff --git a/aio-posix.c b/aio-posix.c
new file mode 100644
index 00000000000..fe4dbb4523e
--- /dev/null
+++ b/aio-posix.c
@@ -0,0 +1,269 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+#include "block/block.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
+
+struct AioHandler
+{
+    GPollFD pfd;
+    IOHandler *io_read;
+    IOHandler *io_write;
+    AioFlushHandler *io_flush;
+    int deleted;
+    void *opaque;
+    QLIST_ENTRY(AioHandler) node;
+};
+
+static AioHandler *find_aio_handler(AioContext *ctx, int fd)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.fd == fd)
+            if (!node->deleted)
+                return node;
+    }
+
+    return NULL;
+}
+
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        AioFlushHandler *io_flush,
+                        void *opaque)
+{
+    AioHandler *node;
+
+    node = find_aio_handler(ctx, fd);
+
+    /* Are we deleting the fd handler? */
+    if (!io_read && !io_write) {
+        if (node) {
+            g_source_remove_poll(&ctx->source, &node->pfd);
+
+            /* If the lock is held, just mark the node as deleted */
+            if (ctx->walking_handlers) {
+                node->deleted = 1;
+                node->pfd.revents = 0;
+            } else {
+                /* Otherwise, delete it for real.  We can't just mark it as
+                 * deleted because deleted nodes are only cleaned up after
+                 * releasing the walking_handlers lock.
+                 */
+                QLIST_REMOVE(node, node);
+                g_free(node);
+            }
+        }
+    } else {
+        if (node == NULL) {
+            /* Alloc and insert if it's not already there */
+            node = g_malloc0(sizeof(AioHandler));
+            node->pfd.fd = fd;
+            QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
+
+            g_source_add_poll(&ctx->source, &node->pfd);
+        }
+        /* Update handler with latest information */
+        node->io_read = io_read;
+        node->io_write = io_write;
+        node->io_flush = io_flush;
+        node->opaque = opaque;
+
+        node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP : 0);
+        node->pfd.events |= (io_write ? G_IO_OUT : 0);
+    }
+
+    aio_notify(ctx);
+}
+
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *notifier,
+                            EventNotifierHandler *io_read,
+                            AioFlushEventNotifierHandler *io_flush)
+{
+    aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
+                       (IOHandler *)io_read, NULL,
+                       (AioFlushHandler *)io_flush, notifier);
+}
+
+bool aio_pending(AioContext *ctx)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        int revents;
+
+        /*
+         * FIXME: right now we cannot get G_IO_HUP and G_IO_ERR because
+         * main-loop.c is still select based (due to the slirp legacy).
+         * If main-loop.c ever switches to poll, G_IO_ERR should be
+         * tested too.  Dispatching G_IO_ERR to both handlers should be
+         * okay, since handlers need to be ready for spurious wakeups.
+         */
+        revents = node->pfd.revents & node->pfd.events;
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+            return true;
+        }
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+bool aio_poll(AioContext *ctx, bool blocking)
+{
+    static struct timeval tv0;
+    AioHandler *node;
+    fd_set rdfds, wrfds;
+    int max_fd = -1;
+    int ret;
+    bool busy, progress;
+
+    progress = false;
+
+    /*
+     * If there are callbacks left that have been queued, we need to call then.
+     * Do not call select in this case, because it is possible that the caller
+     * does not need a complete flush (as is the case for qemu_aio_wait loops).
+     */
+    if (aio_bh_poll(ctx)) {
+        blocking = false;
+        progress = true;
+    }
+
+    /*
+     * Then dispatch any pending callbacks from the GSource.
+     *
+     * We have to walk very carefully in case qemu_aio_set_fd_handler is
+     * called while we're walking.
+     */
+    node = QLIST_FIRST(&ctx->aio_handlers);
+    while (node) {
+        AioHandler *tmp;
+        int revents;
+
+        ctx->walking_handlers++;
+
+        revents = node->pfd.revents & node->pfd.events;
+        node->pfd.revents = 0;
+
+        /* See comment in aio_pending.  */
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+            node->io_read(node->opaque);
+            progress = true;
+        }
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+            node->io_write(node->opaque);
+            progress = true;
+        }
+
+        tmp = node;
+        node = QLIST_NEXT(node, node);
+
+        ctx->walking_handlers--;
+
+        if (!ctx->walking_handlers && tmp->deleted) {
+            QLIST_REMOVE(tmp, node);
+            g_free(tmp);
+        }
+    }
+
+    if (progress && !blocking) {
+        return true;
+    }
+
+    ctx->walking_handlers++;
+
+    FD_ZERO(&rdfds);
+    FD_ZERO(&wrfds);
+
+    /* fill fd sets */
+    busy = false;
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        /* If there aren't pending AIO operations, don't invoke callbacks.
+         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
+         * wait indefinitely.
+         */
+        if (!node->deleted && node->io_flush) {
+            if (node->io_flush(node->opaque) == 0) {
+                continue;
+            }
+            busy = true;
+        }
+        if (!node->deleted && node->io_read) {
+            FD_SET(node->pfd.fd, &rdfds);
+            max_fd = MAX(max_fd, node->pfd.fd + 1);
+        }
+        if (!node->deleted && node->io_write) {
+            FD_SET(node->pfd.fd, &wrfds);
+            max_fd = MAX(max_fd, node->pfd.fd + 1);
+        }
+    }
+
+    ctx->walking_handlers--;
+
+    /* No AIO operations?  Get us out of here */
+    if (!busy) {
+        return progress;
+    }
+
+    /* wait until next event */
+    ret = select(max_fd, &rdfds, &wrfds, NULL, blocking ? NULL : &tv0);
+
+    /* if we have any readable fds, dispatch event */
+    if (ret > 0) {
+        /* we have to walk very carefully in case
+         * qemu_aio_set_fd_handler is called while we're walking */
+        node = QLIST_FIRST(&ctx->aio_handlers);
+        while (node) {
+            AioHandler *tmp;
+
+            ctx->walking_handlers++;
+
+            if (!node->deleted &&
+                FD_ISSET(node->pfd.fd, &rdfds) &&
+                node->io_read) {
+                node->io_read(node->opaque);
+                progress = true;
+            }
+            if (!node->deleted &&
+                FD_ISSET(node->pfd.fd, &wrfds) &&
+                node->io_write) {
+                node->io_write(node->opaque);
+                progress = true;
+            }
+
+            tmp = node;
+            node = QLIST_NEXT(node, node);
+
+            ctx->walking_handlers--;
+
+            if (!ctx->walking_handlers && tmp->deleted) {
+                QLIST_REMOVE(tmp, node);
+                g_free(tmp);
+            }
+        }
+    }
+
+    assert(progress || busy);
+    return true;
+}
diff --git a/aio-win32.c b/aio-win32.c
new file mode 100644
index 00000000000..38723bf1d37
--- /dev/null
+++ b/aio-win32.c
@@ -0,0 +1,219 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM Corp., 2008
+ * Copyright Red Hat Inc., 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+#include "block/block.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
+
+struct AioHandler {
+    EventNotifier *e;
+    EventNotifierHandler *io_notify;
+    AioFlushEventNotifierHandler *io_flush;
+    GPollFD pfd;
+    int deleted;
+    QLIST_ENTRY(AioHandler) node;
+};
+
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *e,
+                            EventNotifierHandler *io_notify,
+                            AioFlushEventNotifierHandler *io_flush)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->e == e && !node->deleted) {
+            break;
+        }
+    }
+
+    /* Are we deleting the fd handler? */
+    if (!io_notify) {
+        if (node) {
+            g_source_remove_poll(&ctx->source, &node->pfd);
+
+            /* If the lock is held, just mark the node as deleted */
+            if (ctx->walking_handlers) {
+                node->deleted = 1;
+                node->pfd.revents = 0;
+            } else {
+                /* Otherwise, delete it for real.  We can't just mark it as
+                 * deleted because deleted nodes are only cleaned up after
+                 * releasing the walking_handlers lock.
+                 */
+                QLIST_REMOVE(node, node);
+                g_free(node);
+            }
+        }
+    } else {
+        if (node == NULL) {
+            /* Alloc and insert if it's not already there */
+            node = g_malloc0(sizeof(AioHandler));
+            node->e = e;
+            node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
+            node->pfd.events = G_IO_IN;
+            QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
+
+            g_source_add_poll(&ctx->source, &node->pfd);
+        }
+        /* Update handler with latest information */
+        node->io_notify = io_notify;
+        node->io_flush = io_flush;
+    }
+
+    aio_notify(ctx);
+}
+
+bool aio_pending(AioContext *ctx)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.revents && node->io_notify) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+bool aio_poll(AioContext *ctx, bool blocking)
+{
+    AioHandler *node;
+    HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
+    bool busy, progress;
+    int count;
+
+    progress = false;
+
+    /*
+     * If there are callbacks left that have been queued, we need to call then.
+     * Do not call select in this case, because it is possible that the caller
+     * does not need a complete flush (as is the case for qemu_aio_wait loops).
+     */
+    if (aio_bh_poll(ctx)) {
+        blocking = false;
+        progress = true;
+    }
+
+    /*
+     * Then dispatch any pending callbacks from the GSource.
+     *
+     * We have to walk very carefully in case qemu_aio_set_fd_handler is
+     * called while we're walking.
+     */
+    node = QLIST_FIRST(&ctx->aio_handlers);
+    while (node) {
+        AioHandler *tmp;
+
+        ctx->walking_handlers++;
+
+        if (node->pfd.revents && node->io_notify) {
+            node->pfd.revents = 0;
+            node->io_notify(node->e);
+            progress = true;
+        }
+
+        tmp = node;
+        node = QLIST_NEXT(node, node);
+
+        ctx->walking_handlers--;
+
+        if (!ctx->walking_handlers && tmp->deleted) {
+            QLIST_REMOVE(tmp, node);
+            g_free(tmp);
+        }
+    }
+
+    if (progress && !blocking) {
+        return true;
+    }
+
+    ctx->walking_handlers++;
+
+    /* fill fd sets */
+    busy = false;
+    count = 0;
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        /* If there aren't pending AIO operations, don't invoke callbacks.
+         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
+         * wait indefinitely.
+         */
+        if (!node->deleted && node->io_flush) {
+            if (node->io_flush(node->e) == 0) {
+                continue;
+            }
+            busy = true;
+        }
+        if (!node->deleted && node->io_notify) {
+            events[count++] = event_notifier_get_handle(node->e);
+        }
+    }
+
+    ctx->walking_handlers--;
+
+    /* No AIO operations?  Get us out of here */
+    if (!busy) {
+        return progress;
+    }
+
+    /* wait until next event */
+    while (count > 0) {
+        int timeout = blocking ? INFINITE : 0;
+        int ret = WaitForMultipleObjects(count, events, FALSE, timeout);
+
+        /* if we have any signaled events, dispatch event */
+        if ((DWORD) (ret - WAIT_OBJECT_0) >= count) {
+            break;
+        }
+
+        blocking = false;
+
+        /* we have to walk very carefully in case
+         * qemu_aio_set_fd_handler is called while we're walking */
+        node = QLIST_FIRST(&ctx->aio_handlers);
+        while (node) {
+            AioHandler *tmp;
+
+            ctx->walking_handlers++;
+
+            if (!node->deleted &&
+                event_notifier_get_handle(node->e) == events[ret - WAIT_OBJECT_0] &&
+                node->io_notify) {
+                node->io_notify(node->e);
+                progress = true;
+            }
+
+            tmp = node;
+            node = QLIST_NEXT(node, node);
+
+            ctx->walking_handlers--;
+
+            if (!ctx->walking_handlers && tmp->deleted) {
+                QLIST_REMOVE(tmp, node);
+                g_free(tmp);
+            }
+        }
+
+        /* Try again, but only call each handler once.  */
+        events[ret - WAIT_OBJECT_0] = events[--count];
+    }
+
+    assert(progress || busy);
+    return true;
+}
diff --git a/aio.c b/aio.c
deleted file mode 100644
index 0a9eb10c76d..00000000000
--- a/aio.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * QEMU aio implementation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "block.h"
-#include "qemu-queue.h"
-#include "qemu_socket.h"
-
-typedef struct AioHandler AioHandler;
-
-/* The list of registered AIO handlers */
-static QLIST_HEAD(, AioHandler) aio_handlers;
-
-/* This is a simple lock used to protect the aio_handlers list.  Specifically,
- * it's used to ensure that no callbacks are removed while we're walking and
- * dispatching callbacks.
- */
-static int walking_handlers;
-
-struct AioHandler
-{
-    int fd;
-    IOHandler *io_read;
-    IOHandler *io_write;
-    AioFlushHandler *io_flush;
-    int deleted;
-    void *opaque;
-    QLIST_ENTRY(AioHandler) node;
-};
-
-static AioHandler *find_aio_handler(int fd)
-{
-    AioHandler *node;
-
-    QLIST_FOREACH(node, &aio_handlers, node) {
-        if (node->fd == fd)
-            if (!node->deleted)
-                return node;
-    }
-
-    return NULL;
-}
-
-int qemu_aio_set_fd_handler(int fd,
-                            IOHandler *io_read,
-                            IOHandler *io_write,
-                            AioFlushHandler *io_flush,
-                            void *opaque)
-{
-    AioHandler *node;
-
-    node = find_aio_handler(fd);
-
-    /* Are we deleting the fd handler? */
-    if (!io_read && !io_write) {
-        if (node) {
-            /* If the lock is held, just mark the node as deleted */
-            if (walking_handlers)
-                node->deleted = 1;
-            else {
-                /* Otherwise, delete it for real.  We can't just mark it as
-                 * deleted because deleted nodes are only cleaned up after
-                 * releasing the walking_handlers lock.
-                 */
-                QLIST_REMOVE(node, node);
-                g_free(node);
-            }
-        }
-    } else {
-        if (node == NULL) {
-            /* Alloc and insert if it's not already there */
-            node = g_malloc0(sizeof(AioHandler));
-            node->fd = fd;
-            QLIST_INSERT_HEAD(&aio_handlers, node, node);
-        }
-        /* Update handler with latest information */
-        node->io_read = io_read;
-        node->io_write = io_write;
-        node->io_flush = io_flush;
-        node->opaque = opaque;
-    }
-
-    qemu_set_fd_handler2(fd, NULL, io_read, io_write, opaque);
-
-    return 0;
-}
-
-void qemu_aio_flush(void)
-{
-    while (qemu_aio_wait());
-}
-
-bool qemu_aio_wait(void)
-{
-    AioHandler *node;
-    fd_set rdfds, wrfds;
-    int max_fd = -1;
-    int ret;
-    bool busy;
-
-    /*
-     * If there are callbacks left that have been queued, we need to call then.
-     * Do not call select in this case, because it is possible that the caller
-     * does not need a complete flush (as is the case for qemu_aio_wait loops).
-     */
-    if (qemu_bh_poll()) {
-        return true;
-    }
-
-    walking_handlers = 1;
-
-    FD_ZERO(&rdfds);
-    FD_ZERO(&wrfds);
-
-    /* fill fd sets */
-    busy = false;
-    QLIST_FOREACH(node, &aio_handlers, node) {
-        /* If there aren't pending AIO operations, don't invoke callbacks.
-         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
-         * wait indefinitely.
-         */
-        if (node->io_flush) {
-            if (node->io_flush(node->opaque) == 0) {
-                continue;
-            }
-            busy = true;
-        }
-        if (!node->deleted && node->io_read) {
-            FD_SET(node->fd, &rdfds);
-            max_fd = MAX(max_fd, node->fd + 1);
-        }
-        if (!node->deleted && node->io_write) {
-            FD_SET(node->fd, &wrfds);
-            max_fd = MAX(max_fd, node->fd + 1);
-        }
-    }
-
-    walking_handlers = 0;
-
-    /* No AIO operations?  Get us out of here */
-    if (!busy) {
-        return false;
-    }
-
-    /* wait until next event */
-    ret = select(max_fd, &rdfds, &wrfds, NULL, NULL);
-
-    /* if we have any readable fds, dispatch event */
-    if (ret > 0) {
-        walking_handlers = 1;
-
-        /* we have to walk very carefully in case
-         * qemu_aio_set_fd_handler is called while we're walking */
-        node = QLIST_FIRST(&aio_handlers);
-        while (node) {
-            AioHandler *tmp;
-
-            if (!node->deleted &&
-                FD_ISSET(node->fd, &rdfds) &&
-                node->io_read) {
-                node->io_read(node->opaque);
-            }
-            if (!node->deleted &&
-                FD_ISSET(node->fd, &wrfds) &&
-                node->io_write) {
-                node->io_write(node->opaque);
-            }
-
-            tmp = node;
-            node = QLIST_NEXT(node, node);
-
-            if (tmp->deleted) {
-                QLIST_REMOVE(tmp, node);
-                g_free(tmp);
-            }
-        }
-
-        walking_handlers = 0;
-    }
-
-    return true;
-}
diff --git a/arch_init.c b/arch_init.c
index 5a1173e29a5..dada6ded1aa 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -29,22 +29,26 @@
 #include <sys/mman.h>
 #endif
 #include "config.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "arch_init.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
+#include "sysemu/arch_init.h"
 #include "audio/audio.h"
 #include "hw/pc.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/audiodev.h"
-#include "kvm.h"
-#include "migration.h"
-#include "net.h"
-#include "gdbstub.h"
+#include "sysemu/kvm.h"
+#include "migration/migration.h"
+#include "exec/gdbstub.h"
 #include "hw/smbios.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw/pcspk.h"
-#include "qemu/page_cache.h"
+#include "migration/page_cache.h"
+#include "qemu/config-file.h"
 #include "qmp-commands.h"
+#include "trace.h"
+#include "exec/cpu-all.h"
 
 #ifdef DEBUG_ARCH_INIT
 #define DPRINTF(fmt, ...) \
@@ -136,7 +140,6 @@ static struct defconfig_file {
     /* Indicates it is an user config file (disabled by -no-user-config) */
     bool userconfig;
 } default_config_files[] = {
-    { CONFIG_QEMU_DATADIR "/cpus-" TARGET_ARCH ".conf",  false },
     { CONFIG_QEMU_CONFDIR "/qemu.conf",                   true },
     { CONFIG_QEMU_CONFDIR "/target-" TARGET_ARCH ".conf", true },
     { NULL }, /* end of list */
@@ -262,16 +265,21 @@ uint64_t xbzrle_mig_pages_overflow(void)
     return acct_info.xbzrle_overflows;
 }
 
-static void save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
-        int cont, int flag)
+static size_t save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
+                             int cont, int flag)
 {
-        qemu_put_be64(f, offset | cont | flag);
-        if (!cont) {
-                qemu_put_byte(f, strlen(block->idstr));
-                qemu_put_buffer(f, (uint8_t *)block->idstr,
-                                strlen(block->idstr));
-        }
+    size_t size;
+
+    qemu_put_be64(f, offset | cont | flag);
+    size = 8;
 
+    if (!cont) {
+        qemu_put_byte(f, strlen(block->idstr));
+        qemu_put_buffer(f, (uint8_t *)block->idstr,
+                        strlen(block->idstr));
+        size += 1 + strlen(block->idstr);
+    }
+    return size;
 }
 
 #define ENCODING_FLAG_XBZRLE 0x1
@@ -318,56 +326,147 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t *current_data,
     }
 
     /* Send XBZRLE based compressed page */
-    save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_XBZRLE);
+    bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_XBZRLE);
     qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
     qemu_put_be16(f, encoded_len);
     qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
-    bytes_sent = encoded_len + 1 + 2;
+    bytes_sent += encoded_len + 1 + 2;
     acct_info.xbzrle_pages++;
     acct_info.xbzrle_bytes += bytes_sent;
 
     return bytes_sent;
 }
 
-static RAMBlock *last_block;
+
+/* This is the last block that we have visited serching for dirty pages
+ */
+static RAMBlock *last_seen_block;
+/* This is the last block from where we have sent data */
+static RAMBlock *last_sent_block;
 static ram_addr_t last_offset;
+static unsigned long *migration_bitmap;
+static uint64_t migration_dirty_pages;
+static uint32_t last_version;
+
+static inline
+ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
+                                                 ram_addr_t start)
+{
+    unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
+    unsigned long nr = base + (start >> TARGET_PAGE_BITS);
+    unsigned long size = base + (int128_get64(mr->size) >> TARGET_PAGE_BITS);
+
+    unsigned long next = find_next_bit(migration_bitmap, size, nr);
+
+    if (next < size) {
+        clear_bit(next, migration_bitmap);
+        migration_dirty_pages--;
+    }
+    return (next - base) << TARGET_PAGE_BITS;
+}
+
+static inline bool migration_bitmap_set_dirty(MemoryRegion *mr,
+                                              ram_addr_t offset)
+{
+    bool ret;
+    int nr = (mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+
+    ret = test_and_set_bit(nr, migration_bitmap);
+
+    if (!ret) {
+        migration_dirty_pages++;
+    }
+    return ret;
+}
+
+static void migration_bitmap_sync(void)
+{
+    RAMBlock *block;
+    ram_addr_t addr;
+    uint64_t num_dirty_pages_init = migration_dirty_pages;
+    MigrationState *s = migrate_get_current();
+    static int64_t start_time;
+    static int64_t num_dirty_pages_period;
+    int64_t end_time;
+
+    if (!start_time) {
+        start_time = qemu_get_clock_ms(rt_clock);
+    }
+
+    trace_migration_bitmap_sync_start();
+    memory_global_sync_dirty_bitmap(get_system_memory());
+
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) {
+            if (memory_region_test_and_clear_dirty(block->mr,
+                                                   addr, TARGET_PAGE_SIZE,
+                                                   DIRTY_MEMORY_MIGRATION)) {
+                migration_bitmap_set_dirty(block->mr, addr);
+            }
+        }
+    }
+    trace_migration_bitmap_sync_end(migration_dirty_pages
+                                    - num_dirty_pages_init);
+    num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
+    end_time = qemu_get_clock_ms(rt_clock);
+
+    /* more than 1 second = 1000 millisecons */
+    if (end_time > start_time + 1000) {
+        s->dirty_pages_rate = num_dirty_pages_period * 1000
+            / (end_time - start_time);
+        start_time = end_time;
+        num_dirty_pages_period = 0;
+    }
+}
 
 /*
  * ram_save_block: Writes a page of memory to the stream f
  *
- * Returns:  0: if the page hasn't changed
- *          -1: if there are no more dirty pages
- *           n: the amount of bytes written in other case
+ * Returns:  The number of bytes written.
+ *           0 means no dirty pages
  */
 
 static int ram_save_block(QEMUFile *f, bool last_stage)
 {
-    RAMBlock *block = last_block;
+    RAMBlock *block = last_seen_block;
     ram_addr_t offset = last_offset;
-    int bytes_sent = -1;
+    bool complete_round = false;
+    int bytes_sent = 0;
     MemoryRegion *mr;
     ram_addr_t current_addr;
 
     if (!block)
-        block = QLIST_FIRST(&ram_list.blocks);
+        block = QTAILQ_FIRST(&ram_list.blocks);
 
-    do {
+    while (true) {
         mr = block->mr;
-        if (memory_region_get_dirty(mr, offset, TARGET_PAGE_SIZE,
-                                    DIRTY_MEMORY_MIGRATION)) {
+        offset = migration_bitmap_find_and_reset_dirty(mr, offset);
+        if (complete_round && block == last_seen_block &&
+            offset >= last_offset) {
+            break;
+        }
+        if (offset >= block->length) {
+            offset = 0;
+            block = QTAILQ_NEXT(block, next);
+            if (!block) {
+                block = QTAILQ_FIRST(&ram_list.blocks);
+                complete_round = true;
+            }
+        } else {
             uint8_t *p;
-            int cont = (block == last_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
-
-            memory_region_reset_dirty(mr, offset, TARGET_PAGE_SIZE,
-                                      DIRTY_MEMORY_MIGRATION);
+            int cont = (block == last_sent_block) ?
+                RAM_SAVE_FLAG_CONTINUE : 0;
 
             p = memory_region_get_ram_ptr(mr) + offset;
 
+            /* In doubt sent page as normal */
+            bytes_sent = -1;
             if (is_dup_page(p)) {
                 acct_info.dup_pages++;
-                save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_COMPRESS);
+                bytes_sent = save_block_hdr(f, block, offset, cont,
+                                            RAM_SAVE_FLAG_COMPRESS);
                 qemu_put_byte(f, *p);
-                bytes_sent = 1;
+                bytes_sent += 1;
             } else if (migrate_use_xbzrle()) {
                 current_addr = block->offset + offset;
                 bytes_sent = save_xbzrle_page(f, p, current_addr, block,
@@ -377,30 +476,22 @@ static int ram_save_block(QEMUFile *f, bool last_stage)
                 }
             }
 
-            /* either we didn't send yet (we may have had XBZRLE overflow) */
+            /* XBZRLE overflow or normal page */
             if (bytes_sent == -1) {
-                save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);
+                bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);
                 qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
-                bytes_sent = TARGET_PAGE_SIZE;
+                bytes_sent += TARGET_PAGE_SIZE;
                 acct_info.norm_pages++;
             }
 
             /* if page is unmodified, continue to the next */
-            if (bytes_sent != 0) {
+            if (bytes_sent > 0) {
+                last_sent_block = block;
                 break;
             }
         }
-
-        offset += TARGET_PAGE_SIZE;
-        if (offset >= block->length) {
-            offset = 0;
-            block = QLIST_NEXT(block, next);
-            if (!block)
-                block = QLIST_FIRST(&ram_list.blocks);
-        }
-    } while (block != last_block || offset != last_offset);
-
-    last_block = block;
+    }
+    last_seen_block = block;
     last_offset = offset;
 
     return bytes_sent;
@@ -410,7 +501,7 @@ static uint64_t bytes_transferred;
 
 static ram_addr_t ram_save_remaining(void)
 {
-    return ram_list.dirty_pages;
+    return migration_dirty_pages;
 }
 
 uint64_t ram_bytes_remaining(void)
@@ -428,46 +519,21 @@ uint64_t ram_bytes_total(void)
     RAMBlock *block;
     uint64_t total = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next)
+    QTAILQ_FOREACH(block, &ram_list.blocks, next)
         total += block->length;
 
     return total;
 }
 
-static int block_compar(const void *a, const void *b)
-{
-    RAMBlock * const *ablock = a;
-    RAMBlock * const *bblock = b;
-
-    return strcmp((*ablock)->idstr, (*bblock)->idstr);
-}
-
-static void sort_ram_list(void)
-{
-    RAMBlock *block, *nblock, **blocks;
-    int n;
-    n = 0;
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
-        ++n;
-    }
-    blocks = g_malloc(n * sizeof *blocks);
-    n = 0;
-    QLIST_FOREACH_SAFE(block, &ram_list.blocks, next, nblock) {
-        blocks[n++] = block;
-        QLIST_REMOVE(block, next);
-    }
-    qsort(blocks, n, sizeof *blocks, block_compar);
-    while (--n >= 0) {
-        QLIST_INSERT_HEAD(&ram_list.blocks, blocks[n], next);
-    }
-    g_free(blocks);
-}
-
 static void migration_end(void)
 {
-    memory_global_dirty_log_stop();
+    if (migration_bitmap) {
+        memory_global_dirty_log_stop();
+        g_free(migration_bitmap);
+        migration_bitmap = NULL;
+    }
 
-    if (migrate_use_xbzrle()) {
+    if (XBZRLE.cache) {
         cache_fini(XBZRLE.cache);
         g_free(XBZRLE.cache);
         g_free(XBZRLE.encoded_buf);
@@ -482,17 +548,28 @@ static void ram_migration_cancel(void *opaque)
     migration_end();
 }
 
+static void reset_ram_globals(void)
+{
+    last_seen_block = NULL;
+    last_sent_block = NULL;
+    last_offset = 0;
+    last_version = ram_list.version;
+}
+
 #define MAX_WAIT 50 /* ms, half buffered_file limit */
 
 static int ram_save_setup(QEMUFile *f, void *opaque)
 {
-    ram_addr_t addr;
     RAMBlock *block;
+    int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    migration_bitmap = bitmap_new(ram_pages);
+    bitmap_set(migration_bitmap, 0, ram_pages);
+    migration_dirty_pages = ram_pages;
 
+    qemu_mutex_lock_ramlist();
     bytes_transferred = 0;
-    last_block = NULL;
-    last_offset = 0;
-    sort_ram_list();
+    reset_ram_globals();
 
     if (migrate_use_xbzrle()) {
         XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
@@ -507,26 +584,18 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
         acct_clear();
     }
 
-    /* Make sure all dirty bits are set */
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
-        for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) {
-            if (!memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE,
-                                         DIRTY_MEMORY_MIGRATION)) {
-                memory_region_set_dirty(block->mr, addr, TARGET_PAGE_SIZE);
-            }
-        }
-    }
-
     memory_global_dirty_log_start();
+    migration_bitmap_sync();
 
     qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         qemu_put_byte(f, strlen(block->idstr));
         qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
         qemu_put_be64(f, block->length);
     }
 
+    qemu_mutex_unlock_ramlist();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
     return 0;
@@ -534,25 +603,28 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
 
 static int ram_save_iterate(QEMUFile *f, void *opaque)
 {
-    uint64_t bytes_transferred_last;
-    double bwidth = 0;
     int ret;
     int i;
-    uint64_t expected_time;
+    int64_t t0;
+    int total_sent = 0;
 
-    bytes_transferred_last = bytes_transferred;
-    bwidth = qemu_get_clock_ns(rt_clock);
+    qemu_mutex_lock_ramlist();
 
+    if (ram_list.version != last_version) {
+        reset_ram_globals();
+    }
+
+    t0 = qemu_get_clock_ns(rt_clock);
     i = 0;
     while ((ret = qemu_file_rate_limit(f)) == 0) {
         int bytes_sent;
 
         bytes_sent = ram_save_block(f, false);
         /* no more blocks to sent */
-        if (bytes_sent < 0) {
+        if (bytes_sent == 0) {
             break;
         }
-        bytes_transferred += bytes_sent;
+        total_sent += bytes_sent;
         acct_info.iterations++;
         /* we want to check in the 1st loop, just in case it was the 1st time
            and we had to sync the dirty bitmap.
@@ -560,9 +632,9 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
            iterations
         */
         if ((i & 63) == 0) {
-            uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000;
+            uint64_t t1 = (qemu_get_clock_ns(rt_clock) - t0) / 1000000;
             if (t1 > MAX_WAIT) {
-                DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n",
+                DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
                         t1, i);
                 break;
             }
@@ -570,38 +642,24 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
         i++;
     }
 
+    qemu_mutex_unlock_ramlist();
+
     if (ret < 0) {
+        bytes_transferred += total_sent;
         return ret;
     }
 
-    bwidth = qemu_get_clock_ns(rt_clock) - bwidth;
-    bwidth = (bytes_transferred - bytes_transferred_last) / bwidth;
-
-    /* if we haven't transferred anything this round, force expected_time to a
-     * a very high value, but without crashing */
-    if (bwidth == 0) {
-        bwidth = 0.000001;
-    }
-
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+    total_sent += 8;
+    bytes_transferred += total_sent;
 
-    expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
-
-    DPRINTF("ram_save_live: expected(" PRIu64 ") <= max(" PRIu64 ")?\n",
-            expected_time, migrate_max_downtime());
-
-    if (expected_time <= migrate_max_downtime()) {
-        memory_global_sync_dirty_bitmap(get_system_memory());
-        expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
-
-        return expected_time <= migrate_max_downtime();
-    }
-    return 0;
+    return total_sent;
 }
 
 static int ram_save_complete(QEMUFile *f, void *opaque)
 {
-    memory_global_sync_dirty_bitmap(get_system_memory());
+    qemu_mutex_lock_ramlist();
+    migration_bitmap_sync();
 
     /* try transferring iterative blocks of memory */
 
@@ -611,18 +669,32 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
         bytes_sent = ram_save_block(f, true);
         /* no more blocks to sent */
-        if (bytes_sent < 0) {
+        if (bytes_sent == 0) {
             break;
         }
         bytes_transferred += bytes_sent;
     }
-    memory_global_dirty_log_stop();
+    migration_end();
 
+    qemu_mutex_unlock_ramlist();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
     return 0;
 }
 
+static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+{
+    uint64_t remaining_size;
+
+    remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+
+    if (remaining_size < max_size) {
+        migration_bitmap_sync();
+        remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+    }
+    return remaining_size;
+}
+
 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
 {
     int ret, rc = 0;
@@ -685,7 +757,7 @@ static inline void *host_from_stream_offset(QEMUFile *f,
     qemu_get_buffer(f, (uint8_t *)id, len);
     id[len] = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (!strncmp(id, block->idstr, sizeof(id)))
             return memory_region_get_ram_ptr(block->mr) + offset;
     }
@@ -729,7 +801,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                     id[len] = 0;
                     length = qemu_get_be64(f);
 
-                    QLIST_FOREACH(block, &ram_list.blocks, next) {
+                    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
                         if (!strncmp(id, block->idstr, sizeof(id))) {
                             if (block->length != length) {
                                 ret =  -EINVAL;
@@ -764,7 +836,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
             memset(host, ch, TARGET_PAGE_SIZE);
 #ifndef _WIN32
             if (ch == 0 &&
-                (!kvm_enabled() || kvm_has_sync_mmu())) {
+                (!kvm_enabled() || kvm_has_sync_mmu()) &&
+                getpagesize() <= TARGET_PAGE_SIZE) {
                 qemu_madvise(host, TARGET_PAGE_SIZE, QEMU_MADV_DONTNEED);
             }
 #endif
@@ -799,8 +872,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
     } while (!(flags & RAM_SAVE_FLAG_EOS));
 
 done:
-    DPRINTF("Completed load of VM with exit code %d seq iteration " PRIu64 "\n",
-            ret, seq_iter);
+    DPRINTF("Completed load of VM with exit code %d seq iteration "
+            "%" PRIu64 "\n", ret, seq_iter);
     return ret;
 }
 
@@ -808,6 +881,7 @@ SaveVMHandlers savevm_ram_handlers = {
     .save_live_setup = ram_save_setup,
     .save_live_iterate = ram_save_iterate,
     .save_live_complete = ram_save_complete,
+    .save_live_pending = ram_save_pending,
     .load_state = ram_load,
     .cancel = ram_migration_cancel,
 };
@@ -922,11 +996,16 @@ void select_soundhw(const char *optarg)
     if (is_help_option(optarg)) {
     show_valid_cards:
 
+#ifdef HAS_AUDIO_CHOICE
         printf("Valid sound card names (comma separated):\n");
         for (c = soundhw; c->name; ++c) {
             printf ("%-11s %s\n", c->name, c->descr);
         }
         printf("\n-soundhw all will enable all of the above\n");
+#else
+        printf("Machine has no user-selectable audio hardware "
+               "(it may or may not have always-present audio hardware).\n");
+#endif
         exit(!is_help_option(optarg));
     }
     else {
diff --git a/async.c b/async.c
index 85cc6410c50..72d268ae352 100644
--- a/async.c
+++ b/async.c
@@ -23,16 +23,14 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-aio.h"
-#include "main-loop.h"
-
-/* Anchor of the list of Bottom Halves belonging to the context */
-static struct QEMUBH *first_bh;
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 /***********************************************************/
 /* bottom halves (can be seen as timers which expire ASAP) */
 
 struct QEMUBH {
+    AioContext *ctx;
     QEMUBHFunc *cb;
     void *opaque;
     QEMUBH *next;
@@ -41,27 +39,27 @@ struct QEMUBH {
     bool deleted;
 };
 
-QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque)
+QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
 {
     QEMUBH *bh;
     bh = g_malloc0(sizeof(QEMUBH));
+    bh->ctx = ctx;
     bh->cb = cb;
     bh->opaque = opaque;
-    bh->next = first_bh;
-    first_bh = bh;
+    bh->next = ctx->first_bh;
+    ctx->first_bh = bh;
     return bh;
 }
 
-int qemu_bh_poll(void)
+int aio_bh_poll(AioContext *ctx)
 {
     QEMUBH *bh, **bhp, *next;
     int ret;
-    static int nesting = 0;
 
-    nesting++;
+    ctx->walking_bh++;
 
     ret = 0;
-    for (bh = first_bh; bh; bh = next) {
+    for (bh = ctx->first_bh; bh; bh = next) {
         next = bh->next;
         if (!bh->deleted && bh->scheduled) {
             bh->scheduled = 0;
@@ -72,11 +70,11 @@ int qemu_bh_poll(void)
         }
     }
 
-    nesting--;
+    ctx->walking_bh--;
 
     /* remove deleted bhs */
-    if (!nesting) {
-        bhp = &first_bh;
+    if (!ctx->walking_bh) {
+        bhp = &ctx->first_bh;
         while (*bhp) {
             bh = *bhp;
             if (bh->deleted) {
@@ -105,8 +103,7 @@ void qemu_bh_schedule(QEMUBH *bh)
         return;
     bh->scheduled = 1;
     bh->idle = 0;
-    /* stop the currently executing CPU to execute the BH ASAP */
-    qemu_notify_event();
+    aio_notify(bh->ctx);
 }
 
 void qemu_bh_cancel(QEMUBH *bh)
@@ -120,23 +117,101 @@ void qemu_bh_delete(QEMUBH *bh)
     bh->deleted = 1;
 }
 
-void qemu_bh_update_timeout(uint32_t *timeout)
+static gboolean
+aio_ctx_prepare(GSource *source, gint    *timeout)
 {
+    AioContext *ctx = (AioContext *) source;
     QEMUBH *bh;
 
-    for (bh = first_bh; bh; bh = bh->next) {
+    for (bh = ctx->first_bh; bh; bh = bh->next) {
         if (!bh->deleted && bh->scheduled) {
             if (bh->idle) {
                 /* idle bottom halves will be polled at least
                  * every 10ms */
-                *timeout = MIN(10, *timeout);
+                *timeout = 10;
             } else {
                 /* non-idle bottom halves will be executed
                  * immediately */
                 *timeout = 0;
-                break;
+                return true;
             }
         }
     }
+
+    return false;
+}
+
+static gboolean
+aio_ctx_check(GSource *source)
+{
+    AioContext *ctx = (AioContext *) source;
+    QEMUBH *bh;
+
+    for (bh = ctx->first_bh; bh; bh = bh->next) {
+        if (!bh->deleted && bh->scheduled) {
+            return true;
+	}
+    }
+    return aio_pending(ctx);
+}
+
+static gboolean
+aio_ctx_dispatch(GSource     *source,
+                 GSourceFunc  callback,
+                 gpointer     user_data)
+{
+    AioContext *ctx = (AioContext *) source;
+
+    assert(callback == NULL);
+    aio_poll(ctx, false);
+    return true;
+}
+
+static void
+aio_ctx_finalize(GSource     *source)
+{
+    AioContext *ctx = (AioContext *) source;
+
+    aio_set_event_notifier(ctx, &ctx->notifier, NULL, NULL);
+    event_notifier_cleanup(&ctx->notifier);
 }
 
+static GSourceFuncs aio_source_funcs = {
+    aio_ctx_prepare,
+    aio_ctx_check,
+    aio_ctx_dispatch,
+    aio_ctx_finalize
+};
+
+GSource *aio_get_g_source(AioContext *ctx)
+{
+    g_source_ref(&ctx->source);
+    return &ctx->source;
+}
+
+void aio_notify(AioContext *ctx)
+{
+    event_notifier_set(&ctx->notifier);
+}
+
+AioContext *aio_context_new(void)
+{
+    AioContext *ctx;
+    ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    event_notifier_init(&ctx->notifier, false);
+    aio_set_event_notifier(ctx, &ctx->notifier, 
+                           (EventNotifierHandler *)
+                           event_notifier_test_and_clear, NULL);
+
+    return ctx;
+}
+
+void aio_context_ref(AioContext *ctx)
+{
+    g_source_ref(&ctx->source);
+}
+
+void aio_context_unref(AioContext *ctx)
+{
+    g_source_unref(&ctx->source);
+}
diff --git a/audio/Makefile.objs b/audio/Makefile.objs
index 0f2932d1b38..d71a8772494 100644
--- a/audio/Makefile.objs
+++ b/audio/Makefile.objs
@@ -12,3 +12,6 @@ common-obj-$(CONFIG_WINWAVE) += winwaveaudio.o
 common-obj-$(CONFIG_AUDIO_PT_INT) += audio_pt_int.o
 common-obj-$(CONFIG_AUDIO_WIN_INT) += audio_win_int.o
 common-obj-y += wavcapture.o
+
+$(obj)/audio.o $(obj)/fmodaudio.o: QEMU_CFLAGS += $(FMOD_CFLAGS)
+$(obj)/sdlaudio.o: QEMU_CFLAGS += $(SDL_CFLAGS)
diff --git a/audio/alsaaudio.c b/audio/alsaaudio.c
index cb45b49c2ae..e4e54426310 100644
--- a/audio/alsaaudio.c
+++ b/audio/alsaaudio.c
@@ -23,7 +23,7 @@
  */
 #include <alsa/asoundlib.h>
 #include "qemu-common.h"
-#include "qemu-char.h"
+#include "qemu/main-loop.h"
 #include "audio.h"
 
 #if QEMU_GNUC_PREREQ(4, 3)
diff --git a/audio/audio.c b/audio/audio.c
index 1c7738930b6..02bb8861f89 100644
--- a/audio/audio.c
+++ b/audio/audio.c
@@ -23,9 +23,9 @@
  */
 #include "hw/hw.h"
 #include "audio.h"
-#include "monitor.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "monitor/monitor.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 #define AUDIO_CAP "audio"
 #include "audio_int.h"
@@ -828,8 +828,9 @@ static int audio_attach_capture (HWVoiceOut *hw)
         QLIST_INSERT_HEAD (&hw_cap->sw_head, sw, entries);
         QLIST_INSERT_HEAD (&hw->cap_head, sc, entries);
 #ifdef DEBUG_CAPTURE
-        asprintf (&sw->name, "for %p %d,%d,%d",
-                  hw, sw->info.freq, sw->info.bits, sw->info.nchannels);
+        sw->name = g_strdup_printf ("for %p %d,%d,%d",
+                                    hw, sw->info.freq, sw->info.bits,
+                                    sw->info.nchannels);
         dolog ("Added %s active = %d\n", sw->name, sw->active);
 #endif
         if (sw->active) {
diff --git a/audio/audio.h b/audio/audio.h
index a70fda97e32..e7ea39777e1 100644
--- a/audio/audio.h
+++ b/audio/audio.h
@@ -25,7 +25,7 @@
 #define QEMU_AUDIO_H
 
 #include "config-host.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef void (*audio_callback_fn) (void *opaque, int avail);
 
diff --git a/audio/audio_template.h b/audio/audio_template.h
index 519432a7c56..16f78807700 100644
--- a/audio/audio_template.h
+++ b/audio/audio_template.h
@@ -410,15 +410,15 @@ SW *glue (AUD_open_, TYPE) (
     SW *old_sw = NULL;
 #endif
 
-    ldebug ("open %s, freq %d, nchannels %d, fmt %d\n",
-            name, as->freq, as->nchannels, as->fmt);
-
     if (audio_bug (AUDIO_FUNC, !card || !name || !callback_fn || !as)) {
         dolog ("card=%p name=%p callback_fn=%p as=%p\n",
                card, name, callback_fn, as);
         goto fail;
     }
 
+    ldebug ("open %s, freq %d, nchannels %d, fmt %d\n",
+            name, as->freq, as->nchannels, as->fmt);
+
     if (audio_bug (AUDIO_FUNC, audio_validate_settings (as))) {
         audio_print_settings (as);
         goto fail;
diff --git a/audio/noaudio.c b/audio/noaudio.c
index 54958f8623f..9f23aa2cb3f 100644
--- a/audio/noaudio.c
+++ b/audio/noaudio.c
@@ -23,7 +23,7 @@
  */
 #include "qemu-common.h"
 #include "audio.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define AUDIO_CAP "noaudio"
 #include "audio_int.h"
diff --git a/audio/ossaudio.c b/audio/ossaudio.c
index df51b7cc587..00be9c91e3b 100644
--- a/audio/ossaudio.c
+++ b/audio/ossaudio.c
@@ -31,8 +31,8 @@
 #include <sys/soundcard.h>
 #endif
 #include "qemu-common.h"
-#include "host-utils.h"
-#include "qemu-char.h"
+#include "qemu/main-loop.h"
+#include "qemu/host-utils.h"
 #include "audio.h"
 
 #define AUDIO_CAP "oss"
diff --git a/audio/spiceaudio.c b/audio/spiceaudio.c
index 6f15591a74f..bc24557de46 100644
--- a/audio/spiceaudio.c
+++ b/audio/spiceaudio.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ui/qemu-spice.h"
 
 #define AUDIO_CAP "spice"
diff --git a/audio/wavaudio.c b/audio/wavaudio.c
index a449b5127e0..950fa8f19c5 100644
--- a/audio/wavaudio.c
+++ b/audio/wavaudio.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "audio.h"
 
 #define AUDIO_CAP "wav"
diff --git a/audio/wavcapture.c b/audio/wavcapture.c
index 4f785f5f49e..9d94623225b 100644
--- a/audio/wavcapture.c
+++ b/audio/wavcapture.c
@@ -1,5 +1,5 @@
 #include "hw/hw.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "audio.h"
 
 typedef struct {
diff --git a/audio/winwaveaudio.c b/audio/winwaveaudio.c
index 72babbf184a..8dbd145ca1f 100644
--- a/audio/winwaveaudio.c
+++ b/audio/winwaveaudio.c
@@ -1,7 +1,7 @@
 /* public domain */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "audio.h"
 
 #define AUDIO_CAP "winwave"
diff --git a/backends/Makefile.objs b/backends/Makefile.objs
new file mode 100644
index 00000000000..883676106b6
--- /dev/null
+++ b/backends/Makefile.objs
@@ -0,0 +1,2 @@
+common-obj-y += rng.o rng-egd.o
+common-obj-$(CONFIG_POSIX) += rng-random.o
diff --git a/backends/rng-egd.c b/backends/rng-egd.c
new file mode 100644
index 00000000000..5e012e9e304
--- /dev/null
+++ b/backends/rng-egd.c
@@ -0,0 +1,224 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng.h"
+#include "char/char.h"
+#include "qapi/qmp/qerror.h"
+#include "hw/qdev.h" /* just for DEFINE_PROP_CHR */
+
+#define TYPE_RNG_EGD "rng-egd"
+#define RNG_EGD(obj) OBJECT_CHECK(RngEgd, (obj), TYPE_RNG_EGD)
+
+typedef struct RngEgd
+{
+    RngBackend parent;
+
+    CharDriverState *chr;
+    char *chr_name;
+
+    GSList *requests;
+} RngEgd;
+
+typedef struct RngRequest
+{
+    EntropyReceiveFunc *receive_entropy;
+    uint8_t *data;
+    void *opaque;
+    size_t offset;
+    size_t size;
+} RngRequest;
+
+static void rng_egd_request_entropy(RngBackend *b, size_t size,
+                                    EntropyReceiveFunc *receive_entropy,
+                                    void *opaque)
+{
+    RngEgd *s = RNG_EGD(b);
+    RngRequest *req;
+
+    req = g_malloc(sizeof(*req));
+
+    req->offset = 0;
+    req->size = size;
+    req->receive_entropy = receive_entropy;
+    req->opaque = opaque;
+    req->data = g_malloc(req->size);
+
+    while (size > 0) {
+        uint8_t header[2];
+        uint8_t len = MIN(size, 255);
+
+        /* synchronous entropy request */
+        header[0] = 0x02;
+        header[1] = len;
+
+        qemu_chr_fe_write(s->chr, header, sizeof(header));
+
+        size -= len;
+    }
+
+    s->requests = g_slist_append(s->requests, req);
+}
+
+static void rng_egd_free_request(RngRequest *req)
+{
+    g_free(req->data);
+    g_free(req);
+}
+
+static int rng_egd_chr_can_read(void *opaque)
+{
+    RngEgd *s = RNG_EGD(opaque);
+    GSList *i;
+    int size = 0;
+
+    for (i = s->requests; i; i = i->next) {
+        RngRequest *req = i->data;
+        size += req->size - req->offset;
+    }
+
+    return size;
+}
+
+static void rng_egd_chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    RngEgd *s = RNG_EGD(opaque);
+
+    while (size > 0 && s->requests) {
+        RngRequest *req = s->requests->data;
+        int len = MIN(size, req->size - req->offset);
+
+        memcpy(req->data + req->offset, buf, len);
+        req->offset += len;
+        size -= len;
+
+        if (req->offset == req->size) {
+            s->requests = g_slist_remove_link(s->requests, s->requests);
+
+            req->receive_entropy(req->opaque, req->data, req->size);
+
+            rng_egd_free_request(req);
+        }
+    }
+}
+
+static void rng_egd_free_requests(RngEgd *s)
+{
+    GSList *i;
+
+    for (i = s->requests; i; i = i->next) {
+        rng_egd_free_request(i->data);
+    }
+
+    g_slist_free(s->requests);
+    s->requests = NULL;
+}
+
+static void rng_egd_cancel_requests(RngBackend *b)
+{
+    RngEgd *s = RNG_EGD(b);
+
+    /* We simply delete the list of pending requests.  If there is data in the 
+     * queue waiting to be read, this is okay, because there will always be
+     * more data than we requested originally
+     */
+    rng_egd_free_requests(s);
+}
+
+static void rng_egd_opened(RngBackend *b, Error **errp)
+{
+    RngEgd *s = RNG_EGD(b);
+
+    if (s->chr_name == NULL) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  "chardev", "a valid character device");
+        return;
+    }
+
+    s->chr = qemu_chr_find(s->chr_name);
+    if (s->chr == NULL) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, s->chr_name);
+        return;
+    }
+
+    /* FIXME we should resubmit pending requests when the CDS reconnects. */
+    qemu_chr_add_handlers(s->chr, rng_egd_chr_can_read, rng_egd_chr_read,
+                          NULL, s);
+}
+
+static void rng_egd_set_chardev(Object *obj, const char *value, Error **errp)
+{
+    RngBackend *b = RNG_BACKEND(obj);
+    RngEgd *s = RNG_EGD(b);
+
+    if (b->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+    } else {
+        g_free(s->chr_name);
+        s->chr_name = g_strdup(value);
+    }
+}
+
+static char *rng_egd_get_chardev(Object *obj, Error **errp)
+{
+    RngEgd *s = RNG_EGD(obj);
+
+    if (s->chr && s->chr->label) {
+        return g_strdup(s->chr->label);
+    }
+
+    return NULL;
+}
+
+static void rng_egd_init(Object *obj)
+{
+    object_property_add_str(obj, "chardev",
+                            rng_egd_get_chardev, rng_egd_set_chardev,
+                            NULL);
+}
+
+static void rng_egd_finalize(Object *obj)
+{
+    RngEgd *s = RNG_EGD(obj);
+
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, NULL, NULL, NULL, NULL);
+    }
+
+    g_free(s->chr_name);
+
+    rng_egd_free_requests(s);
+}
+
+static void rng_egd_class_init(ObjectClass *klass, void *data)
+{
+    RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
+
+    rbc->request_entropy = rng_egd_request_entropy;
+    rbc->cancel_requests = rng_egd_cancel_requests;
+    rbc->opened = rng_egd_opened;
+}
+
+static const TypeInfo rng_egd_info = {
+    .name = TYPE_RNG_EGD,
+    .parent = TYPE_RNG_BACKEND,
+    .instance_size = sizeof(RngEgd),
+    .class_init = rng_egd_class_init,
+    .instance_init = rng_egd_init,
+    .instance_finalize = rng_egd_finalize,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_egd_info);
+}
+
+type_init(register_types);
diff --git a/backends/rng-random.c b/backends/rng-random.c
new file mode 100644
index 00000000000..0d1108811d4
--- /dev/null
+++ b/backends/rng-random.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng-random.h"
+#include "qemu/rng.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/main-loop.h"
+
+struct RndRandom
+{
+    RngBackend parent;
+
+    int fd;
+    char *filename;
+
+    EntropyReceiveFunc *receive_func;
+    void *opaque;
+    size_t size;
+};
+
+/**
+ * A simple and incomplete backend to request entropy from /dev/random.
+ *
+ * This backend exposes an additional "filename" property that can be used to
+ * set the filename to use to open the backend.
+ */
+
+static void entropy_available(void *opaque)
+{
+    RndRandom *s = RNG_RANDOM(opaque);
+    uint8_t buffer[s->size];
+    ssize_t len;
+
+    len = read(s->fd, buffer, s->size);
+    g_assert(len != -1);
+
+    s->receive_func(s->opaque, buffer, len);
+    s->receive_func = NULL;
+
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+}
+
+static void rng_random_request_entropy(RngBackend *b, size_t size,
+                                        EntropyReceiveFunc *receive_entropy,
+                                        void *opaque)
+{
+    RndRandom *s = RNG_RANDOM(b);
+
+    if (s->receive_func) {
+        s->receive_func(s->opaque, NULL, 0);
+    }
+
+    s->receive_func = receive_entropy;
+    s->opaque = opaque;
+    s->size = size;
+
+    qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
+}
+
+static void rng_random_opened(RngBackend *b, Error **errp)
+{
+    RndRandom *s = RNG_RANDOM(b);
+
+    if (s->filename == NULL) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  "filename", "a valid filename");
+    } else {
+        s->fd = open(s->filename, O_RDONLY | O_NONBLOCK);
+
+        if (s->fd == -1) {
+            error_set(errp, QERR_OPEN_FILE_FAILED, s->filename);
+        }
+    }
+}
+
+static char *rng_random_get_filename(Object *obj, Error **errp)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    if (s->filename) {
+        return g_strdup(s->filename);
+    }
+
+    return NULL;
+}
+
+static void rng_random_set_filename(Object *obj, const char *filename,
+                                 Error **errp)
+{
+    RngBackend *b = RNG_BACKEND(obj);
+    RndRandom *s = RNG_RANDOM(obj);
+
+    if (b->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    if (s->filename) {
+        g_free(s->filename);
+    }
+
+    s->filename = g_strdup(filename);
+}
+
+static void rng_random_init(Object *obj)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    object_property_add_str(obj, "filename",
+                            rng_random_get_filename,
+                            rng_random_set_filename,
+                            NULL);
+
+    s->filename = g_strdup("/dev/random");
+}
+
+static void rng_random_finalize(Object *obj)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+
+    if (s->fd != -1) {
+        close(s->fd);
+    }
+
+    g_free(s->filename);
+}
+
+static void rng_random_class_init(ObjectClass *klass, void *data)
+{
+    RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
+
+    rbc->request_entropy = rng_random_request_entropy;
+    rbc->opened = rng_random_opened;
+}
+
+static const TypeInfo rng_random_info = {
+    .name = TYPE_RNG_RANDOM,
+    .parent = TYPE_RNG_BACKEND,
+    .instance_size = sizeof(RndRandom),
+    .class_init = rng_random_class_init,
+    .instance_init = rng_random_init,
+    .instance_finalize = rng_random_finalize,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_random_info);
+}
+
+type_init(register_types);
diff --git a/backends/rng.c b/backends/rng.c
new file mode 100644
index 00000000000..3d3389802e5
--- /dev/null
+++ b/backends/rng.c
@@ -0,0 +1,93 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng.h"
+#include "qapi/qmp/qerror.h"
+
+void rng_backend_request_entropy(RngBackend *s, size_t size,
+                                 EntropyReceiveFunc *receive_entropy,
+                                 void *opaque)
+{
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (k->request_entropy) {
+        k->request_entropy(s, size, receive_entropy, opaque);
+    }
+}
+
+void rng_backend_cancel_requests(RngBackend *s)
+{
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (k->cancel_requests) {
+        k->cancel_requests(s);
+    }
+}
+
+static bool rng_backend_prop_get_opened(Object *obj, Error **errp)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+
+    return s->opened;
+}
+
+void rng_backend_open(RngBackend *s, Error **errp)
+{
+    object_property_set_bool(OBJECT(s), true, "opened", errp);
+}
+
+static void rng_backend_prop_set_opened(Object *obj, bool value, Error **errp)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (value == s->opened) {
+        return;
+    }
+
+    if (!value && s->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    if (k->opened) {
+        k->opened(s, errp);
+    }
+
+    if (!error_is_set(errp)) {
+        s->opened = value;
+    }
+}
+
+static void rng_backend_init(Object *obj)
+{
+    object_property_add_bool(obj, "opened",
+                             rng_backend_prop_get_opened,
+                             rng_backend_prop_set_opened,
+                             NULL);
+}
+
+static const TypeInfo rng_backend_info = {
+    .name = TYPE_RNG_BACKEND,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(RngBackend),
+    .instance_init = rng_backend_init,
+    .class_size = sizeof(RngBackendClass),
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_backend_info);
+}
+
+type_init(register_types);
diff --git a/balloon.c b/balloon.c
index e02ab1c8849..e321f2c6886 100644
--- a/balloon.c
+++ b/balloon.c
@@ -24,13 +24,13 @@
  * THE SOFTWARE.
  */
 
-#include "monitor.h"
-#include "cpu-common.h"
-#include "kvm.h"
-#include "balloon.h"
+#include "monitor/monitor.h"
+#include "exec/cpu-common.h"
+#include "sysemu/kvm.h"
+#include "sysemu/balloon.h"
 #include "trace.h"
 #include "qmp-commands.h"
-#include "qjson.h"
+#include "qapi/qmp/qjson.h"
 
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
diff --git a/block-migration.c b/block-migration.c
index 7def8ab1977..9ac7de6d78f 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -14,16 +14,17 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "hw/hw.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
-#include "block-migration.h"
-#include "migration.h"
-#include "blockdev.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+#include "migration/block.h"
+#include "migration/migration.h"
+#include "sysemu/blockdev.h"
 #include <assert.h>
 
-#define BLOCK_SIZE (BDRV_SECTORS_PER_DIRTY_CHUNK << BDRV_SECTOR_BITS)
+#define BLOCK_SIZE                       (1 << 20)
+#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
 
 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
 #define BLK_MIG_FLAG_EOS                0x02
@@ -77,9 +78,7 @@ typedef struct BlkMigState {
     int64_t total_sector_sum;
     int prev_progress;
     int bulk_completed;
-    long double total_time;
     long double prev_time_offset;
-    int reads;
 } BlkMigState;
 
 static BlkMigState block_mig_state;
@@ -132,12 +131,6 @@ uint64_t blk_mig_bytes_total(void)
     return sum << BDRV_SECTOR_BITS;
 }
 
-static inline long double compute_read_bwidth(void)
-{
-    assert(block_mig_state.total_time != 0);
-    return (block_mig_state.reads / block_mig_state.total_time) * BLOCK_SIZE;
-}
-
 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 {
     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
@@ -191,8 +184,6 @@ static void blk_mig_read_cb(void *opaque, int ret)
 
     blk->ret = ret;
 
-    block_mig_state.reads++;
-    block_mig_state.total_time += (curr_time - block_mig_state.prev_time_offset);
     block_mig_state.prev_time_offset = curr_time;
 
     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
@@ -264,7 +255,7 @@ static void set_dirty_tracking(int enable)
     BlkMigDevState *bmds;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        bdrv_set_dirty_tracking(bmds->bs, enable);
+        bdrv_set_dirty_tracking(bmds->bs, enable ? BLOCK_SIZE : 0);
     }
 }
 
@@ -310,8 +301,6 @@ static void init_blk_migration(QEMUFile *f)
     block_mig_state.total_sector_sum = 0;
     block_mig_state.prev_progress = -1;
     block_mig_state.bulk_completed = 0;
-    block_mig_state.total_time = 0;
-    block_mig_state.reads = 0;
 
     bdrv_iterate(init_blk_migration_it, NULL);
 }
@@ -423,20 +412,23 @@ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 
 error:
     DPRINTF("Error reading sector %" PRId64 "\n", sector);
-    qemu_file_set_error(f, ret);
     g_free(blk->buf);
     g_free(blk);
-    return 0;
+    return ret;
 }
 
+/* return value:
+ * 0: too much data for max_downtime
+ * 1: few enough data for max_downtime
+*/
 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 {
     BlkMigDevState *bmds;
-    int ret = 0;
+    int ret = 1;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        if (mig_save_device_dirty(f, bmds, is_async) == 0) {
-            ret = 1;
+        ret = mig_save_device_dirty(f, bmds, is_async);
+        if (ret <= 0) {
             break;
         }
     }
@@ -444,9 +436,10 @@ static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
     return ret;
 }
 
-static void flush_blks(QEMUFile* f)
+static int flush_blks(QEMUFile *f)
 {
     BlkMigBlock *blk;
+    int ret = 0;
 
     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
@@ -457,7 +450,7 @@ static void flush_blks(QEMUFile* f)
             break;
         }
         if (blk->ret < 0) {
-            qemu_file_set_error(f, blk->ret);
+            ret = blk->ret;
             break;
         }
         blk_send(f, blk);
@@ -474,6 +467,7 @@ static void flush_blks(QEMUFile* f)
     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
             block_mig_state.submitted, block_mig_state.read_done,
             block_mig_state.transferred);
+    return ret;
 }
 
 static int64_t get_remaining_dirty(void)
@@ -485,33 +479,7 @@ static int64_t get_remaining_dirty(void)
         dirty += bdrv_get_dirty_count(bmds->bs);
     }
 
-    return dirty * BLOCK_SIZE;
-}
-
-static int is_stage2_completed(void)
-{
-    int64_t remaining_dirty;
-    long double bwidth;
-
-    if (block_mig_state.bulk_completed == 1) {
-
-        remaining_dirty = get_remaining_dirty();
-        if (remaining_dirty == 0) {
-            return 1;
-        }
-
-        bwidth = compute_read_bwidth();
-
-        if ((remaining_dirty / bwidth) <=
-            migrate_max_downtime()) {
-            /* finish stage2 because we think that we can finish remaining work
-               below max_downtime */
-
-            return 1;
-        }
-    }
-
-    return 0;
+    return dirty << BDRV_SECTOR_BITS;
 }
 
 static void blk_mig_cleanup(void)
@@ -519,6 +487,8 @@ static void blk_mig_cleanup(void)
     BlkMigDevState *bmds;
     BlkMigBlock *blk;
 
+    bdrv_drain_all();
+
     set_dirty_tracking(0);
 
     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
@@ -553,9 +523,7 @@ static int block_save_setup(QEMUFile *f, void *opaque)
     /* start track dirty blocks */
     set_dirty_tracking(1);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -575,9 +543,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -596,16 +562,19 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
                 block_mig_state.bulk_completed = 1;
             }
         } else {
-            if (blk_mig_save_dirty_block(f, 1) == 0) {
+            ret = blk_mig_save_dirty_block(f, 1);
+            if (ret != 0) {
                 /* no more dirty blocks */
                 break;
             }
         }
     }
+    if (ret) {
+        blk_mig_cleanup();
+        return ret;
+    }
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -613,7 +582,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
 
     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 
-    return is_stage2_completed();
+    return 0;
 }
 
 static int block_save_complete(QEMUFile *f, void *opaque)
@@ -623,9 +592,7 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     DPRINTF("Enter save live complete submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -637,18 +604,16 @@ static int block_save_complete(QEMUFile *f, void *opaque)
        all async read completed */
     assert(block_mig_state.submitted == 0);
 
-    while (blk_mig_save_dirty_block(f, 0) != 0) {
-        /* Do nothing */
-    }
-    blk_mig_cleanup();
-
-    /* report completion */
-    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
+    do {
+        ret = blk_mig_save_dirty_block(f, 0);
+    } while (ret == 0);
 
-    ret = qemu_file_get_error(f);
+    blk_mig_cleanup();
     if (ret) {
         return ret;
     }
+    /* report completion */
+    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 
     DPRINTF("Block migration completed\n");
 
@@ -657,6 +622,14 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     return 0;
 }
 
+static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+{
+
+    DPRINTF("Enter save live pending  %ld\n", get_remaining_dirty());
+
+    return get_remaining_dirty();
+}
+
 static int block_load(QEMUFile *f, void *opaque, int version_id)
 {
     static int banner_printed;
@@ -753,6 +726,7 @@ SaveVMHandlers savevm_block_handlers = {
     .save_live_setup = block_save_setup,
     .save_live_iterate = block_save_iterate,
     .save_live_complete = block_save_complete,
+    .save_live_pending = block_save_pending,
     .load_state = block_load,
     .cancel = block_migration_cancel,
     .is_active = block_is_active,
diff --git a/block.c b/block.c
index 470bdcc1f67..ba67c0def2e 100644
--- a/block.c
+++ b/block.c
@@ -24,13 +24,16 @@
 #include "config-host.h"
 #include "qemu-common.h"
 #include "trace.h"
-#include "monitor.h"
-#include "block_int.h"
-#include "module.h"
-#include "qjson.h"
-#include "qemu-coroutine.h"
+#include "monitor/monitor.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qjson.h"
+#include "sysemu/sysemu.h"
+#include "qemu/notify.h"
+#include "block/coroutine.h"
 #include "qmp-commands.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #ifdef CONFIG_BSD
 #include <sys/types.h>
@@ -152,10 +155,6 @@ void bdrv_io_limits_enable(BlockDriverState *bs)
 {
     qemu_co_queue_init(&bs->throttled_reqs);
     bs->block_timer = qemu_new_timer_ns(vm_clock, bdrv_block_timer, bs);
-    bs->slice_time  = 5 * BLOCK_IO_SLICE_TIME;
-    bs->slice_start = qemu_get_clock_ns(vm_clock);
-    bs->slice_end   = bs->slice_start + bs->slice_time;
-    memset(&bs->io_base, 0, sizeof(bs->io_base));
     bs->io_limits_enabled = true;
 }
 
@@ -310,9 +309,16 @@ BlockDriverState *bdrv_new(const char *device_name)
         QTAILQ_INSERT_TAIL(&bdrv_states, bs, list);
     }
     bdrv_iostatus_disable(bs);
+    notifier_list_init(&bs->close_notifiers);
+
     return bs;
 }
 
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
+{
+    notifier_list_add(&bs->close_notifiers, notify);
+}
+
 BlockDriver *bdrv_find_format(const char *format_name)
 {
     BlockDriver *drv1;
@@ -377,7 +383,8 @@ int bdrv_create(BlockDriver *drv, const char* filename,
     };
 
     if (!drv->bdrv_create) {
-        return -ENOTSUP;
+        ret = -ENOTSUP;
+        goto out;
     }
 
     if (qemu_in_coroutine()) {
@@ -392,8 +399,9 @@ int bdrv_create(BlockDriver *drv, const char* filename,
     }
 
     ret = cco.ret;
-    g_free(cco.filename);
 
+out:
+    g_free(cco.filename);
     return ret;
 }
 
@@ -433,7 +441,11 @@ int get_tmp_filename(char *filename, int size)
         return -EOVERFLOW;
     }
     fd = mkstemp(filename);
-    if (fd < 0 || close(fd)) {
+    if (fd < 0) {
+        return -errno;
+    }
+    if (close(fd) != 0) {
+        unlink(filename);
         return -errno;
     }
     return 0;
@@ -502,22 +514,16 @@ BlockDriver *bdrv_find_protocol(const char *filename)
     return NULL;
 }
 
-static int find_image_format(const char *filename, BlockDriver **pdrv)
+static int find_image_format(BlockDriverState *bs, const char *filename,
+                             BlockDriver **pdrv)
 {
-    int ret, score, score_max;
+    int score, score_max;
     BlockDriver *drv1, *drv;
     uint8_t buf[2048];
-    BlockDriverState *bs;
-
-    ret = bdrv_file_open(&bs, filename, 0);
-    if (ret < 0) {
-        *pdrv = NULL;
-        return ret;
-    }
+    int ret = 0;
 
     /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
-    if (bs->sg || !bdrv_is_inserted(bs)) {
-        bdrv_delete(bs);
+    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
         drv = bdrv_find_format("raw");
         if (!drv) {
             ret = -ENOENT;
@@ -527,7 +533,6 @@ static int find_image_format(const char *filename, BlockDriver **pdrv)
     }
 
     ret = bdrv_pread(bs, 0, buf, sizeof(buf));
-    bdrv_delete(bs);
     if (ret < 0) {
         *pdrv = NULL;
         return ret;
@@ -618,10 +623,31 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
     bs->copy_on_read--;
 }
 
+static int bdrv_open_flags(BlockDriverState *bs, int flags)
+{
+    int open_flags = flags | BDRV_O_CACHE_WB;
+
+    /*
+     * Clear flags that are internal to the block layer before opening the
+     * image.
+     */
+    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+    /*
+     * Snapshots should be writable.
+     */
+    if (bs->is_temporary) {
+        open_flags |= BDRV_O_RDWR;
+    }
+
+    return open_flags;
+}
+
 /*
  * Common part for opening disk images and files
  */
-static int bdrv_open_common(BlockDriverState *bs, const char *filename,
+static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
+    const char *filename,
     int flags, BlockDriver *drv)
 {
     int ret, open_flags;
@@ -649,31 +675,22 @@ static int bdrv_open_common(BlockDriverState *bs, const char *filename,
     bs->opaque = g_malloc0(drv->instance_size);
 
     bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
-    open_flags = flags | BDRV_O_CACHE_WB;
-
-    /*
-     * Clear flags that are internal to the block layer before opening the
-     * image.
-     */
-    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
-
-    /*
-     * Snapshots should be writable.
-     */
-    if (bs->is_temporary) {
-        open_flags |= BDRV_O_RDWR;
-    }
+    open_flags = bdrv_open_flags(bs, flags);
 
-    bs->keep_read_only = bs->read_only = !(open_flags & BDRV_O_RDWR);
+    bs->read_only = !(open_flags & BDRV_O_RDWR);
 
     /* Open the image, either directly or using a protocol */
     if (drv->bdrv_file_open) {
-        ret = drv->bdrv_file_open(bs, filename, open_flags);
-    } else {
-        ret = bdrv_file_open(&bs->file, filename, open_flags);
-        if (ret >= 0) {
-            ret = drv->bdrv_open(bs, open_flags);
+        if (file != NULL) {
+            bdrv_swap(file, bs);
+            ret = 0;
+        } else {
+            ret = drv->bdrv_file_open(bs, filename, open_flags);
         }
+    } else {
+        assert(file != NULL);
+        bs->file = file;
+        ret = drv->bdrv_open(bs, open_flags);
     }
 
     if (ret < 0) {
@@ -693,10 +710,7 @@ static int bdrv_open_common(BlockDriverState *bs, const char *filename,
     return 0;
 
 free_and_fail:
-    if (bs->file) {
-        bdrv_delete(bs->file);
-        bs->file = NULL;
-    }
+    bs->file = NULL;
     g_free(bs->opaque);
     bs->opaque = NULL;
     bs->drv = NULL;
@@ -718,7 +732,7 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags)
     }
 
     bs = bdrv_new("");
-    ret = bdrv_open_common(bs, filename, flags, drv);
+    ret = bdrv_open_common(bs, NULL, filename, flags, drv);
     if (ret < 0) {
         bdrv_delete(bs);
         return ret;
@@ -728,6 +742,42 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags)
     return 0;
 }
 
+int bdrv_open_backing_file(BlockDriverState *bs)
+{
+    char backing_filename[PATH_MAX];
+    int back_flags, ret;
+    BlockDriver *back_drv = NULL;
+
+    if (bs->backing_hd != NULL) {
+        return 0;
+    }
+
+    bs->open_flags &= ~BDRV_O_NO_BACKING;
+    if (bs->backing_file[0] == '\0') {
+        return 0;
+    }
+
+    bs->backing_hd = bdrv_new("");
+    bdrv_get_full_backing_filename(bs, backing_filename,
+                                   sizeof(backing_filename));
+
+    if (bs->backing_format[0] != '\0') {
+        back_drv = bdrv_find_format(bs->backing_format);
+    }
+
+    /* backing files always opened read-only */
+    back_flags = bs->open_flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT);
+
+    ret = bdrv_open(bs->backing_hd, backing_filename, back_flags, back_drv);
+    if (ret < 0) {
+        bdrv_delete(bs->backing_hd);
+        bs->backing_hd = NULL;
+        bs->open_flags |= BDRV_O_NO_BACKING;
+        return ret;
+    }
+    return 0;
+}
+
 /*
  * Opens a disk image (raw, qcow2, vmdk, ...)
  */
@@ -735,7 +785,9 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
               BlockDriver *drv)
 {
     int ret;
-    char tmp_filename[PATH_MAX];
+    /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
+    char tmp_filename[PATH_MAX + 1];
+    BlockDriverState *file = NULL;
 
     if (flags & BDRV_O_SNAPSHOT) {
         BlockDriverState *bs1;
@@ -795,9 +847,19 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
         bs->is_temporary = 1;
     }
 
+    /* Open image file without format layer */
+    if (flags & BDRV_O_RDWR) {
+        flags |= BDRV_O_ALLOW_RDWR;
+    }
+
+    ret = bdrv_file_open(&file, filename, bdrv_open_flags(bs, flags));
+    if (ret < 0) {
+        return ret;
+    }
+
     /* Find the right image format driver */
     if (!drv) {
-        ret = find_image_format(filename, &drv);
+        ret = find_image_format(file, filename, &drv);
     }
 
     if (!drv) {
@@ -805,40 +867,23 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
     }
 
     /* Open the image */
-    ret = bdrv_open_common(bs, filename, flags, drv);
+    ret = bdrv_open_common(bs, file, filename, flags, drv);
     if (ret < 0) {
         goto unlink_and_fail;
     }
 
-    /* If there is a backing file, use it */
-    if ((flags & BDRV_O_NO_BACKING) == 0 && bs->backing_file[0] != '\0') {
-        char backing_filename[PATH_MAX];
-        int back_flags;
-        BlockDriver *back_drv = NULL;
-
-        bs->backing_hd = bdrv_new("");
-        bdrv_get_full_backing_filename(bs, backing_filename,
-                                       sizeof(backing_filename));
-
-        if (bs->backing_format[0] != '\0') {
-            back_drv = bdrv_find_format(bs->backing_format);
-        }
-
-        /* backing files always opened read-only */
-        back_flags =
-            flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+    if (bs->file != file) {
+        bdrv_delete(file);
+        file = NULL;
+    }
 
-        ret = bdrv_open(bs->backing_hd, backing_filename, back_flags, back_drv);
+    /* If there is a backing file, use it */
+    if ((flags & BDRV_O_NO_BACKING) == 0) {
+        ret = bdrv_open_backing_file(bs);
         if (ret < 0) {
             bdrv_close(bs);
             return ret;
         }
-        if (bs->is_temporary) {
-            bs->backing_hd->keep_read_only = !(flags & BDRV_O_RDWR);
-        } else {
-            /* base image inherits from "parent" */
-            bs->backing_hd->keep_read_only = bs->keep_read_only;
-        }
     }
 
     if (!bdrv_key_required(bs)) {
@@ -853,21 +898,257 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
     return 0;
 
 unlink_and_fail:
+    if (file != NULL) {
+        bdrv_delete(file);
+    }
     if (bs->is_temporary) {
         unlink(filename);
     }
     return ret;
 }
 
+typedef struct BlockReopenQueueEntry {
+     bool prepared;
+     BDRVReopenState state;
+     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+} BlockReopenQueueEntry;
+
+/*
+ * Adds a BlockDriverState to a simple queue for an atomic, transactional
+ * reopen of multiple devices.
+ *
+ * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
+ * already performed, or alternatively may be NULL a new BlockReopenQueue will
+ * be created and initialized. This newly created BlockReopenQueue should be
+ * passed back in for subsequent calls that are intended to be of the same
+ * atomic 'set'.
+ *
+ * bs is the BlockDriverState to add to the reopen queue.
+ *
+ * flags contains the open flags for the associated bs
+ *
+ * returns a pointer to bs_queue, which is either the newly allocated
+ * bs_queue, or the existing bs_queue being used.
+ *
+ */
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags)
+{
+    assert(bs != NULL);
+
+    BlockReopenQueueEntry *bs_entry;
+    if (bs_queue == NULL) {
+        bs_queue = g_new0(BlockReopenQueue, 1);
+        QSIMPLEQ_INIT(bs_queue);
+    }
+
+    if (bs->file) {
+        bdrv_reopen_queue(bs_queue, bs->file, flags);
+    }
+
+    bs_entry = g_new0(BlockReopenQueueEntry, 1);
+    QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+
+    bs_entry->state.bs = bs;
+    bs_entry->state.flags = flags;
+
+    return bs_queue;
+}
+
+/*
+ * Reopen multiple BlockDriverStates atomically & transactionally.
+ *
+ * The queue passed in (bs_queue) must have been built up previous
+ * via bdrv_reopen_queue().
+ *
+ * Reopens all BDS specified in the queue, with the appropriate
+ * flags.  All devices are prepared for reopen, and failure of any
+ * device will cause all device changes to be abandonded, and intermediate
+ * data cleaned up.
+ *
+ * If all devices prepare successfully, then the changes are committed
+ * to all devices.
+ *
+ */
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
+{
+    int ret = -1;
+    BlockReopenQueueEntry *bs_entry, *next;
+    Error *local_err = NULL;
+
+    assert(bs_queue != NULL);
+
+    bdrv_drain_all();
+
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
+            error_propagate(errp, local_err);
+            goto cleanup;
+        }
+        bs_entry->prepared = true;
+    }
+
+    /* If we reach this point, we have success and just need to apply the
+     * changes
+     */
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        bdrv_reopen_commit(&bs_entry->state);
+    }
+
+    ret = 0;
+
+cleanup:
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+        if (ret && bs_entry->prepared) {
+            bdrv_reopen_abort(&bs_entry->state);
+        }
+        g_free(bs_entry);
+    }
+    g_free(bs_queue);
+    return ret;
+}
+
+
+/* Reopen a single BlockDriverState with the specified flags. */
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+
+    ret = bdrv_reopen_multiple(queue, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
+}
+
+
+/*
+ * Prepares a BlockDriverState for reopen. All changes are staged in the
+ * 'opaque' field of the BDRVReopenState, which is used and allocated by
+ * the block driver layer .bdrv_reopen_prepare()
+ *
+ * bs is the BlockDriverState to reopen
+ * flags are the new open flags
+ * queue is the reopen queue
+ *
+ * Returns 0 on success, non-zero on error.  On error errp will be set
+ * as well.
+ *
+ * On failure, bdrv_reopen_abort() will be called to clean up any data.
+ * It is the responsibility of the caller to then call the abort() or
+ * commit() for any other BDS that have been left in a prepare() state
+ *
+ */
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
+                        Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    assert(reopen_state->bs->drv != NULL);
+    drv = reopen_state->bs->drv;
+
+    /* if we are to stay read-only, do not allow permission change
+     * to r/w */
+    if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
+        reopen_state->flags & BDRV_O_RDWR) {
+        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
+                  reopen_state->bs->device_name);
+        goto error;
+    }
+
+
+    ret = bdrv_flush(reopen_state->bs);
+    if (ret) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
+                  strerror(-ret));
+        goto error;
+    }
+
+    if (drv->bdrv_reopen_prepare) {
+        ret = drv->bdrv_reopen_prepare(reopen_state, queue, &local_err);
+        if (ret) {
+            if (local_err != NULL) {
+                error_propagate(errp, local_err);
+            } else {
+                error_set(errp, QERR_OPEN_FILE_FAILED,
+                          reopen_state->bs->filename);
+            }
+            goto error;
+        }
+    } else {
+        /* It is currently mandatory to have a bdrv_reopen_prepare()
+         * handler for each supported drv. */
+        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+                  drv->format_name, reopen_state->bs->device_name,
+                 "reopening of file");
+        ret = -1;
+        goto error;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+
+/*
+ * Takes the staged changes for the reopen from bdrv_reopen_prepare(), and
+ * makes them final by swapping the staging BlockDriverState contents into
+ * the active BlockDriverState contents.
+ */
+void bdrv_reopen_commit(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    /* If there are any driver level actions to take */
+    if (drv->bdrv_reopen_commit) {
+        drv->bdrv_reopen_commit(reopen_state);
+    }
+
+    /* set BDS specific flags now */
+    reopen_state->bs->open_flags         = reopen_state->flags;
+    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
+                                              BDRV_O_CACHE_WB);
+    reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
+}
+
+/*
+ * Abort the reopen, and delete and free the staged changes in
+ * reopen_state
+ */
+void bdrv_reopen_abort(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    if (drv->bdrv_reopen_abort) {
+        drv->bdrv_reopen_abort(reopen_state);
+    }
+}
+
+
 void bdrv_close(BlockDriverState *bs)
 {
     bdrv_flush(bs);
-    if (bs->drv) {
-        if (bs->job) {
-            block_job_cancel_sync(bs->job);
-        }
-        bdrv_drain_all();
+    if (bs->job) {
+        block_job_cancel_sync(bs->job);
+    }
+    bdrv_drain_all();
+    notifier_list_notify(&bs->close_notifiers, bs);
 
+    if (bs->drv) {
         if (bs == bs_snapshots) {
             bs_snapshots = NULL;
         }
@@ -897,10 +1178,10 @@ void bdrv_close(BlockDriverState *bs)
             bdrv_delete(bs->file);
             bs->file = NULL;
         }
-
-        bdrv_dev_change_media_cb(bs, false);
     }
 
+    bdrv_dev_change_media_cb(bs, false);
+
     /*throttling disk I/O limits*/
     if (bs->io_limits_enabled) {
         bdrv_io_limits_disable(bs);
@@ -1005,7 +1286,6 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
     bs_dest->iostatus           = bs_src->iostatus;
 
     /* dirty bitmap */
-    bs_dest->dirty_count        = bs_src->dirty_count;
     bs_dest->dirty_bitmap       = bs_src->dirty_bitmap;
 
     /* job */
@@ -1152,7 +1432,8 @@ void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
 }
 
 void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               BlockQMPEventAction action, int is_read)
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read)
 {
     QObject *data;
     const char *action_str;
@@ -1175,7 +1456,7 @@ void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
                               bdrv->device_name,
                               action_str,
                               is_read ? "read" : "write");
-    monitor_protocol_event(QEVENT_BLOCK_IO_ERROR, data);
+    monitor_protocol_event(ev, data);
 
     qobject_decref(data);
 }
@@ -1265,13 +1546,11 @@ int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
 int bdrv_commit(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
-    BlockDriver *backing_drv;
     int64_t sector, total_sectors;
     int n, ro, open_flags;
-    int ret = 0, rw_ret = 0;
+    int ret = 0;
     uint8_t *buf;
-    char filename[1024];
-    BlockDriverState *bs_rw, *bs_ro;
+    char filename[PATH_MAX];
 
     if (!drv)
         return -ENOMEDIUM;
@@ -1280,42 +1559,19 @@ int bdrv_commit(BlockDriverState *bs)
         return -ENOTSUP;
     }
 
-    if (bs->backing_hd->keep_read_only) {
-        return -EACCES;
-    }
-
     if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
         return -EBUSY;
     }
 
-    backing_drv = bs->backing_hd->drv;
     ro = bs->backing_hd->read_only;
-    strncpy(filename, bs->backing_hd->filename, sizeof(filename));
+    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
+    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
     open_flags =  bs->backing_hd->open_flags;
 
     if (ro) {
-        /* re-open as RW */
-        bdrv_delete(bs->backing_hd);
-        bs->backing_hd = NULL;
-        bs_rw = bdrv_new("");
-        rw_ret = bdrv_open(bs_rw, filename, open_flags | BDRV_O_RDWR,
-            backing_drv);
-        if (rw_ret < 0) {
-            bdrv_delete(bs_rw);
-            /* try to re-open read-only */
-            bs_ro = bdrv_new("");
-            ret = bdrv_open(bs_ro, filename, open_flags & ~BDRV_O_RDWR,
-                backing_drv);
-            if (ret < 0) {
-                bdrv_delete(bs_ro);
-                /* drive not functional anymore */
-                bs->drv = NULL;
-                return ret;
-            }
-            bs->backing_hd = bs_ro;
-            return rw_ret;
+        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+            return -EACCES;
         }
-        bs->backing_hd = bs_rw;
     }
 
     total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
@@ -1352,20 +1608,8 @@ ro_cleanup:
     g_free(buf);
 
     if (ro) {
-        /* re-open as RO */
-        bdrv_delete(bs->backing_hd);
-        bs->backing_hd = NULL;
-        bs_ro = bdrv_new("");
-        ret = bdrv_open(bs_ro, filename, open_flags & ~BDRV_O_RDWR,
-            backing_drv);
-        if (ret < 0) {
-            bdrv_delete(bs_ro);
-            /* drive not functional anymore */
-            bs->drv = NULL;
-            return ret;
-        }
-        bs->backing_hd = bs_ro;
-        bs->backing_hd->keep_read_only = 0;
+        /* ignoring error return here */
+        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
     }
 
     return ret;
@@ -1429,10 +1673,10 @@ static void tracked_request_begin(BdrvTrackedRequest *req,
 /**
  * Round a region to cluster boundaries
  */
-static void round_to_clusters(BlockDriverState *bs,
-                              int64_t sector_num, int nb_sectors,
-                              int64_t *cluster_sector_num,
-                              int *cluster_nb_sectors)
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors)
 {
     BlockDriverInfo bdi;
 
@@ -1474,8 +1718,8 @@ static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
      * CoR read and write operations are atomic and guest writes cannot
      * interleave between them.
      */
-    round_to_clusters(bs, sector_num, nb_sectors,
-                      &cluster_sector_num, &cluster_nb_sectors);
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
 
     do {
         retry = false;
@@ -1528,6 +1772,149 @@ int bdrv_change_backing_file(BlockDriverState *bs,
     return ret;
 }
 
+/*
+ * Finds the image layer in the chain that has 'bs' as its backing file.
+ *
+ * active is the current topmost image.
+ *
+ * Returns NULL if bs is not found in active's image chain,
+ * or if active == bs.
+ */
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs)
+{
+    BlockDriverState *overlay = NULL;
+    BlockDriverState *intermediate;
+
+    assert(active != NULL);
+    assert(bs != NULL);
+
+    /* if bs is the same as active, then by definition it has no overlay
+     */
+    if (active == bs) {
+        return NULL;
+    }
+
+    intermediate = active;
+    while (intermediate->backing_hd) {
+        if (intermediate->backing_hd == bs) {
+            overlay = intermediate;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+
+    return overlay;
+}
+
+typedef struct BlkIntermediateStates {
+    BlockDriverState *bs;
+    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
+} BlkIntermediateStates;
+
+
+/*
+ * Drops images above 'base' up to and including 'top', and sets the image
+ * above 'top' to have base as its backing file.
+ *
+ * Requires that the overlay to 'top' is opened r/w, so that the backing file
+ * information in 'bs' can be properly updated.
+ *
+ * E.g., this will convert the following chain:
+ * bottom <- base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * bottom <- base <- active
+ *
+ * It is allowed for bottom==base, in which case it converts:
+ *
+ * base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * base <- active
+ *
+ * Error conditions:
+ *  if active == top, that is considered an error
+ *
+ */
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base)
+{
+    BlockDriverState *intermediate;
+    BlockDriverState *base_bs = NULL;
+    BlockDriverState *new_top_bs = NULL;
+    BlkIntermediateStates *intermediate_state, *next;
+    int ret = -EIO;
+
+    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
+    QSIMPLEQ_INIT(&states_to_delete);
+
+    if (!top->drv || !base->drv) {
+        goto exit;
+    }
+
+    new_top_bs = bdrv_find_overlay(active, top);
+
+    if (new_top_bs == NULL) {
+        /* we could not find the image above 'top', this is an error */
+        goto exit;
+    }
+
+    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+     * to do, no intermediate images */
+    if (new_top_bs->backing_hd == base) {
+        ret = 0;
+        goto exit;
+    }
+
+    intermediate = top;
+
+    /* now we will go down through the list, and add each BDS we find
+     * into our deletion queue, until we hit the 'base'
+     */
+    while (intermediate) {
+        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
+        intermediate_state->bs = intermediate;
+        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
+
+        if (intermediate->backing_hd == base) {
+            base_bs = intermediate->backing_hd;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+    if (base_bs == NULL) {
+        /* something went wrong, we did not end at the base. safely
+         * unravel everything, and exit with error */
+        goto exit;
+    }
+
+    /* success - we can delete the intermediate states, and link top->base */
+    ret = bdrv_change_backing_file(new_top_bs, base_bs->filename,
+                                   base_bs->drv ? base_bs->drv->format_name : "");
+    if (ret) {
+        goto exit;
+    }
+    new_top_bs->backing_hd = base_bs;
+
+
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        /* so that bdrv_close() does not recursively close the chain */
+        intermediate_state->bs->backing_hd = NULL;
+        bdrv_delete(intermediate_state->bs);
+    }
+    ret = 0;
+
+exit:
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        g_free(intermediate_state);
+    }
+    return ret;
+}
+
+
 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
                                    size_t size)
 {
@@ -1647,36 +2034,6 @@ int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
     return ret;
 }
 
-#define BITS_PER_LONG  (sizeof(unsigned long) * 8)
-
-static void set_dirty_bitmap(BlockDriverState *bs, int64_t sector_num,
-                             int nb_sectors, int dirty)
-{
-    int64_t start, end;
-    unsigned long val, idx, bit;
-
-    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
-    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    for (; start <= end; start++) {
-        idx = start / BITS_PER_LONG;
-        bit = start % BITS_PER_LONG;
-        val = bs->dirty_bitmap[idx];
-        if (dirty) {
-            if (!(val & (1UL << bit))) {
-                bs->dirty_count++;
-                val |= 1UL << bit;
-            }
-        } else {
-            if (val & (1UL << bit)) {
-                bs->dirty_count--;
-                val &= ~(1UL << bit);
-            }
-        }
-        bs->dirty_bitmap[idx] = val;
-    }
-}
-
 /* Return < 0 if error. Important errors are:
   -EIO         generic I/O error (may happen for all errors)
   -ENOMEDIUM   No media inserted.
@@ -1828,8 +2185,8 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
     /* Cover entire cluster so no additional backing file I/O is required when
      * allocating cluster in the image file.
      */
-    round_to_clusters(bs, sector_num, nb_sectors,
-                      &cluster_sector_num, &cluster_nb_sectors);
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
 
     trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
                                    cluster_sector_num, cluster_nb_sectors);
@@ -2026,7 +2383,7 @@ static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
     }
 
     if (bs->dirty_bitmap) {
-        set_dirty_bitmap(bs, sector_num, nb_sectors, 1);
+        bdrv_set_dirty(bs, sector_num, nb_sectors);
     }
 
     if (bs->wr_highest_sector < sector_num + nb_sectors - 1) {
@@ -2134,18 +2491,51 @@ void bdrv_set_io_limits(BlockDriverState *bs,
     bs->io_limits_enabled = bdrv_io_limits_enabled(bs);
 }
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockErrorAction on_read_error,
-                       BlockErrorAction on_write_error)
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error)
 {
     bs->on_read_error = on_read_error;
     bs->on_write_error = on_write_error;
 }
 
-BlockErrorAction bdrv_get_on_error(BlockDriverState *bs, int is_read)
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
 {
     return is_read ? bs->on_read_error : bs->on_write_error;
 }
 
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
+{
+    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BDRV_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BDRV_ACTION_IGNORE;
+    default:
+        abort();
+    }
+}
+
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error)
+{
+    assert(error >= 0);
+    bdrv_emit_qmp_error_event(bs, QEVENT_BLOCK_IO_ERROR, action, is_read);
+    if (action == BDRV_ACTION_STOP) {
+        vm_stop(RUN_STATE_IO_ERROR);
+        bdrv_iostatus_set_err(bs, error);
+    }
+}
+
 int bdrv_is_read_only(BlockDriverState *bs)
 {
     return bs->read_only;
@@ -2164,6 +2554,13 @@ int bdrv_enable_write_cache(BlockDriverState *bs)
 void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
 {
     bs->enable_write_cache = wce;
+
+    /* so a reopen() will preserve wce */
+    if (wce) {
+        bs->open_flags |= BDRV_O_CACHE_WB;
+    } else {
+        bs->open_flags &= ~BDRV_O_CACHE_WB;
+    }
 }
 
 int bdrv_is_encrypted(BlockDriverState *bs)
@@ -2414,76 +2811,83 @@ int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
     return 0;
 }
 
-BlockInfoList *qmp_query_block(Error **errp)
+BlockInfo *bdrv_query_info(BlockDriverState *bs)
 {
-    BlockInfoList *head = NULL, *cur_item = NULL;
-    BlockDriverState *bs;
+    BlockInfo *info = g_malloc0(sizeof(*info));
+    info->device = g_strdup(bs->device_name);
+    info->type = g_strdup("unknown");
+    info->locked = bdrv_dev_is_medium_locked(bs);
+    info->removable = bdrv_dev_has_removable_media(bs);
 
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        BlockInfoList *info = g_malloc0(sizeof(*info));
+    if (bdrv_dev_has_removable_media(bs)) {
+        info->has_tray_open = true;
+        info->tray_open = bdrv_dev_is_tray_open(bs);
+    }
 
-        info->value = g_malloc0(sizeof(*info->value));
-        info->value->device = g_strdup(bs->device_name);
-        info->value->type = g_strdup("unknown");
-        info->value->locked = bdrv_dev_is_medium_locked(bs);
-        info->value->removable = bdrv_dev_has_removable_media(bs);
+    if (bdrv_iostatus_is_enabled(bs)) {
+        info->has_io_status = true;
+        info->io_status = bs->iostatus;
+    }
+
+    if (bs->dirty_bitmap) {
+        info->has_dirty = true;
+        info->dirty = g_malloc0(sizeof(*info->dirty));
+        info->dirty->count = bdrv_get_dirty_count(bs) * BDRV_SECTOR_SIZE;
+        info->dirty->granularity =
+            ((int64_t) BDRV_SECTOR_SIZE << hbitmap_granularity(bs->dirty_bitmap));
+    }
 
-        if (bdrv_dev_has_removable_media(bs)) {
-            info->value->has_tray_open = true;
-            info->value->tray_open = bdrv_dev_is_tray_open(bs);
+    if (bs->drv) {
+        info->has_inserted = true;
+        info->inserted = g_malloc0(sizeof(*info->inserted));
+        info->inserted->file = g_strdup(bs->filename);
+        info->inserted->ro = bs->read_only;
+        info->inserted->drv = g_strdup(bs->drv->format_name);
+        info->inserted->encrypted = bs->encrypted;
+        info->inserted->encryption_key_missing = bdrv_key_required(bs);
+
+        if (bs->backing_file[0]) {
+            info->inserted->has_backing_file = true;
+            info->inserted->backing_file = g_strdup(bs->backing_file);
         }
 
-        if (bdrv_iostatus_is_enabled(bs)) {
-            info->value->has_io_status = true;
-            info->value->io_status = bs->iostatus;
+        info->inserted->backing_file_depth = bdrv_get_backing_file_depth(bs);
+
+        if (bs->io_limits_enabled) {
+            info->inserted->bps =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
+            info->inserted->bps_rd =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_READ];
+            info->inserted->bps_wr =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_WRITE];
+            info->inserted->iops =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
+            info->inserted->iops_rd =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_READ];
+            info->inserted->iops_wr =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_WRITE];
         }
+    }
+    return info;
+}
 
-        if (bs->drv) {
-            info->value->has_inserted = true;
-            info->value->inserted = g_malloc0(sizeof(*info->value->inserted));
-            info->value->inserted->file = g_strdup(bs->filename);
-            info->value->inserted->ro = bs->read_only;
-            info->value->inserted->drv = g_strdup(bs->drv->format_name);
-            info->value->inserted->encrypted = bs->encrypted;
-            info->value->inserted->encryption_key_missing = bdrv_key_required(bs);
-            if (bs->backing_file[0]) {
-                info->value->inserted->has_backing_file = true;
-                info->value->inserted->backing_file = g_strdup(bs->backing_file);
-            }
+BlockInfoList *qmp_query_block(Error **errp)
+{
+    BlockInfoList *head = NULL, **p_next = &head;
+    BlockDriverState *bs;
 
-            info->value->inserted->backing_file_depth =
-                bdrv_get_backing_file_depth(bs);
-
-            if (bs->io_limits_enabled) {
-                info->value->inserted->bps =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
-                info->value->inserted->bps_rd =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_READ];
-                info->value->inserted->bps_wr =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_WRITE];
-                info->value->inserted->iops =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
-                info->value->inserted->iops_rd =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_READ];
-                info->value->inserted->iops_wr =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_WRITE];
-            }
-        }
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        BlockInfoList *info = g_malloc0(sizeof(*info));
+        info->value = bdrv_query_info(bs);
 
-        /* XXX: waiting for the qapi to support GSList */
-        if (!cur_item) {
-            head = cur_item = info;
-        } else {
-            cur_item->next = info;
-            cur_item = info;
-        }
+        *p_next = info;
+        p_next = &info->next;
     }
 
     return head;
 }
 
-/* Consider exposing this as a full fledged QMP command */
-static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
+BlockStats *bdrv_query_stats(const BlockDriverState *bs)
 {
     BlockStats *s;
 
@@ -2507,7 +2911,7 @@ static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
 
     if (bs->file) {
         s->has_parent = true;
-        s->parent = qmp_query_blockstat(bs->file, NULL);
+        s->parent = bdrv_query_stats(bs->file);
     }
 
     return s;
@@ -2515,20 +2919,15 @@ static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
 
 BlockStatsList *qmp_query_blockstats(Error **errp)
 {
-    BlockStatsList *head = NULL, *cur_item = NULL;
+    BlockStatsList *head = NULL, **p_next = &head;
     BlockDriverState *bs;
 
     QTAILQ_FOREACH(bs, &bdrv_states, list) {
         BlockStatsList *info = g_malloc0(sizeof(*info));
-        info->value = qmp_query_blockstat(bs, NULL);
+        info->value = bdrv_query_stats(bs);
 
-        /* XXX: waiting for the qapi to support GSList */
-        if (!cur_item) {
-            head = cur_item = info;
-        } else {
-            cur_item->next = info;
-            cur_item = info;
-        }
+        *p_next = info;
+        p_next = &info->next;
     }
 
     return head;
@@ -2561,9 +2960,7 @@ int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
     if (bdrv_check_request(bs, sector_num, nb_sectors))
         return -EIO;
 
-    if (bs->dirty_bitmap) {
-        set_dirty_bitmap(bs, sector_num, nb_sectors, 1);
-    }
+    assert(!bs->dirty_bitmap);
 
     return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
 }
@@ -2614,7 +3011,46 @@ void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
     }
 
     drv->bdrv_debug_event(bs, event);
+}
 
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                          const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
+        return bs->drv->bdrv_debug_breakpoint(bs, event, tag);
+    }
+
+    return -ENOTSUP;
+}
+
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_resume) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
+        return bs->drv->bdrv_debug_resume(bs, tag);
+    }
+
+    return -ENOTSUP;
+}
+
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
+        return bs->drv->bdrv_debug_is_suspended(bs, tag);
+    }
+
+    return false;
 }
 
 /**************************************************************/
@@ -2740,22 +3176,70 @@ int bdrv_snapshot_load_tmp(BlockDriverState *bs,
     return -ENOTSUP;
 }
 
+/* backing_file can either be relative, or absolute, or a protocol.  If it is
+ * relative, it must be relative to the chain.  So, passing in bs->filename
+ * from a BDS as backing_file should not be done, as that may be relative to
+ * the CWD rather than the chain. */
 BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
         const char *backing_file)
 {
-    if (!bs->drv) {
+    char *filename_full = NULL;
+    char *backing_file_full = NULL;
+    char *filename_tmp = NULL;
+    int is_protocol = 0;
+    BlockDriverState *curr_bs = NULL;
+    BlockDriverState *retval = NULL;
+
+    if (!bs || !bs->drv || !backing_file) {
         return NULL;
     }
 
-    if (bs->backing_hd) {
-        if (strcmp(bs->backing_file, backing_file) == 0) {
-            return bs->backing_hd;
+    filename_full     = g_malloc(PATH_MAX);
+    backing_file_full = g_malloc(PATH_MAX);
+    filename_tmp      = g_malloc(PATH_MAX);
+
+    is_protocol = path_has_protocol(backing_file);
+
+    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+
+        /* If either of the filename paths is actually a protocol, then
+         * compare unmodified paths; otherwise make paths relative */
+        if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
+            if (strcmp(backing_file, curr_bs->backing_file) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
         } else {
-            return bdrv_find_backing_image(bs->backing_hd, backing_file);
+            /* If not an absolute filename path, make it relative to the current
+             * image's filename path */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         backing_file);
+
+            /* We are going to compare absolute pathnames */
+            if (!realpath(filename_tmp, filename_full)) {
+                continue;
+            }
+
+            /* We need to make sure the backing filename we are comparing against
+             * is relative to the current image filename (or absolute) */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         curr_bs->backing_file);
+
+            if (!realpath(filename_tmp, backing_file_full)) {
+                continue;
+            }
+
+            if (strcmp(backing_file_full, filename_full) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
         }
     }
 
-    return NULL;
+    g_free(filename_full);
+    g_free(backing_file_full);
+    g_free(filename_tmp);
+    return retval;
 }
 
 int bdrv_get_backing_file_depth(BlockDriverState *bs)
@@ -2771,6 +3255,22 @@ int bdrv_get_backing_file_depth(BlockDriverState *bs)
     return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
 }
 
+BlockDriverState *bdrv_find_base(BlockDriverState *bs)
+{
+    BlockDriverState *curr_bs = NULL;
+
+    if (!bs) {
+        return NULL;
+    }
+
+    curr_bs = bs;
+
+    while (curr_bs->backing_hd) {
+        curr_bs = curr_bs->backing_hd;
+    }
+    return curr_bs;
+}
+
 #define NB_SUFFIXES 4
 
 char *get_human_readable_size(char *buf, int buf_size, int64_t size)
@@ -2804,11 +3304,7 @@ char *get_human_readable_size(char *buf, int buf_size, int64_t size)
 char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn)
 {
     char buf1[128], date_buf[128], clock_buf[128];
-#ifdef _WIN32
-    struct tm *ptm;
-#else
     struct tm tm;
-#endif
     time_t ti;
     int64_t secs;
 
@@ -2818,15 +3314,9 @@ char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn)
                  "ID", "TAG", "VM SIZE", "DATE", "VM CLOCK");
     } else {
         ti = sn->date_sec;
-#ifdef _WIN32
-        ptm = localtime(&ti);
-        strftime(date_buf, sizeof(date_buf),
-                 "%Y-%m-%d %H:%M:%S", ptm);
-#else
         localtime_r(&ti, &tm);
         strftime(date_buf, sizeof(date_buf),
                  "%Y-%m-%d %H:%M:%S", &tm);
-#endif
         secs = sn->vm_clock_nsec / 1000000000;
         snprintf(clock_buf, sizeof(clock_buf),
                  "%02d:%02d:%02d.%03d",
@@ -3044,7 +3534,7 @@ int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
 
 void bdrv_aio_cancel(BlockDriverAIOCB *acb)
 {
-    acb->pool->cancel(acb);
+    acb->aiocb_info->cancel(acb);
 }
 
 /* block I/O throttling */
@@ -3234,7 +3724,7 @@ static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb)
     qemu_aio_release(acb);
 }
 
-static AIOPool bdrv_em_aio_pool = {
+static const AIOCBInfo bdrv_em_aiocb_info = {
     .aiocb_size         = sizeof(BlockDriverAIOCBSync),
     .cancel             = bdrv_aio_cancel_em,
 };
@@ -3263,7 +3753,7 @@ static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
 {
     BlockDriverAIOCBSync *acb;
 
-    acb = qemu_aio_get(&bdrv_em_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
     acb->is_write = is_write;
     acb->qiov = qiov;
     acb->bounce = qemu_blockalign(bs, qiov->size);
@@ -3300,15 +3790,23 @@ typedef struct BlockDriverAIOCBCoroutine {
     BlockDriverAIOCB common;
     BlockRequest req;
     bool is_write;
+    bool *done;
     QEMUBH* bh;
 } BlockDriverAIOCBCoroutine;
 
 static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
 {
-    qemu_aio_flush();
+    BlockDriverAIOCBCoroutine *acb =
+        container_of(blockacb, BlockDriverAIOCBCoroutine, common);
+    bool done = false;
+
+    acb->done = &done;
+    while (!done) {
+        qemu_aio_wait();
+    }
 }
 
-static AIOPool bdrv_em_co_aio_pool = {
+static const AIOCBInfo bdrv_em_co_aiocb_info = {
     .aiocb_size         = sizeof(BlockDriverAIOCBCoroutine),
     .cancel             = bdrv_aio_co_cancel_em,
 };
@@ -3318,6 +3816,11 @@ static void bdrv_co_em_bh(void *opaque)
     BlockDriverAIOCBCoroutine *acb = opaque;
 
     acb->common.cb(acb->common.opaque, acb->req.error);
+
+    if (acb->done) {
+        *acb->done = true;
+    }
+
     qemu_bh_delete(acb->bh);
     qemu_aio_release(acb);
 }
@@ -3351,11 +3854,12 @@ static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
     Coroutine *co;
     BlockDriverAIOCBCoroutine *acb;
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
     acb->req.qiov = qiov;
     acb->is_write = is_write;
+    acb->done = NULL;
 
     co = qemu_coroutine_create(bdrv_co_do_rw);
     qemu_coroutine_enter(co, acb);
@@ -3381,7 +3885,9 @@ BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
     Coroutine *co;
     BlockDriverAIOCBCoroutine *acb;
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->done = NULL;
+
     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -3407,9 +3913,10 @@ BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
 
     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
+    acb->done = NULL;
     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -3427,18 +3934,13 @@ void bdrv_init_with_whitelist(void)
     bdrv_init();
 }
 
-void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
                    BlockDriverCompletionFunc *cb, void *opaque)
 {
     BlockDriverAIOCB *acb;
 
-    if (pool->free_aiocb) {
-        acb = pool->free_aiocb;
-        pool->free_aiocb = acb->next;
-    } else {
-        acb = g_malloc0(pool->aiocb_size);
-        acb->pool = pool;
-    }
+    acb = g_slice_alloc(aiocb_info->aiocb_size);
+    acb->aiocb_info = aiocb_info;
     acb->bs = bs;
     acb->cb = cb;
     acb->opaque = opaque;
@@ -3447,10 +3949,8 @@ void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
 
 void qemu_aio_release(void *p)
 {
-    BlockDriverAIOCB *acb = (BlockDriverAIOCB *)p;
-    AIOPool *pool = acb->pool;
-    acb->next = pool->free_aiocb;
-    pool->free_aiocb = acb;
+    BlockDriverAIOCB *acb = p;
+    g_slice_free1(acb->aiocb_info->aiocb_size, acb);
 }
 
 /**************************************************************/
@@ -3640,7 +4140,13 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
         return -EIO;
     } else if (bs->read_only) {
         return -EROFS;
-    } else if (bs->drv->bdrv_co_discard) {
+    }
+
+    if (bs->dirty_bitmap) {
+        bdrv_reset_dirty(bs, sector_num, nb_sectors);
+    }
+
+    if (bs->drv->bdrv_co_discard) {
         return bs->drv->bdrv_co_discard(bs, sector_num, nb_sectors);
     } else if (bs->drv->bdrv_aio_discard) {
         BlockDriverAIOCB *acb;
@@ -3779,22 +4285,36 @@ void *qemu_blockalign(BlockDriverState *bs, size_t size)
     return qemu_memalign((bs && bs->buffer_alignment) ? bs->buffer_alignment : 512, size);
 }
 
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)
+/*
+ * Check if all memory in this vector is sector aligned.
+ */
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
+{
+    int i;
+
+    for (i = 0; i < qiov->niov; i++) {
+        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity)
 {
     int64_t bitmap_size;
 
-    bs->dirty_count = 0;
-    if (enable) {
-        if (!bs->dirty_bitmap) {
-            bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS) +
-                    BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG - 1;
-            bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
+    assert((granularity & (granularity - 1)) == 0);
 
-            bs->dirty_bitmap = g_new0(unsigned long, bitmap_size);
-        }
+    if (granularity) {
+        granularity >>= BDRV_SECTOR_BITS;
+        assert(!bs->dirty_bitmap);
+        bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS);
+        bs->dirty_bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
     } else {
         if (bs->dirty_bitmap) {
-            g_free(bs->dirty_bitmap);
+            hbitmap_free(bs->dirty_bitmap);
             bs->dirty_bitmap = NULL;
         }
     }
@@ -3802,26 +4322,37 @@ void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)
 
 int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
 {
-    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    if (bs->dirty_bitmap &&
-        (sector << BDRV_SECTOR_BITS) < bdrv_getlength(bs)) {
-        return !!(bs->dirty_bitmap[chunk / (sizeof(unsigned long) * 8)] &
-            (1UL << (chunk % (sizeof(unsigned long) * 8))));
+    if (bs->dirty_bitmap) {
+        return hbitmap_get(bs->dirty_bitmap, sector);
     } else {
         return 0;
     }
 }
 
+void bdrv_dirty_iter_init(BlockDriverState *bs, HBitmapIter *hbi)
+{
+    hbitmap_iter_init(hbi, bs->dirty_bitmap, 0);
+}
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
+                    int nr_sectors)
+{
+    hbitmap_set(bs->dirty_bitmap, cur_sector, nr_sectors);
+}
+
 void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
                       int nr_sectors)
 {
-    set_dirty_bitmap(bs, cur_sector, nr_sectors, 0);
+    hbitmap_reset(bs->dirty_bitmap, cur_sector, nr_sectors);
 }
 
 int64_t bdrv_get_dirty_count(BlockDriverState *bs)
 {
-    return bs->dirty_count;
+    if (bs->dirty_bitmap) {
+        return hbitmap_count(bs->dirty_bitmap);
+    } else {
+        return 0;
+    }
 }
 
 void bdrv_set_in_use(BlockDriverState *bs, int in_use)
@@ -3846,9 +4377,9 @@ void bdrv_iostatus_enable(BlockDriverState *bs)
 bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
 {
     return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCK_ERR_STOP_ENOSPC ||
-            bs->on_write_error == BLOCK_ERR_STOP_ANY    ||
-            bs->on_read_error == BLOCK_ERR_STOP_ANY));
+           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
 }
 
 void bdrv_iostatus_disable(BlockDriverState *bs)
@@ -3860,17 +4391,16 @@ void bdrv_iostatus_reset(BlockDriverState *bs)
 {
     if (bdrv_iostatus_is_enabled(bs)) {
         bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (bs->job) {
+            block_job_iostatus_reset(bs->job);
+        }
     }
 }
 
-/* XXX: Today this is set by device models because it makes the implementation
-   quite simple. However, the block layer knows about the error, so it's
-   possible to implement this without device models being involved */
 void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
 {
-    if (bdrv_iostatus_is_enabled(bs) &&
-        bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        assert(error >= 0);
+    assert(bdrv_iostatus_is_enabled(bs));
+    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
         bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
                                          BLOCK_DEVICE_IO_STATUS_FAILED;
     }
@@ -3897,9 +4427,9 @@ bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie)
     bs->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
 }
 
-int bdrv_img_create(const char *filename, const char *fmt,
-                    const char *base_filename, const char *base_fmt,
-                    char *options, uint64_t img_size, int flags)
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags, Error **errp)
 {
     QEMUOptionParameter *param = NULL, *create_options = NULL;
     QEMUOptionParameter *backing_fmt, *backing_file, *size;
@@ -3911,16 +4441,14 @@ int bdrv_img_create(const char *filename, const char *fmt,
     /* Find driver and parse its options */
     drv = bdrv_find_format(fmt);
     if (!drv) {
-        error_report("Unknown file format '%s'", fmt);
-        ret = -EINVAL;
-        goto out;
+        error_setg(errp, "Unknown file format '%s'", fmt);
+        return;
     }
 
     proto_drv = bdrv_find_protocol(filename);
     if (!proto_drv) {
-        error_report("Unknown protocol '%s'", filename);
-        ret = -EINVAL;
-        goto out;
+        error_setg(errp, "Unknown protocol '%s'", filename);
+        return;
     }
 
     create_options = append_option_parameters(create_options,
@@ -3937,8 +4465,7 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (options) {
         param = parse_option_parameters(options, create_options, param);
         if (param == NULL) {
-            error_report("Invalid options for file format '%s'.", fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Invalid options for file format '%s'.", fmt);
             goto out;
         }
     }
@@ -3946,18 +4473,16 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (base_filename) {
         if (set_option_parameter(param, BLOCK_OPT_BACKING_FILE,
                                  base_filename)) {
-            error_report("Backing file not supported for file format '%s'",
-                         fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Backing file not supported for file format '%s'",
+                       fmt);
             goto out;
         }
     }
 
     if (base_fmt) {
         if (set_option_parameter(param, BLOCK_OPT_BACKING_FMT, base_fmt)) {
-            error_report("Backing file format not supported for file "
-                         "format '%s'", fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Backing file format not supported for file "
+                             "format '%s'", fmt);
             goto out;
         }
     }
@@ -3965,9 +4490,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
     backing_file = get_option_parameter(param, BLOCK_OPT_BACKING_FILE);
     if (backing_file && backing_file->value.s) {
         if (!strcmp(filename, backing_file->value.s)) {
-            error_report("Error: Trying to create an image with the "
-                         "same filename as the backing file");
-            ret = -EINVAL;
+            error_setg(errp, "Error: Trying to create an image with the "
+                             "same filename as the backing file");
             goto out;
         }
     }
@@ -3976,9 +4500,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (backing_fmt && backing_fmt->value.s) {
         backing_drv = bdrv_find_format(backing_fmt->value.s);
         if (!backing_drv) {
-            error_report("Unknown backing file format '%s'",
-                         backing_fmt->value.s);
-            ret = -EINVAL;
+            error_setg(errp, "Unknown backing file format '%s'",
+                       backing_fmt->value.s);
             goto out;
         }
     }
@@ -4000,7 +4523,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
 
             ret = bdrv_open(bs, backing_file->value.s, back_flags, backing_drv);
             if (ret < 0) {
-                error_report("Could not open '%s'", backing_file->value.s);
+                error_setg_errno(errp, -ret, "Could not open '%s'",
+                                 backing_file->value.s);
                 goto out;
             }
             bdrv_get_geometry(bs, &size);
@@ -4009,8 +4533,7 @@ int bdrv_img_create(const char *filename, const char *fmt,
             snprintf(buf, sizeof(buf), "%" PRId64, size);
             set_option_parameter(param, BLOCK_OPT_SIZE, buf);
         } else {
-            error_report("Image creation needs a size parameter");
-            ret = -EINVAL;
+            error_setg(errp, "Image creation needs a size parameter");
             goto out;
         }
     }
@@ -4020,17 +4543,16 @@ int bdrv_img_create(const char *filename, const char *fmt,
     puts("");
 
     ret = bdrv_create(drv, filename, param);
-
     if (ret < 0) {
         if (ret == -ENOTSUP) {
-            error_report("Formatting or formatting option not supported for "
-                         "file format '%s'", fmt);
+            error_setg(errp,"Formatting or formatting option not supported for "
+                            "file format '%s'", fmt);
         } else if (ret == -EFBIG) {
-            error_report("The image size is too large for file format '%s'",
-                         fmt);
+            error_setg(errp, "The image size is too large for file format '%s'",
+                       fmt);
         } else {
-            error_report("%s: error while creating %s: %s", filename, fmt,
-                         strerror(-ret));
+            error_setg(errp, "%s: error while creating %s: %s", filename, fmt,
+                       strerror(-ret));
         }
     }
 
@@ -4041,133 +4563,4 @@ out:
     if (bs) {
         bdrv_delete(bs);
     }
-
-    return ret;
-}
-
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
-                       void *opaque, Error **errp)
-{
-    BlockJob *job;
-
-    if (bs->job || bdrv_in_use(bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
-        return NULL;
-    }
-    bdrv_set_in_use(bs, 1);
-
-    job = g_malloc0(job_type->instance_size);
-    job->job_type      = job_type;
-    job->bs            = bs;
-    job->cb            = cb;
-    job->opaque        = opaque;
-    job->busy          = true;
-    bs->job = job;
-
-    /* Only set speed when necessary to avoid NotSupported error */
-    if (speed != 0) {
-        Error *local_err = NULL;
-
-        block_job_set_speed(job, speed, &local_err);
-        if (error_is_set(&local_err)) {
-            bs->job = NULL;
-            g_free(job);
-            bdrv_set_in_use(bs, 0);
-            error_propagate(errp, local_err);
-            return NULL;
-        }
-    }
-    return job;
-}
-
-void block_job_complete(BlockJob *job, int ret)
-{
-    BlockDriverState *bs = job->bs;
-
-    assert(bs->job == job);
-    job->cb(job->opaque, ret);
-    bs->job = NULL;
-    g_free(job);
-    bdrv_set_in_use(bs, 0);
-}
-
-void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
-{
-    Error *local_err = NULL;
-
-    if (!job->job_type->set_speed) {
-        error_set(errp, QERR_NOT_SUPPORTED);
-        return;
-    }
-    job->job_type->set_speed(job, speed, &local_err);
-    if (error_is_set(&local_err)) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    job->speed = speed;
-}
-
-void block_job_cancel(BlockJob *job)
-{
-    job->cancelled = true;
-    if (job->co && !job->busy) {
-        qemu_coroutine_enter(job->co, NULL);
-    }
-}
-
-bool block_job_is_cancelled(BlockJob *job)
-{
-    return job->cancelled;
-}
-
-struct BlockCancelData {
-    BlockJob *job;
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-    bool cancelled;
-    int ret;
-};
-
-static void block_job_cancel_cb(void *opaque, int ret)
-{
-    struct BlockCancelData *data = opaque;
-
-    data->cancelled = block_job_is_cancelled(data->job);
-    data->ret = ret;
-    data->cb(data->opaque, ret);
-}
-
-int block_job_cancel_sync(BlockJob *job)
-{
-    struct BlockCancelData data;
-    BlockDriverState *bs = job->bs;
-
-    assert(bs->job == job);
-
-    /* Set up our own callback to store the result and chain to
-     * the original callback.
-     */
-    data.job = job;
-    data.cb = job->cb;
-    data.opaque = job->opaque;
-    data.ret = -EINPROGRESS;
-    job->cb = block_job_cancel_cb;
-    job->opaque = &data;
-    block_job_cancel(job);
-    while (data.ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
-}
-
-void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns)
-{
-    /* Check cancellation *before* setting busy = false, too!  */
-    if (!block_job_is_cancelled(job)) {
-        job->busy = false;
-        co_sleep_ns(clock, ns);
-        job->busy = true;
-    }
 }
diff --git a/block/Makefile.objs b/block/Makefile.objs
index b5754d39bf1..c067f38e1dd 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -2,10 +2,21 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat
 block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o
 block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
-block-obj-y += parallels.o nbd.o blkdebug.o sheepdog.o blkverify.o
-block-obj-y += stream.o
-block-obj-$(CONFIG_WIN32) += raw-win32.o
+block-obj-y += parallels.o blkdebug.o blkverify.o
+block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
 block-obj-$(CONFIG_POSIX) += raw-posix.o
+block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
+
+ifeq ($(CONFIG_POSIX),y)
+block-obj-y += nbd.o sheepdog.o
 block-obj-$(CONFIG_LIBISCSI) += iscsi.o
 block-obj-$(CONFIG_CURL) += curl.o
 block-obj-$(CONFIG_RBD) += rbd.o
+block-obj-$(CONFIG_GLUSTERFS) += gluster.o
+endif
+
+common-obj-y += stream.o
+common-obj-y += commit.o
+common-obj-y += mirror.o
+
+$(obj)/curl.o: QEMU_CFLAGS+=$(CURL_CFLAGS)
diff --git a/block/blkdebug.c b/block/blkdebug.c
index 59dcea06505..6f7463772b1 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -23,13 +23,17 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/config-file.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
+    int new_state;
+
     QLIST_HEAD(, BlkdebugRule) rules[BLKDBG_EVENT_MAX];
     QSIMPLEQ_HEAD(, BlkdebugRule) active_rules;
+    QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
 } BDRVBlkdebugState;
 
 typedef struct BlkdebugAIOCB {
@@ -38,9 +42,15 @@ typedef struct BlkdebugAIOCB {
     int ret;
 } BlkdebugAIOCB;
 
+typedef struct BlkdebugSuspendedReq {
+    Coroutine *co;
+    char *tag;
+    QLIST_ENTRY(BlkdebugSuspendedReq) next;
+} BlkdebugSuspendedReq;
+
 static void blkdebug_aio_cancel(BlockDriverAIOCB *blockacb);
 
-static AIOPool blkdebug_aio_pool = {
+static const AIOCBInfo blkdebug_aiocb_info = {
     .aiocb_size = sizeof(BlkdebugAIOCB),
     .cancel     = blkdebug_aio_cancel,
 };
@@ -48,6 +58,7 @@ static AIOPool blkdebug_aio_pool = {
 enum {
     ACTION_INJECT_ERROR,
     ACTION_SET_STATE,
+    ACTION_SUSPEND,
 };
 
 typedef struct BlkdebugRule {
@@ -64,6 +75,9 @@ typedef struct BlkdebugRule {
         struct {
             int new_state;
         } set_state;
+        struct {
+            char *tag;
+        } suspend;
     } options;
     QLIST_ENTRY(BlkdebugRule) next;
     QSIMPLEQ_ENTRY(BlkdebugRule) active_next;
@@ -225,6 +239,11 @@ static int add_rule(QemuOpts *opts, void *opaque)
         rule->options.set_state.new_state =
             qemu_opt_get_number(opts, "new_state", 0);
         break;
+
+    case ACTION_SUSPEND:
+        rule->options.suspend.tag =
+            g_strdup(qemu_opt_get(opts, "tag"));
+        break;
     };
 
     /* Add the rule */
@@ -233,12 +252,32 @@ static int add_rule(QemuOpts *opts, void *opaque)
     return 0;
 }
 
+static void remove_rule(BlkdebugRule *rule)
+{
+    switch (rule->action) {
+    case ACTION_INJECT_ERROR:
+    case ACTION_SET_STATE:
+        break;
+    case ACTION_SUSPEND:
+        g_free(rule->options.suspend.tag);
+        break;
+    }
+
+    QLIST_REMOVE(rule, next);
+    g_free(rule);
+}
+
 static int read_config(BDRVBlkdebugState *s, const char *filename)
 {
     FILE *f;
     int ret;
     struct add_rule_data d;
 
+    /* Allow usage without config file */
+    if (!*filename) {
+        return 0;
+    }
+
     f = fopen(filename, "r");
     if (f == NULL) {
         return -errno;
@@ -334,7 +373,7 @@ static BlockDriverAIOCB *inject_error(BlockDriverState *bs,
         return NULL;
     }
 
-    acb = qemu_aio_get(&blkdebug_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&blkdebug_aiocb_info, bs, cb, opaque);
     acb->ret = -error;
 
     bh = qemu_bh_new(error_callback_bh, acb);
@@ -388,6 +427,7 @@ static BlockDriverAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
     return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
 }
 
+
 static void blkdebug_close(BlockDriverState *bs)
 {
     BDRVBlkdebugState *s = bs->opaque;
@@ -396,19 +436,39 @@ static void blkdebug_close(BlockDriverState *bs)
 
     for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
-            QLIST_REMOVE(rule, next);
-            g_free(rule);
+            remove_rule(rule);
         }
     }
 }
 
+static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq r;
+
+    r = (BlkdebugSuspendedReq) {
+        .co         = qemu_coroutine_self(),
+        .tag        = g_strdup(rule->options.suspend.tag),
+    };
+
+    remove_rule(rule);
+    QLIST_INSERT_HEAD(&s->suspended_reqs, &r, next);
+
+    printf("blkdebug: Suspended request '%s'\n", r.tag);
+    qemu_coroutine_yield();
+    printf("blkdebug: Resuming request '%s'\n", r.tag);
+
+    QLIST_REMOVE(&r, next);
+    g_free(r.tag);
+}
+
 static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
-    int old_state, bool injected)
+    bool injected)
 {
     BDRVBlkdebugState *s = bs->opaque;
 
     /* Only process rules for the current state */
-    if (rule->state && rule->state != old_state) {
+    if (rule->state && rule->state != s->state) {
         return injected;
     }
 
@@ -423,7 +483,11 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
         break;
 
     case ACTION_SET_STATE:
-        s->state = rule->options.set_state.new_state;
+        s->new_state = rule->options.set_state.new_state;
+        break;
+
+    case ACTION_SUSPEND:
+        suspend_request(bs, rule);
         break;
     }
     return injected;
@@ -432,16 +496,70 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
 static void blkdebug_debug_event(BlockDriverState *bs, BlkDebugEvent event)
 {
     BDRVBlkdebugState *s = bs->opaque;
-    struct BlkdebugRule *rule;
-    int old_state = s->state;
+    struct BlkdebugRule *rule, *next;
     bool injected;
 
     assert((int)event >= 0 && event < BLKDBG_EVENT_MAX);
 
     injected = false;
-    QLIST_FOREACH(rule, &s->rules[event], next) {
-        injected = process_rule(bs, rule, old_state, injected);
+    s->new_state = s->state;
+    QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {
+        injected = process_rule(bs, rule, injected);
+    }
+    s->state = s->new_state;
+}
+
+static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,
+                                     const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    struct BlkdebugRule *rule;
+    BlkDebugEvent blkdebug_event;
+
+    if (get_event_by_name(event, &blkdebug_event) < 0) {
+        return -ENOENT;
+    }
+
+
+    rule = g_malloc(sizeof(*rule));
+    *rule = (struct BlkdebugRule) {
+        .event  = blkdebug_event,
+        .action = ACTION_SUSPEND,
+        .state  = 0,
+        .options.suspend.tag = g_strdup(tag),
+    };
+
+    QLIST_INSERT_HEAD(&s->rules[blkdebug_event], rule, next);
+
+    return 0;
+}
+
+static int blkdebug_debug_resume(BlockDriverState *bs, const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq *r;
+
+    QLIST_FOREACH(r, &s->suspended_reqs, next) {
+        if (!strcmp(r->tag, tag)) {
+            qemu_coroutine_enter(r->co, NULL);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+
+static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq *r;
+
+    QLIST_FOREACH(r, &s->suspended_reqs, next) {
+        if (!strcmp(r->tag, tag)) {
+            return true;
+        }
     }
+    return false;
 }
 
 static int64_t blkdebug_getlength(BlockDriverState *bs)
@@ -462,7 +580,10 @@ static BlockDriver bdrv_blkdebug = {
     .bdrv_aio_readv     = blkdebug_aio_readv,
     .bdrv_aio_writev    = blkdebug_aio_writev,
 
-    .bdrv_debug_event   = blkdebug_debug_event,
+    .bdrv_debug_event           = blkdebug_debug_event,
+    .bdrv_debug_breakpoint      = blkdebug_debug_breakpoint,
+    .bdrv_debug_resume          = blkdebug_debug_resume,
+    .bdrv_debug_is_suspended    = blkdebug_debug_is_suspended,
 };
 
 static void bdrv_blkdebug_init(void)
diff --git a/block/blkverify.c b/block/blkverify.c
index 9d5f1ec5b91..a7dd45909b0 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -8,8 +8,8 @@
  */
 
 #include <stdarg.h>
-#include "qemu_socket.h" /* for EINPROGRESS on Windows */
-#include "block_int.h"
+#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
+#include "block/block_int.h"
 
 typedef struct {
     BlockDriverState *test_file;
@@ -48,7 +48,7 @@ static void blkverify_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool blkverify_aio_pool = {
+static const AIOCBInfo blkverify_aiocb_info = {
     .aiocb_size         = sizeof(BlkverifyAIOCB),
     .cancel             = blkverify_aio_cancel,
 };
@@ -233,7 +233,7 @@ static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
                                          BlockDriverCompletionFunc *cb,
                                          void *opaque)
 {
-    BlkverifyAIOCB *acb = qemu_aio_get(&blkverify_aio_pool, bs, cb, opaque);
+    BlkverifyAIOCB *acb = qemu_aio_get(&blkverify_aiocb_info, bs, cb, opaque);
 
     acb->bh = NULL;
     acb->is_write = is_write;
diff --git a/block/bochs.c b/block/bochs.c
index ab7944dc43d..37375834e96 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 
@@ -126,7 +126,7 @@ static int bochs_open(BlockDriverState *bs, int flags)
         strcmp(bochs.subtype, GROWING_TYPE) ||
 	((le32_to_cpu(bochs.version) != HEADER_VERSION) &&
 	(le32_to_cpu(bochs.version) != HEADER_V1))) {
-        goto fail;
+        return -EMEDIUMTYPE;
     }
 
     if (le32_to_cpu(bochs.version) == HEADER_V1) {
diff --git a/block/cloop.c b/block/cloop.c
index 7570eb8e745..5a0d0d805f6 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
 
 typedef struct BDRVCloopState {
diff --git a/block/commit.c b/block/commit.c
new file mode 100644
index 00000000000..553447efe7f
--- /dev/null
+++ b/block/commit.c
@@ -0,0 +1,258 @@
+/*
+ * Live block commit
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Jeff Cody   <jcody@redhat.com>
+ *  Based on stream.c by Stefan Hajnoczi
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+#include "qemu/ratelimit.h"
+
+enum {
+    /*
+     * Size of data buffer for populating the image file.  This should be large
+     * enough to process multiple clusters in a single call, so that populating
+     * contiguous regions of the image is efficient.
+     */
+    COMMIT_BUFFER_SIZE = 512 * 1024, /* in bytes */
+};
+
+#define SLICE_TIME 100000000ULL /* ns */
+
+typedef struct CommitBlockJob {
+    BlockJob common;
+    RateLimit limit;
+    BlockDriverState *active;
+    BlockDriverState *top;
+    BlockDriverState *base;
+    BlockdevOnError on_error;
+    int base_flags;
+    int orig_overlay_flags;
+} CommitBlockJob;
+
+static int coroutine_fn commit_populate(BlockDriverState *bs,
+                                        BlockDriverState *base,
+                                        int64_t sector_num, int nb_sectors,
+                                        void *buf)
+{
+    int ret = 0;
+
+    ret = bdrv_read(bs, sector_num, buf, nb_sectors);
+    if (ret) {
+        return ret;
+    }
+
+    ret = bdrv_write(base, sector_num, buf, nb_sectors);
+    if (ret) {
+        return ret;
+    }
+
+    return 0;
+}
+
+static void coroutine_fn commit_run(void *opaque)
+{
+    CommitBlockJob *s = opaque;
+    BlockDriverState *active = s->active;
+    BlockDriverState *top = s->top;
+    BlockDriverState *base = s->base;
+    BlockDriverState *overlay_bs;
+    int64_t sector_num, end;
+    int ret = 0;
+    int n = 0;
+    void *buf;
+    int bytes_written = 0;
+    int64_t base_len;
+
+    ret = s->common.len = bdrv_getlength(top);
+
+
+    if (s->common.len < 0) {
+        goto exit_restore_reopen;
+    }
+
+    ret = base_len = bdrv_getlength(base);
+    if (base_len < 0) {
+        goto exit_restore_reopen;
+    }
+
+    if (base_len < s->common.len) {
+        ret = bdrv_truncate(base, s->common.len);
+        if (ret) {
+            goto exit_restore_reopen;
+        }
+    }
+
+    end = s->common.len >> BDRV_SECTOR_BITS;
+    buf = qemu_blockalign(top, COMMIT_BUFFER_SIZE);
+
+    for (sector_num = 0; sector_num < end; sector_num += n) {
+        uint64_t delay_ns = 0;
+        bool copy;
+
+wait:
+        /* Note that even when no rate limit is applied we need to yield
+         * with no pending I/O here so that bdrv_drain_all() returns.
+         */
+        block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+        if (block_job_is_cancelled(&s->common)) {
+            break;
+        }
+        /* Copy if allocated above the base */
+        ret = bdrv_co_is_allocated_above(top, base, sector_num,
+                                         COMMIT_BUFFER_SIZE / BDRV_SECTOR_SIZE,
+                                         &n);
+        copy = (ret == 1);
+        trace_commit_one_iteration(s, sector_num, n, ret);
+        if (copy) {
+            if (s->common.speed) {
+                delay_ns = ratelimit_calculate_delay(&s->limit, n);
+                if (delay_ns > 0) {
+                    goto wait;
+                }
+            }
+            ret = commit_populate(top, base, sector_num, n, buf);
+            bytes_written += n * BDRV_SECTOR_SIZE;
+        }
+        if (ret < 0) {
+            if (s->on_error == BLOCKDEV_ON_ERROR_STOP ||
+                s->on_error == BLOCKDEV_ON_ERROR_REPORT||
+                (s->on_error == BLOCKDEV_ON_ERROR_ENOSPC && ret == -ENOSPC)) {
+                goto exit_free_buf;
+            } else {
+                n = 0;
+                continue;
+            }
+        }
+        /* Publish progress */
+        s->common.offset += n * BDRV_SECTOR_SIZE;
+    }
+
+    ret = 0;
+
+    if (!block_job_is_cancelled(&s->common) && sector_num == end) {
+        /* success */
+        ret = bdrv_drop_intermediate(active, top, base);
+    }
+
+exit_free_buf:
+    qemu_vfree(buf);
+
+exit_restore_reopen:
+    /* restore base open flags here if appropriate (e.g., change the base back
+     * to r/o). These reopens do not need to be atomic, since we won't abort
+     * even on failure here */
+    if (s->base_flags != bdrv_get_flags(base)) {
+        bdrv_reopen(base, s->base_flags, NULL);
+    }
+    overlay_bs = bdrv_find_overlay(active, top);
+    if (overlay_bs && s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
+        bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
+    }
+
+    block_job_completed(&s->common, ret);
+}
+
+static void commit_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    CommitBlockJob *s = container_of(job, CommitBlockJob, common);
+
+    if (speed < 0) {
+        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        return;
+    }
+    ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
+}
+
+static BlockJobType commit_job_type = {
+    .instance_size = sizeof(CommitBlockJob),
+    .job_type      = "commit",
+    .set_speed     = commit_set_speed,
+};
+
+void commit_start(BlockDriverState *bs, BlockDriverState *base,
+                  BlockDriverState *top, int64_t speed,
+                  BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp)
+{
+    CommitBlockJob *s;
+    BlockReopenQueue *reopen_queue = NULL;
+    int orig_overlay_flags;
+    int orig_base_flags;
+    BlockDriverState *overlay_bs;
+    Error *local_err = NULL;
+
+    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER_COMBINATION);
+        return;
+    }
+
+    /* Once we support top == active layer, remove this check */
+    if (top == bs) {
+        error_setg(errp,
+                   "Top image as the active layer is currently unsupported");
+        return;
+    }
+
+    if (top == base) {
+        error_setg(errp, "Invalid files for merge: top and base are the same");
+        return;
+    }
+
+    overlay_bs = bdrv_find_overlay(bs, top);
+
+    if (overlay_bs == NULL) {
+        error_setg(errp, "Could not find overlay image for %s:", top->filename);
+        return;
+    }
+
+    orig_base_flags    = bdrv_get_flags(base);
+    orig_overlay_flags = bdrv_get_flags(overlay_bs);
+
+    /* convert base & overlay_bs to r/w, if necessary */
+    if (!(orig_base_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, base,
+                                         orig_base_flags | BDRV_O_RDWR);
+    }
+    if (!(orig_overlay_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs,
+                                         orig_overlay_flags | BDRV_O_RDWR);
+    }
+    if (reopen_queue) {
+        bdrv_reopen_multiple(reopen_queue, &local_err);
+        if (local_err != NULL) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+
+    s = block_job_create(&commit_job_type, bs, speed, cb, opaque, errp);
+    if (!s) {
+        return;
+    }
+
+    s->base   = base;
+    s->top    = top;
+    s->active = bs;
+
+    s->base_flags          = orig_base_flags;
+    s->orig_overlay_flags  = orig_overlay_flags;
+
+    s->on_error = on_error;
+    s->common.co = qemu_coroutine_create(commit_run);
+
+    trace_commit_start(bs, base, top, s, s->common.co, opaque);
+    qemu_coroutine_enter(s->common.co, s);
+}
diff --git a/block/cow.c b/block/cow.c
index a5a00eb9ca5..4baf9042fe0 100644
--- a/block/cow.c
+++ b/block/cow.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 /* COW block driver using file system holes */
@@ -73,7 +73,7 @@ static int cow_open(BlockDriverState *bs, int flags)
     }
 
     if (be32_to_cpu(cow_header.magic) != COW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
 
diff --git a/block/curl.c b/block/curl.c
index e7c3634d354..47df9524ea8 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include <curl/curl.h>
 
 // #define DEBUG
@@ -438,7 +438,7 @@ static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
     // Do we have to implement canceling? Seems to work without...
 }
 
-static AIOPool curl_aio_pool = {
+static const AIOCBInfo curl_aiocb_info = {
     .aiocb_size         = sizeof(CURLAIOCB),
     .cancel             = curl_aio_cancel,
 };
@@ -505,7 +505,7 @@ static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 {
     CURLAIOCB *acb;
 
-    acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&curl_aiocb_info, bs, cb, opaque);
 
     acb->qiov = qiov;
     acb->sector_num = sector_num;
@@ -542,8 +542,7 @@ static void curl_close(BlockDriverState *bs)
     }
     if (s->multi)
         curl_multi_cleanup(s->multi);
-    if (s->url)
-        free(s->url);
+    g_free(s->url);
 }
 
 static int64_t curl_getlength(BlockDriverState *bs)
diff --git a/block/dmg.c b/block/dmg.c
index 37902a4347a..ac397dc8f75 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "bswap.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/bswap.h"
+#include "qemu/module.h"
 #include <zlib.h>
 
 typedef struct BDRVDMGState {
diff --git a/block/gluster.c b/block/gluster.c
new file mode 100644
index 00000000000..0f2c32a3fab
--- /dev/null
+++ b/block/gluster.c
@@ -0,0 +1,624 @@
+/*
+ * GlusterFS backend for QEMU
+ *
+ * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * Pipe handling mechanism in AIO implementation is derived from
+ * block/rbd.c. Hence,
+ *
+ * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>,
+ *                         Josh Durgin <josh.durgin@dreamhost.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include <glusterfs/api/glfs.h>
+#include "block/block_int.h"
+#include "qemu/sockets.h"
+#include "qemu/uri.h"
+
+typedef struct GlusterAIOCB {
+    BlockDriverAIOCB common;
+    int64_t size;
+    int ret;
+    bool *finished;
+    QEMUBH *bh;
+} GlusterAIOCB;
+
+typedef struct BDRVGlusterState {
+    struct glfs *glfs;
+    int fds[2];
+    struct glfs_fd *fd;
+    int qemu_aio_count;
+    int event_reader_pos;
+    GlusterAIOCB *event_acb;
+} BDRVGlusterState;
+
+#define GLUSTER_FD_READ  0
+#define GLUSTER_FD_WRITE 1
+
+typedef struct GlusterConf {
+    char *server;
+    int port;
+    char *volname;
+    char *image;
+    char *transport;
+} GlusterConf;
+
+static void qemu_gluster_gconf_free(GlusterConf *gconf)
+{
+    g_free(gconf->server);
+    g_free(gconf->volname);
+    g_free(gconf->image);
+    g_free(gconf->transport);
+    g_free(gconf);
+}
+
+static int parse_volume_options(GlusterConf *gconf, char *path)
+{
+    char *p, *q;
+
+    if (!path) {
+        return -EINVAL;
+    }
+
+    /* volume */
+    p = q = path + strspn(path, "/");
+    p += strcspn(p, "/");
+    if (*p == '\0') {
+        return -EINVAL;
+    }
+    gconf->volname = g_strndup(q, p - q);
+
+    /* image */
+    p += strspn(p, "/");
+    if (*p == '\0') {
+        return -EINVAL;
+    }
+    gconf->image = g_strdup(p);
+    return 0;
+}
+
+/*
+ * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
+ *
+ * 'gluster' is the protocol.
+ *
+ * 'transport' specifies the transport type used to connect to gluster
+ * management daemon (glusterd). Valid transport types are
+ * tcp, unix and rdma. If a transport type isn't specified, then tcp
+ * type is assumed.
+ *
+ * 'server' specifies the server where the volume file specification for
+ * the given volume resides. This can be either hostname, ipv4 address
+ * or ipv6 address. ipv6 address needs to be within square brackets [ ].
+ * If transport type is 'unix', then 'server' field should not be specifed.
+ * The 'socket' field needs to be populated with the path to unix domain
+ * socket.
+ *
+ * 'port' is the port number on which glusterd is listening. This is optional
+ * and if not specified, QEMU will send 0 which will make gluster to use the
+ * default port. If the transport type is unix, then 'port' should not be
+ * specified.
+ *
+ * 'volname' is the name of the gluster volume which contains the VM image.
+ *
+ * 'image' is the path to the actual VM image that resides on gluster volume.
+ *
+ * Examples:
+ *
+ * file=gluster://1.2.3.4/testvol/a.img
+ * file=gluster+tcp://1.2.3.4/testvol/a.img
+ * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
+ * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
+ * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
+ * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
+ * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
+ * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
+ */
+static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
+{
+    URI *uri;
+    QueryParams *qp = NULL;
+    bool is_unix = false;
+    int ret = 0;
+
+    uri = uri_parse(filename);
+    if (!uri) {
+        return -EINVAL;
+    }
+
+    /* transport */
+    if (!strcmp(uri->scheme, "gluster")) {
+        gconf->transport = g_strdup("tcp");
+    } else if (!strcmp(uri->scheme, "gluster+tcp")) {
+        gconf->transport = g_strdup("tcp");
+    } else if (!strcmp(uri->scheme, "gluster+unix")) {
+        gconf->transport = g_strdup("unix");
+        is_unix = true;
+    } else if (!strcmp(uri->scheme, "gluster+rdma")) {
+        gconf->transport = g_strdup("rdma");
+    } else {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = parse_volume_options(gconf, uri->path);
+    if (ret < 0) {
+        goto out;
+    }
+
+    qp = query_params_parse(uri->query);
+    if (qp->n > 1 || (is_unix && !qp->n) || (!is_unix && qp->n)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (is_unix) {
+        if (uri->server || uri->port) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (strcmp(qp->p[0].name, "socket")) {
+            ret = -EINVAL;
+            goto out;
+        }
+        gconf->server = g_strdup(qp->p[0].value);
+    } else {
+        gconf->server = g_strdup(uri->server);
+        gconf->port = uri->port;
+    }
+
+out:
+    if (qp) {
+        query_params_free(qp);
+    }
+    uri_free(uri);
+    return ret;
+}
+
+static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename)
+{
+    struct glfs *glfs = NULL;
+    int ret;
+    int old_errno;
+
+    ret = qemu_gluster_parseuri(gconf, filename);
+    if (ret < 0) {
+        error_report("Usage: file=gluster[+transport]://[server[:port]]/"
+            "volname/image[?socket=...]");
+        errno = -ret;
+        goto out;
+    }
+
+    glfs = glfs_new(gconf->volname);
+    if (!glfs) {
+        goto out;
+    }
+
+    ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
+            gconf->port);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /*
+     * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
+     * GlusterFS makes GF_LOG_* macros available to libgfapi users.
+     */
+    ret = glfs_set_logging(glfs, "-", 4);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = glfs_init(glfs);
+    if (ret) {
+        error_report("Gluster connection failed for server=%s port=%d "
+             "volume=%s image=%s transport=%s\n", gconf->server, gconf->port,
+             gconf->volname, gconf->image, gconf->transport);
+        goto out;
+    }
+    return glfs;
+
+out:
+    if (glfs) {
+        old_errno = errno;
+        glfs_fini(glfs);
+        errno = old_errno;
+    }
+    return NULL;
+}
+
+static void qemu_gluster_complete_aio(GlusterAIOCB *acb, BDRVGlusterState *s)
+{
+    int ret;
+    bool *finished = acb->finished;
+    BlockDriverCompletionFunc *cb = acb->common.cb;
+    void *opaque = acb->common.opaque;
+
+    if (!acb->ret || acb->ret == acb->size) {
+        ret = 0; /* Success */
+    } else if (acb->ret < 0) {
+        ret = acb->ret; /* Read/Write failed */
+    } else {
+        ret = -EIO; /* Partial read/write - fail it */
+    }
+
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    cb(opaque, ret);
+    if (finished) {
+        *finished = true;
+    }
+}
+
+static void qemu_gluster_aio_event_reader(void *opaque)
+{
+    BDRVGlusterState *s = opaque;
+    ssize_t ret;
+
+    do {
+        char *p = (char *)&s->event_acb;
+
+        ret = read(s->fds[GLUSTER_FD_READ], p + s->event_reader_pos,
+                   sizeof(s->event_acb) - s->event_reader_pos);
+        if (ret > 0) {
+            s->event_reader_pos += ret;
+            if (s->event_reader_pos == sizeof(s->event_acb)) {
+                s->event_reader_pos = 0;
+                qemu_gluster_complete_aio(s->event_acb, s);
+            }
+        }
+    } while (ret < 0 && errno == EINTR);
+}
+
+static int qemu_gluster_aio_flush_cb(void *opaque)
+{
+    BDRVGlusterState *s = opaque;
+
+    return (s->qemu_aio_count > 0);
+}
+
+static int qemu_gluster_open(BlockDriverState *bs, const char *filename,
+    int bdrv_flags)
+{
+    BDRVGlusterState *s = bs->opaque;
+    int open_flags = O_BINARY;
+    int ret = 0;
+    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+
+    s->glfs = qemu_gluster_init(gconf, filename);
+    if (!s->glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    if (bdrv_flags & BDRV_O_RDWR) {
+        open_flags |= O_RDWR;
+    } else {
+        open_flags |= O_RDONLY;
+    }
+
+    if ((bdrv_flags & BDRV_O_NOCACHE)) {
+        open_flags |= O_DIRECT;
+    }
+
+    s->fd = glfs_open(s->glfs, gconf->image, open_flags);
+    if (!s->fd) {
+        ret = -errno;
+        goto out;
+    }
+
+    ret = qemu_pipe(s->fds);
+    if (ret < 0) {
+        ret = -errno;
+        goto out;
+    }
+    fcntl(s->fds[GLUSTER_FD_READ], F_SETFL, O_NONBLOCK);
+    qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ],
+        qemu_gluster_aio_event_reader, NULL, qemu_gluster_aio_flush_cb, s);
+
+out:
+    qemu_gluster_gconf_free(gconf);
+    if (!ret) {
+        return ret;
+    }
+    if (s->fd) {
+        glfs_close(s->fd);
+    }
+    if (s->glfs) {
+        glfs_fini(s->glfs);
+    }
+    return ret;
+}
+
+static int qemu_gluster_create(const char *filename,
+        QEMUOptionParameter *options)
+{
+    struct glfs *glfs;
+    struct glfs_fd *fd;
+    int ret = 0;
+    int64_t total_size = 0;
+    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+
+    glfs = qemu_gluster_init(gconf, filename);
+    if (!glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+            total_size = options->value.n / BDRV_SECTOR_SIZE;
+        }
+        options++;
+    }
+
+    fd = glfs_creat(glfs, gconf->image,
+        O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
+    if (!fd) {
+        ret = -errno;
+    } else {
+        if (glfs_ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) {
+            ret = -errno;
+        }
+        if (glfs_close(fd) != 0) {
+            ret = -errno;
+        }
+    }
+out:
+    qemu_gluster_gconf_free(gconf);
+    if (glfs) {
+        glfs_fini(glfs);
+    }
+    return ret;
+}
+
+static void qemu_gluster_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+    GlusterAIOCB *acb = (GlusterAIOCB *)blockacb;
+    bool finished = false;
+
+    acb->finished = &finished;
+    while (!finished) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo gluster_aiocb_info = {
+    .aiocb_size = sizeof(GlusterAIOCB),
+    .cancel = qemu_gluster_aio_cancel,
+};
+
+static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
+{
+    GlusterAIOCB *acb = (GlusterAIOCB *)arg;
+    BlockDriverState *bs = acb->common.bs;
+    BDRVGlusterState *s = bs->opaque;
+    int retval;
+
+    acb->ret = ret;
+    retval = qemu_write_full(s->fds[GLUSTER_FD_WRITE], &acb, sizeof(acb));
+    if (retval != sizeof(acb)) {
+        /*
+         * Gluster AIO callback thread failed to notify the waiting
+         * QEMU thread about IO completion.
+         *
+         * Complete this IO request and make the disk inaccessible for
+         * subsequent reads and writes.
+         */
+        error_report("Gluster failed to notify QEMU about IO completion");
+
+        qemu_mutex_lock_iothread(); /* We are in gluster thread context */
+        acb->common.cb(acb->common.opaque, -EIO);
+        qemu_aio_release(acb);
+        s->qemu_aio_count--;
+        close(s->fds[GLUSTER_FD_READ]);
+        close(s->fds[GLUSTER_FD_WRITE]);
+        qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL,
+            NULL);
+        bs->drv = NULL; /* Make the disk inaccessible */
+        qemu_mutex_unlock_iothread();
+    }
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_rw(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int write)
+{
+    int ret;
+    GlusterAIOCB *acb;
+    BDRVGlusterState *s = bs->opaque;
+    size_t size;
+    off_t offset;
+
+    offset = sector_num * BDRV_SECTOR_SIZE;
+    size = nb_sectors * BDRV_SECTOR_SIZE;
+    s->qemu_aio_count++;
+
+    acb = qemu_aio_get(&gluster_aiocb_info, bs, cb, opaque);
+    acb->size = size;
+    acb->ret = 0;
+    acb->finished = NULL;
+
+    if (write) {
+        ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
+            &gluster_finish_aiocb, acb);
+    } else {
+        ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
+            &gluster_finish_aiocb, acb);
+    }
+
+    if (ret < 0) {
+        goto out;
+    }
+    return &acb->common;
+
+out:
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    return NULL;
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_readv(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_writev(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_flush(BlockDriverState *bs,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    int ret;
+    GlusterAIOCB *acb;
+    BDRVGlusterState *s = bs->opaque;
+
+    acb = qemu_aio_get(&gluster_aiocb_info, bs, cb, opaque);
+    acb->size = 0;
+    acb->ret = 0;
+    acb->finished = NULL;
+    s->qemu_aio_count++;
+
+    ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
+    if (ret < 0) {
+        goto out;
+    }
+    return &acb->common;
+
+out:
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    return NULL;
+}
+
+static int64_t qemu_gluster_getlength(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+    int64_t ret;
+
+    ret = glfs_lseek(s->fd, 0, SEEK_END);
+    if (ret < 0) {
+        return -errno;
+    } else {
+        return ret;
+    }
+}
+
+static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+    struct stat st;
+    int ret;
+
+    ret = glfs_fstat(s->fd, &st);
+    if (ret < 0) {
+        return -errno;
+    } else {
+        return st.st_blocks * 512;
+    }
+}
+
+static void qemu_gluster_close(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+
+    close(s->fds[GLUSTER_FD_READ]);
+    close(s->fds[GLUSTER_FD_WRITE]);
+    qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL, NULL);
+
+    if (s->fd) {
+        glfs_close(s->fd);
+        s->fd = NULL;
+    }
+    glfs_fini(s->glfs);
+}
+
+static QEMUOptionParameter qemu_gluster_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    { NULL }
+};
+
+static BlockDriver bdrv_gluster = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_tcp = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+tcp",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_unix = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+unix",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_rdma = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+rdma",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static void bdrv_gluster_init(void)
+{
+    bdrv_register(&bdrv_gluster_rdma);
+    bdrv_register(&bdrv_gluster_unix);
+    bdrv_register(&bdrv_gluster_tcp);
+    bdrv_register(&bdrv_gluster);
+}
+
+block_init(bdrv_gluster_init);
diff --git a/block/iscsi.c b/block/iscsi.c
index 0b96165ec2f..deb3b688901 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -27,8 +27,9 @@
 #include <poll.h>
 #include <arpa/inet.h>
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "block_int.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "block/block_int.h"
 #include "trace.h"
 #include "hw/scsi-defs.h"
 
@@ -47,6 +48,7 @@ typedef struct IscsiLun {
     int block_size;
     uint64_t num_blocks;
     int events;
+    QEMUTimer *nop_timer;
 } IscsiLun;
 
 typedef struct IscsiAIOCB {
@@ -65,12 +67,8 @@ typedef struct IscsiAIOCB {
 #endif
 } IscsiAIOCB;
 
-struct IscsiTask {
-    IscsiLun *iscsilun;
-    BlockDriverState *bs;
-    int status;
-    int complete;
-};
+#define NOP_INTERVAL 5000
+#define MAX_NOP_FAILURES 3
 
 static void
 iscsi_bh_cb(void *p)
@@ -79,6 +77,9 @@ iscsi_bh_cb(void *p)
 
     qemu_bh_delete(acb->bh);
 
+    g_free(acb->buf);
+    acb->buf = NULL;
+
     if (acb->canceled == 0) {
         acb->common.cb(acb->common.opaque, acb->status);
     }
@@ -133,7 +134,7 @@ iscsi_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool iscsi_aio_pool = {
+static const AIOCBInfo iscsi_aiocb_info = {
     .aiocb_size         = sizeof(IscsiAIOCB),
     .cancel             = iscsi_aio_cancel,
 };
@@ -167,12 +168,6 @@ iscsi_set_events(IscsiLun *iscsilun)
 
     }
 
-    /* If we just added an event, the callback might be delayed
-     * unless we call qemu_notify_event().
-     */
-    if (ev & ~iscsilun->events) {
-        qemu_notify_event();
-    }
     iscsilun->events = ev;
 }
 
@@ -206,6 +201,7 @@ iscsi_aio_write16_cb(struct iscsi_context *iscsi, int status,
     trace_iscsi_aio_write16_cb(iscsi, status, acb, acb->canceled);
 
     g_free(acb->buf);
+    acb->buf = NULL;
 
     if (acb->canceled != 0) {
         return;
@@ -238,9 +234,12 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     size_t size;
     uint32_t num_sectors;
     uint64_t lba;
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
     struct iscsi_data data;
+#endif
+    int ret;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
     trace_iscsi_aio_writev(iscsi, sector_num, nb_sectors, opaque, acb);
 
     acb->iscsilun = iscsilun;
@@ -249,12 +248,23 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
-    /* XXX we should pass the iovec to write16 to avoid the extra copy */
     /* this will allow us to get rid of 'buf' completely */
     size = nb_sectors * BDRV_SECTOR_SIZE;
-    acb->buf = g_malloc(size);
-    qemu_iovec_to_buf(acb->qiov, 0, acb->buf, size);
+
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
+    data.size = MIN(size, acb->qiov->size);
+
+    /* if the iovec only contains one buffer we can pass it directly */
+    if (acb->qiov->niov == 1) {
+        data.data = acb->qiov->iov[0].iov_base;
+    } else {
+        acb->buf = g_malloc(data.size);
+        qemu_iovec_to_buf(acb->qiov, 0, acb->buf, data.size);
+        data.data = acb->buf;
+    }
+#endif
 
     acb->task = malloc(sizeof(struct scsi_task));
     if (acb->task == NULL) {
@@ -268,10 +278,6 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     acb->task->xfer_dir = SCSI_XFER_WRITE;
     acb->task->cdb_size = 16;
     acb->task->cdb[0] = 0x8a;
-    if (!(bs->open_flags & BDRV_O_CACHE_WB)) {
-        /* set FUA on writes when cache mode is write through */
-        acb->task->cdb[1] |= 0x04;
-    }
     lba = sector_qemu2lun(sector_num, iscsilun);
     *(uint32_t *)&acb->task->cdb[2]  = htonl(lba >> 32);
     *(uint32_t *)&acb->task->cdb[6]  = htonl(lba & 0xffffffff);
@@ -279,19 +285,28 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     *(uint32_t *)&acb->task->cdb[10] = htonl(num_sectors);
     acb->task->expxferlen = size;
 
-    data.data = acb->buf;
-    data.size = size;
-
-    if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
-                                 iscsi_aio_write16_cb,
-                                 &data,
-                                 acb) != 0) {
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_write16_cb,
+                                   NULL,
+                                   acb);
+#else
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_write16_cb,
+                                   &data,
+                                   acb);
+#endif
+    if (ret != 0) {
         scsi_free_scsi_task(acb->task);
         g_free(acb->buf);
         qemu_aio_release(acb);
         return NULL;
     }
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    scsi_task_set_iov_out(acb->task, (struct scsi_iovec*) acb->qiov->iov, acb->qiov->niov);
+#endif
+
     iscsi_set_events(iscsilun);
 
     return &acb->common;
@@ -329,13 +344,16 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
     struct iscsi_context *iscsi = iscsilun->iscsi;
     IscsiAIOCB *acb;
     size_t qemu_read_size;
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
     int i;
+#endif
+    int ret;
     uint64_t lba;
     uint32_t num_sectors;
 
     qemu_read_size = BDRV_SECTOR_SIZE * (size_t)nb_sectors;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
     trace_iscsi_aio_readv(iscsi, sector_num, nb_sectors, opaque, acb);
 
     acb->iscsilun = iscsilun;
@@ -390,21 +408,26 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
         *(uint16_t *)&acb->task->cdb[7] = htons(num_sectors);
         break;
     }
-    
-    if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
-                                 iscsi_aio_read16_cb,
-                                 NULL,
-                                 acb) != 0) {
+
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_read16_cb,
+                                   NULL,
+                                   acb);
+    if (ret != 0) {
         scsi_free_scsi_task(acb->task);
         qemu_aio_release(acb);
         return NULL;
     }
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    scsi_task_set_iov_in(acb->task, (struct scsi_iovec*) acb->qiov->iov, acb->qiov->niov);
+#else
     for (i = 0; i < acb->qiov->niov; i++) {
         scsi_task_add_data_in_buffer(acb->task,
                 acb->qiov->iov[i].iov_len,
                 acb->qiov->iov[i].iov_base);
     }
+#endif
 
     iscsi_set_events(iscsilun);
 
@@ -440,12 +463,13 @@ iscsi_aio_flush(BlockDriverState *bs,
     struct iscsi_context *iscsi = iscsilun->iscsi;
     IscsiAIOCB *acb;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
     acb->task = iscsi_synchronizecache10_task(iscsi, iscsilun->lun,
                                          0, 0, 0, 0,
@@ -493,12 +517,13 @@ iscsi_aio_discard(BlockDriverState *bs,
     IscsiAIOCB *acb;
     struct unmap_list list[1];
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
     list[0].lba = sector_qemu2lun(sector_num, iscsilun);
     list[0].num = nb_sectors * BDRV_SECTOR_SIZE / iscsilun->block_size;
@@ -568,7 +593,7 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
 
     assert(req == SG_IO);
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled    = 0;
@@ -628,9 +653,17 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     return &acb->common;
 }
 
+
+static void ioctl_cb(void *opaque, int status)
+{
+    int *p_status = opaque;
+    *p_status = status;
+}
+
 static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
 {
     IscsiLun *iscsilun = bs->opaque;
+    int status;
 
     switch (req) {
     case SG_GET_VERSION_NUM:
@@ -639,6 +672,15 @@ static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
     case SG_GET_SCSI_ID:
         ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
         break;
+    case SG_IO:
+        status = -EINPROGRESS;
+        iscsi_aio_ioctl(bs, req, buf, ioctl_cb, &status);
+
+        while (status == -EINPROGRESS) {
+            qemu_aio_wait();
+        }
+
+        return 0;
     default:
         return -1;
     }
@@ -658,163 +700,6 @@ iscsi_getlength(BlockDriverState *bs)
     return len;
 }
 
-static void
-iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status,
-                        void *command_data, void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_readcapacity16 *rc16;
-    struct scsi_task *task = command_data;
-
-    if (status != 0) {
-        error_report("iSCSI: Failed to read capacity of iSCSI lun. %s",
-                     iscsi_get_error(iscsi));
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    rc16 = scsi_datain_unmarshall(task);
-    if (rc16 == NULL) {
-        error_report("iSCSI: Failed to unmarshall readcapacity16 data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->block_size = rc16->block_length;
-    itask->iscsilun->num_blocks = rc16->returned_lba + 1;
-    itask->bs->total_sectors    = itask->iscsilun->num_blocks *
-                               itask->iscsilun->block_size / BDRV_SECTOR_SIZE ;
-
-    itask->status   = 0;
-    itask->complete = 1;
-    scsi_free_scsi_task(task);
-}
-
-static void
-iscsi_readcapacity10_cb(struct iscsi_context *iscsi, int status,
-                        void *command_data, void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_readcapacity10 *rc10;
-    struct scsi_task *task = command_data;
-
-    if (status != 0) {
-        error_report("iSCSI: Failed to read capacity of iSCSI lun. %s",
-                     iscsi_get_error(iscsi));
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    rc10 = scsi_datain_unmarshall(task);
-    if (rc10 == NULL) {
-        error_report("iSCSI: Failed to unmarshall readcapacity10 data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->block_size = rc10->block_size;
-    if (rc10->lba == 0) {
-        /* blank disk loaded */
-        itask->iscsilun->num_blocks = 0;
-    } else {
-        itask->iscsilun->num_blocks = rc10->lba + 1;
-    }
-    itask->bs->total_sectors    = itask->iscsilun->num_blocks *
-                               itask->iscsilun->block_size / BDRV_SECTOR_SIZE ;
-
-    itask->status   = 0;
-    itask->complete = 1;
-    scsi_free_scsi_task(task);
-}
-
-static void
-iscsi_inquiry_cb(struct iscsi_context *iscsi, int status, void *command_data,
-                 void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_task *task = command_data;
-    struct scsi_inquiry_standard *inq;
-
-    if (status != 0) {
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    inq = scsi_datain_unmarshall(task);
-    if (inq == NULL) {
-        error_report("iSCSI: Failed to unmarshall inquiry data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->type = inq->periperal_device_type;
-
-    scsi_free_scsi_task(task);
-
-    switch (itask->iscsilun->type) {
-    case TYPE_DISK:
-        task = iscsi_readcapacity16_task(iscsi, itask->iscsilun->lun,
-                                   iscsi_readcapacity16_cb, opaque);
-        if (task == NULL) {
-            error_report("iSCSI: failed to send readcapacity16 command.");
-            itask->status   = 1;
-            itask->complete = 1;
-            return;
-        }
-        break;
-    case TYPE_ROM:
-        task = iscsi_readcapacity10_task(iscsi, itask->iscsilun->lun,
-                                   0, 0,
-                                   iscsi_readcapacity10_cb, opaque);
-        if (task == NULL) {
-            error_report("iSCSI: failed to send readcapacity16 command.");
-            itask->status   = 1;
-            itask->complete = 1;
-            return;
-        }
-        break;
-    default:
-        itask->status   = 0;
-        itask->complete = 1;
-    }
-}
-
-static void
-iscsi_connect_cb(struct iscsi_context *iscsi, int status, void *command_data,
-                 void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_task *task;
-
-    if (status != 0) {
-        itask->status   = 1;
-        itask->complete = 1;
-        return;
-    }
-
-    task = iscsi_inquiry_task(iscsi, itask->iscsilun->lun,
-                              0, 0, 36,
-                              iscsi_inquiry_cb, opaque);
-    if (task == NULL) {
-        error_report("iSCSI: failed to send inquiry command.");
-        itask->status   = 1;
-        itask->complete = 1;
-        return;
-    }
-}
-
 static int parse_chap(struct iscsi_context *iscsi, const char *target)
 {
     QemuOptsList *list;
@@ -918,6 +803,26 @@ static char *parse_initiator_name(const char *target)
     }
 }
 
+#if defined(LIBISCSI_FEATURE_NOP_COUNTER)
+static void iscsi_nop_timed_event(void *opaque)
+{
+    IscsiLun *iscsilun = opaque;
+
+    if (iscsi_get_nops_in_flight(iscsilun->iscsi) > MAX_NOP_FAILURES) {
+        error_report("iSCSI: NOP timeout. Reconnecting...");
+        iscsi_reconnect(iscsilun->iscsi);
+    }
+
+    if (iscsi_nop_out_async(iscsilun->iscsi, NULL, NULL, 0, NULL) != 0) {
+        error_report("iSCSI: failed to sent NOP-Out. Disabling NOP messages.");
+        return;
+    }
+
+    qemu_mod_timer(iscsilun->nop_timer, qemu_get_clock_ms(rt_clock) + NOP_INTERVAL);
+    iscsi_set_events(iscsilun);
+}
+#endif
+
 /*
  * We support iscsi url's on the form
  * iscsi://[<username>%<password>@]<host>[:<port>]/<targetname>/<lun>
@@ -927,7 +832,10 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = NULL;
     struct iscsi_url *iscsi_url = NULL;
-    struct IscsiTask task;
+    struct scsi_task *task = NULL;
+    struct scsi_inquiry_standard *inq = NULL;
+    struct scsi_readcapacity10 *rc10 = NULL;
+    struct scsi_readcapacity16 *rc16 = NULL;
     char *initiator_name = NULL;
     int ret;
 
@@ -940,8 +848,7 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
 
     iscsi_url = iscsi_parse_full_url(iscsi, filename);
     if (iscsi_url == NULL) {
-        error_report("Failed to parse URL : %s %s", filename,
-                     iscsi_get_error(iscsi));
+        error_report("Failed to parse URL : %s", filename);
         ret = -EINVAL;
         goto out;
     }
@@ -991,33 +898,80 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
     /* check if we got HEADER_DIGEST via the options */
     parse_header_digest(iscsi, iscsi_url->target);
 
-    task.iscsilun = iscsilun;
-    task.status = 0;
-    task.complete = 0;
-    task.bs = bs;
+    if (iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun) != 0) {
+        error_report("iSCSI: Failed to connect to LUN : %s",
+            iscsi_get_error(iscsi));
+        ret = -EINVAL;
+        goto out;
+    }
 
     iscsilun->iscsi = iscsi;
     iscsilun->lun   = iscsi_url->lun;
 
-    if (iscsi_full_connect_async(iscsi, iscsi_url->portal, iscsi_url->lun,
-                                 iscsi_connect_cb, &task)
-        != 0) {
-        error_report("iSCSI: Failed to start async connect.");
+    task = iscsi_inquiry_sync(iscsi, iscsilun->lun, 0, 0, 36);
+
+    if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+        error_report("iSCSI: failed to send inquiry command.");
         ret = -EINVAL;
         goto out;
     }
 
-    while (!task.complete) {
-        iscsi_set_events(iscsilun);
-        qemu_aio_wait();
-    }
-    if (task.status != 0) {
-        error_report("iSCSI: Failed to connect to LUN : %s",
-                     iscsi_get_error(iscsi));
+    inq = scsi_datain_unmarshall(task);
+    if (inq == NULL) {
+        error_report("iSCSI: Failed to unmarshall inquiry data.");
         ret = -EINVAL;
         goto out;
     }
 
+    iscsilun->type = inq->periperal_device_type;
+
+    scsi_free_scsi_task(task);
+
+    switch (iscsilun->type) {
+    case TYPE_DISK:
+        task = iscsi_readcapacity16_sync(iscsi, iscsilun->lun);
+        if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+            error_report("iSCSI: failed to send readcapacity16 command.");
+            ret = -EINVAL;
+            goto out;
+        }
+        rc16 = scsi_datain_unmarshall(task);
+        if (rc16 == NULL) {
+            error_report("iSCSI: Failed to unmarshall readcapacity16 data.");
+            ret = -EINVAL;
+            goto out;
+        }
+        iscsilun->block_size = rc16->block_length;
+        iscsilun->num_blocks = rc16->returned_lba + 1;
+        break;
+    case TYPE_ROM:
+        task = iscsi_readcapacity10_sync(iscsi, iscsilun->lun, 0, 0);
+        if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+            error_report("iSCSI: failed to send readcapacity10 command.");
+            ret = -EINVAL;
+            goto out;
+        }
+        rc10 = scsi_datain_unmarshall(task);
+        if (rc10 == NULL) {
+            error_report("iSCSI: Failed to unmarshall readcapacity10 data.");
+            ret = -EINVAL;
+            goto out;
+        }
+        iscsilun->block_size = rc10->block_size;
+        if (rc10->lba == 0) {
+            /* blank disk loaded */
+            iscsilun->num_blocks = 0;
+        } else {
+            iscsilun->num_blocks = rc10->lba + 1;
+        }
+        break;
+    default:
+        break;
+    }
+
+    bs->total_sectors    = iscsilun->num_blocks *
+                           iscsilun->block_size / BDRV_SECTOR_SIZE ;
+
     /* Medium changer or tape. We dont have any emulation for this so this must
      * be sg ioctl compatible. We force it to be sg, otherwise qemu will try
      * to read from the device to guess the image format.
@@ -1029,6 +983,12 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
 
     ret = 0;
 
+#if defined(LIBISCSI_FEATURE_NOP_COUNTER)
+    /* Set up a timer for sending out iSCSI NOPs */
+    iscsilun->nop_timer = qemu_new_timer_ms(rt_clock, iscsi_nop_timed_event, iscsilun);
+    qemu_mod_timer(iscsilun->nop_timer, qemu_get_clock_ms(rt_clock) + NOP_INTERVAL);
+#endif
+
 out:
     if (initiator_name != NULL) {
         g_free(initiator_name);
@@ -1036,6 +996,9 @@ out:
     if (iscsi_url != NULL) {
         iscsi_destroy_url(iscsi_url);
     }
+    if (task != NULL) {
+        scsi_free_scsi_task(task);
+    }
 
     if (ret) {
         if (iscsi != NULL) {
@@ -1051,11 +1014,74 @@ static void iscsi_close(BlockDriverState *bs)
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = iscsilun->iscsi;
 
+    if (iscsilun->nop_timer) {
+        qemu_del_timer(iscsilun->nop_timer);
+        qemu_free_timer(iscsilun->nop_timer);
+    }
     qemu_aio_set_fd_handler(iscsi_get_fd(iscsi), NULL, NULL, NULL, NULL);
     iscsi_destroy_context(iscsi);
     memset(iscsilun, 0, sizeof(IscsiLun));
 }
 
+static int iscsi_has_zero_init(BlockDriverState *bs)
+{
+    return 0;
+}
+
+static int iscsi_create(const char *filename, QEMUOptionParameter *options)
+{
+    int ret = 0;
+    int64_t total_size = 0;
+    BlockDriverState bs;
+    IscsiLun *iscsilun = NULL;
+
+    memset(&bs, 0, sizeof(BlockDriverState));
+
+    /* Read out options */
+    while (options && options->name) {
+        if (!strcmp(options->name, "size")) {
+            total_size = options->value.n / BDRV_SECTOR_SIZE;
+        }
+        options++;
+    }
+
+    bs.opaque = g_malloc0(sizeof(struct IscsiLun));
+    iscsilun = bs.opaque;
+
+    ret = iscsi_open(&bs, filename, 0);
+    if (ret != 0) {
+        goto out;
+    }
+    if (iscsilun->nop_timer) {
+        qemu_del_timer(iscsilun->nop_timer);
+        qemu_free_timer(iscsilun->nop_timer);
+    }
+    if (iscsilun->type != TYPE_DISK) {
+        ret = -ENODEV;
+        goto out;
+    }
+    if (bs.total_sectors < total_size) {
+        ret = -ENOSPC;
+    }
+
+    ret = 0;
+out:
+    if (iscsilun->iscsi != NULL) {
+        iscsi_destroy_context(iscsilun->iscsi);
+    }
+    g_free(bs.opaque);
+    return ret;
+}
+
+static QEMUOptionParameter iscsi_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    { NULL }
+};
+
 static BlockDriver bdrv_iscsi = {
     .format_name     = "iscsi",
     .protocol_name   = "iscsi",
@@ -1063,6 +1089,8 @@ static BlockDriver bdrv_iscsi = {
     .instance_size   = sizeof(IscsiLun),
     .bdrv_file_open  = iscsi_open,
     .bdrv_close      = iscsi_close,
+    .bdrv_create     = iscsi_create,
+    .create_options  = iscsi_create_options,
 
     .bdrv_getlength  = iscsi_getlength,
 
@@ -1071,6 +1099,7 @@ static BlockDriver bdrv_iscsi = {
     .bdrv_aio_flush  = iscsi_aio_flush,
 
     .bdrv_aio_discard = iscsi_aio_discard,
+    .bdrv_has_zero_init = iscsi_has_zero_init,
 
 #ifdef __linux__
     .bdrv_ioctl       = iscsi_ioctl,
@@ -1078,9 +1107,36 @@ static BlockDriver bdrv_iscsi = {
 #endif
 };
 
+static QemuOptsList qemu_iscsi_opts = {
+    .name = "iscsi",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_iscsi_opts.head),
+    .desc = {
+        {
+            .name = "user",
+            .type = QEMU_OPT_STRING,
+            .help = "username for CHAP authentication to target",
+        },{
+            .name = "password",
+            .type = QEMU_OPT_STRING,
+            .help = "password for CHAP authentication to target",
+        },{
+            .name = "header-digest",
+            .type = QEMU_OPT_STRING,
+            .help = "HeaderDigest setting. "
+                    "{CRC32C|CRC32C-NONE|NONE-CRC32C|NONE}",
+        },{
+            .name = "initiator-name",
+            .type = QEMU_OPT_STRING,
+            .help = "Initiator iqn name to use when connecting",
+        },
+        { /* end of list */ }
+    },
+};
+
 static void iscsi_block_init(void)
 {
     bdrv_register(&bdrv_iscsi);
+    qemu_add_opts(&qemu_iscsi_opts);
 }
 
 block_init(iscsi_block_init);
diff --git a/linux-aio.c b/block/linux-aio.c
index ce9b5d4be80..ee0f8d10c92 100644
--- a/linux-aio.c
+++ b/block/linux-aio.c
@@ -8,10 +8,11 @@
  * See the COPYING file in the top-level directory.
  */
 #include "qemu-common.h"
-#include "qemu-aio.h"
-#include "block/raw-posix-aio.h"
+#include "block/aio.h"
+#include "qemu/queue.h"
+#include "block/raw-aio.h"
+#include "qemu/event_notifier.h"
 
-#include <sys/eventfd.h>
 #include <libaio.h>
 
 /*
@@ -37,7 +38,7 @@ struct qemu_laiocb {
 
 struct qemu_laio_state {
     io_context_t ctx;
-    int efd;
+    EventNotifier e;
     int count;
 };
 
@@ -76,29 +77,17 @@ static void qemu_laio_process_completion(struct qemu_laio_state *s,
     qemu_aio_release(laiocb);
 }
 
-static void qemu_laio_completion_cb(void *opaque)
+static void qemu_laio_completion_cb(EventNotifier *e)
 {
-    struct qemu_laio_state *s = opaque;
+    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
 
-    while (1) {
+    while (event_notifier_test_and_clear(&s->e)) {
         struct io_event events[MAX_EVENTS];
-        uint64_t val;
-        ssize_t ret;
         struct timespec ts = { 0 };
         int nevents, i;
 
         do {
-            ret = read(s->efd, &val, sizeof(val));
-        } while (ret == -1 && errno == EINTR);
-
-        if (ret == -1 && errno == EAGAIN)
-            break;
-
-        if (ret != 8)
-            break;
-
-        do {
-            nevents = io_getevents(s->ctx, val, MAX_EVENTS, events, &ts);
+            nevents = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS, events, &ts);
         } while (nevents == -EINTR);
 
         for (i = 0; i < nevents; i++) {
@@ -112,9 +101,9 @@ static void qemu_laio_completion_cb(void *opaque)
     }
 }
 
-static int qemu_laio_flush_cb(void *opaque)
+static int qemu_laio_flush_cb(EventNotifier *e)
 {
-    struct qemu_laio_state *s = opaque;
+    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
 
     return (s->count > 0) ? 1 : 0;
 }
@@ -146,11 +135,12 @@ static void laio_cancel(BlockDriverAIOCB *blockacb)
      * We might be able to do this slightly more optimal by removing the
      * O_NONBLOCK flag.
      */
-    while (laiocb->ret == -EINPROGRESS)
-        qemu_laio_completion_cb(laiocb->ctx);
+    while (laiocb->ret == -EINPROGRESS) {
+        qemu_laio_completion_cb(&laiocb->ctx->e);
+    }
 }
 
-static AIOPool laio_pool = {
+static const AIOCBInfo laio_aiocb_info = {
     .aiocb_size         = sizeof(struct qemu_laiocb),
     .cancel             = laio_cancel,
 };
@@ -164,7 +154,7 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
     struct iocb *iocbs;
     off_t offset = sector_num * 512;
 
-    laiocb = qemu_aio_get(&laio_pool, bs, cb, opaque);
+    laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
     laiocb->nbytes = nb_sectors * 512;
     laiocb->ctx = s;
     laiocb->ret = -EINPROGRESS;
@@ -186,7 +176,7 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
                         __func__, type);
         goto out_free_aiocb;
     }
-    io_set_eventfd(&laiocb->iocb, s->efd);
+    io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
     s->count++;
 
     if (io_submit(s->ctx, 1, &iocbs) < 0)
@@ -205,21 +195,21 @@ void *laio_init(void)
     struct qemu_laio_state *s;
 
     s = g_malloc0(sizeof(*s));
-    s->efd = eventfd(0, 0);
-    if (s->efd == -1)
+    if (event_notifier_init(&s->e, false) < 0) {
         goto out_free_state;
-    fcntl(s->efd, F_SETFL, O_NONBLOCK);
+    }
 
-    if (io_setup(MAX_EVENTS, &s->ctx) != 0)
+    if (io_setup(MAX_EVENTS, &s->ctx) != 0) {
         goto out_close_efd;
+    }
 
-    qemu_aio_set_fd_handler(s->efd, qemu_laio_completion_cb, NULL,
-        qemu_laio_flush_cb, s);
+    qemu_aio_set_event_notifier(&s->e, qemu_laio_completion_cb,
+                                qemu_laio_flush_cb);
 
     return s;
 
 out_close_efd:
-    close(s->efd);
+    event_notifier_cleanup(&s->e);
 out_free_state:
     g_free(s);
     return NULL;
diff --git a/block/mirror.c b/block/mirror.c
new file mode 100644
index 00000000000..a62ad86c28f
--- /dev/null
+++ b/block/mirror.c
@@ -0,0 +1,584 @@
+/*
+ * Image mirroring
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Paolo Bonzini  <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "block/blockjob.h"
+#include "block/block_int.h"
+#include "qemu/ratelimit.h"
+#include "qemu/bitmap.h"
+
+#define SLICE_TIME    100000000ULL /* ns */
+#define MAX_IN_FLIGHT 16
+
+/* The mirroring buffer is a list of granularity-sized chunks.
+ * Free chunks are organized in a list.
+ */
+typedef struct MirrorBuffer {
+    QSIMPLEQ_ENTRY(MirrorBuffer) next;
+} MirrorBuffer;
+
+typedef struct MirrorBlockJob {
+    BlockJob common;
+    RateLimit limit;
+    BlockDriverState *target;
+    MirrorSyncMode mode;
+    BlockdevOnError on_source_error, on_target_error;
+    bool synced;
+    bool should_complete;
+    int64_t sector_num;
+    int64_t granularity;
+    size_t buf_size;
+    unsigned long *cow_bitmap;
+    HBitmapIter hbi;
+    uint8_t *buf;
+    QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
+    int buf_free_count;
+
+    unsigned long *in_flight_bitmap;
+    int in_flight;
+    int ret;
+} MirrorBlockJob;
+
+typedef struct MirrorOp {
+    MirrorBlockJob *s;
+    QEMUIOVector qiov;
+    int64_t sector_num;
+    int nb_sectors;
+} MirrorOp;
+
+static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
+                                            int error)
+{
+    s->synced = false;
+    if (read) {
+        return block_job_error_action(&s->common, s->common.bs,
+                                      s->on_source_error, true, error);
+    } else {
+        return block_job_error_action(&s->common, s->target,
+                                      s->on_target_error, false, error);
+    }
+}
+
+static void mirror_iteration_done(MirrorOp *op, int ret)
+{
+    MirrorBlockJob *s = op->s;
+    struct iovec *iov;
+    int64_t chunk_num;
+    int i, nb_chunks, sectors_per_chunk;
+
+    trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret);
+
+    s->in_flight--;
+    iov = op->qiov.iov;
+    for (i = 0; i < op->qiov.niov; i++) {
+        MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
+        QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
+        s->buf_free_count++;
+    }
+
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    chunk_num = op->sector_num / sectors_per_chunk;
+    nb_chunks = op->nb_sectors / sectors_per_chunk;
+    bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
+    if (s->cow_bitmap && ret >= 0) {
+        bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
+    }
+
+    g_slice_free(MirrorOp, op);
+    qemu_coroutine_enter(s->common.co, NULL);
+}
+
+static void mirror_write_complete(void *opaque, int ret)
+{
+    MirrorOp *op = opaque;
+    MirrorBlockJob *s = op->s;
+    if (ret < 0) {
+        BlockDriverState *source = s->common.bs;
+        BlockErrorAction action;
+
+        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        action = mirror_error_action(s, false, -ret);
+        if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
+            s->ret = ret;
+        }
+    }
+    mirror_iteration_done(op, ret);
+}
+
+static void mirror_read_complete(void *opaque, int ret)
+{
+    MirrorOp *op = opaque;
+    MirrorBlockJob *s = op->s;
+    if (ret < 0) {
+        BlockDriverState *source = s->common.bs;
+        BlockErrorAction action;
+
+        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        action = mirror_error_action(s, true, -ret);
+        if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
+            s->ret = ret;
+        }
+
+        mirror_iteration_done(op, ret);
+        return;
+    }
+    bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors,
+                    mirror_write_complete, op);
+}
+
+static void coroutine_fn mirror_iteration(MirrorBlockJob *s)
+{
+    BlockDriverState *source = s->common.bs;
+    int nb_sectors, sectors_per_chunk, nb_chunks;
+    int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
+    MirrorOp *op;
+
+    s->sector_num = hbitmap_iter_next(&s->hbi);
+    if (s->sector_num < 0) {
+        bdrv_dirty_iter_init(source, &s->hbi);
+        s->sector_num = hbitmap_iter_next(&s->hbi);
+        trace_mirror_restart_iter(s, bdrv_get_dirty_count(source));
+        assert(s->sector_num >= 0);
+    }
+
+    hbitmap_next_sector = s->sector_num;
+    sector_num = s->sector_num;
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    end = s->common.len >> BDRV_SECTOR_BITS;
+
+    /* Extend the QEMUIOVector to include all adjacent blocks that will
+     * be copied in this operation.
+     *
+     * We have to do this if we have no backing file yet in the destination,
+     * and the cluster size is very large.  Then we need to do COW ourselves.
+     * The first time a cluster is copied, copy it entirely.  Note that,
+     * because both the granularity and the cluster size are powers of two,
+     * the number of sectors to copy cannot exceed one cluster.
+     *
+     * We also want to extend the QEMUIOVector to include more adjacent
+     * dirty blocks if possible, to limit the number of I/O operations and
+     * run efficiently even with a small granularity.
+     */
+    nb_chunks = 0;
+    nb_sectors = 0;
+    next_sector = sector_num;
+    next_chunk = sector_num / sectors_per_chunk;
+
+    /* Wait for I/O to this cluster (from a previous iteration) to be done.  */
+    while (test_bit(next_chunk, s->in_flight_bitmap)) {
+        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+        qemu_coroutine_yield();
+    }
+
+    do {
+        int added_sectors, added_chunks;
+
+        if (!bdrv_get_dirty(source, next_sector) ||
+            test_bit(next_chunk, s->in_flight_bitmap)) {
+            assert(nb_sectors > 0);
+            break;
+        }
+
+        added_sectors = sectors_per_chunk;
+        if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) {
+            bdrv_round_to_clusters(s->target,
+                                   next_sector, added_sectors,
+                                   &next_sector, &added_sectors);
+
+            /* On the first iteration, the rounding may make us copy
+             * sectors before the first dirty one.
+             */
+            if (next_sector < sector_num) {
+                assert(nb_sectors == 0);
+                sector_num = next_sector;
+                next_chunk = next_sector / sectors_per_chunk;
+            }
+        }
+
+        added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors));
+        added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk;
+
+        /* When doing COW, it may happen that there is not enough space for
+         * a full cluster.  Wait if that is the case.
+         */
+        while (nb_chunks == 0 && s->buf_free_count < added_chunks) {
+            trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight);
+            qemu_coroutine_yield();
+        }
+        if (s->buf_free_count < nb_chunks + added_chunks) {
+            trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
+            break;
+        }
+
+        /* We have enough free space to copy these sectors.  */
+        bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);
+
+        nb_sectors += added_sectors;
+        nb_chunks += added_chunks;
+        next_sector += added_sectors;
+        next_chunk += added_chunks;
+    } while (next_sector < end);
+
+    /* Allocate a MirrorOp that is used as an AIO callback.  */
+    op = g_slice_new(MirrorOp);
+    op->s = s;
+    op->sector_num = sector_num;
+    op->nb_sectors = nb_sectors;
+
+    /* Now make a QEMUIOVector taking enough granularity-sized chunks
+     * from s->buf_free.
+     */
+    qemu_iovec_init(&op->qiov, nb_chunks);
+    next_sector = sector_num;
+    while (nb_chunks-- > 0) {
+        MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
+        QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
+        s->buf_free_count--;
+        qemu_iovec_add(&op->qiov, buf, s->granularity);
+
+        /* Advance the HBitmapIter in parallel, so that we do not examine
+         * the same sector twice.
+         */
+        if (next_sector > hbitmap_next_sector && bdrv_get_dirty(source, next_sector)) {
+            hbitmap_next_sector = hbitmap_iter_next(&s->hbi);
+        }
+
+        next_sector += sectors_per_chunk;
+    }
+
+    bdrv_reset_dirty(source, sector_num, nb_sectors);
+
+    /* Copy the dirty cluster.  */
+    s->in_flight++;
+    trace_mirror_one_iteration(s, sector_num, nb_sectors);
+    bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
+                   mirror_read_complete, op);
+}
+
+static void mirror_free_init(MirrorBlockJob *s)
+{
+    int granularity = s->granularity;
+    size_t buf_size = s->buf_size;
+    uint8_t *buf = s->buf;
+
+    assert(s->buf_free_count == 0);
+    QSIMPLEQ_INIT(&s->buf_free);
+    while (buf_size != 0) {
+        MirrorBuffer *cur = (MirrorBuffer *)buf;
+        QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
+        s->buf_free_count++;
+        buf_size -= granularity;
+        buf += granularity;
+    }
+}
+
+static void mirror_drain(MirrorBlockJob *s)
+{
+    while (s->in_flight > 0) {
+        qemu_coroutine_yield();
+    }
+}
+
+static void coroutine_fn mirror_run(void *opaque)
+{
+    MirrorBlockJob *s = opaque;
+    BlockDriverState *bs = s->common.bs;
+    int64_t sector_num, end, sectors_per_chunk, length;
+    uint64_t last_pause_ns;
+    BlockDriverInfo bdi;
+    char backing_filename[1024];
+    int ret = 0;
+    int n;
+
+    if (block_job_is_cancelled(&s->common)) {
+        goto immediate_exit;
+    }
+
+    s->common.len = bdrv_getlength(bs);
+    if (s->common.len <= 0) {
+        block_job_completed(&s->common, s->common.len);
+        return;
+    }
+
+    length = (bdrv_getlength(bs) + s->granularity - 1) / s->granularity;
+    s->in_flight_bitmap = bitmap_new(length);
+
+    /* If we have no backing file yet in the destination, we cannot let
+     * the destination do COW.  Instead, we copy sectors around the
+     * dirty data if needed.  We need a bitmap to do that.
+     */
+    bdrv_get_backing_filename(s->target, backing_filename,
+                              sizeof(backing_filename));
+    if (backing_filename[0] && !s->target->backing_hd) {
+        bdrv_get_info(s->target, &bdi);
+        if (s->granularity < bdi.cluster_size) {
+            s->buf_size = MAX(s->buf_size, bdi.cluster_size);
+            s->cow_bitmap = bitmap_new(length);
+        }
+    }
+
+    end = s->common.len >> BDRV_SECTOR_BITS;
+    s->buf = qemu_blockalign(bs, s->buf_size);
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    mirror_free_init(s);
+
+    if (s->mode != MIRROR_SYNC_MODE_NONE) {
+        /* First part, loop on the sectors and initialize the dirty bitmap.  */
+        BlockDriverState *base;
+        base = s->mode == MIRROR_SYNC_MODE_FULL ? NULL : bs->backing_hd;
+        for (sector_num = 0; sector_num < end; ) {
+            int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1;
+            ret = bdrv_co_is_allocated_above(bs, base,
+                                             sector_num, next - sector_num, &n);
+
+            if (ret < 0) {
+                goto immediate_exit;
+            }
+
+            assert(n > 0);
+            if (ret == 1) {
+                bdrv_set_dirty(bs, sector_num, n);
+                sector_num = next;
+            } else {
+                sector_num += n;
+            }
+        }
+    }
+
+    bdrv_dirty_iter_init(bs, &s->hbi);
+    last_pause_ns = qemu_get_clock_ns(rt_clock);
+    for (;;) {
+        uint64_t delay_ns;
+        int64_t cnt;
+        bool should_complete;
+
+        if (s->ret < 0) {
+            ret = s->ret;
+            goto immediate_exit;
+        }
+
+        cnt = bdrv_get_dirty_count(bs);
+
+        /* Note that even when no rate limit is applied we need to yield
+         * periodically with no pending I/O so that qemu_aio_flush() returns.
+         * We do so every SLICE_TIME nanoseconds, or when there is an error,
+         * or when the source is clean, whichever comes first.
+         */
+        if (qemu_get_clock_ns(rt_clock) - last_pause_ns < SLICE_TIME &&
+            s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+            if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 ||
+                (cnt == 0 && s->in_flight > 0)) {
+                trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt);
+                qemu_coroutine_yield();
+                continue;
+            } else if (cnt != 0) {
+                mirror_iteration(s);
+                continue;
+            }
+        }
+
+        should_complete = false;
+        if (s->in_flight == 0 && cnt == 0) {
+            trace_mirror_before_flush(s);
+            ret = bdrv_flush(s->target);
+            if (ret < 0) {
+                if (mirror_error_action(s, false, -ret) == BDRV_ACTION_REPORT) {
+                    goto immediate_exit;
+                }
+            } else {
+                /* We're out of the streaming phase.  From now on, if the job
+                 * is cancelled we will actually complete all pending I/O and
+                 * report completion.  This way, block-job-cancel will leave
+                 * the target in a consistent state.
+                 */
+                s->common.offset = end * BDRV_SECTOR_SIZE;
+                if (!s->synced) {
+                    block_job_ready(&s->common);
+                    s->synced = true;
+                }
+
+                should_complete = s->should_complete ||
+                    block_job_is_cancelled(&s->common);
+                cnt = bdrv_get_dirty_count(bs);
+            }
+        }
+
+        if (cnt == 0 && should_complete) {
+            /* The dirty bitmap is not updated while operations are pending.
+             * If we're about to exit, wait for pending operations before
+             * calling bdrv_get_dirty_count(bs), or we may exit while the
+             * source has dirty data to copy!
+             *
+             * Note that I/O can be submitted by the guest while
+             * mirror_populate runs.
+             */
+            trace_mirror_before_drain(s, cnt);
+            bdrv_drain_all();
+            cnt = bdrv_get_dirty_count(bs);
+        }
+
+        ret = 0;
+        trace_mirror_before_sleep(s, cnt, s->synced);
+        if (!s->synced) {
+            /* Publish progress */
+            s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE;
+
+            if (s->common.speed) {
+                delay_ns = ratelimit_calculate_delay(&s->limit, sectors_per_chunk);
+            } else {
+                delay_ns = 0;
+            }
+
+            block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+            if (block_job_is_cancelled(&s->common)) {
+                break;
+            }
+        } else if (!should_complete) {
+            delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0);
+            block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+        } else if (cnt == 0) {
+            /* The two disks are in sync.  Exit and report successful
+             * completion.
+             */
+            assert(QLIST_EMPTY(&bs->tracked_requests));
+            s->common.cancelled = false;
+            break;
+        }
+        last_pause_ns = qemu_get_clock_ns(rt_clock);
+    }
+
+immediate_exit:
+    if (s->in_flight > 0) {
+        /* We get here only if something went wrong.  Either the job failed,
+         * or it was cancelled prematurely so that we do not guarantee that
+         * the target is a copy of the source.
+         */
+        assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common)));
+        mirror_drain(s);
+    }
+
+    assert(s->in_flight == 0);
+    qemu_vfree(s->buf);
+    g_free(s->cow_bitmap);
+    g_free(s->in_flight_bitmap);
+    bdrv_set_dirty_tracking(bs, 0);
+    bdrv_iostatus_disable(s->target);
+    if (s->should_complete && ret == 0) {
+        if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) {
+            bdrv_reopen(s->target, bdrv_get_flags(s->common.bs), NULL);
+        }
+        bdrv_swap(s->target, s->common.bs);
+    }
+    bdrv_close(s->target);
+    bdrv_delete(s->target);
+    block_job_completed(&s->common, ret);
+}
+
+static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    if (speed < 0) {
+        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        return;
+    }
+    ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
+}
+
+static void mirror_iostatus_reset(BlockJob *job)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    bdrv_iostatus_reset(s->target);
+}
+
+static void mirror_complete(BlockJob *job, Error **errp)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+    int ret;
+
+    ret = bdrv_open_backing_file(s->target);
+    if (ret < 0) {
+        char backing_filename[PATH_MAX];
+        bdrv_get_full_backing_filename(s->target, backing_filename,
+                                       sizeof(backing_filename));
+        error_set(errp, QERR_OPEN_FILE_FAILED, backing_filename);
+        return;
+    }
+    if (!s->synced) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        return;
+    }
+
+    s->should_complete = true;
+    block_job_resume(job);
+}
+
+static BlockJobType mirror_job_type = {
+    .instance_size = sizeof(MirrorBlockJob),
+    .job_type      = "mirror",
+    .set_speed     = mirror_set_speed,
+    .iostatus_reset= mirror_iostatus_reset,
+    .complete      = mirror_complete,
+};
+
+void mirror_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, int64_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp)
+{
+    MirrorBlockJob *s;
+
+    if (granularity == 0) {
+        /* Choose the default granularity based on the target file's cluster
+         * size, clamped between 4k and 64k.  */
+        BlockDriverInfo bdi;
+        if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) {
+            granularity = MAX(4096, bdi.cluster_size);
+            granularity = MIN(65536, granularity);
+        } else {
+            granularity = 65536;
+        }
+    }
+
+    assert ((granularity & (granularity - 1)) == 0);
+
+    if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER, "on-source-error");
+        return;
+    }
+
+    s = block_job_create(&mirror_job_type, bs, speed, cb, opaque, errp);
+    if (!s) {
+        return;
+    }
+
+    s->on_source_error = on_source_error;
+    s->on_target_error = on_target_error;
+    s->target = target;
+    s->mode = mode;
+    s->granularity = granularity;
+    s->buf_size = MAX(buf_size, granularity);
+
+    bdrv_set_dirty_tracking(bs, granularity);
+    bdrv_set_enable_write_cache(s->target, true);
+    bdrv_set_on_error(s->target, on_target_error, on_target_error);
+    bdrv_iostatus_enable(s->target);
+    s->common.co = qemu_coroutine_create(mirror_run);
+    trace_mirror_start(bs, s, s->common.co, opaque);
+    qemu_coroutine_enter(s->common.co, s);
+}
diff --git a/block/nbd.c b/block/nbd.c
index 2bce47bf7a3..a5812948d2b 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -27,10 +27,11 @@
  */
 
 #include "qemu-common.h"
-#include "nbd.h"
-#include "block_int.h"
-#include "module.h"
-#include "qemu_socket.h"
+#include "block/nbd.h"
+#include "qemu/uri.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "qemu/sockets.h"
 
 #include <sys/types.h>
 #include <unistd.h>
@@ -55,7 +56,6 @@ typedef struct BDRVNBDState {
     uint32_t nbdflags;
     off_t size;
     size_t blocksize;
-    char *export_name; /* An NBD server may export several devices */
 
     CoMutex send_mutex;
     CoMutex free_sema;
@@ -65,13 +65,75 @@ typedef struct BDRVNBDState {
     Coroutine *recv_coroutine[MAX_NBD_REQUESTS];
     struct nbd_reply reply;
 
-    /* If it begins with  '/', this is a UNIX domain socket. Otherwise,
-     * it's a string of the form <hostname|ip4|\[ip6\]>:port
-     */
+    int is_unix;
     char *host_spec;
+    char *export_name; /* An NBD server may export several devices */
 } BDRVNBDState;
 
-static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
+static int nbd_parse_uri(BDRVNBDState *s, const char *filename)
+{
+    URI *uri;
+    const char *p;
+    QueryParams *qp = NULL;
+    int ret = 0;
+
+    uri = uri_parse(filename);
+    if (!uri) {
+        return -EINVAL;
+    }
+
+    /* transport */
+    if (!strcmp(uri->scheme, "nbd")) {
+        s->is_unix = false;
+    } else if (!strcmp(uri->scheme, "nbd+tcp")) {
+        s->is_unix = false;
+    } else if (!strcmp(uri->scheme, "nbd+unix")) {
+        s->is_unix = true;
+    } else {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    p = uri->path ? uri->path : "/";
+    p += strspn(p, "/");
+    if (p[0]) {
+        s->export_name = g_strdup(p);
+    }
+
+    qp = query_params_parse(uri->query);
+    if (qp->n > 1 || (s->is_unix && !qp->n) || (!s->is_unix && qp->n)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (s->is_unix) {
+        /* nbd+unix:///export?socket=path */
+        if (uri->server || uri->port || strcmp(qp->p[0].name, "socket")) {
+            ret = -EINVAL;
+            goto out;
+        }
+        s->host_spec = g_strdup(qp->p[0].value);
+    } else {
+        /* nbd[+tcp]://host:port/export */
+        if (!uri->server) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (!uri->port) {
+            uri->port = NBD_DEFAULT_PORT;
+        }
+        s->host_spec = g_strdup_printf("%s:%d", uri->server, uri->port);
+    }
+
+out:
+    if (qp) {
+        query_params_free(qp);
+    }
+    uri_free(uri);
+    return ret;
+}
+
+static int nbd_config(BDRVNBDState *s, const char *filename)
 {
     char *file;
     char *export_name;
@@ -79,6 +141,10 @@ static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
     const char *unixpath;
     int err = -EINVAL;
 
+    if (strstr(filename, "://")) {
+        return nbd_parse_uri(s, filename);
+    }
+
     file = g_strdup(filename);
 
     export_name = strstr(file, EN_OPTSTR);
@@ -98,11 +164,10 @@ static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
 
     /* are we a UNIX or TCP socket? */
     if (strstart(host_spec, "unix:", &unixpath)) {
-        if (unixpath[0] != '/') { /* We demand  an absolute path*/
-            goto out;
-        }
+        s->is_unix = true;
         s->host_spec = g_strdup(unixpath);
     } else {
+        s->is_unix = false;
         s->host_spec = g_strdup(host_spec);
     }
 
@@ -262,7 +327,7 @@ static int nbd_establish_connection(BlockDriverState *bs)
     off_t size;
     size_t blocksize;
 
-    if (s->host_spec[0] == '/') {
+    if (s->is_unix) {
         sock = unix_socket_outgoing(s->host_spec);
     } else {
         sock = tcp_socket_outgoing_spec(s->host_spec);
@@ -320,7 +385,7 @@ static int nbd_open(BlockDriverState *bs, const char* filename, int flags)
     qemu_co_mutex_init(&s->free_sema);
 
     /* Pop the config into our state object. Exit if invalid. */
-    result = nbd_config(s, filename, flags);
+    result = nbd_config(s, filename);
     if (result != 0) {
         return result;
     }
@@ -498,6 +563,33 @@ static int64_t nbd_getlength(BlockDriverState *bs)
 
 static BlockDriver bdrv_nbd = {
     .format_name         = "nbd",
+    .protocol_name       = "nbd",
+    .instance_size       = sizeof(BDRVNBDState),
+    .bdrv_file_open      = nbd_open,
+    .bdrv_co_readv       = nbd_co_readv,
+    .bdrv_co_writev      = nbd_co_writev,
+    .bdrv_close          = nbd_close,
+    .bdrv_co_flush_to_os = nbd_co_flush,
+    .bdrv_co_discard     = nbd_co_discard,
+    .bdrv_getlength      = nbd_getlength,
+};
+
+static BlockDriver bdrv_nbd_tcp = {
+    .format_name         = "nbd",
+    .protocol_name       = "nbd+tcp",
+    .instance_size       = sizeof(BDRVNBDState),
+    .bdrv_file_open      = nbd_open,
+    .bdrv_co_readv       = nbd_co_readv,
+    .bdrv_co_writev      = nbd_co_writev,
+    .bdrv_close          = nbd_close,
+    .bdrv_co_flush_to_os = nbd_co_flush,
+    .bdrv_co_discard     = nbd_co_discard,
+    .bdrv_getlength      = nbd_getlength,
+};
+
+static BlockDriver bdrv_nbd_unix = {
+    .format_name         = "nbd",
+    .protocol_name       = "nbd+unix",
     .instance_size       = sizeof(BDRVNBDState),
     .bdrv_file_open      = nbd_open,
     .bdrv_co_readv       = nbd_co_readv,
@@ -506,12 +598,13 @@ static BlockDriver bdrv_nbd = {
     .bdrv_co_flush_to_os = nbd_co_flush,
     .bdrv_co_discard     = nbd_co_discard,
     .bdrv_getlength      = nbd_getlength,
-    .protocol_name       = "nbd",
 };
 
 static void bdrv_nbd_init(void)
 {
     bdrv_register(&bdrv_nbd);
+    bdrv_register(&bdrv_nbd_tcp);
+    bdrv_register(&bdrv_nbd_unix);
 }
 
 block_init(bdrv_nbd_init);
diff --git a/block/parallels.c b/block/parallels.c
index d30f0ecf777..377375046f1 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -24,8 +24,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 
diff --git a/block/qcow.c b/block/qcow.c
index 7b5ab87d2d0..a7135eea479 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
-#include "aes.h"
-#include "migration.h"
+#include "block/aes.h"
+#include "migration/migration.h"
 
 /**************************************************************/
 /* QEMU COW block driver with compression and encryption support */
@@ -112,7 +112,7 @@ static int qcow_open(BlockDriverState *bs, int flags)
     be64_to_cpus(&header.l1_table_offset);
 
     if (header.magic != QCOW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
     if (header.version != QCOW_VERSION) {
@@ -197,6 +197,15 @@ static int qcow_open(BlockDriverState *bs, int flags)
     return ret;
 }
 
+
+/* We have nothing to do for QCOW reopen, stubs just return
+ * success */
+static int qcow_reopen_prepare(BDRVReopenState *state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static int qcow_set_key(BlockDriverState *bs, const char *key)
 {
     BDRVQcowState *s = bs->opaque;
@@ -868,6 +877,7 @@ static BlockDriver bdrv_qcow = {
     .bdrv_probe		= qcow_probe,
     .bdrv_open		= qcow_open,
     .bdrv_close		= qcow_close,
+    .bdrv_reopen_prepare = qcow_reopen_prepare,
     .bdrv_create	= qcow_create,
 
     .bdrv_co_readv          = qcow_co_readv,
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 2d4322a8dd3..2f3114ecc24 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "block_int.h"
+#include "block/block_int.h"
 #include "qemu-common.h"
 #include "qcow2.h"
 #include "trace.h"
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index e179211c576..56fccf9487e 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -25,7 +25,7 @@
 #include <zlib.h>
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 #include "trace.h"
 
@@ -615,57 +615,67 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
     return cluster_offset;
 }
 
-int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
+static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 {
     BDRVQcowState *s = bs->opaque;
-    int i, j = 0, l2_index, ret;
-    uint64_t *old_cluster, start_sect, *l2_table;
-    uint64_t cluster_offset = m->alloc_offset;
-    bool cow = false;
-
-    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
+    int ret;
 
-    if (m->nb_clusters == 0)
+    if (r->nb_sectors == 0) {
         return 0;
+    }
 
-    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+    qemu_co_mutex_unlock(&s->lock);
+    ret = copy_sectors(bs, m->offset / BDRV_SECTOR_SIZE, m->alloc_offset,
+                       r->offset / BDRV_SECTOR_SIZE,
+                       r->offset / BDRV_SECTOR_SIZE + r->nb_sectors);
+    qemu_co_mutex_lock(&s->lock);
 
-    /* copy content of unmodified sectors */
-    start_sect = (m->offset & ~(s->cluster_size - 1)) >> 9;
-    if (m->n_start) {
-        cow = true;
-        qemu_co_mutex_unlock(&s->lock);
-        ret = copy_sectors(bs, start_sect, cluster_offset, 0, m->n_start);
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0)
-            goto err;
-    }
-
-    if (m->nb_available & (s->cluster_sectors - 1)) {
-        cow = true;
-        qemu_co_mutex_unlock(&s->lock);
-        ret = copy_sectors(bs, start_sect, cluster_offset, m->nb_available,
-                           align_offset(m->nb_available, s->cluster_sectors));
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0)
-            goto err;
+    if (ret < 0) {
+        return ret;
     }
 
     /*
-     * Update L2 table.
-     *
      * Before we update the L2 table to actually point to the new cluster, we
      * need to be sure that the refcounts have been increased and COW was
      * handled.
      */
-    if (cow) {
-        qcow2_cache_depends_on_flush(s->l2_table_cache);
+    qcow2_cache_depends_on_flush(s->l2_table_cache);
+
+    return 0;
+}
+
+int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i, j = 0, l2_index, ret;
+    uint64_t *old_cluster, *l2_table;
+    uint64_t cluster_offset = m->alloc_offset;
+
+    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
+    assert(m->nb_clusters > 0);
+
+    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+
+    /* copy content of unmodified sectors */
+    ret = perform_cow(bs, m, &m->cow_start);
+    if (ret < 0) {
+        goto err;
+    }
+
+    ret = perform_cow(bs, m, &m->cow_end);
+    if (ret < 0) {
+        goto err;
     }
 
+    /* Update L2 table. */
+    if (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS) {
+        qcow2_mark_dirty(bs);
+    }
     if (qcow2_need_accurate_refcounts(s)) {
         qcow2_cache_set_dependency(bs, s->l2_table_cache,
                                    s->refcount_block_cache);
     }
+
     ret = get_cluster_table(bs, m->offset, &l2_table, &l2_index);
     if (ret < 0) {
         goto err;
@@ -743,38 +753,16 @@ out:
 }
 
 /*
- * Allocates new clusters for the given guest_offset.
- *
- * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
- * contain the number of clusters that have been allocated and are contiguous
- * in the image file.
- *
- * If *host_offset is non-zero, it specifies the offset in the image file at
- * which the new clusters must start. *nb_clusters can be 0 on return in this
- * case if the cluster at host_offset is already in use. If *host_offset is
- * zero, the clusters can be allocated anywhere in the image file.
- *
- * *host_offset is updated to contain the offset into the image file at which
- * the first allocated cluster starts.
- *
- * Return 0 on success and -errno in error cases. -EAGAIN means that the
- * function has been waiting for another request and the allocation must be
- * restarted, but the whole request should not be failed.
+ * Check if there already is an AIO write request in flight which allocates
+ * the same cluster. In this case we need to wait until the previous
+ * request has completed and updated the L2 table accordingly.
  */
-static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, unsigned int *nb_clusters)
+static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
+    unsigned int *nb_clusters)
 {
     BDRVQcowState *s = bs->opaque;
     QCowL2Meta *old_alloc;
 
-    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
-                                         *host_offset, *nb_clusters);
-
-    /*
-     * Check if there already is an AIO write request in flight which allocates
-     * the same cluster. In this case we need to wait until the previous
-     * request has completed and updated the L2 table accordingly.
-     */
     QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
 
         uint64_t start = guest_offset >> s->cluster_bits;
@@ -807,6 +795,42 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
         abort();
     }
 
+    return 0;
+}
+
+/*
+ * Allocates new clusters for the given guest_offset.
+ *
+ * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
+ * contain the number of clusters that have been allocated and are contiguous
+ * in the image file.
+ *
+ * If *host_offset is non-zero, it specifies the offset in the image file at
+ * which the new clusters must start. *nb_clusters can be 0 on return in this
+ * case if the cluster at host_offset is already in use. If *host_offset is
+ * zero, the clusters can be allocated anywhere in the image file.
+ *
+ * *host_offset is updated to contain the offset into the image file at which
+ * the first allocated cluster starts.
+ *
+ * Return 0 on success and -errno in error cases. -EAGAIN means that the
+ * function has been waiting for another request and the allocation must be
+ * restarted, but the whole request should not be failed.
+ */
+static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *host_offset, unsigned int *nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
+                                         *host_offset, *nb_clusters);
+
+    ret = handle_dependencies(bs, guest_offset, nb_clusters);
+    if (ret < 0) {
+        return ret;
+    }
+
     /* Allocate new clusters */
     trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
     if (*host_offset == 0) {
@@ -818,7 +842,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
         *host_offset = cluster_offset;
         return 0;
     } else {
-        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
+        ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
         if (ret < 0) {
             return ret;
         }
@@ -847,7 +871,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
  * Return 0 on success and -errno in error cases
  */
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, QCowL2Meta *m)
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m)
 {
     BDRVQcowState *s = bs->opaque;
     int l2_index, ret, sectors;
@@ -919,12 +943,6 @@ again:
     }
 
     /* If there is something left to allocate, do that now */
-    *m = (QCowL2Meta) {
-        .cluster_offset     = cluster_offset,
-        .nb_clusters        = 0,
-    };
-    qemu_co_queue_init(&m->dependent_requests);
-
     if (nb_clusters > 0) {
         uint64_t alloc_offset;
         uint64_t alloc_cluster_offset;
@@ -957,22 +975,40 @@ again:
              *
              * avail_sectors: Number of sectors from the start of the first
              * newly allocated to the end of the last newly allocated cluster.
+             *
+             * nb_sectors: The number of sectors from the start of the first
+             * newly allocated cluster to the end of the aread that the write
+             * request actually writes to (excluding COW at the end)
              */
             int requested_sectors = n_end - keep_clusters * s->cluster_sectors;
             int avail_sectors = nb_clusters
                                 << (s->cluster_bits - BDRV_SECTOR_BITS);
+            int alloc_n_start = keep_clusters == 0 ? n_start : 0;
+            int nb_sectors = MIN(requested_sectors, avail_sectors);
+
+            if (keep_clusters == 0) {
+                cluster_offset = alloc_cluster_offset;
+            }
+
+            *m = g_malloc0(sizeof(**m));
 
-            *m = (QCowL2Meta) {
-                .cluster_offset = keep_clusters == 0 ?
-                                  alloc_cluster_offset : cluster_offset,
+            **m = (QCowL2Meta) {
                 .alloc_offset   = alloc_cluster_offset,
-                .offset         = alloc_offset,
-                .n_start        = keep_clusters == 0 ? n_start : 0,
+                .offset         = alloc_offset & ~(s->cluster_size - 1),
                 .nb_clusters    = nb_clusters,
-                .nb_available   = MIN(requested_sectors, avail_sectors),
+                .nb_available   = nb_sectors,
+
+                .cow_start = {
+                    .offset     = 0,
+                    .nb_sectors = alloc_n_start,
+                },
+                .cow_end = {
+                    .offset     = nb_sectors * BDRV_SECTOR_SIZE,
+                    .nb_sectors = avail_sectors - nb_sectors,
+                },
             };
-            qemu_co_queue_init(&m->dependent_requests);
-            QLIST_INSERT_HEAD(&s->cluster_allocs, m, next_in_flight);
+            qemu_co_queue_init(&(*m)->dependent_requests);
+            QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
         }
     }
 
@@ -984,12 +1020,13 @@ again:
 
     assert(sectors > n_start);
     *num = sectors - n_start;
+    *host_offset = cluster_offset;
 
     return 0;
 
 fail:
-    if (m->nb_clusters > 0) {
-        QLIST_REMOVE(m, next_in_flight);
+    if (*m && (*m)->nb_clusters > 0) {
+        QLIST_REMOVE(*m, next_in_flight);
     }
     return ret;
 }
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 5e3f9153fb9..6a95aa6c92d 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 
 static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size);
@@ -301,7 +301,8 @@ static int alloc_refcount_block(BlockDriverState *bs,
     uint64_t last_table_size;
     uint64_t blocks_clusters;
     do {
-        uint64_t table_clusters = size_to_clusters(s, table_size);
+        uint64_t table_clusters =
+            size_to_clusters(s, table_size * sizeof(uint64_t));
         blocks_clusters = 1 +
             ((table_clusters + refcount_block_clusters - 1)
             / refcount_block_clusters);
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index 4e7c93b8b37..eb8fcd55497 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 
 typedef struct QEMU_PACKED QCowSnapshotHeader {
diff --git a/block/qcow2.c b/block/qcow2.c
index 8f183f14655..7610e569e39 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
-#include "aes.h"
+#include "block/aes.h"
 #include "block/qcow2.h"
-#include "qemu-error.h"
-#include "qerror.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/qerror.h"
 #include "trace.h"
 
 /*
@@ -52,6 +52,7 @@ typedef struct {
     uint32_t magic;
     uint32_t len;
 } QCowExtension;
+
 #define  QCOW2_EXT_MAGIC_END 0
 #define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
 #define  QCOW2_EXT_MAGIC_FEATURE_TABLE 0x6803f857
@@ -221,7 +222,7 @@ static void report_unsupported_feature(BlockDriverState *bs,
  * updated successfully.  Therefore it is not required to check the return
  * value of this function.
  */
-static int qcow2_mark_dirty(BlockDriverState *bs)
+int qcow2_mark_dirty(BlockDriverState *bs)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t val;
@@ -310,7 +311,7 @@ static int qcow2_open(BlockDriverState *bs, int flags)
     be32_to_cpus(&header.nb_snapshots);
 
     if (header.magic != QCOW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
     if (header.version < 2 || header.version > 3) {
@@ -558,6 +559,14 @@ static int qcow2_set_key(BlockDriverState *bs, const char *key)
     return 0;
 }
 
+/* We have nothing to do for QCOW2 reopen, stubs just return
+ * success */
+static int qcow2_reopen_prepare(BDRVReopenState *state,
+                                BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static int coroutine_fn qcow2_co_is_allocated(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, int *pnum)
 {
@@ -736,21 +745,6 @@ fail:
     return ret;
 }
 
-static void run_dependent_requests(BDRVQcowState *s, QCowL2Meta *m)
-{
-    /* Take the request off the list of running requests */
-    if (m->nb_clusters != 0) {
-        QLIST_REMOVE(m, next_in_flight);
-    }
-
-    /* Restart all dependent requests */
-    if (!qemu_co_queue_empty(&m->dependent_requests)) {
-        qemu_co_mutex_unlock(&s->lock);
-        qemu_co_queue_restart_all(&m->dependent_requests);
-        qemu_co_mutex_lock(&s->lock);
-    }
-}
-
 static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                            int64_t sector_num,
                            int remaining_sectors,
@@ -765,15 +759,11 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
     QEMUIOVector hd_qiov;
     uint64_t bytes_done = 0;
     uint8_t *cluster_data = NULL;
-    QCowL2Meta l2meta = {
-        .nb_clusters = 0,
-    };
+    QCowL2Meta *l2meta = NULL;
 
     trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
                                  remaining_sectors);
 
-    qemu_co_queue_init(&l2meta.dependent_requests);
-
     qemu_iovec_init(&hd_qiov, qiov->niov);
 
     s->cluster_cache_offset = -1; /* disable compressed cache */
@@ -782,6 +772,8 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
 
     while (remaining_sectors != 0) {
 
+        l2meta = NULL;
+
         trace_qcow2_writev_start_part(qemu_coroutine_self());
         index_in_cluster = sector_num & (s->cluster_sectors - 1);
         n_end = index_in_cluster + remaining_sectors;
@@ -791,17 +783,11 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
         }
 
         ret = qcow2_alloc_cluster_offset(bs, sector_num << 9,
-            index_in_cluster, n_end, &cur_nr_sectors, &l2meta);
+            index_in_cluster, n_end, &cur_nr_sectors, &cluster_offset, &l2meta);
         if (ret < 0) {
             goto fail;
         }
 
-        if (l2meta.nb_clusters > 0 &&
-            (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS)) {
-            qcow2_mark_dirty(bs);
-        }
-
-        cluster_offset = l2meta.cluster_offset;
         assert((cluster_offset & 511) == 0);
 
         qemu_iovec_reset(&hd_qiov);
@@ -826,8 +812,8 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                 cur_nr_sectors * 512);
         }
 
-        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         qemu_co_mutex_unlock(&s->lock);
+        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         trace_qcow2_writev_data(qemu_coroutine_self(),
                                 (cluster_offset >> 9) + index_in_cluster);
         ret = bdrv_co_writev(bs->file,
@@ -838,12 +824,24 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
             goto fail;
         }
 
-        ret = qcow2_alloc_cluster_link_l2(bs, &l2meta);
-        if (ret < 0) {
-            goto fail;
-        }
+        if (l2meta != NULL) {
+            ret = qcow2_alloc_cluster_link_l2(bs, l2meta);
+            if (ret < 0) {
+                goto fail;
+            }
 
-        run_dependent_requests(s, &l2meta);
+            /* Take the request off the list of running requests */
+            if (l2meta->nb_clusters != 0) {
+                QLIST_REMOVE(l2meta, next_in_flight);
+            }
+
+            qemu_co_mutex_unlock(&s->lock);
+            qemu_co_queue_restart_all(&l2meta->dependent_requests);
+            qemu_co_mutex_lock(&s->lock);
+
+            g_free(l2meta);
+            l2meta = NULL;
+        }
 
         remaining_sectors -= cur_nr_sectors;
         sector_num += cur_nr_sectors;
@@ -853,10 +851,16 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
     ret = 0;
 
 fail:
-    run_dependent_requests(s, &l2meta);
-
     qemu_co_mutex_unlock(&s->lock);
 
+    if (l2meta != NULL) {
+        if (l2meta->nb_clusters != 0) {
+            QLIST_REMOVE(l2meta, next_in_flight);
+        }
+        qemu_co_queue_restart_all(&l2meta->dependent_requests);
+        g_free(l2meta);
+    }
+
     qemu_iovec_destroy(&hd_qiov);
     qemu_vfree(cluster_data);
     trace_qcow2_writev_done_req(qemu_coroutine_self(), ret);
@@ -1087,6 +1091,7 @@ int qcow2_update_header(BlockDriverState *bs)
             goto fail;
         }
 
+        /* Using strncpy is ok here, since buf is not NUL-terminated. */
         strncpy(buf, bs->backing_file, buflen);
 
         header->backing_file_offset = cpu_to_be64(buf - ((char*) header));
@@ -1118,31 +1123,33 @@ static int preallocate(BlockDriverState *bs)
 {
     uint64_t nb_sectors;
     uint64_t offset;
+    uint64_t host_offset = 0;
     int num;
     int ret;
-    QCowL2Meta meta;
+    QCowL2Meta *meta;
 
     nb_sectors = bdrv_getlength(bs) >> 9;
     offset = 0;
-    qemu_co_queue_init(&meta.dependent_requests);
-    meta.cluster_offset = 0;
 
     while (nb_sectors) {
         num = MIN(nb_sectors, INT_MAX >> 9);
-        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num, &meta);
+        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
+                                         &host_offset, &meta);
         if (ret < 0) {
             return ret;
         }
 
-        ret = qcow2_alloc_cluster_link_l2(bs, &meta);
+        ret = qcow2_alloc_cluster_link_l2(bs, meta);
         if (ret < 0) {
-            qcow2_free_any_clusters(bs, meta.cluster_offset, meta.nb_clusters);
+            qcow2_free_any_clusters(bs, meta->alloc_offset, meta->nb_clusters);
             return ret;
         }
 
         /* There are no dependent requests, but we need to remove our request
          * from the list of in-flight requests */
-        run_dependent_requests(bs->opaque, &meta);
+        if (meta != NULL) {
+            QLIST_REMOVE(meta, next_in_flight);
+        }
 
         /* TODO Preallocate data if requested */
 
@@ -1155,10 +1162,10 @@ static int preallocate(BlockDriverState *bs)
      * all of the allocated clusters (otherwise we get failing reads after
      * EOF). Extend the image to the last allocated sector.
      */
-    if (meta.cluster_offset != 0) {
+    if (host_offset != 0) {
         uint8_t buf[512];
         memset(buf, 0, 512);
-        ret = bdrv_write(bs->file, (meta.cluster_offset >> 9) + num - 1, buf, 1);
+        ret = bdrv_write(bs->file, (host_offset >> 9) + num - 1, buf, 1);
         if (ret < 0) {
             return ret;
         }
@@ -1679,6 +1686,7 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_probe         = qcow2_probe,
     .bdrv_open          = qcow2_open,
     .bdrv_close         = qcow2_close,
+    .bdrv_reopen_prepare  = qcow2_reopen_prepare,
     .bdrv_create        = qcow2_create,
     .bdrv_co_is_allocated = qcow2_co_is_allocated,
     .bdrv_set_key       = qcow2_set_key,
diff --git a/block/qcow2.h b/block/qcow2.h
index b4eb65470ed..718b52bacad 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -25,8 +25,8 @@
 #ifndef BLOCK_QCOW2_H
 #define BLOCK_QCOW2_H
 
-#include "aes.h"
-#include "qemu-coroutine.h"
+#include "block/aes.h"
+#include "block/coroutine.h"
 
 //#define DEBUG_ALLOC
 //#define DEBUG_ALLOC2
@@ -196,17 +196,56 @@ typedef struct QCowCreateState {
 
 struct QCowAIOCB;
 
-/* XXX This could be private for qcow2-cluster.c */
+typedef struct Qcow2COWRegion {
+    /**
+     * Offset of the COW region in bytes from the start of the first cluster
+     * touched by the request.
+     */
+    uint64_t    offset;
+
+    /** Number of sectors to copy */
+    int         nb_sectors;
+} Qcow2COWRegion;
+
+/**
+ * Describes an in-flight (part of a) write request that writes to clusters
+ * that are not referenced in their L2 table yet.
+ */
 typedef struct QCowL2Meta
 {
+    /** Guest offset of the first newly allocated cluster */
     uint64_t offset;
-    uint64_t cluster_offset;
+
+    /** Host offset of the first newly allocated cluster */
     uint64_t alloc_offset;
-    int n_start;
+
+    /**
+     * Number of sectors from the start of the first allocated cluster to
+     * the end of the (possibly shortened) request
+     */
     int nb_available;
+
+    /** Number of newly allocated clusters */
     int nb_clusters;
+
+    /**
+     * Requests that overlap with this allocation and wait to be restarted
+     * when the allocating request has completed.
+     */
     CoQueue dependent_requests;
 
+    /**
+     * The COW Region between the start of the first allocated cluster and the
+     * area the guest actually writes to.
+     */
+    Qcow2COWRegion cow_start;
+
+    /**
+     * The COW Region between the area the guest actually writes to and the
+     * end of the last allocated cluster.
+     */
+    Qcow2COWRegion cow_end;
+
     QLIST_ENTRY(QCowL2Meta) next_in_flight;
 } QCowL2Meta;
 
@@ -264,6 +303,8 @@ static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
 /* qcow2.c functions */
 int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
                   int64_t sector_num, int nb_sectors);
+
+int qcow2_mark_dirty(BlockDriverState *bs);
 int qcow2_update_header(BlockDriverState *bs);
 
 /* qcow2-refcount.c functions */
@@ -297,7 +338,7 @@ void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *cluster_offset);
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, QCowL2Meta *m);
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m);
 uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
                                          uint64_t offset,
                                          int compressed_size);
diff --git a/block/qed-table.c b/block/qed-table.c
index ce07b055494..76d2dcccf81 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -13,7 +13,7 @@
  */
 
 #include "trace.h"
-#include "qemu_socket.h" /* for EINPROGRESS on Windows */
+#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "qed.h"
 
 typedef struct {
@@ -103,7 +103,6 @@ static void qed_write_table_cb(void *opaque, int ret)
 out:
     qemu_vfree(write_table_cb->table);
     gencb_complete(&write_table_cb->gencb, ret);
-    return;
 }
 
 /**
diff --git a/block/qed.c b/block/qed.c
index 21cb2398707..b8515e58b3f 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -12,11 +12,11 @@
  *
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "trace.h"
 #include "qed.h"
-#include "qerror.h"
-#include "migration.h"
+#include "qapi/qmp/qerror.h"
+#include "migration/migration.h"
 
 static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
 {
@@ -30,7 +30,7 @@ static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool qed_aio_pool = {
+static const AIOCBInfo qed_aiocb_info = {
     .aiocb_size         = sizeof(QEDAIOCB),
     .cancel             = qed_aio_cancel,
 };
@@ -390,7 +390,7 @@ static int bdrv_qed_open(BlockDriverState *bs, int flags)
     qed_header_le_to_cpu(&le_header, &s->header);
 
     if (s->header.magic != QED_MAGIC) {
-        return -EINVAL;
+        return -EMEDIUMTYPE;
     }
     if (s->header.features & ~QED_FEATURE_MASK) {
         /* image uses unsupported feature bits */
@@ -505,6 +505,14 @@ out:
     return ret;
 }
 
+/* We have nothing to do for QED reopen, stubs just return
+ * success */
+static int bdrv_qed_reopen_prepare(BDRVReopenState *state,
+                                   BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static void bdrv_qed_close(BlockDriverState *bs)
 {
     BDRVQEDState *s = bs->opaque;
@@ -1303,7 +1311,7 @@ static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
                                        BlockDriverCompletionFunc *cb,
                                        void *opaque, int flags)
 {
-    QEDAIOCB *acb = qemu_aio_get(&qed_aio_pool, bs, cb, opaque);
+    QEDAIOCB *acb = qemu_aio_get(&qed_aiocb_info, bs, cb, opaque);
 
     trace_qed_aio_setup(bs->opaque, acb, sector_num, nb_sectors,
                         opaque, flags);
@@ -1564,6 +1572,7 @@ static BlockDriver bdrv_qed = {
     .bdrv_rebind              = bdrv_qed_rebind,
     .bdrv_open                = bdrv_qed_open,
     .bdrv_close               = bdrv_qed_close,
+    .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
     .bdrv_create              = bdrv_qed_create,
     .bdrv_co_is_allocated     = bdrv_qed_co_is_allocated,
     .bdrv_make_empty          = bdrv_qed_make_empty,
diff --git a/block/qed.h b/block/qed.h
index a063bf70aff..2b4ddedf313 100644
--- a/block/qed.h
+++ b/block/qed.h
@@ -15,7 +15,7 @@
 #ifndef BLOCK_QED_H
 #define BLOCK_QED_H
 
-#include "block_int.h"
+#include "block/block_int.h"
 
 /* The layout of a QED file is as follows:
  *
diff --git a/block/raw-posix-aio.h b/block/raw-aio.h
index ba118f616b3..c61f1595d96 100644
--- a/block/raw-posix-aio.h
+++ b/block/raw-aio.h
@@ -1,5 +1,5 @@
 /*
- * QEMU Posix block I/O backend AIO support
+ * Declarations for AIO in the raw protocol
  *
  * Copyright IBM, Corp. 2008
  *
@@ -12,34 +12,40 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#ifndef QEMU_RAW_POSIX_AIO_H
-#define QEMU_RAW_POSIX_AIO_H
+#ifndef QEMU_RAW_AIO_H
+#define QEMU_RAW_AIO_H
 
 /* AIO request types */
 #define QEMU_AIO_READ         0x0001
 #define QEMU_AIO_WRITE        0x0002
 #define QEMU_AIO_IOCTL        0x0004
 #define QEMU_AIO_FLUSH        0x0008
+#define QEMU_AIO_DISCARD      0x0010
 #define QEMU_AIO_TYPE_MASK \
-	(QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH)
+        (QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH| \
+         QEMU_AIO_DISCARD)
 
 /* AIO flags */
 #define QEMU_AIO_MISALIGNED   0x1000
+#define QEMU_AIO_BLKDEV       0x2000
 
 
-/* posix-aio-compat.c - thread pool based implementation */
-int paio_init(void);
-BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type);
-BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
-
 /* linux-aio.c - Linux native implementation */
+#ifdef CONFIG_LINUX_AIO
 void *laio_init(void);
 BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque, int type);
+#endif
+
+#ifdef _WIN32
+typedef struct QEMUWin32AIOState QEMUWin32AIOState;
+QEMUWin32AIOState *win32_aio_init(void);
+int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile);
+BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+        QEMUWin32AIOState *aio, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type);
+#endif
 
-#endif /* QEMU_RAW_POSIX_AIO_H */
+#endif /* QEMU_RAW_AIO_H */
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 6be20b19257..657af956371 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -22,12 +22,14 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "block_int.h"
-#include "module.h"
-#include "block/raw-posix-aio.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "trace.h"
+#include "block/thread-pool.h"
+#include "qemu/iov.h"
+#include "raw-aio.h"
 
 #if defined(__APPLE__) && (__MACH__)
 #include <paths.h>
@@ -57,6 +59,9 @@
 #ifdef CONFIG_FIEMAP
 #include <linux/fiemap.h>
 #endif
+#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+#include <linux/falloc.h>
+#endif
 #if defined (__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <sys/disk.h>
 #include <sys/cdio.h>
@@ -133,16 +138,37 @@ typedef struct BDRVRawState {
     int use_aio;
     void *aio_ctx;
 #endif
-    uint8_t *aligned_buf;
-    unsigned aligned_buf_size;
 #ifdef CONFIG_XFS
     bool is_xfs : 1;
 #endif
+    bool has_discard : 1;
 } BDRVRawState;
 
+typedef struct BDRVRawReopenState {
+    int fd;
+    int open_flags;
+#ifdef CONFIG_LINUX_AIO
+    int use_aio;
+#endif
+} BDRVRawReopenState;
+
 static int fd_open(BlockDriverState *bs);
 static int64_t raw_getlength(BlockDriverState *bs);
 
+typedef struct RawPosixAIOData {
+    BlockDriverState *bs;
+    int aio_fildes;
+    union {
+        struct iovec *aio_iov;
+        void *aio_ioctl_buf;
+    };
+    int aio_niov;
+    uint64_t aio_nbytes;
+#define aio_ioctl_cmd   aio_nbytes /* for QEMU_AIO_IOCTL */
+    off_t aio_offset;
+    int aio_type;
+} RawPosixAIOData;
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static int cdrom_reopen(BlockDriverState *bs);
 #endif
@@ -185,6 +211,57 @@ static int raw_normalize_devicepath(const char **filename)
 }
 #endif
 
+static void raw_parse_flags(int bdrv_flags, int *open_flags)
+{
+    assert(open_flags != NULL);
+
+    *open_flags |= O_BINARY;
+    *open_flags &= ~O_ACCMODE;
+    if (bdrv_flags & BDRV_O_RDWR) {
+        *open_flags |= O_RDWR;
+    } else {
+        *open_flags |= O_RDONLY;
+    }
+
+    /* Use O_DSYNC for write-through caching, no flags for write-back caching,
+     * and O_DIRECT for no caching. */
+    if ((bdrv_flags & BDRV_O_NOCACHE)) {
+        *open_flags |= O_DIRECT;
+    }
+}
+
+#ifdef CONFIG_LINUX_AIO
+static int raw_set_aio(void **aio_ctx, int *use_aio, int bdrv_flags)
+{
+    int ret = -1;
+    assert(aio_ctx != NULL);
+    assert(use_aio != NULL);
+    /*
+     * Currently Linux do AIO only for files opened with O_DIRECT
+     * specified so check NOCACHE flag too
+     */
+    if ((bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
+                      (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) {
+
+        /* if non-NULL, laio_init() has already been run */
+        if (*aio_ctx == NULL) {
+            *aio_ctx = laio_init();
+            if (!*aio_ctx) {
+                goto error;
+            }
+        }
+        *use_aio = 1;
+    } else {
+        *use_aio = 0;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+#endif
+
 static int raw_open_common(BlockDriverState *bs, const char *filename,
                            int bdrv_flags, int open_flags)
 {
@@ -196,20 +273,8 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
         return ret;
     }
 
-    s->open_flags = open_flags | O_BINARY;
-    s->open_flags &= ~O_ACCMODE;
-    if (bdrv_flags & BDRV_O_RDWR) {
-        s->open_flags |= O_RDWR;
-    } else {
-        s->open_flags |= O_RDONLY;
-    }
-
-    /* Use O_DSYNC for write-through caching, no flags for write-back caching,
-     * and O_DIRECT for no caching. */
-    if ((bdrv_flags & BDRV_O_NOCACHE))
-        s->open_flags |= O_DIRECT;
-    if (!(bdrv_flags & BDRV_O_CACHE_WB))
-        s->open_flags |= O_DSYNC;
+    s->open_flags = open_flags;
+    raw_parse_flags(bdrv_flags, &s->open_flags);
 
     s->fd = -1;
     fd = qemu_open(filename, s->open_flags, 0644);
@@ -220,46 +285,15 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
         return ret;
     }
     s->fd = fd;
-    s->aligned_buf = NULL;
-
-    if ((bdrv_flags & BDRV_O_NOCACHE)) {
-        /*
-         * Allocate a buffer for read/modify/write cycles.  Chose the size
-         * pessimistically as we don't know the block size yet.
-         */
-        s->aligned_buf_size = 32 * MAX_BLOCKSIZE;
-        s->aligned_buf = qemu_memalign(MAX_BLOCKSIZE, s->aligned_buf_size);
-        if (s->aligned_buf == NULL) {
-            goto out_close;
-        }
-    }
-
-    /* We're falling back to POSIX AIO in some cases so init always */
-    if (paio_init() < 0) {
-        goto out_free_buf;
-    }
 
 #ifdef CONFIG_LINUX_AIO
-    /*
-     * Currently Linux do AIO only for files opened with O_DIRECT
-     * specified so check NOCACHE flag too
-     */
-    if ((bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
-                      (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) {
-
-        s->aio_ctx = laio_init();
-        if (!s->aio_ctx) {
-            goto out_free_buf;
-        }
-        s->use_aio = 1;
-    } else
-#endif
-    {
-#ifdef CONFIG_LINUX_AIO
-        s->use_aio = 0;
-#endif
+    if (raw_set_aio(&s->aio_ctx, &s->use_aio, bdrv_flags)) {
+        qemu_close(fd);
+        return -errno;
     }
+#endif
 
+    s->has_discard = 1;
 #ifdef CONFIG_XFS
     if (platform_test_xfs_fd(s->fd)) {
         s->is_xfs = 1;
@@ -267,12 +301,6 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
 #endif
 
     return 0;
-
-out_free_buf:
-    qemu_vfree(s->aligned_buf);
-out_close:
-    qemu_close(fd);
-    return -errno;
 }
 
 static int raw_open(BlockDriverState *bs, const char *filename, int flags)
@@ -283,6 +311,113 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
     return raw_open_common(bs, filename, flags, 0);
 }
 
+static int raw_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    BDRVRawState *s;
+    BDRVRawReopenState *raw_s;
+    int ret = 0;
+
+    assert(state != NULL);
+    assert(state->bs != NULL);
+
+    s = state->bs->opaque;
+
+    state->opaque = g_malloc0(sizeof(BDRVRawReopenState));
+    raw_s = state->opaque;
+
+#ifdef CONFIG_LINUX_AIO
+    raw_s->use_aio = s->use_aio;
+
+    /* we can use s->aio_ctx instead of a copy, because the use_aio flag is
+     * valid in the 'false' condition even if aio_ctx is set, and raw_set_aio()
+     * won't override aio_ctx if aio_ctx is non-NULL */
+    if (raw_set_aio(&s->aio_ctx, &raw_s->use_aio, state->flags)) {
+        return -1;
+    }
+#endif
+
+    if (s->type == FTYPE_FD || s->type == FTYPE_CD) {
+        raw_s->open_flags |= O_NONBLOCK;
+    }
+
+    raw_parse_flags(state->flags, &raw_s->open_flags);
+
+    raw_s->fd = -1;
+
+    int fcntl_flags = O_APPEND | O_ASYNC | O_NONBLOCK;
+#ifdef O_NOATIME
+    fcntl_flags |= O_NOATIME;
+#endif
+
+    if ((raw_s->open_flags & ~fcntl_flags) == (s->open_flags & ~fcntl_flags)) {
+        /* dup the original fd */
+        /* TODO: use qemu fcntl wrapper */
+#ifdef F_DUPFD_CLOEXEC
+        raw_s->fd = fcntl(s->fd, F_DUPFD_CLOEXEC, 0);
+#else
+        raw_s->fd = dup(s->fd);
+        if (raw_s->fd != -1) {
+            qemu_set_cloexec(raw_s->fd);
+        }
+#endif
+        if (raw_s->fd >= 0) {
+            ret = fcntl_setfl(raw_s->fd, raw_s->open_flags);
+            if (ret) {
+                qemu_close(raw_s->fd);
+                raw_s->fd = -1;
+            }
+        }
+    }
+
+    /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
+    if (raw_s->fd == -1) {
+        assert(!(raw_s->open_flags & O_CREAT));
+        raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
+        if (raw_s->fd == -1) {
+            ret = -1;
+        }
+    }
+    return ret;
+}
+
+
+static void raw_reopen_commit(BDRVReopenState *state)
+{
+    BDRVRawReopenState *raw_s = state->opaque;
+    BDRVRawState *s = state->bs->opaque;
+
+    s->open_flags = raw_s->open_flags;
+
+    qemu_close(s->fd);
+    s->fd = raw_s->fd;
+#ifdef CONFIG_LINUX_AIO
+    s->use_aio = raw_s->use_aio;
+#endif
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+}
+
+
+static void raw_reopen_abort(BDRVReopenState *state)
+{
+    BDRVRawReopenState *raw_s = state->opaque;
+
+     /* nothing to do if NULL, we didn't get far enough */
+    if (raw_s == NULL) {
+        return;
+    }
+
+    if (raw_s->fd >= 0) {
+        qemu_close(raw_s->fd);
+        raw_s->fd = -1;
+    }
+    g_free(state->opaque);
+    state->opaque = NULL;
+}
+
+
 /* XXX: use host sector size if necessary with:
 #ifdef DIOCGSECTORSIZE
         {
@@ -300,20 +435,326 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 #endif
 */
 
+static ssize_t handle_aiocb_ioctl(RawPosixAIOData *aiocb)
+{
+    int ret;
+
+    ret = ioctl(aiocb->aio_fildes, aiocb->aio_ioctl_cmd, aiocb->aio_ioctl_buf);
+    if (ret == -1) {
+        return -errno;
+    }
+
+    return 0;
+}
+
+static ssize_t handle_aiocb_flush(RawPosixAIOData *aiocb)
+{
+    int ret;
+
+    ret = qemu_fdatasync(aiocb->aio_fildes);
+    if (ret == -1) {
+        return -errno;
+    }
+    return 0;
+}
+
+#ifdef CONFIG_PREADV
+
+static bool preadv_present = true;
+
+static ssize_t
+qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return preadv(fd, iov, nr_iov, offset);
+}
+
+static ssize_t
+qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return pwritev(fd, iov, nr_iov, offset);
+}
+
+#else
+
+static bool preadv_present = false;
+
+static ssize_t
+qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return -ENOSYS;
+}
+
+static ssize_t
+qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return -ENOSYS;
+}
+
+#endif
+
+static ssize_t handle_aiocb_rw_vector(RawPosixAIOData *aiocb)
+{
+    ssize_t len;
+
+    do {
+        if (aiocb->aio_type & QEMU_AIO_WRITE)
+            len = qemu_pwritev(aiocb->aio_fildes,
+                               aiocb->aio_iov,
+                               aiocb->aio_niov,
+                               aiocb->aio_offset);
+         else
+            len = qemu_preadv(aiocb->aio_fildes,
+                              aiocb->aio_iov,
+                              aiocb->aio_niov,
+                              aiocb->aio_offset);
+    } while (len == -1 && errno == EINTR);
+
+    if (len == -1) {
+        return -errno;
+    }
+    return len;
+}
+
 /*
- * Check if all memory in this vector is sector aligned.
+ * Read/writes the data to/from a given linear buffer.
+ *
+ * Returns the number of bytes handles or -errno in case of an error. Short
+ * reads are only returned if the end of the file is reached.
  */
-static int qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
+static ssize_t handle_aiocb_rw_linear(RawPosixAIOData *aiocb, char *buf)
 {
-    int i;
+    ssize_t offset = 0;
+    ssize_t len;
+
+    while (offset < aiocb->aio_nbytes) {
+        if (aiocb->aio_type & QEMU_AIO_WRITE) {
+            len = pwrite(aiocb->aio_fildes,
+                         (const char *)buf + offset,
+                         aiocb->aio_nbytes - offset,
+                         aiocb->aio_offset + offset);
+        } else {
+            len = pread(aiocb->aio_fildes,
+                        buf + offset,
+                        aiocb->aio_nbytes - offset,
+                        aiocb->aio_offset + offset);
+        }
+        if (len == -1 && errno == EINTR) {
+            continue;
+        } else if (len == -1) {
+            offset = -errno;
+            break;
+        } else if (len == 0) {
+            break;
+        }
+        offset += len;
+    }
 
-    for (i = 0; i < qiov->niov; i++) {
-        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
-            return 0;
+    return offset;
+}
+
+static ssize_t handle_aiocb_rw(RawPosixAIOData *aiocb)
+{
+    ssize_t nbytes;
+    char *buf;
+
+    if (!(aiocb->aio_type & QEMU_AIO_MISALIGNED)) {
+        /*
+         * If there is just a single buffer, and it is properly aligned
+         * we can just use plain pread/pwrite without any problems.
+         */
+        if (aiocb->aio_niov == 1) {
+             return handle_aiocb_rw_linear(aiocb, aiocb->aio_iov->iov_base);
+        }
+        /*
+         * We have more than one iovec, and all are properly aligned.
+         *
+         * Try preadv/pwritev first and fall back to linearizing the
+         * buffer if it's not supported.
+         */
+        if (preadv_present) {
+            nbytes = handle_aiocb_rw_vector(aiocb);
+            if (nbytes == aiocb->aio_nbytes ||
+                (nbytes < 0 && nbytes != -ENOSYS)) {
+                return nbytes;
+            }
+            preadv_present = false;
         }
+
+        /*
+         * XXX(hch): short read/write.  no easy way to handle the reminder
+         * using these interfaces.  For now retry using plain
+         * pread/pwrite?
+         */
     }
 
-    return 1;
+    /*
+     * Ok, we have to do it the hard way, copy all segments into
+     * a single aligned buffer.
+     */
+    buf = qemu_blockalign(aiocb->bs, aiocb->aio_nbytes);
+    if (aiocb->aio_type & QEMU_AIO_WRITE) {
+        char *p = buf;
+        int i;
+
+        for (i = 0; i < aiocb->aio_niov; ++i) {
+            memcpy(p, aiocb->aio_iov[i].iov_base, aiocb->aio_iov[i].iov_len);
+            p += aiocb->aio_iov[i].iov_len;
+        }
+    }
+
+    nbytes = handle_aiocb_rw_linear(aiocb, buf);
+    if (!(aiocb->aio_type & QEMU_AIO_WRITE)) {
+        char *p = buf;
+        size_t count = aiocb->aio_nbytes, copy;
+        int i;
+
+        for (i = 0; i < aiocb->aio_niov && count; ++i) {
+            copy = count;
+            if (copy > aiocb->aio_iov[i].iov_len) {
+                copy = aiocb->aio_iov[i].iov_len;
+            }
+            memcpy(aiocb->aio_iov[i].iov_base, p, copy);
+            p     += copy;
+            count -= copy;
+        }
+    }
+    qemu_vfree(buf);
+
+    return nbytes;
+}
+
+#ifdef CONFIG_XFS
+static int xfs_discard(BDRVRawState *s, int64_t offset, uint64_t bytes)
+{
+    struct xfs_flock64 fl;
+
+    memset(&fl, 0, sizeof(fl));
+    fl.l_whence = SEEK_SET;
+    fl.l_start = offset;
+    fl.l_len = bytes;
+
+    if (xfsctl(NULL, s->fd, XFS_IOC_UNRESVSP64, &fl) < 0) {
+        DEBUG_BLOCK_PRINT("cannot punch hole (%s)\n", strerror(errno));
+        return -errno;
+    }
+
+    return 0;
+}
+#endif
+
+static ssize_t handle_aiocb_discard(RawPosixAIOData *aiocb)
+{
+    int ret = -EOPNOTSUPP;
+    BDRVRawState *s = aiocb->bs->opaque;
+
+    if (s->has_discard == 0) {
+        return 0;
+    }
+
+    if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
+#ifdef BLKDISCARD
+        do {
+            uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
+            if (ioctl(aiocb->aio_fildes, BLKDISCARD, range) == 0) {
+                return 0;
+            }
+        } while (errno == EINTR);
+
+        ret = -errno;
+#endif
+    } else {
+#ifdef CONFIG_XFS
+        if (s->is_xfs) {
+            return xfs_discard(s, aiocb->aio_offset, aiocb->aio_nbytes);
+        }
+#endif
+
+#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+        do {
+            if (fallocate(s->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                          aiocb->aio_offset, aiocb->aio_nbytes) == 0) {
+                return 0;
+            }
+        } while (errno == EINTR);
+
+        ret = -errno;
+#endif
+    }
+
+    if (ret == -ENODEV || ret == -ENOSYS || ret == -EOPNOTSUPP ||
+        ret == -ENOTTY) {
+        s->has_discard = 0;
+        ret = 0;
+    }
+    return ret;
+}
+
+static int aio_worker(void *arg)
+{
+    RawPosixAIOData *aiocb = arg;
+    ssize_t ret = 0;
+
+    switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
+    case QEMU_AIO_READ:
+        ret = handle_aiocb_rw(aiocb);
+        if (ret >= 0 && ret < aiocb->aio_nbytes && aiocb->bs->growable) {
+            iov_memset(aiocb->aio_iov, aiocb->aio_niov, ret,
+                      0, aiocb->aio_nbytes - ret);
+
+            ret = aiocb->aio_nbytes;
+        }
+        if (ret == aiocb->aio_nbytes) {
+            ret = 0;
+        } else if (ret >= 0 && ret < aiocb->aio_nbytes) {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_WRITE:
+        ret = handle_aiocb_rw(aiocb);
+        if (ret == aiocb->aio_nbytes) {
+            ret = 0;
+        } else if (ret >= 0 && ret < aiocb->aio_nbytes) {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_FLUSH:
+        ret = handle_aiocb_flush(aiocb);
+        break;
+    case QEMU_AIO_IOCTL:
+        ret = handle_aiocb_ioctl(aiocb);
+        break;
+    case QEMU_AIO_DISCARD:
+        ret = handle_aiocb_discard(aiocb);
+        break;
+    default:
+        fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
+        ret = -EINVAL;
+        break;
+    }
+
+    g_slice_free(RawPosixAIOData, aiocb);
+    return ret;
+}
+
+static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+
+    acb->bs = bs;
+    acb->aio_type = type;
+    acb->aio_fildes = fd;
+
+    if (qiov) {
+        acb->aio_iov = qiov->iov;
+        acb->aio_niov = qiov->niov;
+    }
+    acb->aio_nbytes = nb_sectors * 512;
+    acb->aio_offset = sector_num * 512;
+
+    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
 }
 
 static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
@@ -330,8 +771,8 @@ static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
      * boundary.  Check if this is the case or tell the low-level
      * driver that it needs to copy the buffer.
      */
-    if (s->aligned_buf) {
-        if (!qiov_is_aligned(bs, qiov)) {
+    if ((bs->open_flags & BDRV_O_NOCACHE)) {
+        if (!bdrv_qiov_is_aligned(bs, qiov)) {
             type |= QEMU_AIO_MISALIGNED;
 #ifdef CONFIG_LINUX_AIO
         } else if (s->use_aio) {
@@ -378,8 +819,6 @@ static void raw_close(BlockDriverState *bs)
     if (s->fd >= 0) {
         qemu_close(s->fd);
         s->fd = -1;
-        if (s->aligned_buf != NULL)
-            qemu_vfree(s->aligned_buf);
     }
 }
 
@@ -687,37 +1126,14 @@ static int coroutine_fn raw_co_is_allocated(BlockDriverState *bs,
     }
 }
 
-#ifdef CONFIG_XFS
-static int xfs_discard(BDRVRawState *s, int64_t sector_num, int nb_sectors)
-{
-    struct xfs_flock64 fl;
-
-    memset(&fl, 0, sizeof(fl));
-    fl.l_whence = SEEK_SET;
-    fl.l_start = sector_num << 9;
-    fl.l_len = (int64_t)nb_sectors << 9;
-
-    if (xfsctl(NULL, s->fd, XFS_IOC_UNRESVSP64, &fl) < 0) {
-        DEBUG_BLOCK_PRINT("cannot punch hole (%s)\n", strerror(errno));
-        return -errno;
-    }
-
-    return 0;
-}
-#endif
-
-static coroutine_fn int raw_co_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
+static coroutine_fn BlockDriverAIOCB *raw_aio_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors,
+    BlockDriverCompletionFunc *cb, void *opaque)
 {
-#ifdef CONFIG_XFS
     BDRVRawState *s = bs->opaque;
 
-    if (s->is_xfs) {
-        return xfs_discard(s, sector_num, nb_sectors);
-    }
-#endif
-
-    return 0;
+    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+                       cb, opaque, QEMU_AIO_DISCARD);
 }
 
 static QEMUOptionParameter raw_create_options[] = {
@@ -735,14 +1151,17 @@ static BlockDriver bdrv_file = {
     .instance_size = sizeof(BDRVRawState),
     .bdrv_probe = NULL, /* no probe for protocols */
     .bdrv_file_open = raw_open,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit = raw_reopen_commit,
+    .bdrv_reopen_abort = raw_reopen_abort,
     .bdrv_close = raw_close,
     .bdrv_create = raw_create,
-    .bdrv_co_discard = raw_co_discard,
     .bdrv_co_is_allocated = raw_co_is_allocated,
 
     .bdrv_aio_readv = raw_aio_readv,
     .bdrv_aio_writev = raw_aio_writev,
     .bdrv_aio_flush = raw_aio_flush,
+    .bdrv_aio_discard = raw_aio_discard,
 
     .bdrv_truncate = raw_truncate,
     .bdrv_getlength = raw_getlength,
@@ -937,10 +1356,19 @@ static BlockDriverAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
+    RawPosixAIOData *acb;
 
     if (fd_open(bs) < 0)
         return NULL;
-    return paio_ioctl(bs, s->fd, req, buf, cb, opaque);
+
+    acb = g_slice_new(RawPosixAIOData);
+    acb->bs = bs;
+    acb->aio_type = QEMU_AIO_IOCTL;
+    acb->aio_fildes = s->fd;
+    acb->aio_offset = 0;
+    acb->aio_ioctl_buf = buf;
+    acb->aio_ioctl_cmd = req;
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
 }
 
 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
@@ -962,6 +1390,19 @@ static int fd_open(BlockDriverState *bs)
 
 #endif /* !linux && !FreeBSD */
 
+static coroutine_fn BlockDriverAIOCB *hdev_aio_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors,
+    BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BDRVRawState *s = bs->opaque;
+
+    if (fd_open(bs) < 0) {
+        return NULL;
+    }
+    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+                       cb, opaque, QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
+}
+
 static int hdev_create(const char *filename, QEMUOptionParameter *options)
 {
     int fd;
@@ -1004,6 +1445,9 @@ static BlockDriver bdrv_host_device = {
     .bdrv_probe_device  = hdev_probe_device,
     .bdrv_file_open     = hdev_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1011,6 +1455,7 @@ static BlockDriver bdrv_host_device = {
     .bdrv_aio_readv	= raw_aio_readv,
     .bdrv_aio_writev	= raw_aio_writev,
     .bdrv_aio_flush	= raw_aio_flush,
+    .bdrv_aio_discard   = hdev_aio_discard,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength	= raw_getlength,
@@ -1125,6 +1570,9 @@ static BlockDriver bdrv_host_floppy = {
     .bdrv_probe_device	= floppy_probe_device,
     .bdrv_file_open     = floppy_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1224,6 +1672,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1343,6 +1794,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1363,6 +1817,40 @@ static BlockDriver bdrv_host_cdrom = {
 };
 #endif /* __FreeBSD__ */
 
+#ifdef CONFIG_LINUX_AIO
+/**
+ * Return the file descriptor for Linux AIO
+ *
+ * This function is a layering violation and should be removed when it becomes
+ * possible to call the block layer outside the global mutex.  It allows the
+ * caller to hijack the file descriptor so I/O can be performed outside the
+ * block layer.
+ */
+int raw_get_aio_fd(BlockDriverState *bs)
+{
+    BDRVRawState *s;
+
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    }
+
+    if (bs->drv == bdrv_find_format("raw")) {
+        bs = bs->file;
+    }
+
+    /* raw-posix has several protocols so just check for raw_aio_readv */
+    if (bs->drv->bdrv_aio_readv != raw_aio_readv) {
+        return -ENOTSUP;
+    }
+
+    s = bs->opaque;
+    if (!s->use_aio) {
+        return -ENOTSUP;
+    }
+    return s->fd;
+}
+#endif /* CONFIG_LINUX_AIO */
+
 static void bdrv_file_init(void)
 {
     /*
diff --git a/block/raw-win32.c b/block/raw-win32.c
index c56bf83375e..b89ac19ffaf 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -22,9 +22,13 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/timer.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "raw-aio.h"
+#include "trace.h"
+#include "block/thread-pool.h"
+#include "qemu/iov.h"
 #include <windows.h>
 #include <winioctl.h>
 
@@ -32,12 +36,130 @@
 #define FTYPE_CD     1
 #define FTYPE_HARDDISK 2
 
+static QEMUWin32AIOState *aio;
+
+typedef struct RawWin32AIOData {
+    BlockDriverState *bs;
+    HANDLE hfile;
+    struct iovec *aio_iov;
+    int aio_niov;
+    size_t aio_nbytes;
+    off64_t aio_offset;
+    int aio_type;
+} RawWin32AIOData;
+
 typedef struct BDRVRawState {
     HANDLE hfile;
     int type;
     char drive_path[16]; /* format: "d:\" */
+    QEMUWin32AIOState *aio;
 } BDRVRawState;
 
+/*
+ * Read/writes the data to/from a given linear buffer.
+ *
+ * Returns the number of bytes handles or -errno in case of an error. Short
+ * reads are only returned if the end of the file is reached.
+ */
+static size_t handle_aiocb_rw(RawWin32AIOData *aiocb)
+{
+    size_t offset = 0;
+    int i;
+
+    for (i = 0; i < aiocb->aio_niov; i++) {
+        OVERLAPPED ov;
+        DWORD ret, ret_count, len;
+
+        memset(&ov, 0, sizeof(ov));
+        ov.Offset = (aiocb->aio_offset + offset);
+        ov.OffsetHigh = (aiocb->aio_offset + offset) >> 32;
+        len = aiocb->aio_iov[i].iov_len;
+        if (aiocb->aio_type & QEMU_AIO_WRITE) {
+            ret = WriteFile(aiocb->hfile, aiocb->aio_iov[i].iov_base,
+                            len, &ret_count, &ov);
+        } else {
+            ret = ReadFile(aiocb->hfile, aiocb->aio_iov[i].iov_base,
+                           len, &ret_count, &ov);
+        }
+        if (!ret) {
+            ret_count = 0;
+        }
+        if (ret_count != len) {
+            break;
+        }
+        offset += len;
+    }
+
+    return offset;
+}
+
+static int aio_worker(void *arg)
+{
+    RawWin32AIOData *aiocb = arg;
+    ssize_t ret = 0;
+    size_t count;
+
+    switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
+    case QEMU_AIO_READ:
+        count = handle_aiocb_rw(aiocb);
+        if (count < aiocb->aio_nbytes && aiocb->bs->growable) {
+            /* A short read means that we have reached EOF. Pad the buffer
+             * with zeros for bytes after EOF. */
+            iov_memset(aiocb->aio_iov, aiocb->aio_niov, count,
+                      0, aiocb->aio_nbytes - count);
+
+            count = aiocb->aio_nbytes;
+        }
+        if (count == aiocb->aio_nbytes) {
+            ret = 0;
+        } else {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_WRITE:
+        count = handle_aiocb_rw(aiocb);
+        if (count == aiocb->aio_nbytes) {
+            count = 0;
+        } else {
+            count = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_FLUSH:
+        if (!FlushFileBuffers(aiocb->hfile)) {
+            return -EIO;
+        }
+        break;
+    default:
+        fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
+        ret = -EINVAL;
+        break;
+    }
+
+    g_slice_free(RawWin32AIOData, aiocb);
+    return ret;
+}
+
+static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
+
+    acb->bs = bs;
+    acb->hfile = hfile;
+    acb->aio_type = type;
+
+    if (qiov) {
+        acb->aio_iov = qiov->iov;
+        acb->aio_niov = qiov->niov;
+    }
+    acb->aio_nbytes = nb_sectors * 512;
+    acb->aio_offset = sector_num * 512;
+
+    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
+}
+
 int qemu_ftruncate64(int fd, int64_t length)
 {
     LARGE_INTEGER li;
@@ -77,6 +199,26 @@ static int set_sparse(int fd)
 				 NULL, 0, NULL, 0, &returned, NULL);
 }
 
+static void raw_parse_flags(int flags, int *access_flags, DWORD *overlapped)
+{
+    assert(access_flags != NULL);
+    assert(overlapped != NULL);
+
+    if (flags & BDRV_O_RDWR) {
+        *access_flags = GENERIC_READ | GENERIC_WRITE;
+    } else {
+        *access_flags = GENERIC_READ;
+    }
+
+    *overlapped = FILE_ATTRIBUTE_NORMAL;
+    if (flags & BDRV_O_NATIVE_AIO) {
+        *overlapped |= FILE_FLAG_OVERLAPPED;
+    }
+    if (flags & BDRV_O_NOCACHE) {
+        *overlapped |= FILE_FLAG_NO_BUFFERING;
+    }
+}
+
 static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 {
     BDRVRawState *s = bs->opaque;
@@ -85,17 +227,15 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 
     s->type = FTYPE_FILE;
 
-    if (flags & BDRV_O_RDWR) {
-        access_flags = GENERIC_READ | GENERIC_WRITE;
-    } else {
-        access_flags = GENERIC_READ;
+    raw_parse_flags(flags, &access_flags, &overlapped);
+    
+    if ((flags & BDRV_O_NATIVE_AIO) && aio == NULL) {
+        aio = win32_aio_init();
+        if (aio == NULL) {
+            return -EINVAL;
+        }
     }
 
-    overlapped = FILE_ATTRIBUTE_NORMAL;
-    if (flags & BDRV_O_NOCACHE)
-        overlapped |= FILE_FLAG_NO_BUFFERING;
-    if (!(flags & BDRV_O_CACHE_WB))
-        overlapped |= FILE_FLAG_WRITE_THROUGH;
     s->hfile = CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           OPEN_EXISTING, overlapped, NULL);
@@ -104,64 +244,53 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 
         if (err == ERROR_ACCESS_DENIED)
             return -EACCES;
-        return -1;
+        return -EINVAL;
+    }
+
+    if (flags & BDRV_O_NATIVE_AIO) {
+        int ret = win32_aio_attach(aio, s->hfile);
+        if (ret < 0) {
+            CloseHandle(s->hfile);
+            return ret;
+        }
+        s->aio = aio;
     }
     return 0;
 }
 
-static int raw_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
+                         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+                         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    OVERLAPPED ov;
-    DWORD ret_count;
-    int ret;
-    int64_t offset = sector_num * 512;
-    int count = nb_sectors * 512;
-
-    memset(&ov, 0, sizeof(ov));
-    ov.Offset = offset;
-    ov.OffsetHigh = offset >> 32;
-    ret = ReadFile(s->hfile, buf, count, &ret_count, &ov);
-    if (!ret)
-        return ret_count;
-    if (ret_count == count)
-        ret_count = 0;
-    return ret_count;
+    if (s->aio) {
+        return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
+                                nb_sectors, cb, opaque, QEMU_AIO_READ); 
+    } else {
+        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+                           cb, opaque, QEMU_AIO_READ);
+    }
 }
 
-static int raw_write(BlockDriverState *bs, int64_t sector_num,
-                     const uint8_t *buf, int nb_sectors)
+static BlockDriverAIOCB *raw_aio_writev(BlockDriverState *bs,
+                          int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+                          BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    OVERLAPPED ov;
-    DWORD ret_count;
-    int ret;
-    int64_t offset = sector_num * 512;
-    int count = nb_sectors * 512;
-
-    memset(&ov, 0, sizeof(ov));
-    ov.Offset = offset;
-    ov.OffsetHigh = offset >> 32;
-    ret = WriteFile(s->hfile, buf, count, &ret_count, &ov);
-    if (!ret)
-        return ret_count;
-    if (ret_count == count)
-        ret_count = 0;
-    return ret_count;
+    if (s->aio) {
+        return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
+                                nb_sectors, cb, opaque, QEMU_AIO_WRITE); 
+    } else {
+        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+                           cb, opaque, QEMU_AIO_WRITE);
+    }
 }
 
-static int raw_flush(BlockDriverState *bs)
+static BlockDriverAIOCB *raw_aio_flush(BlockDriverState *bs,
+                         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    int ret;
-
-    ret = FlushFileBuffers(s->hfile);
-    if (ret == 0) {
-        return -EIO;
-    }
-
-    return 0;
+    return paio_submit(bs, s->hfile, 0, NULL, 0, cb, opaque, QEMU_AIO_FLUSH);
 }
 
 static void raw_close(BlockDriverState *bs)
@@ -174,13 +303,24 @@ static int raw_truncate(BlockDriverState *bs, int64_t offset)
 {
     BDRVRawState *s = bs->opaque;
     LONG low, high;
+    DWORD dwPtrLow;
 
     low = offset;
     high = offset >> 32;
-    if (!SetFilePointer(s->hfile, low, &high, FILE_BEGIN))
-	return -EIO;
-    if (!SetEndOfFile(s->hfile))
+
+    /*
+     * An error has occurred if the return value is INVALID_SET_FILE_POINTER
+     * and GetLastError doesn't return NO_ERROR.
+     */
+    dwPtrLow = SetFilePointer(s->hfile, low, &high, FILE_BEGIN);
+    if (dwPtrLow == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) {
+        fprintf(stderr, "SetFilePointer error: %lu\n", GetLastError());
+        return -EIO;
+    }
+    if (SetEndOfFile(s->hfile) == 0) {
+        fprintf(stderr, "SetEndOfFile error: %lu\n", GetLastError());
         return -EIO;
+    }
     return 0;
 }
 
@@ -282,9 +422,9 @@ static BlockDriver bdrv_file = {
     .bdrv_close		= raw_close,
     .bdrv_create	= raw_create,
 
-    .bdrv_read              = raw_read,
-    .bdrv_write             = raw_write,
-    .bdrv_co_flush_to_disk  = raw_flush,
+    .bdrv_aio_readv     = raw_aio_readv,
+    .bdrv_aio_writev    = raw_aio_writev,
+    .bdrv_aio_flush     = raw_aio_flush,
 
     .bdrv_truncate	= raw_truncate,
     .bdrv_getlength	= raw_getlength,
@@ -374,18 +514,10 @@ static int hdev_open(BlockDriverState *bs, const char *filename, int flags)
     }
     s->type = find_device_type(bs, filename);
 
-    if (flags & BDRV_O_RDWR) {
-        access_flags = GENERIC_READ | GENERIC_WRITE;
-    } else {
-        access_flags = GENERIC_READ;
-    }
+    raw_parse_flags(flags, &access_flags, &overlapped);
+
     create_flags = OPEN_EXISTING;
 
-    overlapped = FILE_ATTRIBUTE_NORMAL;
-    if (flags & BDRV_O_NOCACHE)
-        overlapped |= FILE_FLAG_NO_BUFFERING;
-    if (!(flags & BDRV_O_CACHE_WB))
-        overlapped |= FILE_FLAG_WRITE_THROUGH;
     s->hfile = CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           create_flags, overlapped, NULL);
@@ -413,9 +545,9 @@ static BlockDriver bdrv_host_device = {
     .bdrv_close		= raw_close,
     .bdrv_has_zero_init = hdev_has_zero_init,
 
-    .bdrv_read              = raw_read,
-    .bdrv_write             = raw_write,
-    .bdrv_co_flush_to_disk  = raw_flush,
+    .bdrv_aio_readv     = raw_aio_readv,
+    .bdrv_aio_writev    = raw_aio_writev,
+    .bdrv_aio_flush     = raw_aio_flush,
 
     .bdrv_getlength	= raw_getlength,
     .bdrv_get_allocated_file_size
diff --git a/block/raw.c b/block/raw.c
index ff34ea41e7a..75812db3c27 100644
--- a/block/raw.c
+++ b/block/raw.c
@@ -1,7 +1,7 @@
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 static int raw_open(BlockDriverState *bs, int flags)
 {
@@ -9,6 +9,14 @@ static int raw_open(BlockDriverState *bs, int flags)
     return 0;
 }
 
+/* We have nothing to do for raw reopen, stubs just return
+ * success */
+static int raw_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue,  Error **errp)
+{
+    return 0;
+}
+
 static int coroutine_fn raw_co_readv(BlockDriverState *bs, int64_t sector_num,
                                      int nb_sectors, QEMUIOVector *qiov)
 {
@@ -115,6 +123,8 @@ static BlockDriver bdrv_raw = {
     .bdrv_open          = raw_open,
     .bdrv_close         = raw_close,
 
+    .bdrv_reopen_prepare  = raw_reopen_prepare,
+
     .bdrv_co_readv          = raw_co_readv,
     .bdrv_co_writev         = raw_co_writev,
     .bdrv_co_is_allocated   = raw_co_is_allocated,
diff --git a/block/rbd.c b/block/rbd.c
index 5a0f79fc8f3..8cd10a7b594 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -14,8 +14,8 @@
 #include <inttypes.h>
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "block_int.h"
+#include "qemu/error-report.h"
+#include "block/block_int.h"
 
 #include <rbd/librbd.h>
 
@@ -69,7 +69,7 @@ typedef enum {
 typedef struct RBDAIOCB {
     BlockDriverAIOCB common;
     QEMUBH *bh;
-    int ret;
+    int64_t ret;
     QEMUIOVector *qiov;
     char *bounce;
     RBDAIOCmd cmd;
@@ -77,6 +77,7 @@ typedef struct RBDAIOCB {
     int error;
     struct BDRVRBDState *s;
     int cancelled;
+    int status;
 } RBDAIOCB;
 
 typedef struct RADOSCB {
@@ -86,7 +87,7 @@ typedef struct RADOSCB {
     int done;
     int64_t size;
     char *buf;
-    int ret;
+    int64_t ret;
 } RADOSCB;
 
 #define RBD_FD_READ 0
@@ -376,12 +377,6 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
     RBDAIOCB *acb = rcb->acb;
     int64_t r;
 
-    if (acb->cancelled) {
-        qemu_vfree(acb->bounce);
-        qemu_aio_release(acb);
-        goto done;
-    }
-
     r = rcb->ret;
 
     if (acb->cmd == RBD_AIO_WRITE ||
@@ -409,7 +404,6 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
     /* Note that acb->bh can be NULL in case where the aio was cancelled */
     acb->bh = qemu_bh_new(rbd_aio_bh_cb, acb);
     qemu_bh_schedule(acb->bh);
-done:
     g_free(rcb);
 }
 
@@ -487,12 +481,6 @@ static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
         rados_conf_set(s->cluster, "rbd_cache", "false");
     } else {
         rados_conf_set(s->cluster, "rbd_cache", "true");
-        if (!(flags & BDRV_O_CACHE_WB)) {
-            r = rados_conf_set(s->cluster, "rbd_cache_max_dirty", "0");
-            if (r < 0) {
-                rados_conf_set(s->cluster, "rbd_cache", "false");
-            }
-        }
     }
 
     if (strstr(conf, "conf=") == NULL) {
@@ -574,9 +562,15 @@ static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb)
 {
     RBDAIOCB *acb = (RBDAIOCB *) blockacb;
     acb->cancelled = 1;
+
+    while (acb->status == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    qemu_aio_release(acb);
 }
 
-static AIOPool rbd_aio_pool = {
+static const AIOCBInfo rbd_aiocb_info = {
     .aiocb_size = sizeof(RBDAIOCB),
     .cancel = qemu_rbd_aio_cancel,
 };
@@ -645,8 +639,11 @@ static void rbd_aio_bh_cb(void *opaque)
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
     qemu_bh_delete(acb->bh);
     acb->bh = NULL;
+    acb->status = 0;
 
-    qemu_aio_release(acb);
+    if (!acb->cancelled) {
+        qemu_aio_release(acb);
+    }
 }
 
 static int rbd_aio_discard_wrapper(rbd_image_t image,
@@ -678,7 +675,7 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     BDRVRBDState *s = bs->opaque;
 
-    acb = qemu_aio_get(&rbd_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&rbd_aiocb_info, bs, cb, opaque);
     acb->cmd = cmd;
     acb->qiov = qiov;
     if (cmd == RBD_AIO_DISCARD) {
@@ -691,6 +688,7 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->s = s;
     acb->cancelled = 0;
     acb->bh = NULL;
+    acb->status = -EINPROGRESS;
 
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
diff --git a/block/sheepdog.c b/block/sheepdog.c
index df4f44107b3..3e49bb83bbb 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -13,10 +13,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu_socket.h"
-#include "block_int.h"
-#include "bitops.h"
+#include "qemu/error-report.h"
+#include "qemu/sockets.h"
+#include "block/block_int.h"
+#include "qemu/bitops.h"
 
 #define SD_PROTO_VER 0x01
 
@@ -36,7 +36,8 @@
 
 #define SD_FLAG_CMD_WRITE    0x01
 #define SD_FLAG_CMD_COW      0x02
-#define SD_FLAG_CMD_CACHE    0x04
+#define SD_FLAG_CMD_CACHE    0x04 /* Writeback mode for cache */
+#define SD_FLAG_CMD_DIRECT   0x08 /* Don't use cache */
 
 #define SD_RES_SUCCESS       0x00 /* Success */
 #define SD_RES_UNKNOWN       0x01 /* Unknown error */
@@ -201,12 +202,12 @@ static inline uint64_t fnv_64a_buf(void *buf, size_t len, uint64_t hval)
     return hval;
 }
 
-static inline int is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
+static inline bool is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
 {
     return inode->vdi_id == inode->data_vdi_id[idx];
 }
 
-static inline int is_data_obj(uint64_t oid)
+static inline bool is_data_obj(uint64_t oid)
 {
     return !(VDI_BIT & oid);
 }
@@ -231,7 +232,7 @@ static inline uint64_t vid_to_data_oid(uint32_t vid, uint32_t idx)
     return ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
 }
 
-static inline int is_snapshot(struct SheepdogInode *inode)
+static inline bool is_snapshot(struct SheepdogInode *inode)
 {
     return !!inode->snap_ctime;
 }
@@ -265,6 +266,7 @@ typedef struct AIOReq {
 enum AIOCBState {
     AIOCB_WRITE_UDATA,
     AIOCB_READ_UDATA,
+    AIOCB_FLUSH_CACHE,
 };
 
 struct SheepdogAIOCB {
@@ -281,7 +283,7 @@ struct SheepdogAIOCB {
     Coroutine *coroutine;
     void (*aio_done_func)(SheepdogAIOCB *);
 
-    int canceled;
+    bool canceled;
     int nr_pending;
 };
 
@@ -292,13 +294,12 @@ typedef struct BDRVSheepdogState {
     uint32_t max_dirty_data_idx;
 
     char name[SD_MAX_VDI_LEN];
-    int is_snapshot;
-    uint8_t cache_enabled;
+    bool is_snapshot;
+    uint32_t cache_flags;
 
     char *addr;
     char *port;
     int fd;
-    int flush_fd;
 
     CoMutex lock;
     Coroutine *co_send;
@@ -417,21 +418,20 @@ static void sd_aio_cancel(BlockDriverAIOCB *blockacb)
      */
     acb->ret = -EIO;
     qemu_coroutine_enter(acb->coroutine, NULL);
-    acb->canceled = 1;
+    acb->canceled = true;
 }
 
-static AIOPool sd_aio_pool = {
+static const AIOCBInfo sd_aiocb_info = {
     .aiocb_size = sizeof(SheepdogAIOCB),
     .cancel = sd_aio_cancel,
 };
 
 static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
-                                   int64_t sector_num, int nb_sectors,
-                                   BlockDriverCompletionFunc *cb, void *opaque)
+                                   int64_t sector_num, int nb_sectors)
 {
     SheepdogAIOCB *acb;
 
-    acb = qemu_aio_get(&sd_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&sd_aiocb_info, bs, NULL, NULL);
 
     acb->qiov = qiov;
 
@@ -439,7 +439,7 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
     acb->nb_sectors = nb_sectors;
 
     acb->aio_done_func = NULL;
-    acb->canceled = 0;
+    acb->canceled = false;
     acb->coroutine = qemu_coroutine_self();
     acb->ret = 0;
     acb->nr_pending = 0;
@@ -613,7 +613,7 @@ static int do_req(int sockfd, SheepdogReq *hdr, void *data,
 }
 
 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
-                           struct iovec *iov, int niov, int create,
+                           struct iovec *iov, int niov, bool create,
                            enum AIOCBState aiocb_type);
 
 
@@ -646,7 +646,7 @@ static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid)
         QLIST_REMOVE(aio_req, aio_siblings);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
         ret = add_aio_request(s, aio_req, acb->qiov->iov,
-                              acb->qiov->niov, 0, acb->aiocb_type);
+                              acb->qiov->niov, false, acb->aiocb_type);
         if (ret < 0) {
             error_report("add_aio_request is failed");
             free_aio_req(s, aio_req);
@@ -714,16 +714,17 @@ static void coroutine_fn aio_read_response(void *opaque)
              * and max_dirty_data_idx are changed to include updated
              * index between them.
              */
-            s->inode.data_vdi_id[idx] = s->inode.vdi_id;
-            s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
-            s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
-
+            if (rsp.result == SD_RES_SUCCESS) {
+                s->inode.data_vdi_id[idx] = s->inode.vdi_id;
+                s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
+                s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
+            }
             /*
              * Some requests may be blocked because simultaneous
              * create requests are not allowed, so we search the
              * pending requests here.
              */
-            send_pending_req(s, vid_to_data_oid(s->inode.vdi_id, idx));
+            send_pending_req(s, aio_req->oid);
         }
         break;
     case AIOCB_READ_UDATA:
@@ -734,6 +735,13 @@ static void coroutine_fn aio_read_response(void *opaque)
             goto out;
         }
         break;
+    case AIOCB_FLUSH_CACHE:
+        if (rsp.result == SD_RES_INVALID_PARMS) {
+            dprintf("disable cache since the server doesn't support it\n");
+            s->cache_flags = SD_FLAG_CMD_DIRECT;
+            rsp.result = SD_RES_SUCCESS;
+        }
+        break;
     }
 
     if (rsp.result != SD_RES_SUCCESS) {
@@ -866,14 +874,14 @@ static int parse_vdiname(BDRVSheepdogState *s, const char *filename,
         s->port = 0;
     }
 
-    strncpy(vdi, p, SD_MAX_VDI_LEN);
+    pstrcpy(vdi, SD_MAX_VDI_LEN, p);
 
     p = strchr(vdi, ':');
     if (p) {
         *p++ = '\0';
         *snapid = strtoul(p, NULL, 10);
         if (*snapid == 0) {
-            strncpy(tag, p, SD_MAX_VDI_TAG_LEN);
+            pstrcpy(tag, SD_MAX_VDI_TAG_LEN, p);
         }
     } else {
         *snapid = CURRENT_VDI_ID; /* search current vdi */
@@ -900,7 +908,10 @@ static int find_vdi_name(BDRVSheepdogState *s, char *filename, uint32_t snapid,
         return fd;
     }
 
-    memset(buf, 0, sizeof(buf));
+    /* This pair of strncpy calls ensures that the buffer is zero-filled,
+     * which is desirable since we'll soon be sending those bytes, and
+     * don't want the send_req to read uninitialized data.
+     */
     strncpy(buf, filename, SD_MAX_VDI_LEN);
     strncpy(buf + SD_MAX_VDI_LEN, tag, SD_MAX_VDI_TAG_LEN);
 
@@ -940,12 +951,12 @@ out:
 }
 
 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
-                           struct iovec *iov, int niov, int create,
+                           struct iovec *iov, int niov, bool create,
                            enum AIOCBState aiocb_type)
 {
     int nr_copies = s->inode.nr_copies;
     SheepdogObjReq hdr;
-    unsigned int wlen;
+    unsigned int wlen = 0;
     int ret;
     uint64_t oid = aio_req->oid;
     unsigned int datalen = aio_req->data_len;
@@ -959,22 +970,27 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
     memset(&hdr, 0, sizeof(hdr));
 
-    if (aiocb_type == AIOCB_READ_UDATA) {
-        wlen = 0;
+    switch (aiocb_type) {
+    case AIOCB_FLUSH_CACHE:
+        hdr.opcode = SD_OP_FLUSH_VDI;
+        break;
+    case AIOCB_READ_UDATA:
         hdr.opcode = SD_OP_READ_OBJ;
         hdr.flags = flags;
-    } else if (create) {
-        wlen = datalen;
-        hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
-        hdr.flags = SD_FLAG_CMD_WRITE | flags;
-    } else {
+        break;
+    case AIOCB_WRITE_UDATA:
+        if (create) {
+            hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
+        } else {
+            hdr.opcode = SD_OP_WRITE_OBJ;
+        }
         wlen = datalen;
-        hdr.opcode = SD_OP_WRITE_OBJ;
         hdr.flags = SD_FLAG_CMD_WRITE | flags;
+        break;
     }
 
-    if (s->cache_enabled) {
-        hdr.flags |= SD_FLAG_CMD_CACHE;
+    if (s->cache_flags) {
+        hdr.flags |= s->cache_flags;
     }
 
     hdr.oid = oid;
@@ -1019,7 +1035,7 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
 static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
                              unsigned int datalen, uint64_t offset,
-                             int write, int create, uint8_t cache)
+                             bool write, bool create, uint32_t cache_flags)
 {
     SheepdogObjReq hdr;
     SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
@@ -1043,9 +1059,7 @@ static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
         hdr.opcode = SD_OP_READ_OBJ;
     }
 
-    if (cache) {
-        hdr.flags |= SD_FLAG_CMD_CACHE;
-    }
+    hdr.flags |= cache_flags;
 
     hdr.oid = oid;
     hdr.data_length = datalen;
@@ -1068,18 +1082,19 @@ static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
 }
 
 static int read_object(int fd, char *buf, uint64_t oid, int copies,
-                       unsigned int datalen, uint64_t offset, uint8_t cache)
+                       unsigned int datalen, uint64_t offset,
+                       uint32_t cache_flags)
 {
-    return read_write_object(fd, buf, oid, copies, datalen, offset, 0, 0,
-                             cache);
+    return read_write_object(fd, buf, oid, copies, datalen, offset, false,
+                             false, cache_flags);
 }
 
 static int write_object(int fd, char *buf, uint64_t oid, int copies,
-                        unsigned int datalen, uint64_t offset, int create,
-                        uint8_t cache)
+                        unsigned int datalen, uint64_t offset, bool create,
+                        uint32_t cache_flags)
 {
-    return read_write_object(fd, buf, oid, copies, datalen, offset, 1, create,
-                             cache);
+    return read_write_object(fd, buf, oid, copies, datalen, offset, true,
+                             create, cache_flags);
 }
 
 static int sd_open(BlockDriverState *bs, const char *filename, int flags)
@@ -1114,19 +1129,18 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
         goto out;
     }
 
-    if (flags & BDRV_O_CACHE_WB) {
-        s->cache_enabled = 1;
-        s->flush_fd = connect_to_sdog(s->addr, s->port);
-        if (s->flush_fd < 0) {
-            error_report("failed to connect");
-            ret = s->flush_fd;
-            goto out;
-        }
+    /*
+     * QEMU block layer emulates writethrough cache as 'writeback + flush', so
+     * we always set SD_FLAG_CMD_CACHE (writeback cache) as default.
+     */
+    s->cache_flags = SD_FLAG_CMD_CACHE;
+    if (flags & BDRV_O_NOCACHE) {
+        s->cache_flags = SD_FLAG_CMD_DIRECT;
     }
 
     if (snapid || tag[0] != '\0') {
         dprintf("%" PRIx32 " snapshot inode was open.\n", vid);
-        s->is_snapshot = 1;
+        s->is_snapshot = true;
     }
 
     fd = connect_to_sdog(s->addr, s->port);
@@ -1138,7 +1152,7 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
 
     buf = g_malloc(SD_INODE_SIZE);
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), 0, SD_INODE_SIZE, 0,
-                      s->cache_enabled);
+                      s->cache_flags);
 
     closesocket(fd);
 
@@ -1151,7 +1165,7 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
     s->max_dirty_data_idx = 0;
 
     bs->total_sectors = s->inode.vdi_size / SECTOR_SIZE;
-    strncpy(s->name, vdi, sizeof(s->name));
+    pstrcpy(s->name, sizeof(s->name), vdi);
     qemu_co_mutex_init(&s->lock);
     g_free(buf);
     return 0;
@@ -1179,8 +1193,11 @@ static int do_sd_create(char *filename, int64_t vdi_size,
         return fd;
     }
 
+    /* FIXME: would it be better to fail (e.g., return -EIO) when filename
+     * does not fit in buf?  For now, just truncate and avoid buffer overrun.
+     */
     memset(buf, 0, sizeof(buf));
-    strncpy(buf, filename, SD_MAX_VDI_LEN);
+    pstrcpy(buf, sizeof(buf), filename);
 
     memset(&hdr, 0, sizeof(hdr));
     hdr.opcode = SD_OP_NEW_VDI;
@@ -1266,7 +1283,7 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
     BDRVSheepdogState *s;
     char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
     uint32_t snapid;
-    int prealloc = 0;
+    bool prealloc = false;
     const char *vdiname;
 
     s = g_malloc0(sizeof(BDRVSheepdogState));
@@ -1288,9 +1305,9 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
             backing_file = options->value.s;
         } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
             if (!options->value.s || !strcmp(options->value.s, "off")) {
-                prealloc = 0;
+                prealloc = false;
             } else if (!strcmp(options->value.s, "full")) {
-                prealloc = 1;
+                prealloc = true;
             } else {
                 error_report("Invalid preallocation mode: '%s'",
                              options->value.s);
@@ -1382,9 +1399,6 @@ static void sd_close(BlockDriverState *bs)
 
     qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL);
     closesocket(s->fd);
-    if (s->cache_enabled) {
-        closesocket(s->flush_fd);
-    }
     g_free(s->addr);
 }
 
@@ -1418,7 +1432,7 @@ static int sd_truncate(BlockDriverState *bs, int64_t offset)
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
     s->inode.vdi_size = offset;
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_flags);
     close(fd);
 
     if (ret < 0) {
@@ -1457,7 +1471,7 @@ static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
         aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
                                 data_len, offset, 0, 0, offset);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
-        ret = add_aio_request(s, aio_req, &iov, 1, 0, AIOCB_WRITE_UDATA);
+        ret = add_aio_request(s, aio_req, &iov, 1, false, AIOCB_WRITE_UDATA);
         if (ret) {
             free_aio_req(s, aio_req);
             acb->ret = -EIO;
@@ -1501,7 +1515,7 @@ static int sd_create_branch(BDRVSheepdogState *s)
     }
 
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
-                      SD_INODE_SIZE, 0, s->cache_enabled);
+                      SD_INODE_SIZE, 0, s->cache_flags);
 
     closesocket(fd);
 
@@ -1511,7 +1525,7 @@ static int sd_create_branch(BDRVSheepdogState *s)
 
     memcpy(&s->inode, buf, sizeof(s->inode));
 
-    s->is_snapshot = 0;
+    s->is_snapshot = false;
     ret = 0;
     dprintf("%" PRIx32 " was newly created.\n", s->inode.vdi_id);
 
@@ -1566,7 +1580,7 @@ static int coroutine_fn sd_co_rw_vector(void *p)
     while (done != total) {
         uint8_t flags = 0;
         uint64_t old_oid = 0;
-        int create = 0;
+        bool create = false;
 
         oid = vid_to_data_oid(inode->data_vdi_id[idx], idx);
 
@@ -1581,10 +1595,10 @@ static int coroutine_fn sd_co_rw_vector(void *p)
             break;
         case AIOCB_WRITE_UDATA:
             if (!inode->data_vdi_id[idx]) {
-                create = 1;
+                create = true;
             } else if (!is_data_obj_writable(inode, idx)) {
                 /* Copy-On-Write */
-                create = 1;
+                create = true;
                 old_oid = oid;
                 flags = SD_FLAG_CMD_COW;
             }
@@ -1657,7 +1671,7 @@ static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
         bs->total_sectors = sector_num + nb_sectors;
     }
 
-    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
+    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors);
     acb->aio_done_func = sd_write_done;
     acb->aiocb_type = AIOCB_WRITE_UDATA;
 
@@ -1678,7 +1692,7 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
     SheepdogAIOCB *acb;
     int ret;
 
-    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
+    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors);
     acb->aiocb_type = AIOCB_READ_UDATA;
     acb->aio_done_func = sd_finish_aiocb;
 
@@ -1696,39 +1710,31 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
 static int coroutine_fn sd_co_flush_to_disk(BlockDriverState *bs)
 {
     BDRVSheepdogState *s = bs->opaque;
-    SheepdogObjReq hdr = { 0 };
-    SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
-    SheepdogInode *inode = &s->inode;
+    SheepdogAIOCB *acb;
+    AIOReq *aio_req;
     int ret;
-    unsigned int wlen = 0, rlen = 0;
 
-    if (!s->cache_enabled) {
+    if (s->cache_flags != SD_FLAG_CMD_CACHE) {
         return 0;
     }
 
-    hdr.opcode = SD_OP_FLUSH_VDI;
-    hdr.oid = vid_to_vdi_oid(inode->vdi_id);
+    acb = sd_aio_setup(bs, NULL, 0, 0);
+    acb->aiocb_type = AIOCB_FLUSH_CACHE;
+    acb->aio_done_func = sd_finish_aiocb;
 
-    ret = do_req(s->flush_fd, (SheepdogReq *)&hdr, NULL, &wlen, &rlen);
-    if (ret) {
-        error_report("failed to send a request to the sheep");
+    aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
+                            0, 0, 0, 0, 0);
+    QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
+    ret = add_aio_request(s, aio_req, NULL, 0, false, acb->aiocb_type);
+    if (ret < 0) {
+        error_report("add_aio_request is failed");
+        free_aio_req(s, aio_req);
+        qemu_aio_release(acb);
         return ret;
     }
 
-    if (rsp->result == SD_RES_INVALID_PARMS) {
-        dprintf("disable write cache since the server doesn't support it\n");
-
-        s->cache_enabled = 0;
-        closesocket(s->flush_fd);
-        return 0;
-    }
-
-    if (rsp->result != SD_RES_SUCCESS) {
-        error_report("%s", sd_strerror(rsp->result));
-        return -EIO;
-    }
-
-    return 0;
+    qemu_coroutine_yield();
+    return acb->ret;
 }
 
 static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
@@ -1754,6 +1760,9 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 
     s->inode.vm_state_size = sn_info->vm_state_size;
     s->inode.vm_clock_nsec = sn_info->vm_clock_nsec;
+    /* It appears that inode.tag does not require a NUL terminator,
+     * which means this use of strncpy is ok.
+     */
     strncpy(s->inode.tag, sn_info->name, sizeof(s->inode.tag));
     /* we don't need to update entire object */
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
@@ -1766,7 +1775,7 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     }
 
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_flags);
     if (ret < 0) {
         error_report("failed to write snapshot's inode.");
         goto cleanup;
@@ -1783,7 +1792,7 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     inode = (SheepdogInode *)g_malloc(datalen);
 
     ret = read_object(fd, (char *)inode, vid_to_vdi_oid(new_vid),
-                      s->inode.nr_copies, datalen, 0, s->cache_enabled);
+                      s->inode.nr_copies, datalen, 0, s->cache_flags);
 
     if (ret < 0) {
         error_report("failed to read new inode info. %s", strerror(errno));
@@ -1813,13 +1822,13 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 
     memcpy(old_s, s, sizeof(BDRVSheepdogState));
 
-    memset(vdi, 0, sizeof(vdi));
-    strncpy(vdi, s->name, sizeof(vdi));
+    pstrcpy(vdi, sizeof(vdi), s->name);
 
-    memset(tag, 0, sizeof(tag));
     snapid = strtoul(snapshot_id, NULL, 10);
-    if (!snapid) {
-        strncpy(tag, s->name, sizeof(tag));
+    if (snapid) {
+        tag[0] = 0;
+    } else {
+        pstrcpy(tag, sizeof(tag), s->name);
     }
 
     ret = find_vdi_name(s, vdi, snapid, tag, &vid, 1);
@@ -1837,7 +1846,7 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 
     buf = g_malloc(SD_INODE_SIZE);
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
-                      SD_INODE_SIZE, 0, s->cache_enabled);
+                      SD_INODE_SIZE, 0, s->cache_flags);
 
     closesocket(fd);
 
@@ -1853,7 +1862,7 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto out;
     }
 
-    s->is_snapshot = 1;
+    s->is_snapshot = true;
 
     g_free(buf);
     g_free(old_s);
@@ -1934,7 +1943,7 @@ static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
         /* we don't need to read entire object */
         ret = read_object(fd, (char *)&inode, vid_to_vdi_oid(vid),
                           0, SD_INODE_SIZE - sizeof(inode.data_vdi_id), 0,
-                          s->cache_enabled);
+                          s->cache_flags);
 
         if (ret) {
             continue;
@@ -1948,8 +1957,9 @@ static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 
             snprintf(sn_tab[found].id_str, sizeof(sn_tab[found].id_str), "%u",
                      inode.snap_id);
-            strncpy(sn_tab[found].name, inode.tag,
-                    MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)));
+            pstrcpy(sn_tab[found].name,
+                    MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)),
+                    inode.tag);
             found++;
         }
     }
@@ -1970,8 +1980,8 @@ out:
 static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
                                 int64_t pos, int size, int load)
 {
-    int fd, create;
-    int ret = 0, remaining = size;
+    bool create;
+    int fd, ret = 0, remaining = size;
     unsigned int data_len;
     uint64_t vmstate_oid;
     uint32_t vdi_index;
@@ -1986,7 +1996,7 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         vdi_index = pos / SD_DATA_OBJ_SIZE;
         offset = pos % SD_DATA_OBJ_SIZE;
 
-        data_len = MIN(remaining, SD_DATA_OBJ_SIZE);
+        data_len = MIN(remaining, SD_DATA_OBJ_SIZE - offset);
 
         vmstate_oid = vid_to_vmstate_oid(s->inode.vdi_id, vdi_index);
 
@@ -1994,11 +2004,11 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         if (load) {
             ret = read_object(fd, (char *)data, vmstate_oid,
                               s->inode.nr_copies, data_len, offset,
-                              s->cache_enabled);
+                              s->cache_flags);
         } else {
             ret = write_object(fd, (char *)data, vmstate_oid,
                                s->inode.nr_copies, data_len, offset, create,
-                               s->cache_enabled);
+                               s->cache_flags);
         }
 
         if (ret < 0) {
@@ -2007,6 +2017,7 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         }
 
         pos += data_len;
+        data += data_len;
         remaining -= data_len;
     }
     ret = size;
diff --git a/block/stream.c b/block/stream.c
index c4f87dd5b6f..d6df06f35a1 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -12,7 +12,8 @@
  */
 
 #include "trace.h"
-#include "block_int.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
 #include "qemu/ratelimit.h"
 
 enum {
@@ -30,6 +31,7 @@ typedef struct StreamBlockJob {
     BlockJob common;
     RateLimit limit;
     BlockDriverState *base;
+    BlockdevOnError on_error;
     char backing_file_id[1024];
 } StreamBlockJob;
 
@@ -77,13 +79,14 @@ static void coroutine_fn stream_run(void *opaque)
     BlockDriverState *bs = s->common.bs;
     BlockDriverState *base = s->base;
     int64_t sector_num, end;
+    int error = 0;
     int ret = 0;
     int n = 0;
     void *buf;
 
     s->common.len = bdrv_getlength(bs);
     if (s->common.len < 0) {
-        block_job_complete(&s->common, s->common.len);
+        block_job_completed(&s->common, s->common.len);
         return;
     }
 
@@ -105,7 +108,7 @@ static void coroutine_fn stream_run(void *opaque)
 
 wait:
         /* Note that even when no rate limit is applied we need to yield
-         * with no pending I/O here so that qemu_aio_flush() returns.
+         * with no pending I/O here so that bdrv_drain_all() returns.
          */
         block_job_sleep_ns(&s->common, rt_clock, delay_ns);
         if (block_job_is_cancelled(&s->common)) {
@@ -141,7 +144,19 @@ wait:
             ret = stream_populate(bs, sector_num, n, buf);
         }
         if (ret < 0) {
-            break;
+            BlockErrorAction action =
+                block_job_error_action(&s->common, s->common.bs, s->on_error,
+                                       true, -ret);
+            if (action == BDRV_ACTION_STOP) {
+                n = 0;
+                continue;
+            }
+            if (error == 0) {
+                error = ret;
+            }
+            if (action == BDRV_ACTION_REPORT) {
+                break;
+            }
         }
         ret = 0;
 
@@ -153,6 +168,9 @@ wait:
         bdrv_disable_copy_on_read(bs);
     }
 
+    /* Do not remove the backing file if an error was there but ignored.  */
+    ret = error;
+
     if (!block_job_is_cancelled(&s->common) && sector_num == end && ret == 0) {
         const char *base_id = NULL, *base_fmt = NULL;
         if (base) {
@@ -166,7 +184,7 @@ wait:
     }
 
     qemu_vfree(buf);
-    block_job_complete(&s->common, ret);
+    block_job_completed(&s->common, ret);
 }
 
 static void stream_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -188,11 +206,19 @@ static BlockJobType stream_job_type = {
 
 void stream_start(BlockDriverState *bs, BlockDriverState *base,
                   const char *base_id, int64_t speed,
+                  BlockdevOnError on_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     StreamBlockJob *s;
 
+    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER, "on-error");
+        return;
+    }
+
     s = block_job_create(&stream_job_type, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
@@ -203,6 +229,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
         pstrcpy(s->backing_file_id, sizeof(s->backing_file_id), base_id);
     }
 
+    s->on_error = on_error;
     s->common.co = qemu_coroutine_create(stream_run);
     trace_stream_start(bs, base, s, s->common.co, opaque);
     qemu_coroutine_enter(s->common.co, s);
diff --git a/block/vdi.c b/block/vdi.c
index c4f1529db9b..257a592ea9e 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -50,19 +50,16 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
 #else
 /* TODO: move uuid emulation to some central place in QEMU. */
-#include "sysemu.h"     /* UUID_FMT */
+#include "sysemu/sysemu.h"     /* UUID_FMT */
 typedef unsigned char uuid_t[16];
-void uuid_generate(uuid_t out);
-int uuid_is_null(const uuid_t uu);
-void uuid_unparse(const uuid_t uu, char *out);
 #endif
 
 /* Code configuration options. */
@@ -124,18 +121,18 @@ void uuid_unparse(const uuid_t uu, char *out);
 #define VDI_IS_ALLOCATED(X) ((X) < VDI_DISCARDED)
 
 #if !defined(CONFIG_UUID)
-void uuid_generate(uuid_t out)
+static inline void uuid_generate(uuid_t out)
 {
     memset(out, 0, sizeof(uuid_t));
 }
 
-int uuid_is_null(const uuid_t uu)
+static inline int uuid_is_null(const uuid_t uu)
 {
     uuid_t null_uuid = { 0 };
     return memcmp(uu, null_uuid, sizeof(uuid_t)) == 0;
 }
 
-void uuid_unparse(const uuid_t uu, char *out)
+static inline void uuid_unparse(const uuid_t uu, char *out)
 {
     snprintf(out, 37, UUID_FMT,
             uu[0], uu[1], uu[2], uu[3], uu[4], uu[5], uu[6], uu[7],
@@ -249,7 +246,7 @@ static void vdi_header_print(VdiHeader *header)
 {
     char uuid[37];
     logout("text        %s", header->text);
-    logout("signature   0x%04x\n", header->signature);
+    logout("signature   0x%08x\n", header->signature);
     logout("header size 0x%04x\n", header->header_size);
     logout("image type  0x%04x\n", header->image_type);
     logout("image flags 0x%04x\n", header->image_flags);
@@ -372,10 +369,12 @@ static int vdi_open(BlockDriverState *bs, int flags)
     BDRVVdiState *s = bs->opaque;
     VdiHeader header;
     size_t bmap_size;
+    int ret;
 
     logout("\n");
 
-    if (bdrv_read(bs->file, 0, (uint8_t *)&header, 1) < 0) {
+    ret = bdrv_read(bs->file, 0, (uint8_t *)&header, 1);
+    if (ret < 0) {
         goto fail;
     }
 
@@ -393,33 +392,45 @@ static int vdi_open(BlockDriverState *bs, int flags)
         header.disk_size &= ~(SECTOR_SIZE - 1);
     }
 
-    if (header.version != VDI_VERSION_1_1) {
+    if (header.signature != VDI_SIGNATURE) {
+        logout("bad vdi signature %08x\n", header.signature);
+        ret = -EMEDIUMTYPE;
+        goto fail;
+    } else if (header.version != VDI_VERSION_1_1) {
         logout("unsupported version %u.%u\n",
                header.version >> 16, header.version & 0xffff);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.offset_bmap % SECTOR_SIZE != 0) {
         /* We only support block maps which start on a sector boundary. */
         logout("unsupported block map offset 0x%x B\n", header.offset_bmap);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.offset_data % SECTOR_SIZE != 0) {
         /* We only support data blocks which start on a sector boundary. */
         logout("unsupported data offset 0x%x B\n", header.offset_data);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.sector_size != SECTOR_SIZE) {
         logout("unsupported sector size %u B\n", header.sector_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.block_size != 1 * MiB) {
         logout("unsupported block size %u B\n", header.block_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.disk_size >
                (uint64_t)header.blocks_in_image * header.block_size) {
         logout("unsupported disk size %" PRIu64 " B\n", header.disk_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (!uuid_is_null(header.uuid_link)) {
         logout("link uuid != 0, unsupported\n");
+        ret = -ENOTSUP;
         goto fail;
     } else if (!uuid_is_null(header.uuid_parent)) {
         logout("parent uuid != 0, unsupported\n");
+        ret = -ENOTSUP;
         goto fail;
     }
 
@@ -435,7 +446,8 @@ static int vdi_open(BlockDriverState *bs, int flags)
     if (bmap_size > 0) {
         s->bmap = g_malloc(bmap_size * SECTOR_SIZE);
     }
-    if (bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size) < 0) {
+    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
+    if (ret < 0) {
         goto fail_free_bmap;
     }
 
@@ -451,7 +463,13 @@ static int vdi_open(BlockDriverState *bs, int flags)
     g_free(s->bmap);
 
  fail:
-    return -1;
+    return ret;
+}
+
+static int vdi_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
 }
 
 static int coroutine_fn vdi_co_is_allocated(BlockDriverState *bs,
@@ -628,7 +646,6 @@ static int vdi_create(const char *filename, QEMUOptionParameter *options)
     VdiHeader header;
     size_t i;
     size_t bmap_size;
-    uint32_t *bmap;
 
     logout("\n");
 
@@ -693,21 +710,21 @@ static int vdi_create(const char *filename, QEMUOptionParameter *options)
         result = -errno;
     }
 
-    bmap = NULL;
     if (bmap_size > 0) {
-        bmap = (uint32_t *)g_malloc0(bmap_size);
-    }
-    for (i = 0; i < blocks; i++) {
-        if (image_type == VDI_TYPE_STATIC) {
-            bmap[i] = i;
-        } else {
-            bmap[i] = VDI_UNALLOCATED;
+        uint32_t *bmap = g_malloc0(bmap_size);
+        for (i = 0; i < blocks; i++) {
+            if (image_type == VDI_TYPE_STATIC) {
+                bmap[i] = i;
+            } else {
+                bmap[i] = VDI_UNALLOCATED;
+            }
         }
+        if (write(fd, bmap, bmap_size) < 0) {
+            result = -errno;
+        }
+        g_free(bmap);
     }
-    if (write(fd, bmap, bmap_size) < 0) {
-        result = -errno;
-    }
-    g_free(bmap);
+
     if (image_type == VDI_TYPE_STATIC) {
         if (ftruncate(fd, sizeof(header) + bmap_size + blocks * block_size)) {
             result = -errno;
@@ -762,6 +779,7 @@ static BlockDriver bdrv_vdi = {
     .bdrv_probe = vdi_probe,
     .bdrv_open = vdi_open,
     .bdrv_close = vdi_close,
+    .bdrv_reopen_prepare = vdi_reopen_prepare,
     .bdrv_create = vdi_create,
     .bdrv_co_is_allocated = vdi_co_is_allocated,
     .bdrv_make_empty = vdi_make_empty,
diff --git a/block/vmdk.c b/block/vmdk.c
index bba4c61a742..8333afb5e38 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -24,9 +24,9 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 #include <zlib.h>
 
 #define VMDK3_MAGIC (('C' << 24) | ('O' << 16) | ('W' << 8) | 'D')
@@ -300,6 +300,40 @@ static int vmdk_is_cid_valid(BlockDriverState *bs)
     return 1;
 }
 
+/* Queue extents, if any, for reopen() */
+static int vmdk_reopen_prepare(BDRVReopenState *state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    BDRVVmdkState *s;
+    int ret = -1;
+    int i;
+    VmdkExtent *e;
+
+    assert(state != NULL);
+    assert(state->bs != NULL);
+
+    if (queue == NULL) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
+                 "No reopen queue for VMDK extents");
+        goto exit;
+    }
+
+    s = state->bs->opaque;
+
+    assert(s != NULL);
+
+    for (i = 0; i < s->num_extents; i++) {
+        e = &s->extents[i];
+        if (e->file != state->bs->file) {
+            bdrv_reopen_queue(queue, e->file, state->flags);
+        }
+    }
+    ret = 0;
+
+exit:
+    return ret;
+}
+
 static int vmdk_parent_open(BlockDriverState *bs)
 {
     char *p_name;
@@ -582,7 +616,7 @@ static int vmdk_open_sparse(BlockDriverState *bs,
             return vmdk_open_vmdk4(bs, file, flags);
             break;
         default:
-            return -EINVAL;
+            return -EMEDIUMTYPE;
             break;
     }
 }
@@ -684,7 +718,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags,
     }
     buf[2047] = '\0';
     if (vmdk_parse_description(buf, "createType", ct, sizeof(ct))) {
-        return -EINVAL;
+        return -EMEDIUMTYPE;
     }
     if (strcmp(ct, "monolithicFlat") &&
         strcmp(ct, "twoGbMaxExtentSparse") &&
@@ -1058,6 +1092,7 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
     BDRVVmdkState *s = bs->opaque;
     int ret;
     uint64_t n, index_in_cluster;
+    uint64_t extent_begin_sector, extent_relative_sector_num;
     VmdkExtent *extent = NULL;
     uint64_t cluster_offset;
 
@@ -1069,7 +1104,9 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
         ret = get_cluster_offset(
                             bs, extent, NULL,
                             sector_num << 9, 0, &cluster_offset);
-        index_in_cluster = sector_num % extent->cluster_sectors;
+        extent_begin_sector = extent->end_sector - extent->sectors;
+        extent_relative_sector_num = sector_num - extent_begin_sector;
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
         n = extent->cluster_sectors - index_in_cluster;
         if (n > nb_sectors) {
             n = nb_sectors;
@@ -1120,6 +1157,7 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
     VmdkExtent *extent = NULL;
     int n, ret;
     int64_t index_in_cluster;
+    uint64_t extent_begin_sector, extent_relative_sector_num;
     uint64_t cluster_offset;
     VmdkMetaData m_data;
 
@@ -1162,7 +1200,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
         if (ret) {
             return -EINVAL;
         }
-        index_in_cluster = sector_num % extent->cluster_sectors;
+        extent_begin_sector = extent->end_sector - extent->sectors;
+        extent_relative_sector_num = sector_num - extent_begin_sector;
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
         n = extent->cluster_sectors - index_in_cluster;
         if (n > nb_sectors) {
             n = nb_sectors;
@@ -1374,8 +1414,7 @@ static int relative_path(char *dest, int dest_size,
         return -1;
     }
     if (path_is_absolute(target)) {
-        dest[dest_size - 1] = '\0';
-        strncpy(dest, target, dest_size - 1);
+        pstrcpy(dest, dest_size, target);
         return 0;
     }
     while (base[i] == target[i]) {
@@ -1646,6 +1685,7 @@ static BlockDriver bdrv_vmdk = {
     .instance_size  = sizeof(BDRVVmdkState),
     .bdrv_probe     = vmdk_probe,
     .bdrv_open      = vmdk_open,
+    .bdrv_reopen_prepare = vmdk_reopen_prepare,
     .bdrv_read      = vmdk_co_read,
     .bdrv_write     = vmdk_co_write,
     .bdrv_close     = vmdk_close,
diff --git a/block/vpc.c b/block/vpc.c
index c0b82c4f574..7948609e505 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -23,9 +23,12 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
+#if defined(CONFIG_UUID)
+#include <uuid/uuid.h>
+#endif
 
 /**************************************************************/
 
@@ -198,7 +201,8 @@ static int vpc_open(BlockDriverState *bs, int flags)
     bs->total_sectors = (int64_t)
         be16_to_cpu(footer->cyls) * footer->heads * footer->secs_per_cyl;
 
-    if (bs->total_sectors >= 65535 * 16 * 255) {
+    /* Allow a maximum disk size of approximately 2 TB */
+    if (bs->total_sectors >= 65535LL * 255 * 255) {
         err = -EFBIG;
         goto fail;
     }
@@ -265,6 +269,12 @@ static int vpc_open(BlockDriverState *bs, int flags)
     return err;
 }
 
+static int vpc_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 /*
  * Returns the absolute byte offset of the given sector in the image file.
  * If the sector is not allocated, -1 is returned instead.
@@ -518,19 +528,27 @@ static coroutine_fn int vpc_co_write(BlockDriverState *bs, int64_t sector_num,
  * Note that the geometry doesn't always exactly match total_sectors but
  * may round it down.
  *
- * Returns 0 on success, -EFBIG if the size is larger than 127 GB
+ * Returns 0 on success, -EFBIG if the size is larger than ~2 TB. Override
+ * the hardware EIDE and ATA-2 limit of 16 heads (max disk size of 127 GB)
+ * and instead allow up to 255 heads.
  */
 static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
     uint8_t* heads, uint8_t* secs_per_cyl)
 {
     uint32_t cyls_times_heads;
 
-    if (total_sectors > 65535 * 16 * 255)
+    /* Allow a maximum disk size of approximately 2 TB */
+    if (total_sectors > 65535LL * 255 * 255) {
         return -EFBIG;
+    }
 
     if (total_sectors > 65535 * 16 * 63) {
         *secs_per_cyl = 255;
-        *heads = 16;
+        if (total_sectors > 65535 * 16 * 255) {
+            *heads = 255;
+        } else {
+            *heads = 16;
+        }
         cyls_times_heads = total_sectors / *secs_per_cyl;
     } else {
         *secs_per_cyl = 17;
@@ -733,7 +751,9 @@ static int vpc_create(const char *filename, QEMUOptionParameter *options)
 
     footer->type = be32_to_cpu(disk_type);
 
-    /* TODO uuid is missing */
+#if defined(CONFIG_UUID)
+    uuid_generate(footer->uuid);
+#endif
 
     footer->checksum = be32_to_cpu(vpc_checksum(buf, HEADER_SIZE));
 
@@ -783,6 +803,7 @@ static BlockDriver bdrv_vpc = {
     .bdrv_probe     = vpc_probe,
     .bdrv_open      = vpc_open,
     .bdrv_close     = vpc_close,
+    .bdrv_reopen_prepare = vpc_reopen_prepare,
     .bdrv_create    = vpc_create,
 
     .bdrv_read              = vpc_co_read,
diff --git a/block/vvfat.c b/block/vvfat.c
index 59d3c5b8ac2..06e6654824a 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -25,9 +25,9 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 
 #ifndef S_IWGRP
 #define S_IWGRP 0
@@ -529,13 +529,9 @@ static inline uint8_t fat_chksum(const direntry_t* entry)
 /* if return_time==0, this returns the fat_date, else the fat_time */
 static uint16_t fat_datetime(time_t time,int return_time) {
     struct tm* t;
-#ifdef _WIN32
-    t=localtime(&time); /* this is not thread safe */
-#else
     struct tm t1;
     t = &t1;
     localtime_r(&time,t);
-#endif
     if(return_time)
 	return cpu_to_le16((t->tm_sec/2)|(t->tm_min<<5)|(t->tm_hour<<11));
     return cpu_to_le16((t->tm_mday)|((t->tm_mon+1)<<5)|((t->tm_year-80)<<9));
diff --git a/block/win32-aio.c b/block/win32-aio.c
new file mode 100644
index 00000000000..5d0fbbfb7db
--- /dev/null
+++ b/block/win32-aio.c
@@ -0,0 +1,215 @@
+/*
+ * Block driver for RAW files (win32)
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "qemu/timer.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "qemu-common.h"
+#include "block/aio.h"
+#include "raw-aio.h"
+#include "qemu/event_notifier.h"
+#include "qemu/iov.h"
+#include <windows.h>
+#include <winioctl.h>
+
+#define FTYPE_FILE 0
+#define FTYPE_CD     1
+#define FTYPE_HARDDISK 2
+
+struct QEMUWin32AIOState {
+    HANDLE hIOCP;
+    EventNotifier e;
+    int count;
+};
+
+typedef struct QEMUWin32AIOCB {
+    BlockDriverAIOCB common;
+    struct QEMUWin32AIOState *ctx;
+    int nbytes;
+    OVERLAPPED ov;
+    QEMUIOVector *qiov;
+    void *buf;
+    bool is_read;
+    bool is_linear;
+} QEMUWin32AIOCB;
+
+/*
+ * Completes an AIO request (calls the callback and frees the ACB).
+ */
+static void win32_aio_process_completion(QEMUWin32AIOState *s,
+    QEMUWin32AIOCB *waiocb, DWORD count)
+{
+    int ret;
+    s->count--;
+
+    if (waiocb->ov.Internal != 0) {
+        ret = -EIO;
+    } else {
+        ret = 0;
+        if (count < waiocb->nbytes) {
+            /* Short reads mean EOF, pad with zeros. */
+            if (waiocb->is_read) {
+                qemu_iovec_memset(waiocb->qiov, count, 0,
+                    waiocb->qiov->size - count);
+            } else {
+                ret = -EINVAL;
+            }
+       }
+    }
+
+    if (!waiocb->is_linear) {
+        if (ret == 0 && waiocb->is_read) {
+            QEMUIOVector *qiov = waiocb->qiov;
+            iov_from_buf(qiov->iov, qiov->niov, 0, waiocb->buf, qiov->size);
+        }
+        qemu_vfree(waiocb->buf);
+    }
+
+
+    waiocb->common.cb(waiocb->common.opaque, ret);
+    qemu_aio_release(waiocb);
+}
+
+static void win32_aio_completion_cb(EventNotifier *e)
+{
+    QEMUWin32AIOState *s = container_of(e, QEMUWin32AIOState, e);
+    DWORD count;
+    ULONG_PTR key;
+    OVERLAPPED *ov;
+
+    event_notifier_test_and_clear(&s->e);
+    while (GetQueuedCompletionStatus(s->hIOCP, &count, &key, &ov, 0)) {
+        QEMUWin32AIOCB *waiocb = container_of(ov, QEMUWin32AIOCB, ov);
+
+        win32_aio_process_completion(s, waiocb, count);
+    }
+}
+
+static int win32_aio_flush_cb(EventNotifier *e)
+{
+    QEMUWin32AIOState *s = container_of(e, QEMUWin32AIOState, e);
+
+    return (s->count > 0) ? 1 : 0;
+}
+
+static void win32_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+    QEMUWin32AIOCB *waiocb = (QEMUWin32AIOCB *)blockacb;
+
+    /*
+     * CancelIoEx is only supported in Vista and newer.  For now, just
+     * wait for completion.
+     */
+    while (!HasOverlappedIoCompleted(&waiocb->ov)) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo win32_aiocb_info = {
+    .aiocb_size         = sizeof(QEMUWin32AIOCB),
+    .cancel             = win32_aio_cancel,
+};
+
+BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+        QEMUWin32AIOState *aio, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    struct QEMUWin32AIOCB *waiocb;
+    uint64_t offset = sector_num * 512;
+    DWORD rc;
+
+    waiocb = qemu_aio_get(&win32_aiocb_info, bs, cb, opaque);
+    waiocb->nbytes = nb_sectors * 512;
+    waiocb->qiov = qiov;
+    waiocb->is_read = (type == QEMU_AIO_READ);
+
+    if (qiov->niov > 1) {
+        waiocb->buf = qemu_blockalign(bs, qiov->size);
+        if (type & QEMU_AIO_WRITE) {
+            iov_to_buf(qiov->iov, qiov->niov, 0, waiocb->buf, qiov->size);
+        }
+        waiocb->is_linear = false;
+    } else {
+        waiocb->buf = qiov->iov[0].iov_base;
+        waiocb->is_linear = true;
+    }
+
+    memset(&waiocb->ov, 0, sizeof(waiocb->ov));
+    waiocb->ov.Offset = (DWORD)offset;
+    waiocb->ov.OffsetHigh = (DWORD)(offset >> 32);
+    waiocb->ov.hEvent = event_notifier_get_handle(&aio->e);
+
+    aio->count++;
+
+    if (type & QEMU_AIO_READ) {
+        rc = ReadFile(hfile, waiocb->buf, waiocb->nbytes, NULL, &waiocb->ov);
+    } else {
+        rc = WriteFile(hfile, waiocb->buf, waiocb->nbytes, NULL, &waiocb->ov);
+    }
+    if(rc == 0 && GetLastError() != ERROR_IO_PENDING) {
+        goto out_dec_count;
+    }
+    return &waiocb->common;
+
+out_dec_count:
+    aio->count--;
+    qemu_aio_release(waiocb);
+    return NULL;
+}
+
+int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile)
+{
+    if (CreateIoCompletionPort(hfile, aio->hIOCP, (ULONG_PTR) 0, 0) == NULL) {
+        return -EINVAL;
+    } else {
+        return 0;
+    }
+}
+
+QEMUWin32AIOState *win32_aio_init(void)
+{
+    QEMUWin32AIOState *s;
+
+    s = g_malloc0(sizeof(*s));
+    if (event_notifier_init(&s->e, false) < 0) {
+        goto out_free_state;
+    }
+
+    s->hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
+    if (s->hIOCP == NULL) {
+        goto out_close_efd;
+    }
+
+    qemu_aio_set_event_notifier(&s->e, win32_aio_completion_cb,
+                                win32_aio_flush_cb);
+
+    return s;
+
+out_close_efd:
+    event_notifier_cleanup(&s->e);
+out_free_state:
+    g_free(s);
+    return NULL;
+}
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
new file mode 100644
index 00000000000..dc4e9a2462c
--- /dev/null
+++ b/blockdev-nbd.c
@@ -0,0 +1,133 @@
+/*
+ * Serving QEMU block devices via NBD
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "sysemu/blockdev.h"
+#include "hw/block-common.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/sysemu.h"
+#include "qmp-commands.h"
+#include "trace.h"
+#include "block/nbd.h"
+#include "qemu/sockets.h"
+
+static int server_fd = -1;
+
+static void nbd_accept(void *opaque)
+{
+    struct sockaddr_in addr;
+    socklen_t addr_len = sizeof(addr);
+
+    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
+    if (fd >= 0) {
+        nbd_client_new(NULL, fd, nbd_client_put);
+    }
+}
+
+void qmp_nbd_server_start(SocketAddress *addr, Error **errp)
+{
+    if (server_fd != -1) {
+        error_setg(errp, "NBD server already running");
+        return;
+    }
+
+    server_fd = socket_listen(addr, errp);
+    if (server_fd != -1) {
+        qemu_set_fd_handler2(server_fd, NULL, nbd_accept, NULL, NULL);
+    }
+}
+
+/* Hook into the BlockDriverState notifiers to close the export when
+ * the file is closed.
+ */
+typedef struct NBDCloseNotifier {
+    Notifier n;
+    NBDExport *exp;
+    QTAILQ_ENTRY(NBDCloseNotifier) next;
+} NBDCloseNotifier;
+
+static QTAILQ_HEAD(, NBDCloseNotifier) close_notifiers =
+    QTAILQ_HEAD_INITIALIZER(close_notifiers);
+
+static void nbd_close_notifier(Notifier *n, void *data)
+{
+    NBDCloseNotifier *cn = DO_UPCAST(NBDCloseNotifier, n, n);
+
+    notifier_remove(&cn->n);
+    QTAILQ_REMOVE(&close_notifiers, cn, next);
+
+    nbd_export_close(cn->exp);
+    nbd_export_put(cn->exp);
+    g_free(cn);
+}
+
+static void nbd_server_put_ref(NBDExport *exp)
+{
+    BlockDriverState *bs = nbd_export_get_blockdev(exp);
+    drive_put_ref(drive_get_by_blockdev(bs));
+}
+
+void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
+                        Error **errp)
+{
+    BlockDriverState *bs;
+    NBDExport *exp;
+    NBDCloseNotifier *n;
+
+    if (server_fd == -1) {
+        error_setg(errp, "NBD server not running");
+        return;
+    }
+
+    if (nbd_export_find(device)) {
+        error_setg(errp, "NBD server already exporting device '%s'", device);
+        return;
+    }
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (!has_writable) {
+        writable = false;
+    }
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    exp = nbd_export_new(bs, 0, -1, writable ? 0 : NBD_FLAG_READ_ONLY,
+                         nbd_server_put_ref);
+
+    nbd_export_set_name(exp, device);
+    drive_get_ref(drive_get_by_blockdev(bs));
+
+    n = g_malloc0(sizeof(NBDCloseNotifier));
+    n->n.notify = nbd_close_notifier;
+    n->exp = exp;
+    bdrv_add_close_notifier(bs, &n->n);
+    QTAILQ_INSERT_TAIL(&close_notifiers, n, next);
+}
+
+void qmp_nbd_server_stop(Error **errp)
+{
+    while (!QTAILQ_EMPTY(&close_notifiers)) {
+        NBDCloseNotifier *cn = QTAILQ_FIRST(&close_notifiers);
+        nbd_close_notifier(&cn->n, nbd_export_get_blockdev(cn->exp));
+    }
+
+    if (server_fd != -1) {
+        qemu_set_fd_handler2(server_fd, NULL, NULL, NULL, NULL);
+        close(server_fd);
+        server_fd = -1;
+    }
+}
diff --git a/blockdev.c b/blockdev.c
index 7c83baa353c..63e6f1eafa8 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -7,18 +7,19 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "monitor.h"
-#include "qerror.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-objects.h"
-#include "sysemu.h"
-#include "block_int.h"
+#include "block/blockjob.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qapi/qmp/types.h"
+#include "sysemu/sysemu.h"
+#include "block/block_int.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
 
 static QTAILQ_HEAD(drivelist, DriveInfo) drives = QTAILQ_HEAD_INITIALIZER(drives);
 
@@ -237,16 +238,16 @@ static void drive_put_ref_bh_schedule(DriveInfo *dinfo)
     qemu_bh_schedule(s->bh);
 }
 
-static int parse_block_error_action(const char *buf, int is_read)
+static int parse_block_error_action(const char *buf, bool is_read)
 {
     if (!strcmp(buf, "ignore")) {
-        return BLOCK_ERR_IGNORE;
+        return BLOCKDEV_ON_ERROR_IGNORE;
     } else if (!is_read && !strcmp(buf, "enospc")) {
-        return BLOCK_ERR_STOP_ENOSPC;
+        return BLOCKDEV_ON_ERROR_ENOSPC;
     } else if (!strcmp(buf, "stop")) {
-        return BLOCK_ERR_STOP_ANY;
+        return BLOCKDEV_ON_ERROR_STOP;
     } else if (!strcmp(buf, "report")) {
-        return BLOCK_ERR_REPORT;
+        return BLOCKDEV_ON_ERROR_REPORT;
     } else {
         error_report("'%s' invalid %s error action",
                      buf, is_read ? "read" : "write");
@@ -274,7 +275,7 @@ static bool do_check_io_limits(BlockIOLimit *io_limits)
     return true;
 }
 
-DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
+DriveInfo *drive_init(QemuOpts *opts, BlockInterfaceType block_default_type)
 {
     const char *buf;
     const char *file = NULL;
@@ -324,7 +325,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
             return NULL;
 	}
     } else {
-        type = default_to_scsi ? IF_SCSI : IF_IDE;
+        type = block_default_type;
     }
 
     max_devs = if_max_devs[type];
@@ -432,7 +433,13 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         return NULL;
     }
 
-    on_write_error = BLOCK_ERR_STOP_ENOSPC;
+    if (qemu_opt_get(opts, "boot") != NULL) {
+        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
+                "ignored. Future versions will reject this parameter. Please "
+                "update your scripts.\n");
+    }
+
+    on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
     if ((buf = qemu_opt_get(opts, "werror")) != NULL) {
         if (type != IF_IDE && type != IF_SCSI && type != IF_VIRTIO && type != IF_NONE) {
             error_report("werror is not supported by this bus type");
@@ -445,7 +452,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         }
     }
 
-    on_read_error = BLOCK_ERR_REPORT;
+    on_read_error = BLOCKDEV_ON_ERROR_REPORT;
     if ((buf = qemu_opt_get(opts, "rerror")) != NULL) {
         if (type != IF_IDE && type != IF_VIRTIO && type != IF_SCSI && type != IF_NONE) {
             error_report("rerror is not supported by this bus type");
@@ -527,6 +534,8 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
                      if_name[type], mediastr, unit_id);
     }
     dinfo->bdrv = bdrv_new(dinfo->id);
+    dinfo->bdrv->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
+    dinfo->bdrv->read_only = ro;
     dinfo->devaddr = devaddr;
     dinfo->type = type;
     dinfo->bus = bus_id;
@@ -559,7 +568,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         break;
     case IF_VIRTIO:
         /* add virtio block device */
-        opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+        opts = qemu_opts_create_nofail(qemu_find_opts("device"));
         if (arch_type == QEMU_ARCH_S390X) {
             qemu_opt_set(opts, "driver", "virtio-blk-s390");
         } else {
@@ -608,8 +617,13 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
 
     ret = bdrv_open(dinfo->bdrv, file, bdrv_flags, drv);
     if (ret < 0) {
-        error_report("could not open disk image %s: %s",
-                     file, strerror(-ret));
+        if (ret == -EMEDIUMTYPE) {
+            error_report("could not open disk image %s: not in %s format",
+                         file, drv->format_name);
+        } else {
+            error_report("could not open disk image %s: %s",
+                         file, strerror(-ret));
+        }
         goto err;
     }
 
@@ -633,21 +647,17 @@ void do_commit(Monitor *mon, const QDict *qdict)
 
     if (!strcmp(device, "all")) {
         ret = bdrv_commit_all();
-        if (ret == -EBUSY) {
-            qerror_report(QERR_DEVICE_IN_USE, device);
-            return;
-        }
     } else {
         bs = bdrv_find(device);
         if (!bs) {
-            qerror_report(QERR_DEVICE_NOT_FOUND, device);
+            monitor_printf(mon, "Device '%s' not found\n", device);
             return;
         }
         ret = bdrv_commit(bs);
-        if (ret == -EBUSY) {
-            qerror_report(QERR_DEVICE_IN_USE, device);
-            return;
-        }
+    }
+    if (ret < 0) {
+        monitor_printf(mon, "'commit' error for '%s': %s\n", device,
+                       strerror(-ret));
     }
 }
 
@@ -698,6 +708,7 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
     int ret = 0;
     BlockdevActionList *dev_entry = dev_list;
     BlkTransactionStates *states, *next;
+    Error *local_err = NULL;
 
     QSIMPLEQ_HEAD(snap_bdrv_states, BlkTransactionStates) snap_bdrv_states;
     QSIMPLEQ_INIT(&snap_bdrv_states);
@@ -777,12 +788,12 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
 
         /* create new image w/backing file */
         if (mode != NEW_IMAGE_MODE_EXISTING) {
-            ret = bdrv_img_create(new_image_file, format,
-                                  states->old_bs->filename,
-                                  states->old_bs->drv->format_name,
-                                  NULL, -1, flags);
-            if (ret) {
-                error_set(errp, QERR_OPEN_FILE_FAILED, new_image_file);
+            bdrv_img_create(new_image_file, format,
+                            states->old_bs->filename,
+                            states->old_bs->drv->format_name,
+                            NULL, -1, flags, &local_err);
+            if (error_is_set(&local_err)) {
+                error_propagate(errp, local_err);
                 goto delete_and_fail;
             }
         }
@@ -803,6 +814,11 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
     QSIMPLEQ_FOREACH(states, &snap_bdrv_states, entry) {
         /* This removes our old bs from the bdrv_states, and adds the new bs */
         bdrv_append(states->new_bs, states->old_bs);
+        /* We don't need (or want) to use the transactional
+         * bdrv_reopen_multiple() across all the entries at once, because we
+         * don't want to abort all of them if one of them fails the reopen */
+        bdrv_reopen(states->new_bs, states->new_bs->open_flags & ~BDRV_O_RDWR,
+                    NULL);
     }
 
     /* success */
@@ -822,7 +838,6 @@ exit:
     QSIMPLEQ_FOREACH_SAFE(states, &snap_bdrv_states, entry, next) {
         g_free(states);
     }
-    return;
 }
 
 
@@ -1049,26 +1064,12 @@ void qmp_block_resize(const char *device, int64_t size, Error **errp)
     }
 }
 
-static QObject *qobject_from_block_job(BlockJob *job)
-{
-    return qobject_from_jsonf("{ 'type': %s,"
-                              "'device': %s,"
-                              "'len': %" PRId64 ","
-                              "'offset': %" PRId64 ","
-                              "'speed': %" PRId64 " }",
-                              job->job_type->job_type,
-                              bdrv_get_device_name(job->bs),
-                              job->len,
-                              job->offset,
-                              job->speed);
-}
-
-static void block_stream_cb(void *opaque, int ret)
+static void block_job_cb(void *opaque, int ret)
 {
     BlockDriverState *bs = opaque;
     QObject *obj;
 
-    trace_block_stream_cb(bs, bs->job, ret);
+    trace_block_job_cb(bs, bs->job, ret);
 
     assert(bs->job);
     obj = qobject_from_block_job(bs->job);
@@ -1088,13 +1089,18 @@ static void block_stream_cb(void *opaque, int ret)
 }
 
 void qmp_block_stream(const char *device, bool has_base,
-                      const char *base, bool has_speed,
-                      int64_t speed, Error **errp)
+                      const char *base, bool has_speed, int64_t speed,
+                      bool has_on_error, BlockdevOnError on_error,
+                      Error **errp)
 {
     BlockDriverState *bs;
     BlockDriverState *base_bs = NULL;
     Error *local_err = NULL;
 
+    if (!has_on_error) {
+        on_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+
     bs = bdrv_find(device);
     if (!bs) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, device);
@@ -1110,7 +1116,7 @@ void qmp_block_stream(const char *device, bool has_base,
     }
 
     stream_start(bs, base_bs, base, has_speed ? speed : 0,
-                 block_stream_cb, bs, &local_err);
+                 on_error, block_job_cb, bs, &local_err);
     if (error_is_set(&local_err)) {
         error_propagate(errp, local_err);
         return;
@@ -1124,6 +1130,210 @@ void qmp_block_stream(const char *device, bool has_base,
     trace_qmp_block_stream(bs, bs->job);
 }
 
+void qmp_block_commit(const char *device,
+                      bool has_base, const char *base, const char *top,
+                      bool has_speed, int64_t speed,
+                      Error **errp)
+{
+    BlockDriverState *bs;
+    BlockDriverState *base_bs, *top_bs;
+    Error *local_err = NULL;
+    /* This will be part of the QMP command, if/when the
+     * BlockdevOnError change for blkmirror makes it in
+     */
+    BlockdevOnError on_error = BLOCKDEV_ON_ERROR_REPORT;
+
+    /* drain all i/o before commits */
+    bdrv_drain_all();
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    /* default top_bs is the active layer */
+    top_bs = bs;
+
+    if (top) {
+        if (strcmp(bs->filename, top) != 0) {
+            top_bs = bdrv_find_backing_image(bs, top);
+        }
+    }
+
+    if (top_bs == NULL) {
+        error_setg(errp, "Top image file %s not found", top ? top : "NULL");
+        return;
+    }
+
+    if (has_base && base) {
+        base_bs = bdrv_find_backing_image(top_bs, base);
+    } else {
+        base_bs = bdrv_find_base(top_bs);
+    }
+
+    if (base_bs == NULL) {
+        error_set(errp, QERR_BASE_NOT_FOUND, base ? base : "NULL");
+        return;
+    }
+
+    commit_start(bs, base_bs, top_bs, speed, on_error, block_job_cb, bs,
+                &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    /* Grab a reference so hotplug does not delete the BlockDriverState from
+     * underneath us.
+     */
+    drive_get_ref(drive_get_by_blockdev(bs));
+}
+
+#define DEFAULT_MIRROR_BUF_SIZE   (10 << 20)
+
+void qmp_drive_mirror(const char *device, const char *target,
+                      bool has_format, const char *format,
+                      enum MirrorSyncMode sync,
+                      bool has_mode, enum NewImageMode mode,
+                      bool has_speed, int64_t speed,
+                      bool has_granularity, uint32_t granularity,
+                      bool has_buf_size, int64_t buf_size,
+                      bool has_on_source_error, BlockdevOnError on_source_error,
+                      bool has_on_target_error, BlockdevOnError on_target_error,
+                      Error **errp)
+{
+    BlockDriverState *bs;
+    BlockDriverState *source, *target_bs;
+    BlockDriver *proto_drv;
+    BlockDriver *drv = NULL;
+    Error *local_err = NULL;
+    int flags;
+    uint64_t size;
+    int ret;
+
+    if (!has_speed) {
+        speed = 0;
+    }
+    if (!has_on_source_error) {
+        on_source_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+    if (!has_on_target_error) {
+        on_target_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+    if (!has_mode) {
+        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+    }
+    if (!has_granularity) {
+        granularity = 0;
+    }
+    if (!has_buf_size) {
+        buf_size = DEFAULT_MIRROR_BUF_SIZE;
+    }
+
+    if (granularity != 0 && (granularity < 512 || granularity > 1048576 * 64)) {
+        error_set(errp, QERR_INVALID_PARAMETER, device);
+        return;
+    }
+    if (granularity & (granularity - 1)) {
+        error_set(errp, QERR_INVALID_PARAMETER, device);
+        return;
+    }
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (!bdrv_is_inserted(bs)) {
+        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        return;
+    }
+
+    if (!has_format) {
+        format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
+    }
+    if (format) {
+        drv = bdrv_find_format(format);
+        if (!drv) {
+            error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
+            return;
+        }
+    }
+
+    if (bdrv_in_use(bs)) {
+        error_set(errp, QERR_DEVICE_IN_USE, device);
+        return;
+    }
+
+    flags = bs->open_flags | BDRV_O_RDWR;
+    source = bs->backing_hd;
+    if (!source && sync == MIRROR_SYNC_MODE_TOP) {
+        sync = MIRROR_SYNC_MODE_FULL;
+    }
+
+    proto_drv = bdrv_find_protocol(target);
+    if (!proto_drv) {
+        error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
+        return;
+    }
+
+    bdrv_get_geometry(bs, &size);
+    size *= 512;
+    if (sync == MIRROR_SYNC_MODE_FULL && mode != NEW_IMAGE_MODE_EXISTING) {
+        /* create new image w/o backing file */
+        assert(format && drv);
+        bdrv_img_create(target, format,
+                        NULL, NULL, NULL, size, flags, &local_err);
+    } else {
+        switch (mode) {
+        case NEW_IMAGE_MODE_EXISTING:
+            ret = 0;
+            break;
+        case NEW_IMAGE_MODE_ABSOLUTE_PATHS:
+            /* create new image with backing file */
+            bdrv_img_create(target, format,
+                            source->filename,
+                            source->drv->format_name,
+                            NULL, size, flags, &local_err);
+            break;
+        default:
+            abort();
+        }
+    }
+
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Mirroring takes care of copy-on-write using the source's backing
+     * file.
+     */
+    target_bs = bdrv_new("");
+    ret = bdrv_open(target_bs, target, flags | BDRV_O_NO_BACKING, drv);
+
+    if (ret < 0) {
+        bdrv_delete(target_bs);
+        error_set(errp, QERR_OPEN_FILE_FAILED, target);
+        return;
+    }
+
+    mirror_start(bs, target_bs, speed, granularity, buf_size, sync,
+                 on_source_error, on_target_error,
+                 block_job_cb, bs, &local_err);
+    if (local_err != NULL) {
+        bdrv_delete(target_bs);
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Grab a reference so hotplug does not delete the BlockDriverState from
+     * underneath us.
+     */
+    drive_get_ref(drive_get_by_blockdev(bs));
+}
+
 static BlockJob *find_block_job(const char *device)
 {
     BlockDriverState *bs;
@@ -1140,19 +1350,28 @@ void qmp_block_job_set_speed(const char *device, int64_t speed, Error **errp)
     BlockJob *job = find_block_job(device);
 
     if (!job) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, device);
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
 
     block_job_set_speed(job, speed, errp);
 }
 
-void qmp_block_job_cancel(const char *device, Error **errp)
+void qmp_block_job_cancel(const char *device,
+                          bool has_force, bool force, Error **errp)
 {
     BlockJob *job = find_block_job(device);
 
+    if (!has_force) {
+        force = false;
+    }
+
     if (!job) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, device);
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+    if (job->paused && !force) {
+        error_set(errp, QERR_BLOCK_JOB_PAUSED, device);
         return;
     }
 
@@ -1160,25 +1379,53 @@ void qmp_block_job_cancel(const char *device, Error **errp)
     block_job_cancel(job);
 }
 
+void qmp_block_job_pause(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_pause(job);
+    block_job_pause(job);
+}
+
+void qmp_block_job_resume(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_resume(job);
+    block_job_resume(job);
+}
+
+void qmp_block_job_complete(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_complete(job);
+    block_job_complete(job, errp);
+}
+
 static void do_qmp_query_block_jobs_one(void *opaque, BlockDriverState *bs)
 {
     BlockJobInfoList **prev = opaque;
     BlockJob *job = bs->job;
 
     if (job) {
-        BlockJobInfoList *elem;
-        BlockJobInfo *info = g_new(BlockJobInfo, 1);
-        *info = (BlockJobInfo){
-            .type   = g_strdup(job->job_type->job_type),
-            .device = g_strdup(bdrv_get_device_name(bs)),
-            .len    = job->len,
-            .offset = job->offset,
-            .speed  = job->speed,
-        };
-
-        elem = g_new0(BlockJobInfoList, 1);
-        elem->value = info;
-
+        BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
+        elem->value = block_job_query(bs->job);
         (*prev)->next = elem;
         *prev = elem;
     }
@@ -1192,3 +1439,121 @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
     bdrv_iterate(do_qmp_query_block_jobs_one, &prev);
     return dummy.next;
 }
+
+QemuOptsList qemu_drive_opts = {
+    .name = "drive",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_drive_opts.head),
+    .desc = {
+        {
+            .name = "bus",
+            .type = QEMU_OPT_NUMBER,
+            .help = "bus number",
+        },{
+            .name = "unit",
+            .type = QEMU_OPT_NUMBER,
+            .help = "unit number (i.e. lun for scsi)",
+        },{
+            .name = "if",
+            .type = QEMU_OPT_STRING,
+            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
+        },{
+            .name = "index",
+            .type = QEMU_OPT_NUMBER,
+            .help = "index number",
+        },{
+            .name = "cyls",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of cylinders (ide disk geometry)",
+        },{
+            .name = "heads",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of heads (ide disk geometry)",
+        },{
+            .name = "secs",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of sectors (ide disk geometry)",
+        },{
+            .name = "trans",
+            .type = QEMU_OPT_STRING,
+            .help = "chs translation (auto, lba. none)",
+        },{
+            .name = "media",
+            .type = QEMU_OPT_STRING,
+            .help = "media type (disk, cdrom)",
+        },{
+            .name = "snapshot",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable snapshot mode",
+        },{
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+            .help = "disk image",
+        },{
+            .name = "cache",
+            .type = QEMU_OPT_STRING,
+            .help = "host cache usage (none, writeback, writethrough, "
+                    "directsync, unsafe)",
+        },{
+            .name = "aio",
+            .type = QEMU_OPT_STRING,
+            .help = "host AIO implementation (threads, native)",
+        },{
+            .name = "format",
+            .type = QEMU_OPT_STRING,
+            .help = "disk format (raw, qcow2, ...)",
+        },{
+            .name = "serial",
+            .type = QEMU_OPT_STRING,
+            .help = "disk serial number",
+        },{
+            .name = "rerror",
+            .type = QEMU_OPT_STRING,
+            .help = "read error action",
+        },{
+            .name = "werror",
+            .type = QEMU_OPT_STRING,
+            .help = "write error action",
+        },{
+            .name = "addr",
+            .type = QEMU_OPT_STRING,
+            .help = "pci address (virtio only)",
+        },{
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
+            .name = "iops",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit total I/O operations per second",
+        },{
+            .name = "iops_rd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit read operations per second",
+        },{
+            .name = "iops_wr",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit write operations per second",
+        },{
+            .name = "bps",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit total bytes per second",
+        },{
+            .name = "bps_rd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit read bytes per second",
+        },{
+            .name = "bps_wr",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit write bytes per second",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },{
+            .name = "boot",
+            .type = QEMU_OPT_BOOL,
+            .help = "(deprecated, ignored)",
+        },
+        { /* end of list */ }
+    },
+};
diff --git a/blockjob.c b/blockjob.c
new file mode 100644
index 00000000000..ca80df1d0e5
--- /dev/null
+++ b/blockjob.c
@@ -0,0 +1,283 @@
+/*
+ * QEMU System Emulator block driver
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "config-host.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "monitor/monitor.h"
+#include "block/block.h"
+#include "block/blockjob.h"
+#include "block/block_int.h"
+#include "qapi/qmp/qjson.h"
+#include "block/coroutine.h"
+#include "qmp-commands.h"
+#include "qemu/timer.h"
+
+void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       void *opaque, Error **errp)
+{
+    BlockJob *job;
+
+    if (bs->job || bdrv_in_use(bs)) {
+        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
+        return NULL;
+    }
+    bdrv_set_in_use(bs, 1);
+
+    job = g_malloc0(job_type->instance_size);
+    job->job_type      = job_type;
+    job->bs            = bs;
+    job->cb            = cb;
+    job->opaque        = opaque;
+    job->busy          = true;
+    bs->job = job;
+
+    /* Only set speed when necessary to avoid NotSupported error */
+    if (speed != 0) {
+        Error *local_err = NULL;
+
+        block_job_set_speed(job, speed, &local_err);
+        if (error_is_set(&local_err)) {
+            bs->job = NULL;
+            g_free(job);
+            bdrv_set_in_use(bs, 0);
+            error_propagate(errp, local_err);
+            return NULL;
+        }
+    }
+    return job;
+}
+
+void block_job_completed(BlockJob *job, int ret)
+{
+    BlockDriverState *bs = job->bs;
+
+    assert(bs->job == job);
+    job->cb(job->opaque, ret);
+    bs->job = NULL;
+    g_free(job);
+    bdrv_set_in_use(bs, 0);
+}
+
+void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    Error *local_err = NULL;
+
+    if (!job->job_type->set_speed) {
+        error_set(errp, QERR_NOT_SUPPORTED);
+        return;
+    }
+    job->job_type->set_speed(job, speed, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    job->speed = speed;
+}
+
+void block_job_complete(BlockJob *job, Error **errp)
+{
+    if (job->paused || job->cancelled || !job->job_type->complete) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        return;
+    }
+
+    job->job_type->complete(job, errp);
+}
+
+void block_job_pause(BlockJob *job)
+{
+    job->paused = true;
+}
+
+bool block_job_is_paused(BlockJob *job)
+{
+    return job->paused;
+}
+
+void block_job_resume(BlockJob *job)
+{
+    job->paused = false;
+    block_job_iostatus_reset(job);
+    if (job->co && !job->busy) {
+        qemu_coroutine_enter(job->co, NULL);
+    }
+}
+
+void block_job_cancel(BlockJob *job)
+{
+    job->cancelled = true;
+    block_job_resume(job);
+}
+
+bool block_job_is_cancelled(BlockJob *job)
+{
+    return job->cancelled;
+}
+
+void block_job_iostatus_reset(BlockJob *job)
+{
+    job->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+    if (job->job_type->iostatus_reset) {
+        job->job_type->iostatus_reset(job);
+    }
+}
+
+struct BlockCancelData {
+    BlockJob *job;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+    bool cancelled;
+    int ret;
+};
+
+static void block_job_cancel_cb(void *opaque, int ret)
+{
+    struct BlockCancelData *data = opaque;
+
+    data->cancelled = block_job_is_cancelled(data->job);
+    data->ret = ret;
+    data->cb(data->opaque, ret);
+}
+
+int block_job_cancel_sync(BlockJob *job)
+{
+    struct BlockCancelData data;
+    BlockDriverState *bs = job->bs;
+
+    assert(bs->job == job);
+
+    /* Set up our own callback to store the result and chain to
+     * the original callback.
+     */
+    data.job = job;
+    data.cb = job->cb;
+    data.opaque = job->opaque;
+    data.ret = -EINPROGRESS;
+    job->cb = block_job_cancel_cb;
+    job->opaque = &data;
+    block_job_cancel(job);
+    while (data.ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
+}
+
+void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns)
+{
+    assert(job->busy);
+
+    /* Check cancellation *before* setting busy = false, too!  */
+    if (block_job_is_cancelled(job)) {
+        return;
+    }
+
+    job->busy = false;
+    if (block_job_is_paused(job)) {
+        qemu_coroutine_yield();
+    } else {
+        co_sleep_ns(clock, ns);
+    }
+    job->busy = true;
+}
+
+BlockJobInfo *block_job_query(BlockJob *job)
+{
+    BlockJobInfo *info = g_new0(BlockJobInfo, 1);
+    info->type      = g_strdup(job->job_type->job_type);
+    info->device    = g_strdup(bdrv_get_device_name(job->bs));
+    info->len       = job->len;
+    info->busy      = job->busy;
+    info->paused    = job->paused;
+    info->offset    = job->offset;
+    info->speed     = job->speed;
+    info->io_status = job->iostatus;
+    return info;
+}
+
+static void block_job_iostatus_set_err(BlockJob *job, int error)
+{
+    if (job->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        job->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                          BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
+}
+
+
+QObject *qobject_from_block_job(BlockJob *job)
+{
+    return qobject_from_jsonf("{ 'type': %s,"
+                              "'device': %s,"
+                              "'len': %" PRId64 ","
+                              "'offset': %" PRId64 ","
+                              "'speed': %" PRId64 " }",
+                              job->job_type->job_type,
+                              bdrv_get_device_name(job->bs),
+                              job->len,
+                              job->offset,
+                              job->speed);
+}
+
+void block_job_ready(BlockJob *job)
+{
+    QObject *data = qobject_from_block_job(job);
+    monitor_protocol_event(QEVENT_BLOCK_JOB_READY, data);
+    qobject_decref(data);
+}
+
+BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
+                                        BlockdevOnError on_err,
+                                        int is_read, int error)
+{
+    BlockErrorAction action;
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        action = (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
+        break;
+    case BLOCKDEV_ON_ERROR_STOP:
+        action = BDRV_ACTION_STOP;
+        break;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        action = BDRV_ACTION_REPORT;
+        break;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        action = BDRV_ACTION_IGNORE;
+        break;
+    default:
+        abort();
+    }
+    bdrv_emit_qmp_error_event(job->bs, QEVENT_BLOCK_JOB_ERROR, action, is_read);
+    if (action == BDRV_ACTION_STOP) {
+        block_job_pause(job);
+        block_job_iostatus_set_err(job, error);
+        if (bs != job->bs) {
+            bdrv_iostatus_set_err(bs, error);
+        }
+    }
+    return action;
+}
diff --git a/bsd-user/elfload.c b/bsd-user/elfload.c
index 55b213609e8..a6cd3ab4f36 100644
--- a/bsd-user/elfload.c
+++ b/bsd-user/elfload.c
@@ -10,7 +10,7 @@
 #include <string.h>
 
 #include "qemu.h"
-#include "disas.h"
+#include "disas/disas.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
diff --git a/bsd-user/main.c b/bsd-user/main.c
index 095ae8eaaae..1dc033046bf 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -31,8 +31,8 @@
 /* For tb_lock */
 #include "cpu.h"
 #include "tcg.h"
-#include "qemu-timer.h"
-#include "envlist.h"
+#include "qemu/timer.h"
+#include "qemu/envlist.h"
 
 #define DEBUG_LOGFILE "/tmp/qemu.log"
 
diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
index 5d6cffc4585..aae8ea10bee 100644
--- a/bsd-user/mmap.c
+++ b/bsd-user/mmap.c
@@ -74,7 +74,7 @@ void mmap_unlock(void)
 }
 #endif
 
-void *qemu_vmalloc(size_t size)
+static void *bsd_vmalloc(size_t size)
 {
     void *p;
     mmap_lock();
@@ -98,7 +98,7 @@ void *g_malloc(size_t size)
 {
     char * p;
     size += 16;
-    p = qemu_vmalloc(size);
+    p = bsd_vmalloc(size);
     *(size_t *)p = size;
     return p + 16;
 }
diff --git a/bsd-user/qemu-types.h b/bsd-user/qemu-types.h
deleted file mode 100644
index 1adda9fbdb9..00000000000
--- a/bsd-user/qemu-types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#ifndef QEMU_TYPES_H
-#define QEMU_TYPES_H
-#include "cpu.h"
-
-#ifdef TARGET_ABI32
-typedef uint32_t abi_ulong;
-typedef int32_t abi_long;
-#define TARGET_ABI_FMT_lx "%08x"
-#define TARGET_ABI_FMT_ld "%d"
-#define TARGET_ABI_FMT_lu "%u"
-#define TARGET_ABI_BITS 32
-#else
-typedef target_ulong abi_ulong;
-typedef target_long abi_long;
-#define TARGET_ABI_FMT_lx TARGET_FMT_lx
-#define TARGET_ABI_FMT_ld TARGET_FMT_ld
-#define TARGET_ABI_FMT_lu TARGET_FMT_lu
-#define TARGET_ABI_BITS TARGET_LONG_BITS
-/* for consistency, define ABI32 too */
-#if TARGET_ABI_BITS == 32
-#define TARGET_ABI32 1
-#endif
-#endif
-#endif
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index 8a5ee3d81f7..a826086dab6 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -11,7 +11,7 @@
 #include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
 enum BSDType {
     target_freebsd,
@@ -23,7 +23,7 @@ extern enum BSDType bsd_type;
 #include "syscall_defs.h"
 #include "syscall.h"
 #include "target_signal.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 #if defined(CONFIG_USE_NPTL)
 #define THREAD __thread
@@ -146,7 +146,7 @@ int get_osversion(void);
 void fork_start(void);
 void fork_end(int child);
 
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 /* strace.c */
 void
diff --git a/bswap.h b/bswap.h
deleted file mode 100644
index cc7f84d30f9..00000000000
--- a/bswap.h
+++ /dev/null
@@ -1,713 +0,0 @@
-#ifndef BSWAP_H
-#define BSWAP_H
-
-#include "config-host.h"
-
-#include <inttypes.h>
-#include "softfloat.h"
-
-#ifdef CONFIG_MACHINE_BSWAP_H
-#include <sys/endian.h>
-#include <sys/types.h>
-#include <machine/bswap.h>
-#else
-
-#ifdef CONFIG_BYTESWAP_H
-#include <byteswap.h>
-#else
-
-#define bswap_16(x) \
-({ \
-	uint16_t __x = (x); \
-	((uint16_t)( \
-		(((uint16_t)(__x) & (uint16_t)0x00ffU) << 8) | \
-		(((uint16_t)(__x) & (uint16_t)0xff00U) >> 8) )); \
-})
-
-#define bswap_32(x) \
-({ \
-	uint32_t __x = (x); \
-	((uint32_t)( \
-		(((uint32_t)(__x) & (uint32_t)0x000000ffUL) << 24) | \
-		(((uint32_t)(__x) & (uint32_t)0x0000ff00UL) <<  8) | \
-		(((uint32_t)(__x) & (uint32_t)0x00ff0000UL) >>  8) | \
-		(((uint32_t)(__x) & (uint32_t)0xff000000UL) >> 24) )); \
-})
-
-#define bswap_64(x) \
-({ \
-	uint64_t __x = (x); \
-	((uint64_t)( \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000000000ffULL) << 56) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000000000ff00ULL) << 40) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000ff000000ULL) <<  8) | \
-	        (uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000ff00000000ULL) >>  8) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0xff00000000000000ULL) >> 56) )); \
-})
-
-#endif /* !CONFIG_BYTESWAP_H */
-
-static inline uint16_t bswap16(uint16_t x)
-{
-    return bswap_16(x);
-}
-
-static inline uint32_t bswap32(uint32_t x)
-{
-    return bswap_32(x);
-}
-
-static inline uint64_t bswap64(uint64_t x)
-{
-    return bswap_64(x);
-}
-
-#endif /* ! CONFIG_MACHINE_BSWAP_H */
-
-static inline void bswap16s(uint16_t *s)
-{
-    *s = bswap16(*s);
-}
-
-static inline void bswap32s(uint32_t *s)
-{
-    *s = bswap32(*s);
-}
-
-static inline void bswap64s(uint64_t *s)
-{
-    *s = bswap64(*s);
-}
-
-#if defined(HOST_WORDS_BIGENDIAN)
-#define be_bswap(v, size) (v)
-#define le_bswap(v, size) bswap ## size(v)
-#define be_bswaps(v, size)
-#define le_bswaps(p, size) *p = bswap ## size(*p);
-#else
-#define le_bswap(v, size) (v)
-#define be_bswap(v, size) bswap ## size(v)
-#define le_bswaps(v, size)
-#define be_bswaps(p, size) *p = bswap ## size(*p);
-#endif
-
-#define CPU_CONVERT(endian, size, type)\
-static inline type endian ## size ## _to_cpu(type v)\
-{\
-    return endian ## _bswap(v, size);\
-}\
-\
-static inline type cpu_to_ ## endian ## size(type v)\
-{\
-    return endian ## _bswap(v, size);\
-}\
-\
-static inline void endian ## size ## _to_cpus(type *p)\
-{\
-    endian ## _bswaps(p, size)\
-}\
-\
-static inline void cpu_to_ ## endian ## size ## s(type *p)\
-{\
-    endian ## _bswaps(p, size)\
-}\
-\
-static inline type endian ## size ## _to_cpup(const type *p)\
-{\
-    return endian ## size ## _to_cpu(*p);\
-}\
-\
-static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
-{\
-     *p = cpu_to_ ## endian ## size(v);\
-}
-
-CPU_CONVERT(be, 16, uint16_t)
-CPU_CONVERT(be, 32, uint32_t)
-CPU_CONVERT(be, 64, uint64_t)
-
-CPU_CONVERT(le, 16, uint16_t)
-CPU_CONVERT(le, 32, uint32_t)
-CPU_CONVERT(le, 64, uint64_t)
-
-/* unaligned versions (optimized for frequent unaligned accesses)*/
-
-#if defined(__i386__) || defined(_ARCH_PPC)
-
-#define cpu_to_le16wu(p, v) cpu_to_le16w(p, v)
-#define cpu_to_le32wu(p, v) cpu_to_le32w(p, v)
-#define le16_to_cpupu(p) le16_to_cpup(p)
-#define le32_to_cpupu(p) le32_to_cpup(p)
-#define be32_to_cpupu(p) be32_to_cpup(p)
-
-#define cpu_to_be16wu(p, v) cpu_to_be16w(p, v)
-#define cpu_to_be32wu(p, v) cpu_to_be32w(p, v)
-#define cpu_to_be64wu(p, v) cpu_to_be64w(p, v)
-
-#else
-
-static inline void cpu_to_le16wu(uint16_t *p, uint16_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v & 0xff;
-    p1[1] = v >> 8;
-}
-
-static inline void cpu_to_le32wu(uint32_t *p, uint32_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v & 0xff;
-    p1[1] = v >> 8;
-    p1[2] = v >> 16;
-    p1[3] = v >> 24;
-}
-
-static inline uint16_t le16_to_cpupu(const uint16_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[0] | (p1[1] << 8);
-}
-
-static inline uint32_t le32_to_cpupu(const uint32_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[0] | (p1[1] << 8) | (p1[2] << 16) | (p1[3] << 24);
-}
-
-static inline uint32_t be32_to_cpupu(const uint32_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[3] | (p1[2] << 8) | (p1[1] << 16) | (p1[0] << 24);
-}
-
-static inline void cpu_to_be16wu(uint16_t *p, uint16_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 8;
-    p1[1] = v & 0xff;
-}
-
-static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 24;
-    p1[1] = v >> 16;
-    p1[2] = v >> 8;
-    p1[3] = v & 0xff;
-}
-
-static inline void cpu_to_be64wu(uint64_t *p, uint64_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 56;
-    p1[1] = v >> 48;
-    p1[2] = v >> 40;
-    p1[3] = v >> 32;
-    p1[4] = v >> 24;
-    p1[5] = v >> 16;
-    p1[6] = v >> 8;
-    p1[7] = v & 0xff;
-}
-
-#endif
-
-#ifdef HOST_WORDS_BIGENDIAN
-#define cpu_to_32wu cpu_to_be32wu
-#define leul_to_cpu(v) glue(glue(le,HOST_LONG_BITS),_to_cpu)(v)
-#else
-#define cpu_to_32wu cpu_to_le32wu
-#define leul_to_cpu(v) (v)
-#endif
-
-#undef le_bswap
-#undef be_bswap
-#undef le_bswaps
-#undef be_bswaps
-
-/* len must be one of 1, 2, 4 */
-static inline uint32_t qemu_bswap_len(uint32_t value, int len)
-{
-    return bswap32(value) >> (32 - 8 * len);
-}
-
-typedef union {
-    float32 f;
-    uint32_t l;
-} CPU_FloatU;
-
-typedef union {
-    float64 d;
-#if defined(HOST_WORDS_BIGENDIAN)
-    struct {
-        uint32_t upper;
-        uint32_t lower;
-    } l;
-#else
-    struct {
-        uint32_t lower;
-        uint32_t upper;
-    } l;
-#endif
-    uint64_t ll;
-} CPU_DoubleU;
-
-typedef union {
-     floatx80 d;
-     struct {
-         uint64_t lower;
-         uint16_t upper;
-     } l;
-} CPU_LDoubleU;
-
-typedef union {
-    float128 q;
-#if defined(HOST_WORDS_BIGENDIAN)
-    struct {
-        uint32_t upmost;
-        uint32_t upper;
-        uint32_t lower;
-        uint32_t lowest;
-    } l;
-    struct {
-        uint64_t upper;
-        uint64_t lower;
-    } ll;
-#else
-    struct {
-        uint32_t lowest;
-        uint32_t lower;
-        uint32_t upper;
-        uint32_t upmost;
-    } l;
-    struct {
-        uint64_t lower;
-        uint64_t upper;
-    } ll;
-#endif
-} CPU_QuadU;
-
-/* unaligned/endian-independent pointer access */
-
-/*
- * the generic syntax is:
- *
- * load: ld{type}{sign}{size}{endian}_p(ptr)
- *
- * store: st{type}{size}{endian}_p(ptr, val)
- *
- * Note there are small differences with the softmmu access API!
- *
- * type is:
- * (empty): integer access
- *   f    : float access
- *
- * sign is:
- * (empty): for floats or 32 bit size
- *   u    : unsigned
- *   s    : signed
- *
- * size is:
- *   b: 8 bits
- *   w: 16 bits
- *   l: 32 bits
- *   q: 64 bits
- *
- * endian is:
- * (empty): 8 bit access
- *   be   : big endian
- *   le   : little endian
- */
-static inline int ldub_p(const void *ptr)
-{
-    return *(uint8_t *)ptr;
-}
-
-static inline int ldsb_p(const void *ptr)
-{
-    return *(int8_t *)ptr;
-}
-
-static inline void stb_p(void *ptr, int v)
-{
-    *(uint8_t *)ptr = v;
-}
-
-/* NOTE: on arm, putting 2 in /proc/sys/debug/alignment so that the
-   kernel handles unaligned load/stores may give better results, but
-   it is a system wide setting : bad */
-#if defined(HOST_WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
-
-/* conservative code for little endian unaligned accesses */
-static inline int lduw_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return val;
-#else
-    const uint8_t *p = ptr;
-    return p[0] | (p[1] << 8);
-#endif
-}
-
-static inline int ldsw_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return (int16_t)val;
-#else
-    const uint8_t *p = ptr;
-    return (int16_t)(p[0] | (p[1] << 8));
-#endif
-}
-
-static inline int ldl_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lwbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return val;
-#else
-    const uint8_t *p = ptr;
-    return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
-#endif
-}
-
-static inline uint64_t ldq_le_p(const void *ptr)
-{
-    const uint8_t *p = ptr;
-    uint32_t v1, v2;
-    v1 = ldl_le_p(p);
-    v2 = ldl_le_p(p + 4);
-    return v1 | ((uint64_t)v2 << 32);
-}
-
-static inline void stw_le_p(void *ptr, int v)
-{
-#ifdef _ARCH_PPC
-    __asm__ __volatile__ ("sthbrx %1,0,%2" : "=m" (*(uint16_t *)ptr) : "r" (v), "r" (ptr));
-#else
-    uint8_t *p = ptr;
-    p[0] = v;
-    p[1] = v >> 8;
-#endif
-}
-
-static inline void stl_le_p(void *ptr, int v)
-{
-#ifdef _ARCH_PPC
-    __asm__ __volatile__ ("stwbrx %1,0,%2" : "=m" (*(uint32_t *)ptr) : "r" (v), "r" (ptr));
-#else
-    uint8_t *p = ptr;
-    p[0] = v;
-    p[1] = v >> 8;
-    p[2] = v >> 16;
-    p[3] = v >> 24;
-#endif
-}
-
-static inline void stq_le_p(void *ptr, uint64_t v)
-{
-    uint8_t *p = ptr;
-    stl_le_p(p, (uint32_t)v);
-    stl_le_p(p + 4, v >> 32);
-}
-
-/* float access */
-
-static inline float32 ldfl_le_p(const void *ptr)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.i = ldl_le_p(ptr);
-    return u.f;
-}
-
-static inline void stfl_le_p(void *ptr, float32 v)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.f = v;
-    stl_le_p(ptr, u.i);
-}
-
-static inline float64 ldfq_le_p(const void *ptr)
-{
-    CPU_DoubleU u;
-    u.l.lower = ldl_le_p(ptr);
-    u.l.upper = ldl_le_p(ptr + 4);
-    return u.d;
-}
-
-static inline void stfq_le_p(void *ptr, float64 v)
-{
-    CPU_DoubleU u;
-    u.d = v;
-    stl_le_p(ptr, u.l.lower);
-    stl_le_p(ptr + 4, u.l.upper);
-}
-
-#else
-
-static inline int lduw_le_p(const void *ptr)
-{
-    return *(uint16_t *)ptr;
-}
-
-static inline int ldsw_le_p(const void *ptr)
-{
-    return *(int16_t *)ptr;
-}
-
-static inline int ldl_le_p(const void *ptr)
-{
-    return *(uint32_t *)ptr;
-}
-
-static inline uint64_t ldq_le_p(const void *ptr)
-{
-    return *(uint64_t *)ptr;
-}
-
-static inline void stw_le_p(void *ptr, int v)
-{
-    *(uint16_t *)ptr = v;
-}
-
-static inline void stl_le_p(void *ptr, int v)
-{
-    *(uint32_t *)ptr = v;
-}
-
-static inline void stq_le_p(void *ptr, uint64_t v)
-{
-    *(uint64_t *)ptr = v;
-}
-
-/* float access */
-
-static inline float32 ldfl_le_p(const void *ptr)
-{
-    return *(float32 *)ptr;
-}
-
-static inline float64 ldfq_le_p(const void *ptr)
-{
-    return *(float64 *)ptr;
-}
-
-static inline void stfl_le_p(void *ptr, float32 v)
-{
-    *(float32 *)ptr = v;
-}
-
-static inline void stfq_le_p(void *ptr, float64 v)
-{
-    *(float64 *)ptr = v;
-}
-#endif
-
-#if !defined(HOST_WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
-
-static inline int lduw_be_p(const void *ptr)
-{
-#if defined(__i386__)
-    int val;
-    asm volatile ("movzwl %1, %0\n"
-                  "xchgb %b0, %h0\n"
-                  : "=q" (val)
-                  : "m" (*(uint16_t *)ptr));
-    return val;
-#else
-    const uint8_t *b = ptr;
-    return ((b[0] << 8) | b[1]);
-#endif
-}
-
-static inline int ldsw_be_p(const void *ptr)
-{
-#if defined(__i386__)
-    int val;
-    asm volatile ("movzwl %1, %0\n"
-                  "xchgb %b0, %h0\n"
-                  : "=q" (val)
-                  : "m" (*(uint16_t *)ptr));
-    return (int16_t)val;
-#else
-    const uint8_t *b = ptr;
-    return (int16_t)((b[0] << 8) | b[1]);
-#endif
-}
-
-static inline int ldl_be_p(const void *ptr)
-{
-#if defined(__i386__) || defined(__x86_64__)
-    int val;
-    asm volatile ("movl %1, %0\n"
-                  "bswap %0\n"
-                  : "=r" (val)
-                  : "m" (*(uint32_t *)ptr));
-    return val;
-#else
-    const uint8_t *b = ptr;
-    return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3];
-#endif
-}
-
-static inline uint64_t ldq_be_p(const void *ptr)
-{
-    uint32_t a,b;
-    a = ldl_be_p(ptr);
-    b = ldl_be_p((uint8_t *)ptr + 4);
-    return (((uint64_t)a<<32)|b);
-}
-
-static inline void stw_be_p(void *ptr, int v)
-{
-#if defined(__i386__)
-    asm volatile ("xchgb %b0, %h0\n"
-                  "movw %w0, %1\n"
-                  : "=q" (v)
-                  : "m" (*(uint16_t *)ptr), "0" (v));
-#else
-    uint8_t *d = (uint8_t *) ptr;
-    d[0] = v >> 8;
-    d[1] = v;
-#endif
-}
-
-static inline void stl_be_p(void *ptr, int v)
-{
-#if defined(__i386__) || defined(__x86_64__)
-    asm volatile ("bswap %0\n"
-                  "movl %0, %1\n"
-                  : "=r" (v)
-                  : "m" (*(uint32_t *)ptr), "0" (v));
-#else
-    uint8_t *d = (uint8_t *) ptr;
-    d[0] = v >> 24;
-    d[1] = v >> 16;
-    d[2] = v >> 8;
-    d[3] = v;
-#endif
-}
-
-static inline void stq_be_p(void *ptr, uint64_t v)
-{
-    stl_be_p(ptr, v >> 32);
-    stl_be_p((uint8_t *)ptr + 4, v);
-}
-
-/* float access */
-
-static inline float32 ldfl_be_p(const void *ptr)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.i = ldl_be_p(ptr);
-    return u.f;
-}
-
-static inline void stfl_be_p(void *ptr, float32 v)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.f = v;
-    stl_be_p(ptr, u.i);
-}
-
-static inline float64 ldfq_be_p(const void *ptr)
-{
-    CPU_DoubleU u;
-    u.l.upper = ldl_be_p(ptr);
-    u.l.lower = ldl_be_p((uint8_t *)ptr + 4);
-    return u.d;
-}
-
-static inline void stfq_be_p(void *ptr, float64 v)
-{
-    CPU_DoubleU u;
-    u.d = v;
-    stl_be_p(ptr, u.l.upper);
-    stl_be_p((uint8_t *)ptr + 4, u.l.lower);
-}
-
-#else
-
-static inline int lduw_be_p(const void *ptr)
-{
-    return *(uint16_t *)ptr;
-}
-
-static inline int ldsw_be_p(const void *ptr)
-{
-    return *(int16_t *)ptr;
-}
-
-static inline int ldl_be_p(const void *ptr)
-{
-    return *(uint32_t *)ptr;
-}
-
-static inline uint64_t ldq_be_p(const void *ptr)
-{
-    return *(uint64_t *)ptr;
-}
-
-static inline void stw_be_p(void *ptr, int v)
-{
-    *(uint16_t *)ptr = v;
-}
-
-static inline void stl_be_p(void *ptr, int v)
-{
-    *(uint32_t *)ptr = v;
-}
-
-static inline void stq_be_p(void *ptr, uint64_t v)
-{
-    *(uint64_t *)ptr = v;
-}
-
-/* float access */
-
-static inline float32 ldfl_be_p(const void *ptr)
-{
-    return *(float32 *)ptr;
-}
-
-static inline float64 ldfq_be_p(const void *ptr)
-{
-    return *(float64 *)ptr;
-}
-
-static inline void stfl_be_p(void *ptr, float32 v)
-{
-    *(float32 *)ptr = v;
-}
-
-static inline void stfq_be_p(void *ptr, float64 v)
-{
-    *(float64 *)ptr = v;
-}
-
-#endif
-
-#endif /* BSWAP_H */
diff --git a/bt-host.c b/bt-host.c
index 0d3ad28e8c0..2092754530a 100644
--- a/bt-host.c
+++ b/bt-host.c
@@ -18,9 +18,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "net.h"
-#include "bt-host.h"
+#include "bt/bt.h"
+#include "qemu/main-loop.h"
 
 #ifndef _WIN32
 # include <errno.h>
diff --git a/bt-host.h b/bt-host.h
deleted file mode 100644
index f1eff65f452..00000000000
--- a/bt-host.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef BT_HOST_H
-#define BT_HOST_H
-
-struct HCIInfo;
-
-/* bt-host.c */
-struct HCIInfo *bt_host_hci(const char *id);
-
-#endif
diff --git a/bt-vhci.c b/bt-vhci.c
index bbc10298543..a6a7ab03295 100644
--- a/bt-vhci.c
+++ b/bt-vhci.c
@@ -18,9 +18,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "hw/bt.h"
+#include "qemu/main-loop.h"
 
 #define VHCI_DEV	"/dev/vhci"
 #define VHCI_UDEV	"/dev/hci_vhci"
diff --git a/buffered_file.c b/buffered_file.c
deleted file mode 100644
index f170aa046f5..00000000000
--- a/buffered_file.c
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * QEMU buffered QEMUFile
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "hw/hw.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-
-//#define DEBUG_BUFFERED_FILE
-
-typedef struct QEMUFileBuffered
-{
-    BufferedPutFunc *put_buffer;
-    BufferedPutReadyFunc *put_ready;
-    BufferedWaitForUnfreezeFunc *wait_for_unfreeze;
-    BufferedCloseFunc *close;
-    void *opaque;
-    QEMUFile *file;
-    int freeze_output;
-    size_t bytes_xfer;
-    size_t xfer_limit;
-    uint8_t *buffer;
-    size_t buffer_size;
-    size_t buffer_capacity;
-    QEMUTimer *timer;
-} QEMUFileBuffered;
-
-#ifdef DEBUG_BUFFERED_FILE
-#define DPRINTF(fmt, ...) \
-    do { printf("buffered-file: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-static void buffered_append(QEMUFileBuffered *s,
-                            const uint8_t *buf, size_t size)
-{
-    if (size > (s->buffer_capacity - s->buffer_size)) {
-        void *tmp;
-
-        DPRINTF("increasing buffer capacity from %zu by %zu\n",
-                s->buffer_capacity, size + 1024);
-
-        s->buffer_capacity += size + 1024;
-
-        tmp = g_realloc(s->buffer, s->buffer_capacity);
-        if (tmp == NULL) {
-            fprintf(stderr, "qemu file buffer expansion failed\n");
-            exit(1);
-        }
-
-        s->buffer = tmp;
-    }
-
-    memcpy(s->buffer + s->buffer_size, buf, size);
-    s->buffer_size += size;
-}
-
-static void buffered_flush(QEMUFileBuffered *s)
-{
-    size_t offset = 0;
-    int error;
-
-    error = qemu_file_get_error(s->file);
-    if (error != 0) {
-        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
-        return;
-    }
-
-    DPRINTF("flushing %zu byte(s) of data\n", s->buffer_size);
-
-    while (offset < s->buffer_size) {
-        ssize_t ret;
-
-        ret = s->put_buffer(s->opaque, s->buffer + offset,
-                            s->buffer_size - offset);
-        if (ret == -EAGAIN) {
-            DPRINTF("backend not ready, freezing\n");
-            s->freeze_output = 1;
-            break;
-        }
-
-        if (ret <= 0) {
-            DPRINTF("error flushing data, %zd\n", ret);
-            qemu_file_set_error(s->file, ret);
-            break;
-        } else {
-            DPRINTF("flushed %zd byte(s)\n", ret);
-            offset += ret;
-        }
-    }
-
-    DPRINTF("flushed %zu of %zu byte(s)\n", offset, s->buffer_size);
-    memmove(s->buffer, s->buffer + offset, s->buffer_size - offset);
-    s->buffer_size -= offset;
-}
-
-static int buffered_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileBuffered *s = opaque;
-    int offset = 0, error;
-    ssize_t ret;
-
-    DPRINTF("putting %d bytes at %" PRId64 "\n", size, pos);
-
-    error = qemu_file_get_error(s->file);
-    if (error) {
-        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
-        return error;
-    }
-
-    DPRINTF("unfreezing output\n");
-    s->freeze_output = 0;
-
-    buffered_flush(s);
-
-    while (!s->freeze_output && offset < size) {
-        if (s->bytes_xfer > s->xfer_limit) {
-            DPRINTF("transfer limit exceeded when putting\n");
-            break;
-        }
-
-        ret = s->put_buffer(s->opaque, buf + offset, size - offset);
-        if (ret == -EAGAIN) {
-            DPRINTF("backend not ready, freezing\n");
-            s->freeze_output = 1;
-            break;
-        }
-
-        if (ret <= 0) {
-            DPRINTF("error putting\n");
-            qemu_file_set_error(s->file, ret);
-            offset = -EINVAL;
-            break;
-        }
-
-        DPRINTF("put %zd byte(s)\n", ret);
-        offset += ret;
-        s->bytes_xfer += ret;
-    }
-
-    if (offset >= 0) {
-        DPRINTF("buffering %d bytes\n", size - offset);
-        buffered_append(s, buf + offset, size - offset);
-        offset = size;
-    }
-
-    if (pos == 0 && size == 0) {
-        DPRINTF("file is ready\n");
-        if (s->bytes_xfer <= s->xfer_limit) {
-            DPRINTF("notifying client\n");
-            s->put_ready(s->opaque);
-        }
-    }
-
-    return offset;
-}
-
-static int buffered_close(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-    int ret;
-
-    DPRINTF("closing\n");
-
-    while (!qemu_file_get_error(s->file) && s->buffer_size) {
-        buffered_flush(s);
-        if (s->freeze_output)
-            s->wait_for_unfreeze(s->opaque);
-    }
-
-    ret = s->close(s->opaque);
-
-    qemu_del_timer(s->timer);
-    qemu_free_timer(s->timer);
-    g_free(s->buffer);
-    g_free(s);
-
-    return ret;
-}
-
-/*
- * The meaning of the return values is:
- *   0: We can continue sending
- *   1: Time to stop
- *   negative: There has been an error
- */
-static int buffered_rate_limit(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-    int ret;
-
-    ret = qemu_file_get_error(s->file);
-    if (ret) {
-        return ret;
-    }
-    if (s->freeze_output)
-        return 1;
-
-    if (s->bytes_xfer > s->xfer_limit)
-        return 1;
-
-    return 0;
-}
-
-static int64_t buffered_set_rate_limit(void *opaque, int64_t new_rate)
-{
-    QEMUFileBuffered *s = opaque;
-    if (qemu_file_get_error(s->file)) {
-        goto out;
-    }
-    if (new_rate > SIZE_MAX) {
-        new_rate = SIZE_MAX;
-    }
-
-    s->xfer_limit = new_rate / 10;
-    
-out:
-    return s->xfer_limit;
-}
-
-static int64_t buffered_get_rate_limit(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-  
-    return s->xfer_limit;
-}
-
-static void buffered_rate_tick(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-
-    if (qemu_file_get_error(s->file)) {
-        buffered_close(s);
-        return;
-    }
-
-    qemu_mod_timer(s->timer, qemu_get_clock_ms(rt_clock) + 100);
-
-    if (s->freeze_output)
-        return;
-
-    s->bytes_xfer = 0;
-
-    buffered_flush(s);
-
-    /* Add some checks around this */
-    s->put_ready(s->opaque);
-}
-
-QEMUFile *qemu_fopen_ops_buffered(void *opaque,
-                                  size_t bytes_per_sec,
-                                  BufferedPutFunc *put_buffer,
-                                  BufferedPutReadyFunc *put_ready,
-                                  BufferedWaitForUnfreezeFunc *wait_for_unfreeze,
-                                  BufferedCloseFunc *close)
-{
-    QEMUFileBuffered *s;
-
-    s = g_malloc0(sizeof(*s));
-
-    s->opaque = opaque;
-    s->xfer_limit = bytes_per_sec / 10;
-    s->put_buffer = put_buffer;
-    s->put_ready = put_ready;
-    s->wait_for_unfreeze = wait_for_unfreeze;
-    s->close = close;
-
-    s->file = qemu_fopen_ops(s, buffered_put_buffer, NULL,
-                             buffered_close, buffered_rate_limit,
-                             buffered_set_rate_limit,
-			     buffered_get_rate_limit);
-
-    s->timer = qemu_new_timer_ms(rt_clock, buffered_rate_tick, s);
-
-    qemu_mod_timer(s->timer, qemu_get_clock_ms(rt_clock) + 100);
-
-    return s->file;
-}
diff --git a/buffered_file.h b/buffered_file.h
deleted file mode 100644
index 98d358baeab..00000000000
--- a/buffered_file.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * QEMU buffered QEMUFile
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_BUFFERED_FILE_H
-#define QEMU_BUFFERED_FILE_H
-
-#include "hw/hw.h"
-
-typedef ssize_t (BufferedPutFunc)(void *opaque, const void *data, size_t size);
-typedef void (BufferedPutReadyFunc)(void *opaque);
-typedef void (BufferedWaitForUnfreezeFunc)(void *opaque);
-typedef int (BufferedCloseFunc)(void *opaque);
-
-QEMUFile *qemu_fopen_ops_buffered(void *opaque, size_t xfer_limit,
-                                  BufferedPutFunc *put_buffer,
-                                  BufferedPutReadyFunc *put_ready,
-                                  BufferedWaitForUnfreezeFunc *wait_for_unfreeze,
-                                  BufferedCloseFunc *close);
-
-#endif
diff --git a/cmd.c b/cmd.c
index f40f09bbb76..10a8688b2d2 100644
--- a/cmd.c
+++ b/cmd.c
@@ -24,8 +24,8 @@
 #include <getopt.h>
 
 #include "cmd.h"
-#include "qemu-aio.h"
-#include "main-loop.h"
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 #define _(x)	x	/* not gettext support yet */
 
diff --git a/configure b/configure
index d97fd817096..b7635e4fec9 100755
--- a/configure
+++ b/configure
@@ -111,14 +111,12 @@ source_path=`dirname "$0"`
 cpu=""
 interp_prefix="/usr/gnemul/qemu-%M"
 static="no"
-sparc_cpu=""
 cross_prefix=""
 audio_drv_list=""
 audio_card_list="ac97 es1370 sb16 hda"
 audio_possible_cards="ac97 es1370 sb16 cs4231a adlib gus hda"
 block_drv_whitelist=""
-host_cc="gcc"
-helper_cflags=""
+host_cc="cc"
 libs_softmmu=""
 libs_tools=""
 audio_pt_int=""
@@ -127,7 +125,8 @@ cc_i386=i386-pc-linux-gnu-gcc
 libs_qga=""
 debug_info="yes"
 
-target_list=""
+# Don't accept a target_list environment variable.
+unset target_list
 
 # Default value for a variable defining feature "foo".
 #  * foo="no"  feature will only be used if --enable-foo arg is given
@@ -148,6 +147,7 @@ curses=""
 docs=""
 fdt=""
 nptl=""
+pixman=""
 sdl=""
 virtfs=""
 vnc="yes"
@@ -158,6 +158,7 @@ vnc_tls=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
+vnc_ws=""
 xen=""
 xen_ctrl_version=""
 xen_pci_passthrough=""
@@ -176,6 +177,8 @@ strip_opt="yes"
 tcg_interpreter="no"
 bigendian="no"
 mingw32="no"
+gcov="no"
+gcov_tool="gcov"
 EXESUF=""
 prefix="/usr/local"
 mandir="\${prefix}/share/man"
@@ -183,8 +186,10 @@ datadir="\${prefix}/share"
 qemu_docdir="\${prefix}/share/doc/qemu"
 bindir="\${prefix}/bin"
 libdir="\${prefix}/lib"
+libexecdir="\${prefix}/libexec"
 includedir="\${prefix}/include"
 sysconfdir="\${prefix}/etc"
+local_statedir="\${prefix}/var"
 confsuffix="/qemu"
 slirp="yes"
 fmod_lib=""
@@ -198,7 +203,7 @@ cocoa="no"
 softmmu="yes"
 linux_user="no"
 bsd_user="no"
-guest_base=""
+guest_base="yes"
 uname_release=""
 mixemu="no"
 aix="no"
@@ -210,15 +215,17 @@ trace_backend="nop"
 trace_file="trace"
 spice=""
 rbd=""
-smartcard=""
 smartcard_nss=""
 usb_redir=""
 opengl=""
 zlib="yes"
 guest_agent="yes"
+want_tools="yes"
 libiscsi=""
 coroutine=""
 seccomp=""
+glusterfs=""
+virtio_blk_data_plane=""
 
 # parse CC options first
 for opt do
@@ -233,36 +240,34 @@ for opt do
   --cpu=*) cpu="$optarg"
   ;;
   --extra-cflags=*) QEMU_CFLAGS="$optarg $QEMU_CFLAGS"
+                    EXTRA_CFLAGS="$optarg"
   ;;
   --extra-ldflags=*) LDFLAGS="$optarg $LDFLAGS"
+                     EXTRA_LDFLAGS="$optarg"
   ;;
   --enable-debug-info) debug_info="yes"
   ;;
   --disable-debug-info) debug_info="no"
   ;;
-  --sparc_cpu=*)
-    sparc_cpu="$optarg"
-    case $sparc_cpu in
-    v7|v8|v8plus|v8plusa)
-      cpu="sparc"
-    ;;
-    v9)
-      cpu="sparc64"
-    ;;
-    *)
-      echo "undefined SPARC architecture. Exiting";
-      exit 1
-    ;;
-    esac
-  ;;
   esac
 done
 # OS specific
 # Using uname is really, really broken.  Once we have the right set of checks
 # we can eliminate its usage altogether.
 
-cc="${CC-${cross_prefix}gcc}"
+# Preferred compiler:
+#  ${CC} (if set)
+#  ${cross_prefix}gcc (if cross-prefix specified)
+#  system compiler
+if test -z "${CC}${cross_prefix}"; then
+  cc="$host_cc"
+else
+  cc="${CC-${cross_prefix}gcc}"
+fi
+
 ar="${AR-${cross_prefix}ar}"
+as="${AS-${cross_prefix}as}"
+cpp="${CPP-$cc -E}"
 objcopy="${OBJCOPY-${cross_prefix}objcopy}"
 ld="${LD-${cross_prefix}ld}"
 libtool="${LIBTOOL-${cross_prefix}libtool}"
@@ -280,7 +285,7 @@ QEMU_CFLAGS="-fno-strict-aliasing $QEMU_CFLAGS"
 QEMU_CFLAGS="-Wall -Wundef -Wwrite-strings -Wmissing-prototypes $QEMU_CFLAGS"
 QEMU_CFLAGS="-Wstrict-prototypes -Wredundant-decls $QEMU_CFLAGS"
 QEMU_CFLAGS="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE $QEMU_CFLAGS"
-QEMU_INCLUDES="-I. -I\$(SRC_PATH) -I\$(SRC_PATH)/fpu"
+QEMU_INCLUDES="-I. -I\$(SRC_PATH) -I\$(SRC_PATH)/include"
 if test "$debug_info" = "yes"; then
     CFLAGS="-g $CFLAGS"
     LDFLAGS="-g $LDFLAGS"
@@ -342,8 +347,6 @@ elif check_define __i386__ ; then
 elif check_define __x86_64__ ; then
   cpu="x86_64"
 elif check_define __sparc__ ; then
-  # We can't check for 64 bit (when gcc is biarch) or V8PLUSA
-  # They must be specified using --sparc_cpu
   if check_define __arch64__ ; then
     cpu="sparc64"
   else
@@ -569,6 +572,7 @@ EOF
   qemu_docdir="\${prefix}"
   bindir="\${prefix}"
   sysconfdir="\${prefix}"
+  local_statedir="\${prefix}"
   confsuffix=""
   libs_qga="-lws2_32 -lwinmm -lpowrprof $libs_qga"
 fi
@@ -602,6 +606,8 @@ for opt do
   ;;
   --python=*) python="$optarg"
   ;;
+  --gcov=*) gcov_tool="$optarg"
+  ;;
   --smbd=*) smbd="$optarg"
   ;;
   --extra-cflags=*)
@@ -622,6 +628,8 @@ for opt do
   ;;
   --enable-gprof) gprof="yes"
   ;;
+  --enable-gcov) gcov="yes"
+  ;;
   --static)
     static="yes"
     LDFLAGS="-static $LDFLAGS"
@@ -633,6 +641,8 @@ for opt do
   ;;
   --libdir=*) libdir="$optarg"
   ;;
+  --libexecdir=*) libexecdir="$optarg"
+  ;;
   --includedir=*) includedir="$optarg"
   ;;
   --datadir=*) datadir="$optarg"
@@ -643,7 +653,9 @@ for opt do
   ;;
   --sysconfdir=*) sysconfdir="$optarg"
   ;;
-  --sbindir=*|--libexecdir=*|--sharedstatedir=*|--localstatedir=*|\
+  --localstatedir=*) local_statedir="$optarg"
+  ;;
+  --sbindir=*|--sharedstatedir=*|\
   --oldincludedir=*|--datarootdir=*|--infodir=*|--localedir=*|\
   --htmldir=*|--dvidir=*|--pdfdir=*|--psdir=*)
     # These switches are silently ignored, for compatibility with
@@ -651,6 +663,12 @@ for opt do
     # configure to be used by RPM and similar macros that set
     # lots of directory switches by default.
   ;;
+  --with-system-pixman) pixman="system"
+  ;;
+  --without-system-pixman) pixman="internal"
+  ;;
+  --without-pixman) pixman="none"
+  ;;
   --disable-sdl) sdl="no"
   ;;
   --enable-sdl) sdl="yes"
@@ -707,6 +725,10 @@ for opt do
   ;;
   --enable-vnc-png) vnc_png="yes"
   ;;
+  --disable-vnc-ws) vnc_ws="no"
+  ;;
+  --enable-vnc-ws) vnc_ws="yes"
+  ;;
   --disable-slirp) slirp="no"
   ;;
   --disable-uuid) uuid="no"
@@ -789,8 +811,6 @@ for opt do
   ;;
   --enable-uname-release=*) uname_release="$optarg"
   ;;
-  --sparc_cpu=*)
-  ;;
   --enable-werror) werror="yes"
   ;;
   --disable-werror) werror="no"
@@ -847,10 +867,6 @@ for opt do
   ;;
   --enable-xfsctl) xfs="yes"
   ;;
-  --disable-smartcard) smartcard="no"
-  ;;
-  --enable-smartcard) smartcard="yes"
-  ;;
   --disable-smartcard-nss) smartcard_nss="no"
   ;;
   --enable-smartcard-nss) smartcard_nss="yes"
@@ -865,92 +881,57 @@ for opt do
   ;;
   --disable-guest-agent) guest_agent="no"
   ;;
+  --enable-tools) want_tools="yes"
+  ;;
+  --disable-tools) want_tools="no"
+  ;;
   --enable-seccomp) seccomp="yes"
   ;;
   --disable-seccomp) seccomp="no"
   ;;
+  --disable-glusterfs) glusterfs="no"
+  ;;
+  --enable-glusterfs) glusterfs="yes"
+  ;;
+  --disable-virtio-blk-data-plane) virtio_blk_data_plane="no"
+  ;;
+  --enable-virtio-blk-data-plane) virtio_blk_data_plane="yes"
+  ;;
   *) echo "ERROR: unknown option $opt"; show_help="yes"
   ;;
   esac
 done
 
-#
-# If cpu ~= sparc and  sparc_cpu hasn't been defined, plug in the right
-# QEMU_CFLAGS/LDFLAGS (assume sparc_v8plus for 32-bit and sparc_v9 for 64-bit)
-#
-host_guest_base="no"
 case "$cpu" in
-    sparc) case $sparc_cpu in
-           v7|v8)
-             QEMU_CFLAGS="-mcpu=${sparc_cpu} -D__sparc_${sparc_cpu}__ $QEMU_CFLAGS"
-           ;;
-           v8plus|v8plusa)
-             QEMU_CFLAGS="-mcpu=ultrasparc -D__sparc_${sparc_cpu}__ $QEMU_CFLAGS"
-           ;;
-           *) # sparc_cpu not defined in the command line
-             QEMU_CFLAGS="-mcpu=ultrasparc -D__sparc_v8plus__ $QEMU_CFLAGS"
-           esac
+    sparc)
            LDFLAGS="-m32 $LDFLAGS"
-           QEMU_CFLAGS="-m32 -ffixed-g2 -ffixed-g3 $QEMU_CFLAGS"
-           if test "$solaris" = "no" ; then
-             QEMU_CFLAGS="-ffixed-g1 -ffixed-g6 $QEMU_CFLAGS"
-             helper_cflags="-ffixed-i0"
-           fi
+           QEMU_CFLAGS="-m32 -mcpu=ultrasparc $QEMU_CFLAGS"
            ;;
     sparc64)
-           QEMU_CFLAGS="-m64 -mcpu=ultrasparc -D__sparc_v9__ $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
-           QEMU_CFLAGS="-ffixed-g5 -ffixed-g6 -ffixed-g7 $QEMU_CFLAGS"
-           if test "$solaris" != "no" ; then
-             QEMU_CFLAGS="-ffixed-g1 $QEMU_CFLAGS"
-           fi
+           QEMU_CFLAGS="-m64 -mcpu=ultrasparc $QEMU_CFLAGS"
            ;;
     s390)
            QEMU_CFLAGS="-m31 -march=z990 $QEMU_CFLAGS"
            LDFLAGS="-m31 $LDFLAGS"
-           host_guest_base="yes"
            ;;
     s390x)
            QEMU_CFLAGS="-m64 -march=z990 $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
-           host_guest_base="yes"
            ;;
     i386)
            QEMU_CFLAGS="-m32 $QEMU_CFLAGS"
            LDFLAGS="-m32 $LDFLAGS"
            cc_i386='$(CC) -m32'
-           helper_cflags="-fomit-frame-pointer"
-           host_guest_base="yes"
            ;;
     x86_64)
            QEMU_CFLAGS="-m64 $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
            cc_i386='$(CC) -m32'
-           host_guest_base="yes"
-           ;;
-    arm*)
-           host_guest_base="yes"
-           ;;
-    ppc*)
-           host_guest_base="yes"
-           ;;
-    mips*)
-           host_guest_base="yes"
-           ;;
-    ia64*)
-           host_guest_base="yes"
-           ;;
-    hppa*)
-           host_guest_base="yes"
-           ;;
-    unicore32*)
-           host_guest_base="yes"
            ;;
+    # No special flags required for other host CPUs
 esac
 
-[ -z "$guest_base" ] && guest_base="$host_guest_base"
-
-
 default_target_list=""
 
 # these targets are portable
@@ -1055,6 +1036,7 @@ echo "  --datadir=PATH           install firmware in PATH$confsuffix"
 echo "  --docdir=PATH            install documentation in PATH$confsuffix"
 echo "  --bindir=PATH            install binaries in PATH"
 echo "  --sysconfdir=PATH        install config in PATH$confsuffix"
+echo "  --localstatedir=PATH     install local state in PATH"
 echo "  --with-confsuffix=SUFFIX suffix for QEMU data inside datadir and sysconfdir [$confsuffix]"
 echo "  --enable-debug-tcg       enable TCG debugging"
 echo "  --disable-debug-tcg      disable TCG debugging (default)"
@@ -1092,6 +1074,8 @@ echo "  --disable-vnc-jpeg       disable JPEG lossy compression for VNC server"
 echo "  --enable-vnc-jpeg        enable JPEG lossy compression for VNC server"
 echo "  --disable-vnc-png        disable PNG compression for VNC server (default)"
 echo "  --enable-vnc-png         enable PNG compression for VNC server"
+echo "  --disable-vnc-ws         disable Websockets support for VNC server"
+echo "  --enable-vnc-ws          enable Websockets support for VNC server"
 echo "  --disable-curses         disable curses output"
 echo "  --enable-curses          enable curses output"
 echo "  --disable-curl           disable curl connectivity"
@@ -1124,7 +1108,6 @@ echo "  --fmod-inc               path to FMOD includes"
 echo "  --oss-lib                path to OSS library"
 echo "  --enable-uname-release=R Return R for uname -r in usermode emulation"
 echo "  --cpu=CPU                Build for host CPU [$cpu]"
-echo "  --sparc_cpu=V            Build qemu for Sparc architecture v7, v8, v8plus, v8plusa, v9"
 echo "  --disable-uuid           disable uuid support"
 echo "  --enable-uuid            enable uuid support"
 echo "  --disable-vde            disable support for vde network"
@@ -1149,8 +1132,6 @@ echo "  --enable-spice           enable spice"
 echo "  --enable-rbd             enable building the rados block device (rbd)"
 echo "  --disable-libiscsi       disable iscsi support"
 echo "  --enable-libiscsi        enable iscsi support"
-echo "  --disable-smartcard      disable smartcard support"
-echo "  --enable-smartcard       enable smartcard support"
 echo "  --disable-smartcard-nss  disable smartcard nss support"
 echo "  --enable-smartcard-nss   enable smartcard nss support"
 echo "  --disable-usb-redir      disable usb network redirection support"
@@ -1161,6 +1142,10 @@ echo "  --disable-seccomp        disable seccomp support"
 echo "  --enable-seccomp         enables seccomp support"
 echo "  --with-coroutine=BACKEND coroutine backend. Supported options:"
 echo "                           gthread, ucontext, sigaltstack, windows"
+echo "  --enable-glusterfs       enable GlusterFS backend"
+echo "  --disable-glusterfs      disable GlusterFS backend"
+echo "  --enable-gcov            enable test coverage analysis with gcov"
+echo "  --gcov=GCOV              use specified gcov [$gcov_tool]"
 echo ""
 echo "NOTE: The object files are built at the place where configure is launched"
 exit 1
@@ -1207,6 +1192,7 @@ gcc_flags="-Wold-style-declaration -Wold-style-definition -Wtype-limits"
 gcc_flags="-Wformat-security -Wformat-y2k -Winit-self -Wignored-qualifiers $gcc_flags"
 gcc_flags="-Wmissing-include-dirs -Wempty-body -Wnested-externs $gcc_flags"
 gcc_flags="-fstack-protector-all -Wendif-labels $gcc_flags"
+gcc_flags="-Wno-initializer-overrides $gcc_flags"
 # Note that we do not add -Werror to gcc_flags here, because that would
 # enable it for all configure tests. If a configure test failed due
 # to -Werror this would just silently disable some features,
@@ -1215,11 +1201,30 @@ cat > $TMPC << EOF
 int main(void) { return 0; }
 EOF
 for flag in $gcc_flags; do
-    if compile_prog "-Werror $flag" "" ; then
+    # Use the positive sense of the flag when testing for -Wno-wombat
+    # support (gcc will happily accept the -Wno- form of unknown
+    # warning options).
+    optflag="$(echo $flag | sed -e 's/^-Wno-/-W/')"
+    if compile_prog "-Werror $optflag" "" ; then
 	QEMU_CFLAGS="$QEMU_CFLAGS $flag"
     fi
 done
 
+# Workaround for http://gcc.gnu.org/PR55489.  Happens with -fPIE/-fPIC and
+# large functions that use global variables.  The bug is in all releases of
+# GCC, but it became particularly acute in 4.6.x and 4.7.x.  It is fixed in
+# 4.7.3 and 4.8.0.  We should be able to delete this at the end of 2013.
+cat > $TMPC << EOF
+#if __GNUC__ == 4 && (__GNUC_MINOR__ == 6 || (__GNUC_MINOR__ == 7 && __GNUC_PATCHLEVEL__ <= 2))
+int main(void) { return 0; }
+#else
+#error No bug in this compiler.
+#endif
+EOF
+if compile_prog "-Werror -fno-gcse" "" ; then
+  TRANSLATE_OPT_CFLAGS=-fno-gcse
+fi
+
 if test "$static" = "yes" ; then
   if test "$pie" = "yes" ; then
     echo "static and pie are mutually incompatible"
@@ -1314,15 +1319,11 @@ if ! "$python" -c 'import sys; sys.exit(sys.version_info < (2,4) or sys.version_
   exit 1
 fi
 
-if test -z "$target_list" ; then
+if test -z "${target_list+xxx}" ; then
     target_list="$default_target_list"
 else
     target_list=`echo "$target_list" | sed -e 's/,/ /g'`
 fi
-if test -z "$target_list" ; then
-    echo "No targets enabled"
-    exit 1
-fi
 # see if system emulation was really requested
 case " $target_list " in
   *"-softmmu "*) softmmu=yes
@@ -1347,7 +1348,7 @@ if test -z "$cross_prefix" ; then
 # big/little endian test
 cat > $TMPC << EOF
 #include <inttypes.h>
-int main(int argc, char ** argv){
+int main(void) {
         volatile uint32_t i=0x01234567;
         return (*((uint8_t*)(&i))) == 0x67;
 }
@@ -1377,6 +1378,14 @@ esac
 fi
 
 ##########################################
+# pkg-config probe
+
+if ! has "$pkg_config_exe"; then
+  echo "Error: pkg-config binary '$pkg_config_exe' not found"
+  exit 1
+fi
+
+##########################################
 # NPTL probe
 
 if test "$nptl" != "no" ; then
@@ -1424,14 +1433,14 @@ fi
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if $pkg_config libseccomp --modversion >/dev/null 2>&1; then
-        LIBS=`$pkg_config --libs libseccomp`
+    if $pkg_config --atleast-version=1.0.0 libseccomp --modversion >/dev/null 2>&1; then
+        libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
 	seccomp="yes"
     else
-	seccomp="no"
 	if test "$seccomp" = "yes"; then
             feature_not_found "libseccomp"
 	fi
+	seccomp="no"
     fi
 fi
 ##########################################
@@ -1607,14 +1616,6 @@ if test "$xen_pci_passthrough" != "no"; then
 fi
 
 ##########################################
-# pkg-config probe
-
-if ! has "$pkg_config_exe"; then
-  echo "Error: pkg-config binary '$pkg_config_exe' not found"
-  exit 1
-fi
-
-##########################################
 # libtool probe
 
 if ! has $libtool; then
@@ -1718,8 +1719,8 @@ EOF
 fi
 
 ##########################################
-# VNC TLS detection
-if test "$vnc" = "yes" -a "$vnc_tls" != "no" ; then
+# VNC TLS/WS detection
+if test "$vnc" = "yes" -a \( "$vnc_tls" != "no" -o "$vnc_ws" != "no" \) ; then
   cat > $TMPC <<EOF
 #include <gnutls/gnutls.h>
 int main(void) { gnutls_session_t s; gnutls_init(&s, GNUTLS_SERVER); return 0; }
@@ -1727,13 +1728,23 @@ EOF
   vnc_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
   vnc_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
   if compile_prog "$vnc_tls_cflags" "$vnc_tls_libs" ; then
-    vnc_tls=yes
+    if test "$vnc_tls" != "no" ; then
+      vnc_tls=yes
+    fi
+    if test "$vnc_ws" != "no" ; then
+      vnc_ws=yes
+    fi
     libs_softmmu="$vnc_tls_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_tls_cflags"
   else
     if test "$vnc_tls" = "yes" ; then
       feature_not_found "vnc-tls"
     fi
+    if test "$vnc_ws" = "yes" ; then
+      feature_not_found "vnc-ws"
+    fi
     vnc_tls=no
+    vnc_ws=no
   fi
 fi
 
@@ -1751,6 +1762,7 @@ EOF
   if compile_prog "$vnc_sasl_cflags" "$vnc_sasl_libs" ; then
     vnc_sasl=yes
     libs_softmmu="$vnc_sasl_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_sasl_cflags"
   else
     if test "$vnc_sasl" = "yes" ; then
       feature_not_found "vnc-sasl"
@@ -1772,6 +1784,7 @@ EOF
   if compile_prog "$vnc_jpeg_cflags" "$vnc_jpeg_libs" ; then
     vnc_jpeg=yes
     libs_softmmu="$vnc_jpeg_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_jpeg_cflags"
   else
     if test "$vnc_jpeg" = "yes" ; then
       feature_not_found "vnc-jpeg"
@@ -2042,7 +2055,7 @@ fi
 if test "$mingw32" = "yes" ; then
     curses_list="-lpdcurses"
 else
-    curses_list="-lncurses -lcurses"
+    curses_list="-lncurses:-lcurses:$($pkg_config --libs ncurses 2>/dev/null)"
 fi
 
 if test "$curses" != "no" ; then
@@ -2055,13 +2068,16 @@ int main(void) {
   return s != 0;
 }
 EOF
+  IFS=:
   for curses_lib in $curses_list; do
+    unset IFS
     if compile_prog "" "$curses_lib" ; then
       curses_found=yes
       libs_softmmu="$curses_lib $libs_softmmu"
       break
     fi
   done
+  unset IFS
   if test "$curses_found" = "yes" ; then
     curses=yes
   else
@@ -2141,6 +2157,45 @@ else
 fi
 
 ##########################################
+# pixman support probe
+
+if test "$pixman" = ""; then
+  if test "$want_tools" = "no" -a "$softmmu" = "no"; then
+    pixman="none"
+  elif $pkg_config pixman-1 > /dev/null 2>&1; then
+    pixman="system"
+  else
+    pixman="internal"
+  fi
+fi
+if test "$pixman" = "none"; then
+  if test "$want_tools" != "no" -o "$softmmu" != "no"; then
+    echo "ERROR: pixman disabled but system emulation or tools build"
+    echo "       enabled.  You can turn off pixman only if you also"
+    echo "       disable all system emulation targets and the tools"
+    echo "       build with '--disable-tools --disable-system'."
+    exit 1
+  fi
+  pixman_cflags=
+  pixman_libs=
+elif test "$pixman" = "system"; then
+  pixman_cflags=`$pkg_config --cflags pixman-1 2>/dev/null`
+  pixman_libs=`$pkg_config --libs pixman-1 2>/dev/null`
+else
+  if test ! -d ${source_path}/pixman/pixman; then
+    echo "ERROR: pixman not present. Your options:"
+    echo "  (1) Preferred: Install the pixman devel package (any recent"
+    echo "      distro should have packages as Xorg needs pixman too)."
+    echo "  (2) Fetch the pixman submodule, using:"
+    echo "      git submodule update --init pixman"
+    exit 1
+  fi
+  mkdir -p pixman/pixman
+  pixman_cflags="-I\$(SRC_PATH)/pixman/pixman -I\$(BUILD_DIR)/pixman/pixman"
+  pixman_libs="-L\$(BUILD_DIR)/pixman/pixman/.libs -lpixman-1"
+fi
+
+##########################################
 # libcap probe
 
 if test "$cap" != "no" ; then
@@ -2247,6 +2302,17 @@ EOF
 fi
 
 ##########################################
+# adjust virtio-blk-data-plane based on linux-aio
+
+if test "$virtio_blk_data_plane" = "yes" -a \
+	"$linux_aio" != "yes" ; then
+  echo "Error: virtio-blk-data-plane requires Linux AIO, please try --enable-linux-aio"
+  exit 1
+elif test -z "$virtio_blk_data_plane" ; then
+  virtio_blk_data_plane=$linux_aio
+fi
+
+##########################################
 # attr probe
 
 if test "$attr" != "no" ; then
@@ -2340,6 +2406,29 @@ EOF
   fi
 fi
 
+##########################################
+# glusterfs probe
+if test "$glusterfs" != "no" ; then
+  cat > $TMPC <<EOF
+#include <glusterfs/api/glfs.h>
+int main(void) {
+    (void) glfs_new("volume");
+    return 0;
+}
+EOF
+  glusterfs_libs="-lgfapi -lgfrpc -lgfxdr"
+  if compile_prog "" "$glusterfs_libs" ; then
+    glusterfs=yes
+    libs_tools="$glusterfs_libs $libs_tools"
+    libs_softmmu="$glusterfs_libs $libs_softmmu"
+  else
+    if test "$glusterfs" = "yes" ; then
+      feature_not_found "GlusterFS backend support"
+    fi
+    glusterfs=no
+  fi
+fi
+
 #
 # Check for xxxat() functions when we are building linux-user
 # emulator.  This is done because older glibc versions don't
@@ -2426,8 +2515,7 @@ cat > $TMPC << EOF
 int main(void)
 {
     int pipefd[2];
-    pipe2(pipefd, O_CLOEXEC);
-    return 0;
+    return pipe2(pipefd, O_CLOEXEC);
 }
 EOF
 if compile_prog "" "" ; then
@@ -2512,6 +2600,22 @@ if compile_prog "" "" ; then
   fallocate=yes
 fi
 
+# check for fallocate hole punching
+fallocate_punch_hole=no
+cat > $TMPC << EOF
+#include <fcntl.h>
+#include <linux/falloc.h>
+
+int main(void)
+{
+    fallocate(0, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, 0);
+    return 0;
+}
+EOF
+if compile_prog "" "" ; then
+  fallocate_punch_hole=yes
+fi
+
 # check for sync_file_range
 sync_file_range=no
 cat > $TMPC << EOF
@@ -2633,7 +2737,7 @@ if compile_prog "" "" ; then
   byteswap_h=yes
 fi
 
-# Search for bswap_32 function
+# Search for bswap32 function
 bswap_h=no
 cat > $TMPC << EOF
 #include <sys/endian.h>
@@ -2668,17 +2772,44 @@ fi
 
 
 ##########################################
+# Do we need libm
+cat > $TMPC << EOF
+#include <math.h>
+int main(void) { return isnan(sin(0.0)); }
+EOF
+if compile_prog "" "" ; then
+  :
+elif compile_prog "" "-lm" ; then
+  LIBS="-lm $LIBS"
+  libs_qga="-lm $libs_qga"
+else
+  echo
+  echo "Error: libm check failed"
+  echo
+  exit 1
+fi
+
+##########################################
 # Do we need librt
+# uClibc provides 2 versions of clock_gettime(), one with realtime
+# support and one without. This means that the clock_gettime() don't
+# need -lrt. We still need it for timer_create() so we check for this
+# function in addition.
 cat > $TMPC <<EOF
 #include <signal.h>
 #include <time.h>
-int main(void) { return clock_gettime(CLOCK_REALTIME, NULL); }
+int main(void) {
+  timer_create(CLOCK_REALTIME, NULL, NULL);
+  return clock_gettime(CLOCK_REALTIME, NULL);
+}
 EOF
 
 if compile_prog "" "" ; then
   :
-elif compile_prog "" "-lrt" ; then
+# we need pthread for static linking. use previous pthread test result
+elif compile_prog "" "-lrt $pthread_lib" ; then
   LIBS="-lrt $LIBS"
+  libs_qga="-lrt $libs_qga"
 fi
 
 if test "$darwin" != "yes" -a "$mingw32" != "yes" -a "$solaris" != yes -a \
@@ -2695,12 +2826,14 @@ int main(void) { spice_server_new(); return 0; }
 EOF
   spice_cflags=$($pkg_config --cflags spice-protocol spice-server 2>/dev/null)
   spice_libs=$($pkg_config --libs spice-protocol spice-server 2>/dev/null)
-  if $pkg_config --atleast-version=0.8.2 spice-server >/dev/null 2>&1 && \
-     $pkg_config --atleast-version=0.8.1 spice-protocol > /dev/null 2>&1 && \
+  if $pkg_config --atleast-version=0.12.0 spice-server >/dev/null 2>&1 && \
+     $pkg_config --atleast-version=0.12.2 spice-protocol > /dev/null 2>&1 && \
      compile_prog "$spice_cflags" "$spice_libs" ; then
     spice="yes"
     libs_softmmu="$libs_softmmu $spice_libs"
     QEMU_CFLAGS="$QEMU_CFLAGS $spice_cflags"
+    spice_protocol_version=$($pkg_config --modversion spice-protocol)
+    spice_server_version=$($pkg_config --modversion spice-server)
   else
     if test "$spice" = "yes" ; then
       feature_not_found "spice"
@@ -2710,52 +2843,47 @@ EOF
 fi
 
 # check for libcacard for smartcard support
-if test "$smartcard" != "no" ; then
-    smartcard="yes"
-    smartcard_cflags=""
-    # TODO - what's the minimal nss version we support?
-    if test "$smartcard_nss" != "no"; then
-      cat > $TMPC << EOF
+smartcard_cflags=""
+# TODO - what's the minimal nss version we support?
+if test "$smartcard_nss" != "no"; then
+  cat > $TMPC << EOF
 #include <pk11pub.h>
 int main(void) { PK11_FreeSlot(0); return 0; }
 EOF
-        smartcard_includes="-I\$(SRC_PATH)/libcacard"
-        libcacard_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
-        libcacard_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
-        test_cflags="$libcacard_cflags"
-        # The header files in nss < 3.13.3 have a bug which causes them to
-        # emit a warning. If we're going to compile QEMU with -Werror, then
-        # test that the headers don't have this bug. Otherwise we would pass
-        # the configure test but fail to compile QEMU later.
-        if test "$werror" = "yes"; then
-            test_cflags="-Werror $test_cflags"
-        fi
-        if $pkg_config --atleast-version=3.12.8 nss >/dev/null 2>&1 && \
-          compile_prog "$test_cflags" "$libcacard_libs"; then
-            smartcard_nss="yes"
-            QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
-            QEMU_INCLUDES="$QEMU_INCLUDES $smartcard_includes"
-            libs_softmmu="$libcacard_libs $libs_softmmu"
-        else
-            if test "$smartcard_nss" = "yes"; then
-                feature_not_found "nss"
-            fi
-            smartcard_nss="no"
+    smartcard_includes="-I\$(SRC_PATH)/libcacard"
+    libcacard_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
+    libcacard_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
+    test_cflags="$libcacard_cflags"
+    # The header files in nss < 3.13.3 have a bug which causes them to
+    # emit a warning. If we're going to compile QEMU with -Werror, then
+    # test that the headers don't have this bug. Otherwise we would pass
+    # the configure test but fail to compile QEMU later.
+    if test "$werror" = "yes"; then
+        test_cflags="-Werror $test_cflags"
+    fi
+    if test -n "$libtool" &&
+            $pkg_config --atleast-version=3.12.8 nss >/dev/null 2>&1 && \
+      compile_prog "$test_cflags" "$libcacard_libs"; then
+        smartcard_nss="yes"
+        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
+        QEMU_INCLUDES="$QEMU_INCLUDES $smartcard_includes"
+        libs_softmmu="$libcacard_libs $libs_softmmu"
+    else
+        if test "$smartcard_nss" = "yes"; then
+            feature_not_found "nss"
         fi
+        smartcard_nss="no"
     fi
 fi
-if test "$smartcard" = "no" ; then
-    smartcard_nss="no"
-fi
 
 # check for usbredirparser for usb network redirection support
 if test "$usb_redir" != "no" ; then
-    if $pkg_config --atleast-version=0.3.4 libusbredirparser >/dev/null 2>&1 ; then
+    if $pkg_config --atleast-version=0.6 libusbredirparser-0.5 >/dev/null 2>&1 ; then
         usb_redir="yes"
-        usb_redir_cflags=$($pkg_config --cflags libusbredirparser 2>/dev/null)
-        usb_redir_libs=$($pkg_config --libs libusbredirparser 2>/dev/null)
+        usb_redir_cflags=$($pkg_config --cflags libusbredirparser-0.5 2>/dev/null)
+        usb_redir_libs=$($pkg_config --libs libusbredirparser-0.5 2>/dev/null)
         QEMU_CFLAGS="$QEMU_CFLAGS $usb_redir_cflags"
-        LIBS="$LIBS $usb_redir_libs"
+        libs_softmmu="$libs_softmmu $usb_redir_libs"
     else
         if test "$usb_redir" = "yes"; then
             feature_not_found "usb-redir"
@@ -2812,6 +2940,24 @@ if compile_prog "" "" ; then
 fi
 
 ##########################################
+# check if we have usable SIGEV_THREAD_ID
+
+sigev_thread_id=no
+cat > $TMPC << EOF
+#include <signal.h>
+int main(void) {
+  struct sigevent ev;
+  ev.sigev_notify = SIGEV_THREAD_ID;
+  ev._sigev_un._tid = 0;
+  asm volatile("" : : "g"(&ev));
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    sigev_thread_id=yes
+fi
+
+##########################################
 # check if trace backend exists
 
 $python "$source_path/scripts/tracetool.py" "--backend=$trace_backend" --check-backend  > /dev/null 2> /dev/null
@@ -2868,7 +3014,7 @@ static int sfaa(int *ptr)
   return __sync_fetch_and_and(ptr, 0);
 }
 
-int main(int argc, char **argv)
+int main(void)
 {
   int val = 42;
   sfaa(&val);
@@ -2899,8 +3045,6 @@ EOF
     else
 	coroutine_backend=gthread
     fi
-  else
-    echo "Silently falling back into gthread backend under darwin"
   fi
 elif test "$coroutine" = "gthread" ; then
   coroutine_backend=gthread
@@ -2959,15 +3103,16 @@ int main(void) {
 }
 EOF
 if compile_prog "-Werror" "" ; then
-    pragma_disable_unused_but_set=yes
+    pragma_diagnostic_available=yes
 fi
 
 ########################################
-# check if we have valgrind/valgrind.h
+# check if we have valgrind/valgrind.h and valgrind/memcheck.h
 
 valgrind_h=no
 cat > $TMPC << EOF
 #include <valgrind/valgrind.h>
+#include <valgrind/memcheck.h>
 int main(void) {
   return 0;
 }
@@ -2991,14 +3136,33 @@ if compile_prog "" "" ; then
     has_environ=yes
 fi
 
+########################################
+# check if cpuid.h is usable.
+
+cpuid_h=no
+cat > $TMPC << EOF
+#include <cpuid.h>
+int main(void) {
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    cpuid_h=yes
+fi
+
+
 ##########################################
 # End of CC checks
 # After here, no more $cc or $ld runs
 
-if test "$debug" = "no" ; then
+if test "$gcov" = "yes" ; then
+  CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
+  LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
+elif test "$debug" = "no" ; then
   CFLAGS="-O2 -D_FORTIFY_SOURCE=2 $CFLAGS"
 fi
 
+
 # Disable zero malloc errors for official releases unless explicitly told to
 # enable/disable
 if test -z "$zero_malloc" ; then
@@ -3039,9 +3203,14 @@ fi
 qemu_confdir=$sysconfdir$confsuffix
 qemu_datadir=$datadir$confsuffix
 
-tools=
-if test "$softmmu" = yes ; then
+tools=""
+if test "$want_tools" = "yes" ; then
   tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) $tools"
+  if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
+    tools="qemu-nbd\$(EXESUF) $tools"
+  fi
+fi
+if test "$softmmu" = yes ; then
   if test "$virtfs" != no ; then
     if test "$cap" = yes && test "$linux" = yes && test "$attr" = yes ; then
       virtfs=yes
@@ -3055,15 +3224,11 @@ if test "$softmmu" = yes ; then
     fi
   fi
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
-      tools="qemu-nbd\$(EXESUF) $tools"
     if [ "$guest_agent" = "yes" ]; then
       tools="qemu-ga\$(EXESUF) $tools"
     fi
   fi
 fi
-if test "$smartcard_nss" = "yes" ; then
-  tools="vscclient\$(EXESUF) $tools"
-fi
 
 # Mac OS X ships with a broken assembler
 roms=
@@ -3076,12 +3241,18 @@ if test "$cpu" = "ppc64" -a "$targetos" != "Darwin" ; then
   roms="$roms spapr-rtas"
 fi
 
+# add pixman flags after all config tests are done
+QEMU_CFLAGS="$QEMU_CFLAGS $pixman_cflags"
+libs_softmmu="$libs_softmmu $pixman_libs"
+
 echo "Install prefix    $prefix"
 echo "BIOS directory    `eval echo $qemu_datadir`"
 echo "binary directory  `eval echo $bindir`"
 echo "library directory `eval echo $libdir`"
+echo "libexec directory `eval echo $libexecdir`"
 echo "include directory `eval echo $includedir`"
 echo "config directory  `eval echo $sysconfdir`"
+echo "local state directory   `eval echo $local_statedir`"
 if test "$mingw32" = "no" ; then
 echo "Manual directory  `eval echo $mandir`"
 echo "ELF interp prefix $interp_prefix"
@@ -3112,6 +3283,7 @@ echo "-Werror enabled   $werror"
 if test "$darwin" = "yes" ; then
     echo "Cocoa support     $cocoa"
 fi
+echo "pixman            $pixman"
 echo "SDL support       $sdl"
 echo "curses support    $curses"
 echo "curl support      $curl"
@@ -3127,6 +3299,7 @@ if test "$vnc" = "yes" ; then
     echo "VNC SASL support  $vnc_sasl"
     echo "VNC JPEG support  $vnc_jpeg"
     echo "VNC PNG support   $vnc_png"
+    echo "VNC WS support    $vnc_ws"
 fi
 if test -n "$sparc_cpu"; then
     echo "Target Sparc Arch $sparc_cpu"
@@ -3151,12 +3324,13 @@ echo "preadv support    $preadv"
 echo "fdatasync         $fdatasync"
 echo "madvise           $madvise"
 echo "posix_madvise     $posix_madvise"
+echo "sigev_thread_id   $sigev_thread_id"
 echo "uuid support      $uuid"
 echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "Trace backend     $trace_backend"
 echo "Trace output file $trace_file-<pid>"
-echo "spice support     $spice"
+echo "spice support     $spice ($spice_protocol_version/$spice_server_version)"
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
 echo "nss used          $smartcard_nss"
@@ -3166,6 +3340,10 @@ echo "libiscsi support  $libiscsi"
 echo "build guest agent $guest_agent"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine_backend"
+echo "GlusterFS support $glusterfs"
+echo "virtio-blk-data-plane $virtio_blk_data_plane"
+echo "gcov              $gcov_tool"
+echo "gcov enabled      $gcov"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -3174,6 +3352,8 @@ fi
 config_host_mak="config-host.mak"
 config_host_ld="config-host.ld"
 
+echo "# Automatically generated by configure - do not modify" >config-all-disas.mak
+
 echo "# Automatically generated by configure - do not modify" > $config_host_mak
 printf "# Configured with:" >> $config_host_mak
 printf " '%s'" "$0" "$@" >> $config_host_mak
@@ -3183,14 +3363,17 @@ echo all: >> $config_host_mak
 echo "prefix=$prefix" >> $config_host_mak
 echo "bindir=$bindir" >> $config_host_mak
 echo "libdir=$libdir" >> $config_host_mak
+echo "libexecdir=$libexecdir" >> $config_host_mak
 echo "includedir=$includedir" >> $config_host_mak
 echo "mandir=$mandir" >> $config_host_mak
 echo "sysconfdir=$sysconfdir" >> $config_host_mak
 echo "qemu_confdir=$qemu_confdir" >> $config_host_mak
 echo "qemu_datadir=$qemu_datadir" >> $config_host_mak
 echo "qemu_docdir=$qemu_docdir" >> $config_host_mak
-echo "libexecdir=\${prefix}/libexec" >> $config_host_mak
-echo "CONFIG_QEMU_HELPERDIR=\"$prefix/libexec\"" >> $config_host_mak
+echo "qemu_localstatedir=$local_statedir" >> $config_host_mak
+echo "qemu_helperdir=$libexecdir" >> $config_host_mak
+echo "extra_cflags=$EXTRA_CFLAGS" >> $config_host_mak
+echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
 if test "$debug_tcg" = "yes" ; then
@@ -3251,7 +3434,6 @@ fi
 if test "$slirp" = "yes" ; then
   echo "CONFIG_SLIRP=y" >> $config_host_mak
   echo "CONFIG_SMBD_COMMAND=\"$smbd\"" >> $config_host_mak
-  QEMU_INCLUDES="-I\$(SRC_PATH)/slirp $QEMU_INCLUDES"
 fi
 if test "$vde" = "yes" ; then
   echo "CONFIG_VDE=y" >> $config_host_mak
@@ -3286,19 +3468,19 @@ if test "$vnc" = "yes" ; then
 fi
 if test "$vnc_tls" = "yes" ; then
   echo "CONFIG_VNC_TLS=y" >> $config_host_mak
-  echo "VNC_TLS_CFLAGS=$vnc_tls_cflags" >> $config_host_mak
 fi
 if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
-  echo "VNC_SASL_CFLAGS=$vnc_sasl_cflags" >> $config_host_mak
 fi
 if test "$vnc_jpeg" = "yes" ; then
   echo "CONFIG_VNC_JPEG=y" >> $config_host_mak
-  echo "VNC_JPEG_CFLAGS=$vnc_jpeg_cflags" >> $config_host_mak
 fi
 if test "$vnc_png" = "yes" ; then
   echo "CONFIG_VNC_PNG=y" >> $config_host_mak
-  echo "VNC_PNG_CFLAGS=$vnc_png_cflags" >> $config_host_mak
+fi
+if test "$vnc_ws" = "yes" ; then
+  echo "CONFIG_VNC_WS=y" >> $config_host_mak
+  echo "VNC_WS_CFLAGS=$vnc_ws_cflags" >> $config_host_mak
 fi
 if test "$fnmatch" = "yes" ; then
   echo "CONFIG_FNMATCH=y" >> $config_host_mak
@@ -3348,6 +3530,9 @@ fi
 if test "$fallocate" = "yes" ; then
   echo "CONFIG_FALLOCATE=y" >> $config_host_mak
 fi
+if test "$fallocate_punch_hole" = "yes" ; then
+  echo "CONFIG_FALLOCATE_PUNCH_HOLE=y" >> $config_host_mak
+fi
 if test "$sync_file_range" = "yes" ; then
   echo "CONFIG_SYNC_FILE_RANGE=y" >> $config_host_mak
 fi
@@ -3433,15 +3618,14 @@ fi
 if test "$posix_madvise" = "yes" ; then
   echo "CONFIG_POSIX_MADVISE=y" >> $config_host_mak
 fi
+if test "$sigev_thread_id" = "yes" ; then
+  echo "CONFIG_SIGEV_THREAD_ID=y" >> $config_host_mak
+fi
 
 if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
 fi
 
-if test "$smartcard" = "yes" ; then
-  echo "CONFIG_SMARTCARD=y" >> $config_host_mak
-fi
-
 if test "$smartcard_nss" = "yes" ; then
   echo "CONFIG_SMARTCARD_NSS=y" >> $config_host_mak
   echo "libcacard_libs=$libcacard_libs" >> $config_host_mak
@@ -3492,8 +3676,8 @@ if test "$linux_magic_h" = "yes" ; then
   echo "CONFIG_LINUX_MAGIC_H=y" >> $config_host_mak
 fi
 
-if test "$pragma_disable_unused_but_set" = "yes" ; then
-  echo "CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET=y" >> $config_host_mak
+if test "$pragma_diagnostic_available" = "yes" ; then
+  echo "CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE=y" >> $config_host_mak
 fi
 
 if test "$valgrind_h" = "yes" ; then
@@ -3504,6 +3688,18 @@ if test "$has_environ" = "yes" ; then
   echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak
 fi
 
+if test "$cpuid_h" = "yes" ; then
+  echo "CONFIG_CPUID_H=y" >> $config_host_mak
+fi
+
+if test "$glusterfs" = "yes" ; then
+  echo "CONFIG_GLUSTERFS=y" >> $config_host_mak
+fi
+
+if test "$virtio_blk_data_plane" = "yes" ; then
+  echo "CONFIG_VIRTIO_BLK_DATA_PLANE=y" >> $config_host_mak
+fi
+
 # USB host support
 case "$usb" in
 linux)
@@ -3553,13 +3749,21 @@ echo "MAKE=$make" >> $config_host_mak
 echo "INSTALL=$install" >> $config_host_mak
 echo "INSTALL_DIR=$install -d -m 0755" >> $config_host_mak
 echo "INSTALL_DATA=$install -c -m 0644" >> $config_host_mak
-echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+if test -n "$libtool"; then
+  echo "INSTALL_PROG=\$(LIBTOOL) --mode=install $install -c -m 0755" >> $config_host_mak
+  echo "INSTALL_LIB=\$(LIBTOOL) --mode=install $install -c -m 0644" >> $config_host_mak
+else
+  echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+  echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
+fi
 echo "PYTHON=$python" >> $config_host_mak
 echo "CC=$cc" >> $config_host_mak
 echo "CC_I386=$cc_i386" >> $config_host_mak
 echo "HOST_CC=$host_cc" >> $config_host_mak
 echo "OBJCC=$objcc" >> $config_host_mak
 echo "AR=$ar" >> $config_host_mak
+echo "AS=$as" >> $config_host_mak
+echo "CPP=$cpp" >> $config_host_mak
 echo "OBJCOPY=$objcopy" >> $config_host_mak
 echo "LD=$ld" >> $config_host_mak
 echo "WINDRES=$windres" >> $config_host_mak
@@ -3572,7 +3776,11 @@ if test "$sparse" = "yes" ; then
   echo "HOST_CC      := REAL_CC=\"\$(HOST_CC)\" cgcc"  >> $config_host_mak
   echo "QEMU_CFLAGS  += -Wbitwise -Wno-transparent-union -Wno-old-initializer -Wno-non-pointer-null" >> $config_host_mak
 fi
-echo "HELPER_CFLAGS=$helper_cflags" >> $config_host_mak
+if test "$cross_prefix" != ""; then
+  echo "AUTOCONF_HOST := --host=${cross_prefix%-}"     >> $config_host_mak
+else
+  echo "AUTOCONF_HOST := "                             >> $config_host_mak
+fi
 echo "LDFLAGS=$LDFLAGS" >> $config_host_mak
 echo "ARLIBS_BEGIN=$arlibs_begin" >> $config_host_mak
 echo "ARLIBS_END=$arlibs_end" >> $config_host_mak
@@ -3581,6 +3789,11 @@ echo "LIBS_TOOLS+=$libs_tools" >> $config_host_mak
 echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
+echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
+if test "$gcov" = "yes" ; then
+  echo "CONFIG_GCOV=y" >> $config_host_mak
+  echo "GCOV=$gcov_tool" >> $config_host_mak
+fi
 
 # generate list of library paths for linker script
 
@@ -3594,11 +3807,6 @@ if test -f ${config_host_ld}~ ; then
   fi
 fi
 
-for d in libdis libdis-user; do
-    symlink "$source_path/Makefile.dis" "$d/Makefile"
-    echo > $d/config.mak
-done
-
 # use included Linux headers
 if test "$linux" = "yes" ; then
   mkdir -p linux-headers
@@ -3683,15 +3891,12 @@ TARGET_ABI_DIR=""
 
 case "$target_arch2" in
   i386)
-    target_phys_bits=64
   ;;
   x86_64)
     TARGET_BASE_ARCH=i386
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   alpha)
-    target_phys_bits=64
     target_long_alignment=8
     target_nptl="yes"
   ;;
@@ -3700,22 +3905,18 @@ case "$target_arch2" in
     bflt="yes"
     target_nptl="yes"
     gdb_xml_files="arm-core.xml arm-vfp.xml arm-vfp3.xml arm-neon.xml"
-    target_phys_bits=64
     target_llong_alignment=4
     target_libs_softmmu="$fdt_libs"
   ;;
   cris)
     target_nptl="yes"
-    target_phys_bits=32
   ;;
   lm32)
-    target_phys_bits=32
     target_libs_softmmu="$opengl_libs"
   ;;
   m68k)
     bflt="yes"
     gdb_xml_files="cf-core.xml cf-fp.xml"
-    target_phys_bits=32
     target_int_alignment=2
     target_long_alignment=2
     target_llong_alignment=2
@@ -3724,36 +3925,30 @@ case "$target_arch2" in
     TARGET_ARCH=microblaze
     bflt="yes"
     target_nptl="yes"
-    target_phys_bits=32
     target_libs_softmmu="$fdt_libs"
   ;;
   mips|mipsel)
     TARGET_ARCH=mips
     echo "TARGET_ABI_MIPSO32=y" >> $config_target_mak
     target_nptl="yes"
-    target_phys_bits=64
   ;;
   mipsn32|mipsn32el)
     TARGET_ARCH=mipsn32
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN32=y" >> $config_target_mak
-    target_phys_bits=64
   ;;
   mips64|mips64el)
     TARGET_ARCH=mips64
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN64=y" >> $config_target_mak
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   or32)
     TARGET_ARCH=openrisc
     TARGET_BASE_ARCH=openrisc
-    target_phys_bits=32
   ;;
   ppc)
     gdb_xml_files="power-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_nptl="yes"
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3761,7 +3956,6 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     gdb_xml_files="power-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_nptl="yes"
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3769,7 +3963,6 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_long_alignment=8
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3779,21 +3972,17 @@ case "$target_arch2" in
     TARGET_ABI_DIR=ppc
     echo "TARGET_ABI32=y" >> $config_target_mak
     gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_libs_softmmu="$fdt_libs"
   ;;
   sh4|sh4eb)
     TARGET_ARCH=sh4
     bflt="yes"
     target_nptl="yes"
-    target_phys_bits=32
   ;;
   sparc)
-    target_phys_bits=64
   ;;
   sparc64)
     TARGET_BASE_ARCH=sparc
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   sparc32plus)
@@ -3801,19 +3990,15 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=sparc
     TARGET_ABI_DIR=sparc
     echo "TARGET_ABI32=y" >> $config_target_mak
-    target_phys_bits=64
   ;;
   s390x)
     target_nptl="yes"
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   unicore32)
-    target_phys_bits=32
   ;;
   xtensa|xtensaeb)
     TARGET_ARCH=xtensa
-    target_phys_bits=32
   ;;
   *)
     echo "Unsupported target CPU"
@@ -3827,17 +4012,19 @@ fi
 
 symlink "$source_path/Makefile.target" "$target_dir/Makefile"
 
-
-case "$target_arch2" in
-  alpha | i386 | or32 | sparc* | x86_64 | xtensa* | ppc*)
-    echo "CONFIG_TCG_PASS_AREG0=y" >> $config_target_mak
-  ;;
-esac
-
 upper() {
     echo "$@"| LC_ALL=C tr '[a-z]' '[A-Z]'
 }
 
+case "$cpu" in
+  i386|x86_64|ppc)
+    # The TCG interpreter currently does not support ld/st optimization.
+    if test "$tcg_interpreter" = "no" ; then
+        echo "CONFIG_QEMU_LDST_OPTIMIZATION=y" >> $config_target_mak
+    fi
+  ;;
+esac
+
 echo "TARGET_SHORT_ALIGNMENT=$target_short_alignment" >> $config_target_mak
 echo "TARGET_INT_ALIGNMENT=$target_int_alignment" >> $config_target_mak
 echo "TARGET_LONG_ALIGNMENT=$target_long_alignment" >> $config_target_mak
@@ -3855,7 +4042,6 @@ echo "TARGET_ABI_DIR=$TARGET_ABI_DIR" >> $config_target_mak
 case "$target_arch2" in
   i386|x86_64)
     if test "$xen" = "yes" -a "$target_softmmu" = "yes" ; then
-      target_phys_bits=64
       echo "CONFIG_XEN=y" >> $config_target_mak
       if test "$xen_pci_passthrough" = yes; then
         echo "CONFIG_XEN_PCI_PASSTHROUGH=y" >> "$config_target_mak"
@@ -3895,14 +4081,8 @@ if test "$target_bigendian" = "yes" ; then
   echo "TARGET_WORDS_BIGENDIAN=y" >> $config_target_mak
 fi
 if test "$target_softmmu" = "yes" ; then
-  echo "TARGET_PHYS_ADDR_BITS=$target_phys_bits" >> $config_target_mak
   echo "CONFIG_SOFTMMU=y" >> $config_target_mak
   echo "LIBS+=$libs_softmmu $target_libs_softmmu" >> $config_target_mak
-  echo "HWDIR=../libhw$target_phys_bits" >> $config_target_mak
-  echo "subdir-$target: subdir-libhw$target_phys_bits" >> $config_host_mak
-  if test "$smartcard_nss" = "yes" ; then
-    echo "subdir-$target: subdir-libcacard" >> $config_host_mak
-  fi
   case "$target_arch2" in
     i386|x86_64)
       echo "CONFIG_HAVE_CORE_DUMP=y" >> $config_target_mak
@@ -3965,83 +4145,77 @@ if test "$linux" = "yes" ; then
   includes="-I\$(SRC_PATH)/linux-headers $includes"
 fi
 
-if test "$target_user_only" = "yes" ; then
-    libdis_config_mak=libdis-user/config.mak
-else
-    libdis_config_mak=libdis/config.mak
-fi
-
 for i in $ARCH $TARGET_BASE_ARCH ; do
   case "$i" in
   alpha)
     echo "CONFIG_ALPHA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ALPHA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_ALPHA_DIS=y"  >> config-all-disas.mak
   ;;
   arm)
     echo "CONFIG_ARM_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ARM_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_ARM_DIS=y"  >> config-all-disas.mak
   ;;
   cris)
     echo "CONFIG_CRIS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_CRIS_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_CRIS_DIS=y"  >> config-all-disas.mak
   ;;
   hppa)
     echo "CONFIG_HPPA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_HPPA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_HPPA_DIS=y"  >> config-all-disas.mak
   ;;
   i386|x86_64)
     echo "CONFIG_I386_DIS=y"  >> $config_target_mak
-    echo "CONFIG_I386_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_I386_DIS=y"  >> config-all-disas.mak
   ;;
   ia64*)
     echo "CONFIG_IA64_DIS=y"  >> $config_target_mak
-    echo "CONFIG_IA64_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_IA64_DIS=y"  >> config-all-disas.mak
   ;;
   lm32)
     echo "CONFIG_LM32_DIS=y"  >> $config_target_mak
-    echo "CONFIG_LM32_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_LM32_DIS=y"  >> config-all-disas.mak
   ;;
   m68k)
     echo "CONFIG_M68K_DIS=y"  >> $config_target_mak
-    echo "CONFIG_M68K_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_M68K_DIS=y"  >> config-all-disas.mak
   ;;
   microblaze*)
     echo "CONFIG_MICROBLAZE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_MICROBLAZE_DIS=y"  >> config-all-disas.mak
   ;;
   mips*)
     echo "CONFIG_MIPS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MIPS_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_MIPS_DIS=y"  >> config-all-disas.mak
   ;;
   or32)
     echo "CONFIG_OPENRISC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_OPENRISC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_OPENRISC_DIS=y"  >> config-all-disas.mak
   ;;
   ppc*)
     echo "CONFIG_PPC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_PPC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_PPC_DIS=y"  >> config-all-disas.mak
   ;;
   s390*)
     echo "CONFIG_S390_DIS=y"  >> $config_target_mak
-    echo "CONFIG_S390_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_S390_DIS=y"  >> config-all-disas.mak
   ;;
   sh4)
     echo "CONFIG_SH4_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SH4_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_SH4_DIS=y"  >> config-all-disas.mak
   ;;
   sparc*)
     echo "CONFIG_SPARC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SPARC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_SPARC_DIS=y"  >> config-all-disas.mak
   ;;
   xtensa*)
     echo "CONFIG_XTENSA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_XTENSA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_XTENSA_DIS=y"  >> config-all-disas.mak
   ;;
   esac
 done
 if test "$tcg_interpreter" = "yes" ; then
   echo "CONFIG_TCI_DIS=y"  >> $config_target_mak
-  echo "CONFIG_TCI_DIS=y"  >> $libdis_config_mak
+  echo "CONFIG_TCI_DIS=y"  >> config-all-disas.mak
 fi
 
 case "$ARCH" in
@@ -4080,15 +4254,11 @@ fi
 if test "$ARCH" = "tci"; then
   linker_script=""
 else
-  linker_script="-Wl,-T../config-host.ld -Wl,-T,\$(SRC_PATH)/\$(ARCH).ld"
+  linker_script="-Wl,-T../config-host.ld -Wl,-T,\$(SRC_PATH)/ldscripts/\$(ARCH).ld"
 fi
 
 if test "$target_linux_user" = "yes" -o "$target_bsd_user" = "yes" ; then
   case "$ARCH" in
-  sparc)
-    # -static is used to avoid g1/g3 usage by the dynamic linker
-    ldflags="$linker_script -static $ldflags"
-    ;;
   alpha | s390x)
     # The default placement of the application is fine.
     ;;
@@ -4104,20 +4274,24 @@ echo "QEMU_INCLUDES+=$includes" >> $config_target_mak
 
 done # for target in $targets
 
+if [ "$pixman" = "internal" ]; then
+  echo "config-host.h: subdir-pixman" >> $config_host_mak
+fi
+
 # build tree in object directory in case the source is not in the current directory
 DIRS="tests tests/tcg tests/tcg/cris tests/tcg/lm32"
 DIRS="$DIRS pc-bios/optionrom pc-bios/spapr-rtas"
 DIRS="$DIRS roms/seabios roms/vgabios"
 DIRS="$DIRS qapi-generated"
-DIRS="$DIRS libcacard libcacard/libcacard libcacard/trace"
 FILES="Makefile tests/tcg/Makefile qdict-test-data.txt"
 FILES="$FILES tests/tcg/cris/Makefile tests/tcg/cris/.gdbinit"
-FILES="$FILES tests/tcg/lm32/Makefile libcacard/Makefile"
+FILES="$FILES tests/tcg/lm32/Makefile"
 FILES="$FILES pc-bios/optionrom/Makefile pc-bios/keymaps"
 FILES="$FILES pc-bios/spapr-rtas/Makefile"
 FILES="$FILES roms/seabios/Makefile roms/vgabios/Makefile"
 for bios_file in \
     $source_path/pc-bios/*.bin \
+    $source_path/pc-bios/*.aml \
     $source_path/pc-bios/*.rom \
     $source_path/pc-bios/*.dtb \
     $source_path/pc-bios/openbios-* \
@@ -4137,23 +4311,15 @@ for rom in seabios vgabios ; do
     config_mak=roms/$rom/config.mak
     echo "# Automatically generated by configure - do not modify" > $config_mak
     echo "SRC_PATH=$source_path/roms/$rom" >> $config_mak
+    echo "AS=$as" >> $config_mak
     echo "CC=$cc" >> $config_mak
     echo "BCC=bcc" >> $config_mak
-    echo "CPP=${cross_prefix}cpp" >> $config_mak
+    echo "CPP=$cpp" >> $config_mak
     echo "OBJCOPY=objcopy" >> $config_mak
     echo "IASL=iasl" >> $config_mak
     echo "LD=$ld" >> $config_mak
 done
 
-for hwlib in 32 64; do
-  d=libhw$hwlib
-  symlink "$source_path/Makefile.hw" "$d/Makefile"
-  echo "QEMU_CFLAGS+=-DTARGET_PHYS_ADDR_BITS=$hwlib" > $d/config.mak
-done
-
-d=libuser
-symlink "$source_path/Makefile.user" "$d/Makefile"
-
 if test "$docs" = "yes" ; then
   mkdir -p QMP
 fi
diff --git a/coroutine-gthread.c b/coroutine-gthread.c
index 30c24c94b87..d3e5b991f77 100644
--- a/coroutine-gthread.c
+++ b/coroutine-gthread.c
@@ -20,7 +20,7 @@
 
 #include <glib.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/coroutine-sigaltstack.c b/coroutine-sigaltstack.c
index 861e87805af..e37ebac9c44 100644
--- a/coroutine-sigaltstack.c
+++ b/coroutine-sigaltstack.c
@@ -31,7 +31,7 @@
 #include <pthread.h>
 #include <signal.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 enum {
     /* Maximum free pool size prevents holding too many freed coroutines */
@@ -171,8 +171,8 @@ static Coroutine *coroutine_new(void)
     CoroutineThreadState *coTS;
     struct sigaction sa;
     struct sigaction osa;
-    struct sigaltstack ss;
-    struct sigaltstack oss;
+    stack_t ss;
+    stack_t oss;
     sigset_t sigs;
     sigset_t osigs;
     jmp_buf old_env;
diff --git a/coroutine-ucontext.c b/coroutine-ucontext.c
index 784081ab18a..a9c30e9df41 100644
--- a/coroutine-ucontext.c
+++ b/coroutine-ucontext.c
@@ -28,7 +28,7 @@
 #include <pthread.h>
 #include <ucontext.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 #ifdef CONFIG_VALGRIND_H
 #include <valgrind/valgrind.h>
@@ -200,7 +200,7 @@ Coroutine *qemu_coroutine_new(void)
 }
 
 #ifdef CONFIG_VALGRIND_H
-#ifdef CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 /* Work around an unused variable in the valgrind.h macro... */
 #pragma GCC diagnostic ignored "-Wunused-but-set-variable"
 #endif
@@ -208,7 +208,7 @@ static inline void valgrind_stack_deregister(CoroutineUContext *co)
 {
     VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
 }
-#ifdef CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 #pragma GCC diagnostic error "-Wunused-but-set-variable"
 #endif
 #endif
diff --git a/coroutine-win32.c b/coroutine-win32.c
index 4179609eecd..edc1f72c182 100644
--- a/coroutine-win32.c
+++ b/coroutine-win32.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 typedef struct
 {
diff --git a/cpu-common.h b/cpu-common.h
deleted file mode 100644
index 85548de5ea9..00000000000
--- a/cpu-common.h
+++ /dev/null
@@ -1,117 +0,0 @@
-#ifndef CPU_COMMON_H
-#define CPU_COMMON_H 1
-
-/* CPU interfaces that are target independent.  */
-
-#include "targphys.h"
-
-#ifndef NEED_CPU_H
-#include "poison.h"
-#endif
-
-#include "bswap.h"
-#include "qemu-queue.h"
-
-#if !defined(CONFIG_USER_ONLY)
-
-enum device_endian {
-    DEVICE_NATIVE_ENDIAN,
-    DEVICE_BIG_ENDIAN,
-    DEVICE_LITTLE_ENDIAN,
-};
-
-/* address in the RAM (different from a physical address) */
-#if defined(CONFIG_XEN_BACKEND) && TARGET_PHYS_ADDR_BITS == 64
-typedef uint64_t ram_addr_t;
-#  define RAM_ADDR_MAX UINT64_MAX
-#  define RAM_ADDR_FMT "%" PRIx64
-#else
-typedef uintptr_t ram_addr_t;
-#  define RAM_ADDR_MAX UINTPTR_MAX
-#  define RAM_ADDR_FMT "%" PRIxPTR
-#endif
-
-/* memory API */
-
-typedef void CPUWriteMemoryFunc(void *opaque, target_phys_addr_t addr, uint32_t value);
-typedef uint32_t CPUReadMemoryFunc(void *opaque, target_phys_addr_t addr);
-
-void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
-/* This should only be used for ram local to a device.  */
-void *qemu_get_ram_ptr(ram_addr_t addr);
-void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size);
-/* Same but slower, to use for migration, where the order of
- * RAMBlocks must not change. */
-void *qemu_safe_ram_ptr(ram_addr_t addr);
-void qemu_put_ram_ptr(void *addr);
-/* This should not be used by devices.  */
-int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
-ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr);
-void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
-
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
-                            int len, int is_write);
-static inline void cpu_physical_memory_read(target_phys_addr_t addr,
-                                            void *buf, int len)
-{
-    cpu_physical_memory_rw(addr, buf, len, 0);
-}
-static inline void cpu_physical_memory_write(target_phys_addr_t addr,
-                                             const void *buf, int len)
-{
-    cpu_physical_memory_rw(addr, (void *)buf, len, 1);
-}
-void *cpu_physical_memory_map(target_phys_addr_t addr,
-                              target_phys_addr_t *plen,
-                              int is_write);
-void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
-                               int is_write, target_phys_addr_t access_len);
-void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
-void cpu_unregister_map_client(void *cookie);
-
-bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr);
-
-/* Coalesced MMIO regions are areas where write operations can be reordered.
- * This usually implies that write operations are side-effect free.  This allows
- * batching which can make a major impact on performance when using
- * virtualization.
- */
-void qemu_flush_coalesced_mmio_buffer(void);
-
-uint32_t ldub_phys(target_phys_addr_t addr);
-uint32_t lduw_le_phys(target_phys_addr_t addr);
-uint32_t lduw_be_phys(target_phys_addr_t addr);
-uint32_t ldl_le_phys(target_phys_addr_t addr);
-uint32_t ldl_be_phys(target_phys_addr_t addr);
-uint64_t ldq_le_phys(target_phys_addr_t addr);
-uint64_t ldq_be_phys(target_phys_addr_t addr);
-void stb_phys(target_phys_addr_t addr, uint32_t val);
-void stw_le_phys(target_phys_addr_t addr, uint32_t val);
-void stw_be_phys(target_phys_addr_t addr, uint32_t val);
-void stl_le_phys(target_phys_addr_t addr, uint32_t val);
-void stl_be_phys(target_phys_addr_t addr, uint32_t val);
-void stq_le_phys(target_phys_addr_t addr, uint64_t val);
-void stq_be_phys(target_phys_addr_t addr, uint64_t val);
-
-#ifdef NEED_CPU_H
-uint32_t lduw_phys(target_phys_addr_t addr);
-uint32_t ldl_phys(target_phys_addr_t addr);
-uint64_t ldq_phys(target_phys_addr_t addr);
-void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val);
-void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val);
-void stw_phys(target_phys_addr_t addr, uint32_t val);
-void stl_phys(target_phys_addr_t addr, uint32_t val);
-void stq_phys(target_phys_addr_t addr, uint64_t val);
-#endif
-
-void cpu_physical_memory_write_rom(target_phys_addr_t addr,
-                                   const uint8_t *buf, int len);
-
-extern struct MemoryRegion io_mem_ram;
-extern struct MemoryRegion io_mem_rom;
-extern struct MemoryRegion io_mem_unassigned;
-extern struct MemoryRegion io_mem_notdirty;
-
-#endif
-
-#endif /* !CPU_COMMON_H */
diff --git a/cpu-exec.c b/cpu-exec.c
index 134b3c4fcfb..19ebb4a924c 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -18,18 +18,18 @@
  */
 #include "config.h"
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
-#include "qemu-barrier.h"
-#include "qtest.h"
+#include "qemu/atomic.h"
+#include "sysemu/qtest.h"
 
 int tb_invalidated_flag;
 
 //#define CONFIG_DEBUG_EXEC
 
-bool qemu_cpu_has_work(CPUArchState *env)
+bool qemu_cpu_has_work(CPUState *cpu)
 {
-    return cpu_has_work(env);
+    return cpu_has_work(cpu);
 }
 
 void cpu_loop_exit(CPUArchState *env)
@@ -181,16 +181,14 @@ volatile sig_atomic_t exit_request;
 
 int cpu_exec(CPUArchState *env)
 {
-#ifdef TARGET_PPC
     CPUState *cpu = ENV_GET_CPU(env);
-#endif
     int ret, interrupt_request;
     TranslationBlock *tb;
     uint8_t *tc_ptr;
     tcg_target_ulong next_tb;
 
     if (env->halted) {
-        if (!cpu_has_work(env)) {
+        if (!cpu_has_work(cpu)) {
             return EXCP_HALTED;
         }
 
@@ -552,7 +550,7 @@ int cpu_exec(CPUArchState *env)
 #if defined(TARGET_I386)
                     env->eflags = env->eflags | cpu_cc_compute_all(env, CC_OP)
                         | (DF & DF_MASK);
-                    log_cpu_state(env, X86_DUMP_CCOP);
+                    log_cpu_state(env, CPU_DUMP_CCOP);
                     env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
 #elif defined(TARGET_M68K)
                     cpu_m68k_flush_flags(env, env->cc_op);
diff --git a/cpus.c b/cpus.c
index e476a3cd5e0..41779eb02e7 100644
--- a/cpus.c
+++ b/cpus.c
@@ -25,21 +25,21 @@
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
 
-#include "monitor.h"
-#include "sysemu.h"
-#include "gdbstub.h"
-#include "dma.h"
-#include "kvm.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
+#include "sysemu/dma.h"
+#include "sysemu/kvm.h"
 #include "qmp-commands.h"
 
-#include "qemu-thread.h"
-#include "cpus.h"
-#include "qtest.h"
-#include "main-loop.h"
-#include "bitmap.h"
+#include "qemu/thread.h"
+#include "sysemu/cpus.h"
+#include "sysemu/qtest.h"
+#include "qemu/main-loop.h"
+#include "qemu/bitmap.h"
 
 #ifndef _WIN32
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 #endif
 
 #ifdef CONFIG_LINUX
@@ -64,13 +64,15 @@ static CPUArchState *next_cpu;
 
 static bool cpu_thread_is_idle(CPUArchState *env)
 {
-    if (env->stop || env->queued_work_first) {
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    if (cpu->stop || cpu->queued_work_first) {
         return false;
     }
-    if (env->stopped || !runstate_is_running()) {
+    if (cpu->stopped || !runstate_is_running()) {
         return true;
     }
-    if (!env->halted || qemu_cpu_has_work(env) ||
+    if (!env->halted || qemu_cpu_has_work(cpu) ||
         kvm_async_interrupts_enabled()) {
         return false;
     }
@@ -388,18 +390,16 @@ void hw_error(const char *fmt, ...)
 {
     va_list ap;
     CPUArchState *env;
+    CPUState *cpu;
 
     va_start(ap, fmt);
     fprintf(stderr, "qemu: hardware error: ");
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        fprintf(stderr, "CPU #%d:\n", env->cpu_index);
-#ifdef TARGET_I386
-        cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
-#else
-        cpu_dump_state(env, stderr, fprintf, 0);
-#endif
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
+        fprintf(stderr, "CPU #%d:\n", cpu->cpu_index);
+        cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU);
     }
     va_end(ap);
     abort();
@@ -432,9 +432,9 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
-int cpu_is_stopped(CPUArchState *env)
+bool cpu_is_stopped(CPUState *cpu)
 {
-    return !runstate_is_running() || env->stopped;
+    return !runstate_is_running() || cpu->stopped;
 }
 
 static void do_vm_stop(RunState state)
@@ -450,22 +450,24 @@ static void do_vm_stop(RunState state)
     }
 }
 
-static int cpu_can_run(CPUArchState *env)
+static bool cpu_can_run(CPUState *cpu)
 {
-    if (env->stop) {
-        return 0;
+    if (cpu->stop) {
+        return false;
     }
-    if (env->stopped || !runstate_is_running()) {
-        return 0;
+    if (cpu->stopped || !runstate_is_running()) {
+        return false;
     }
-    return 1;
+    return true;
 }
 
 static void cpu_handle_guest_debug(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     gdb_set_stop_cpu(env);
     qemu_system_debug_request();
-    env->stopped = 1;
+    cpu->stopped = true;
 }
 
 static void cpu_signal(int sig)
@@ -515,7 +517,7 @@ static void qemu_init_sigbus(void)
     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 }
 
-static void qemu_kvm_eat_signals(CPUArchState *env)
+static void qemu_kvm_eat_signals(CPUState *cpu)
 {
     struct timespec ts = { 0, 0 };
     siginfo_t siginfo;
@@ -536,7 +538,7 @@ static void qemu_kvm_eat_signals(CPUArchState *env)
 
         switch (r) {
         case SIGBUS:
-            if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
+            if (kvm_on_sigbus_vcpu(cpu, siginfo.si_code, siginfo.si_addr)) {
                 sigbus_reraise();
             }
             break;
@@ -558,7 +560,7 @@ static void qemu_init_sigbus(void)
 {
 }
 
-static void qemu_kvm_eat_signals(CPUArchState *env)
+static void qemu_kvm_eat_signals(CPUState *cpu)
 {
 }
 #endif /* !CONFIG_LINUX */
@@ -613,7 +615,7 @@ static void qemu_tcg_init_cpu_signals(void)
 }
 #endif /* _WIN32 */
 
-QemuMutex qemu_global_mutex;
+static QemuMutex qemu_global_mutex;
 static QemuCond qemu_io_proceeded_cond;
 static bool iothread_requesting_mutex;
 
@@ -640,27 +642,27 @@ void qemu_init_cpu_loop(void)
     qemu_thread_get_self(&io_thread);
 }
 
-void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data)
+void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
 {
     struct qemu_work_item wi;
 
-    if (qemu_cpu_is_self(env)) {
+    if (qemu_cpu_is_self(cpu)) {
         func(data);
         return;
     }
 
     wi.func = func;
     wi.data = data;
-    if (!env->queued_work_first) {
-        env->queued_work_first = &wi;
+    if (cpu->queued_work_first == NULL) {
+        cpu->queued_work_first = &wi;
     } else {
-        env->queued_work_last->next = &wi;
+        cpu->queued_work_last->next = &wi;
     }
-    env->queued_work_last = &wi;
+    cpu->queued_work_last = &wi;
     wi.next = NULL;
     wi.done = false;
 
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(cpu);
     while (!wi.done) {
         CPUArchState *self_env = cpu_single_env;
 
@@ -669,33 +671,31 @@ void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data)
     }
 }
 
-static void flush_queued_work(CPUArchState *env)
+static void flush_queued_work(CPUState *cpu)
 {
     struct qemu_work_item *wi;
 
-    if (!env->queued_work_first) {
+    if (cpu->queued_work_first == NULL) {
         return;
     }
 
-    while ((wi = env->queued_work_first)) {
-        env->queued_work_first = wi->next;
+    while ((wi = cpu->queued_work_first)) {
+        cpu->queued_work_first = wi->next;
         wi->func(wi->data);
         wi->done = true;
     }
-    env->queued_work_last = NULL;
+    cpu->queued_work_last = NULL;
     qemu_cond_broadcast(&qemu_work_cond);
 }
 
-static void qemu_wait_io_event_common(CPUArchState *env)
+static void qemu_wait_io_event_common(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
-
-    if (env->stop) {
-        env->stop = 0;
-        env->stopped = 1;
+    if (cpu->stop) {
+        cpu->stop = false;
+        cpu->stopped = true;
         qemu_cond_signal(&qemu_pause_cond);
     }
-    flush_queued_work(env);
+    flush_queued_work(cpu);
     cpu->thread_kicked = false;
 }
 
@@ -715,18 +715,20 @@ static void qemu_tcg_wait_io_event(void)
     }
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        qemu_wait_io_event_common(env);
+        qemu_wait_io_event_common(ENV_GET_CPU(env));
     }
 }
 
 static void qemu_kvm_wait_io_event(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     while (cpu_thread_is_idle(env)) {
-        qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
-    qemu_kvm_eat_signals(env);
-    qemu_wait_io_event_common(env);
+    qemu_kvm_eat_signals(cpu);
+    qemu_wait_io_event_common(cpu);
 }
 
 static void *qemu_kvm_cpu_thread_fn(void *arg)
@@ -737,10 +739,10 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
 
     qemu_mutex_lock(&qemu_global_mutex);
     qemu_thread_get_self(cpu->thread);
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
     cpu_single_env = env;
 
-    r = kvm_init_vcpu(env);
+    r = kvm_init_vcpu(cpu);
     if (r < 0) {
         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
         exit(1);
@@ -749,11 +751,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     qemu_kvm_init_cpu_signals(env);
 
     /* signal CPU creation */
-    env->created = 1;
+    cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
     while (1) {
-        if (cpu_can_run(env)) {
+        if (cpu_can_run(cpu)) {
             r = kvm_cpu_exec(env);
             if (r == EXCP_DEBUG) {
                 cpu_handle_guest_debug(env);
@@ -778,13 +780,13 @@ static void *qemu_dummy_cpu_thread_fn(void *arg)
 
     qemu_mutex_lock_iothread();
     qemu_thread_get_self(cpu->thread);
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
 
     sigemptyset(&waitset);
     sigaddset(&waitset, SIG_IPI);
 
     /* signal CPU creation */
-    env->created = 1;
+    cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
     cpu_single_env = env;
@@ -801,7 +803,7 @@ static void *qemu_dummy_cpu_thread_fn(void *arg)
         }
         qemu_mutex_lock_iothread();
         cpu_single_env = env;
-        qemu_wait_io_event_common(env);
+        qemu_wait_io_event_common(cpu);
     }
 
     return NULL;
@@ -812,8 +814,8 @@ static void tcg_exec_all(void);
 
 static void *qemu_tcg_cpu_thread_fn(void *arg)
 {
-    CPUArchState *env = arg;
-    CPUState *cpu = ENV_GET_CPU(env);
+    CPUState *cpu = arg;
+    CPUArchState *env;
 
     qemu_tcg_init_cpu_signals();
     qemu_thread_get_self(cpu->thread);
@@ -821,18 +823,19 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     /* signal CPU creation */
     qemu_mutex_lock(&qemu_global_mutex);
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->thread_id = qemu_get_thread_id();
-        env->created = 1;
+        cpu = ENV_GET_CPU(env);
+        cpu->thread_id = qemu_get_thread_id();
+        cpu->created = true;
     }
     qemu_cond_signal(&qemu_cpu_cond);
 
     /* wait for initial kick-off after machine start */
-    while (first_cpu->stopped) {
+    while (ENV_GET_CPU(first_cpu)->stopped) {
         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 
         /* process any pending work */
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            qemu_wait_io_event_common(env);
+            qemu_wait_io_event_common(ENV_GET_CPU(env));
         }
     }
 
@@ -847,9 +850,8 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     return NULL;
 }
 
-static void qemu_cpu_kick_thread(CPUArchState *env)
+static void qemu_cpu_kick_thread(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
 #ifndef _WIN32
     int err;
 
@@ -859,7 +861,7 @@ static void qemu_cpu_kick_thread(CPUArchState *env)
         exit(1);
     }
 #else /* _WIN32 */
-    if (!qemu_cpu_is_self(env)) {
+    if (!qemu_cpu_is_self(cpu)) {
         SuspendThread(cpu->hThread);
         cpu_signal(0);
         ResumeThread(cpu->hThread);
@@ -867,14 +869,11 @@ static void qemu_cpu_kick_thread(CPUArchState *env)
 #endif
 }
 
-void qemu_cpu_kick(void *_env)
+void qemu_cpu_kick(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
-    qemu_cond_broadcast(env->halt_cond);
+    qemu_cond_broadcast(cpu->halt_cond);
     if (!tcg_enabled() && !cpu->thread_kicked) {
-        qemu_cpu_kick_thread(env);
+        qemu_cpu_kick_thread(cpu);
         cpu->thread_kicked = true;
     }
 }
@@ -886,7 +885,7 @@ void qemu_cpu_kick_self(void)
     CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
 
     if (!cpu_single_cpu->thread_kicked) {
-        qemu_cpu_kick_thread(cpu_single_env);
+        qemu_cpu_kick_thread(cpu_single_cpu);
         cpu_single_cpu->thread_kicked = true;
     }
 #else
@@ -894,14 +893,16 @@ void qemu_cpu_kick_self(void)
 #endif
 }
 
-int qemu_cpu_is_self(void *_env)
+bool qemu_cpu_is_self(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
     return qemu_thread_is_self(cpu->thread);
 }
 
+static bool qemu_in_vcpu_thread(void)
+{
+    return cpu_single_env && qemu_cpu_is_self(ENV_GET_CPU(cpu_single_env));
+}
+
 void qemu_mutex_lock_iothread(void)
 {
     if (!tcg_enabled()) {
@@ -909,7 +910,7 @@ void qemu_mutex_lock_iothread(void)
     } else {
         iothread_requesting_mutex = true;
         if (qemu_mutex_trylock(&qemu_global_mutex)) {
-            qemu_cpu_kick_thread(first_cpu);
+            qemu_cpu_kick_thread(ENV_GET_CPU(first_cpu));
             qemu_mutex_lock(&qemu_global_mutex);
         }
         iothread_requesting_mutex = false;
@@ -927,7 +928,8 @@ static int all_vcpus_paused(void)
     CPUArchState *penv = first_cpu;
 
     while (penv) {
-        if (!penv->stopped) {
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        if (!pcpu->stopped) {
             return 0;
         }
         penv = penv->next_cpu;
@@ -942,17 +944,19 @@ void pause_all_vcpus(void)
 
     qemu_clock_enable(vm_clock, false);
     while (penv) {
-        penv->stop = 1;
-        qemu_cpu_kick(penv);
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        pcpu->stop = true;
+        qemu_cpu_kick(pcpu);
         penv = penv->next_cpu;
     }
 
-    if (!qemu_thread_is_self(&io_thread)) {
+    if (qemu_in_vcpu_thread()) {
         cpu_stop_current();
         if (!kvm_enabled()) {
             while (penv) {
-                penv->stop = 0;
-                penv->stopped = 1;
+                CPUState *pcpu = ENV_GET_CPU(penv);
+                pcpu->stop = 0;
+                pcpu->stopped = true;
                 penv = penv->next_cpu;
             }
             return;
@@ -963,7 +967,7 @@ void pause_all_vcpus(void)
         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
         penv = first_cpu;
         while (penv) {
-            qemu_cpu_kick(penv);
+            qemu_cpu_kick(ENV_GET_CPU(penv));
             penv = penv->next_cpu;
         }
     }
@@ -975,36 +979,34 @@ void resume_all_vcpus(void)
 
     qemu_clock_enable(vm_clock, true);
     while (penv) {
-        penv->stop = 0;
-        penv->stopped = 0;
-        qemu_cpu_kick(penv);
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        pcpu->stop = false;
+        pcpu->stopped = false;
+        qemu_cpu_kick(pcpu);
         penv = penv->next_cpu;
     }
 }
 
-static void qemu_tcg_init_vcpu(void *_env)
+static void qemu_tcg_init_vcpu(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
     /* share a single thread for all cpus with TCG */
     if (!tcg_cpu_thread) {
         cpu->thread = g_malloc0(sizeof(QemuThread));
-        env->halt_cond = g_malloc0(sizeof(QemuCond));
-        qemu_cond_init(env->halt_cond);
-        tcg_halt_cond = env->halt_cond;
-        qemu_thread_create(cpu->thread, qemu_tcg_cpu_thread_fn, env,
+        cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+        qemu_cond_init(cpu->halt_cond);
+        tcg_halt_cond = cpu->halt_cond;
+        qemu_thread_create(cpu->thread, qemu_tcg_cpu_thread_fn, cpu,
                            QEMU_THREAD_JOINABLE);
 #ifdef _WIN32
         cpu->hThread = qemu_thread_get_handle(cpu->thread);
 #endif
-        while (env->created == 0) {
+        while (!cpu->created) {
             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
         }
         tcg_cpu_thread = cpu->thread;
     } else {
         cpu->thread = tcg_cpu_thread;
-        env->halt_cond = tcg_halt_cond;
+        cpu->halt_cond = tcg_halt_cond;
     }
 }
 
@@ -1013,11 +1015,11 @@ static void qemu_kvm_start_vcpu(CPUArchState *env)
     CPUState *cpu = ENV_GET_CPU(env);
 
     cpu->thread = g_malloc0(sizeof(QemuThread));
-    env->halt_cond = g_malloc0(sizeof(QemuCond));
-    qemu_cond_init(env->halt_cond);
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
     qemu_thread_create(cpu->thread, qemu_kvm_cpu_thread_fn, env,
                        QEMU_THREAD_JOINABLE);
-    while (env->created == 0) {
+    while (!cpu->created) {
         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
@@ -1027,11 +1029,11 @@ static void qemu_dummy_start_vcpu(CPUArchState *env)
     CPUState *cpu = ENV_GET_CPU(env);
 
     cpu->thread = g_malloc0(sizeof(QemuThread));
-    env->halt_cond = g_malloc0(sizeof(QemuCond));
-    qemu_cond_init(env->halt_cond);
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
     qemu_thread_create(cpu->thread, qemu_dummy_cpu_thread_fn, env,
                        QEMU_THREAD_JOINABLE);
-    while (env->created == 0) {
+    while (!cpu->created) {
         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
@@ -1039,14 +1041,15 @@ static void qemu_dummy_start_vcpu(CPUArchState *env)
 void qemu_init_vcpu(void *_env)
 {
     CPUArchState *env = _env;
+    CPUState *cpu = ENV_GET_CPU(env);
 
-    env->nr_cores = smp_cores;
-    env->nr_threads = smp_threads;
-    env->stopped = 1;
+    cpu->nr_cores = smp_cores;
+    cpu->nr_threads = smp_threads;
+    cpu->stopped = true;
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(env);
     } else if (tcg_enabled()) {
-        qemu_tcg_init_vcpu(env);
+        qemu_tcg_init_vcpu(cpu);
     } else {
         qemu_dummy_start_vcpu(env);
     }
@@ -1055,8 +1058,9 @@ void qemu_init_vcpu(void *_env)
 void cpu_stop_current(void)
 {
     if (cpu_single_env) {
-        cpu_single_env->stop = 0;
-        cpu_single_env->stopped = 1;
+        CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+        cpu_single_cpu->stop = false;
+        cpu_single_cpu->stopped = true;
         cpu_exit(cpu_single_env);
         qemu_cond_signal(&qemu_pause_cond);
     }
@@ -1064,7 +1068,7 @@ void cpu_stop_current(void)
 
 void vm_stop(RunState state)
 {
-    if (!qemu_thread_is_self(&io_thread)) {
+    if (qemu_in_vcpu_thread()) {
         qemu_system_vmstop_request(state);
         /*
          * FIXME: should not return to device code in case
@@ -1137,17 +1141,18 @@ static void tcg_exec_all(void)
     }
     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
         CPUArchState *env = next_cpu;
+        CPUState *cpu = ENV_GET_CPU(env);
 
         qemu_clock_enable(vm_clock,
                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
 
-        if (cpu_can_run(env)) {
+        if (cpu_can_run(cpu)) {
             r = tcg_cpu_exec(env);
             if (r == EXCP_DEBUG) {
                 cpu_handle_guest_debug(env);
                 break;
             }
-        } else if (env->stop || env->stopped) {
+        } else if (cpu->stop || cpu->stopped) {
             break;
         }
     }
@@ -1157,12 +1162,14 @@ static void tcg_exec_all(void)
 void set_numa_modes(void)
 {
     CPUArchState *env;
+    CPUState *cpu;
     int i;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
         for (i = 0; i < nb_numa_nodes; i++) {
-            if (test_bit(env->cpu_index, node_cpumask[i])) {
-                env->numa_node = i;
+            if (test_bit(cpu->cpu_index, node_cpumask[i])) {
+                cpu->numa_node = i;
             }
         }
     }
@@ -1192,10 +1199,8 @@ void set_cpu_log_filename(const char *optarg)
 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
 {
     /* XXX: implement xxx_cpu_list for targets that still miss it */
-#if defined(cpu_list_id)
-    cpu_list_id(f, cpu_fprintf, optarg);
-#elif defined(cpu_list)
-    cpu_list(f, cpu_fprintf); /* deprecated */
+#if defined(cpu_list)
+    cpu_list(f, cpu_fprintf);
 #endif
 }
 
@@ -1204,17 +1209,18 @@ CpuInfoList *qmp_query_cpus(Error **errp)
     CpuInfoList *head = NULL, *cur_item = NULL;
     CPUArchState *env;
 
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        CPUState *cpu = ENV_GET_CPU(env);
         CpuInfoList *info;
 
         cpu_synchronize_state(env);
 
         info = g_malloc0(sizeof(*info));
         info->value = g_malloc0(sizeof(*info->value));
-        info->value->CPU = env->cpu_index;
+        info->value->CPU = cpu->cpu_index;
         info->value->current = (env == first_cpu);
         info->value->halted = env->halted;
-        info->value->thread_id = env->thread_id;
+        info->value->thread_id = cpu->thread_id;
 #if defined(TARGET_I386)
         info->value->has_pc = true;
         info->value->pc = env->eip + env->segs[R_CS].base;
@@ -1249,6 +1255,7 @@ void qmp_memsave(int64_t addr, int64_t size, const char *filename,
     FILE *f;
     uint32_t l;
     CPUArchState *env;
+    CPUState *cpu;
     uint8_t buf[1024];
 
     if (!has_cpu) {
@@ -1256,7 +1263,8 @@ void qmp_memsave(int64_t addr, int64_t size, const char *filename,
     }
 
     for (env = first_cpu; env; env = env->next_cpu) {
-        if (cpu_index == env->cpu_index) {
+        cpu = ENV_GET_CPU(env);
+        if (cpu_index == cpu->cpu_index) {
             break;
         }
     }
diff --git a/cputlb.c b/cputlb.c
index d3e7b257ae3..88239c494ab 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -19,13 +19,13 @@
 
 #include "config.h"
 #include "cpu.h"
-#include "exec-all.h"
-#include "memory.h"
+#include "exec/exec-all.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
-#include "cputlb.h"
+#include "exec/cputlb.h"
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "exec/memory-internal.h"
 
 //#define DEBUG_TLB
 //#define DEBUG_TLB_CHECK
@@ -237,7 +237,7 @@ static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
    is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
    supplied size is only used by tlb_flush_page.  */
 void tlb_set_page(CPUArchState *env, target_ulong vaddr,
-                  target_phys_addr_t paddr, int prot,
+                  hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size)
 {
     MemoryRegionSection *section;
@@ -246,13 +246,13 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr,
     target_ulong code_address;
     uintptr_t addend;
     CPUTLBEntry *te;
-    target_phys_addr_t iotlb;
+    hwaddr iotlb;
 
     assert(size >= TARGET_PAGE_SIZE);
     if (size != TARGET_PAGE_SIZE) {
         tlb_add_large_page(env, vaddr, size);
     }
-    section = phys_page_find(paddr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, paddr >> TARGET_PAGE_BITS);
 #if defined(DEBUG_TLB)
     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
            " prot=%x idx=%d pd=0x%08lx\n",
@@ -325,11 +325,7 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
     mmu_idx = cpu_mmu_index(env1);
     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                  (addr & TARGET_PAGE_MASK))) {
-#ifdef CONFIG_TCG_PASS_AREG0
         cpu_ldub_code(env1, addr);
-#else
-        ldub_code(addr);
-#endif
     }
     pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK;
     mr = iotlb_to_region(pd);
@@ -348,19 +344,18 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 #define MMUSUFFIX _cmmu
 #undef GETPC
 #define GETPC() ((uintptr_t)0)
-#define env cpu_single_env
 #define SOFTMMU_CODE_ACCESS
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #undef env
diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index f335a725f04..2f1a5c99430 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -22,6 +22,7 @@ CONFIG_ADS7846=y
 CONFIG_MAX111X=y
 CONFIG_SSI=y
 CONFIG_SSI_SD=y
+CONFIG_SSI_M25P80=y
 CONFIG_LAN9118=y
 CONFIG_SMC91C111=y
 CONFIG_DS1338=y
diff --git a/default-configs/microblaze-softmmu.mak b/default-configs/microblaze-softmmu.mak
index 64c9485de44..2f442e5aea0 100644
--- a/default-configs/microblaze-softmmu.mak
+++ b/default-configs/microblaze-softmmu.mak
@@ -5,3 +5,5 @@ CONFIG_PFLASH_CFI01=y
 CONFIG_SERIAL=y
 CONFIG_XILINX=y
 CONFIG_XILINX_AXI=y
+CONFIG_SSI=y
+CONFIG_SSI_M25P80=y
diff --git a/default-configs/microblazeel-softmmu.mak b/default-configs/microblazeel-softmmu.mak
index a9622760c9e..af9a3cde0d8 100644
--- a/default-configs/microblazeel-softmmu.mak
+++ b/default-configs/microblazeel-softmmu.mak
@@ -5,3 +5,5 @@ CONFIG_PFLASH_CFI01=y
 CONFIG_SERIAL=y
 CONFIG_XILINX=y
 CONFIG_XILINX_AXI=y
+CONFIG_SSI=y
+CONFIG_SSI_M25P80=y
diff --git a/default-configs/pci.mak b/default-configs/pci.mak
index 69e18f14282..ee2d18d5f28 100644
--- a/default-configs/pci.mak
+++ b/default-configs/pci.mak
@@ -19,3 +19,6 @@ CONFIG_IDE_PCI=y
 CONFIG_AHCI=y
 CONFIG_ESP=y
 CONFIG_ESP_PCI=y
+CONFIG_SERIAL=y
+CONFIG_SERIAL_PCI=y
+CONFIG_IPACK=y
diff --git a/default-configs/ppc-softmmu.mak b/default-configs/ppc-softmmu.mak
index d0fde7b93d2..1f4a1cff61d 100644
--- a/default-configs/ppc-softmmu.mak
+++ b/default-configs/ppc-softmmu.mak
@@ -8,6 +8,7 @@ CONFIG_M48T59=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
 CONFIG_I8254=y
 CONFIG_PCKBD=y
 CONFIG_FDC=y
@@ -16,6 +17,7 @@ CONFIG_I82374=y
 CONFIG_OPENPIC=y
 CONFIG_PREP_PCI=y
 CONFIG_I82378=y
+CONFIG_PC87312=y
 CONFIG_MACIO=y
 CONFIG_PCSPK=y
 CONFIG_CUDA=y
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index e4265b49787..5ff406caa51 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -8,13 +8,18 @@ CONFIG_M48T59=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
 CONFIG_I8254=y
 CONFIG_PCKBD=y
 CONFIG_FDC=y
 CONFIG_DMA=y
+CONFIG_I82374=y
 CONFIG_OPENPIC=y
 CONFIG_PREP_PCI=y
+CONFIG_I82378=y
+CONFIG_PC87312=y
 CONFIG_MACIO=y
+CONFIG_PCSPK=y
 CONFIG_CUDA=y
 CONFIG_ADB=y
 CONFIG_MAC_NVRAM=y
diff --git a/default-configs/sparc64-softmmu.mak b/default-configs/sparc64-softmmu.mak
index c9a36c1e46b..03e8b427124 100644
--- a/default-configs/sparc64-softmmu.mak
+++ b/default-configs/sparc64-softmmu.mak
@@ -6,7 +6,6 @@ CONFIG_M48T59=y
 CONFIG_PTIMER=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
-CONFIG_VGA_CIRRUS=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_PCKBD=y
diff --git a/device_tree.c b/device_tree.c
index d7a9b6bb89c..56af24b3972 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -20,10 +20,10 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "hw/loader.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 
 #include <libfdt.h>
 
@@ -304,3 +304,18 @@ int qemu_devtree_add_subnode(void *fdt, const char *name)
     g_free(dupname);
     return retval;
 }
+
+void qemu_devtree_dumpdtb(void *fdt, int size)
+{
+    QemuOpts *machine_opts;
+
+    machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (machine_opts) {
+        const char *dumpdtb = qemu_opt_get(machine_opts, "dumpdtb");
+        if (dumpdtb) {
+            /* Dump the dtb to a file and quit */
+            exit(g_file_set_contents(dumpdtb, fdt, size, NULL) ? 0 : 1);
+        }
+    }
+
+}
diff --git a/disas.c b/disas.c
index 7b2acc99430..a46faeed803 100644
--- a/disas.c
+++ b/disas.c
@@ -1,11 +1,16 @@
 /* General "disassemble this chunk" code.  Used for debugging. */
 #include "config.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include "elf.h"
 #include <errno.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
+
+typedef struct CPUDebug {
+    struct disassemble_info info;
+    CPUArchState *env;
+} CPUDebug;
 
 /* Filled in by elfload.c.  Simplistic, but will do for now. */
 struct syminfo *syminfos = NULL;
@@ -32,7 +37,9 @@ target_read_memory (bfd_vma memaddr,
                     int length,
                     struct disassemble_info *info)
 {
-    cpu_memory_rw_debug(cpu_single_env, memaddr, myaddr, length, 0);
+    CPUDebug *s = container_of(info, CPUDebug, info);
+
+    cpu_memory_rw_debug(s->env, memaddr, myaddr, length, 0);
     return 0;
 }
 
@@ -158,32 +165,35 @@ print_insn_thumb1(bfd_vma pc, disassemble_info *info)
     ppc  - nonzero means little endian
     other targets - unused
  */
-void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
+void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+                  target_ulong size, int flags)
 {
     target_ulong pc;
     int count;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, out, fprintf);
+    INIT_DISASSEMBLE_INFO(s.info, out, fprintf);
 
-    disasm_info.read_memory_func = target_read_memory;
-    disasm_info.buffer_vma = code;
-    disasm_info.buffer_length = size;
-    disasm_info.print_address_func = generic_print_target_address;
+    s.env = env;
+    s.info.read_memory_func = target_read_memory;
+    s.info.buffer_vma = code;
+    s.info.buffer_length = size;
+    s.info.print_address_func = generic_print_target_address;
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(TARGET_I386)
-    if (flags == 2)
-        disasm_info.mach = bfd_mach_x86_64;
-    else if (flags == 1)
-        disasm_info.mach = bfd_mach_i386_i8086;
-    else
-        disasm_info.mach = bfd_mach_i386_i386;
+    if (flags == 2) {
+        s.info.mach = bfd_mach_x86_64;
+    } else if (flags == 1) {
+        s.info.mach = bfd_mach_i386_i8086;
+    } else {
+        s.info.mach = bfd_mach_i386_i386;
+    }
     print_insn = print_insn_i386;
 #elif defined(TARGET_ARM)
     if (flags & 1) {
@@ -193,27 +203,28 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
     }
     if (flags & 2) {
 #ifdef TARGET_WORDS_BIGENDIAN
-        disasm_info.endian = BFD_ENDIAN_LITTLE;
+        s.info.endian = BFD_ENDIAN_LITTLE;
 #else
-        disasm_info.endian = BFD_ENDIAN_BIG;
+        s.info.endian = BFD_ENDIAN_BIG;
 #endif
     }
 #elif defined(TARGET_SPARC)
     print_insn = print_insn_sparc;
 #ifdef TARGET_SPARC64
-    disasm_info.mach = bfd_mach_sparc_v9b;
+    s.info.mach = bfd_mach_sparc_v9b;
 #endif
 #elif defined(TARGET_PPC)
-    if (flags >> 16)
-        disasm_info.endian = BFD_ENDIAN_LITTLE;
+    if (flags >> 16) {
+        s.info.endian = BFD_ENDIAN_LITTLE;
+    }
     if (flags & 0xFFFF) {
         /* If we have a precise definitions of the instructions set, use it */
-        disasm_info.mach = flags & 0xFFFF;
+        s.info.mach = flags & 0xFFFF;
     } else {
 #ifdef TARGET_PPC64
-        disasm_info.mach = bfd_mach_ppc64;
+        s.info.mach = bfd_mach_ppc64;
 #else
-        disasm_info.mach = bfd_mach_ppc;
+        s.info.mach = bfd_mach_ppc;
 #endif
     }
     print_insn = print_insn_ppc;
@@ -226,27 +237,27 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
     print_insn = print_insn_little_mips;
 #endif
 #elif defined(TARGET_SH4)
-    disasm_info.mach = bfd_mach_sh4;
+    s.info.mach = bfd_mach_sh4;
     print_insn = print_insn_sh;
 #elif defined(TARGET_ALPHA)
-    disasm_info.mach = bfd_mach_alpha_ev6;
+    s.info.mach = bfd_mach_alpha_ev6;
     print_insn = print_insn_alpha;
 #elif defined(TARGET_CRIS)
     if (flags != 32) {
-        disasm_info.mach = bfd_mach_cris_v0_v10;
+        s.info.mach = bfd_mach_cris_v0_v10;
         print_insn = print_insn_crisv10;
     } else {
-        disasm_info.mach = bfd_mach_cris_v32;
+        s.info.mach = bfd_mach_cris_v32;
         print_insn = print_insn_crisv32;
     }
 #elif defined(TARGET_S390X)
-    disasm_info.mach = bfd_mach_s390_64;
+    s.info.mach = bfd_mach_s390_64;
     print_insn = print_insn_s390;
 #elif defined(TARGET_MICROBLAZE)
-    disasm_info.mach = bfd_arch_microblaze;
+    s.info.mach = bfd_arch_microblaze;
     print_insn = print_insn_microblaze;
 #elif defined(TARGET_LM32)
-    disasm_info.mach = bfd_mach_lm32;
+    s.info.mach = bfd_mach_lm32;
     print_insn = print_insn_lm32;
 #else
     fprintf(out, "0x" TARGET_FMT_lx
@@ -256,14 +267,14 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
 
     for (pc = code; size > 0; pc += count, size -= count) {
 	fprintf(out, "0x" TARGET_FMT_lx ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+	count = print_insn(pc, &s.info);
 #if 0
         {
             int i;
             uint8_t b;
             fprintf(out, " {");
             for(i = 0; i < count; i++) {
-                target_read_memory(pc + i, &b, 1, &disasm_info);
+                target_read_memory(pc + i, &b, 1, &s.info);
                 fprintf(out, " %02x", b);
             }
             fprintf(out, " }");
@@ -287,28 +298,28 @@ void disas(FILE *out, void *code, unsigned long size)
 {
     uintptr_t pc;
     int count;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, out, fprintf);
-    disasm_info.print_address_func = generic_print_host_address;
+    INIT_DISASSEMBLE_INFO(s.info, out, fprintf);
+    s.info.print_address_func = generic_print_host_address;
 
-    disasm_info.buffer = code;
-    disasm_info.buffer_vma = (uintptr_t)code;
-    disasm_info.buffer_length = size;
+    s.info.buffer = code;
+    s.info.buffer_vma = (uintptr_t)code;
+    s.info.buffer_length = size;
 
 #ifdef HOST_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(CONFIG_TCG_INTERPRETER)
     print_insn = print_insn_tci;
 #elif defined(__i386__)
-    disasm_info.mach = bfd_mach_i386_i386;
+    s.info.mach = bfd_mach_i386_i386;
     print_insn = print_insn_i386;
 #elif defined(__x86_64__)
-    disasm_info.mach = bfd_mach_x86_64;
+    s.info.mach = bfd_mach_x86_64;
     print_insn = print_insn_i386;
 #elif defined(_ARCH_PPC)
     print_insn = print_insn_ppc;
@@ -316,9 +327,7 @@ void disas(FILE *out, void *code, unsigned long size)
     print_insn = print_insn_alpha;
 #elif defined(__sparc__)
     print_insn = print_insn_sparc;
-#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || defined(__sparc_v9__)
-    disasm_info.mach = bfd_mach_sparc_v9b;
-#endif
+    s.info.mach = bfd_mach_sparc_v9b;
 #elif defined(__arm__)
     print_insn = print_insn_arm;
 #elif defined(__MIPSEB__)
@@ -340,7 +349,7 @@ void disas(FILE *out, void *code, unsigned long size)
 #endif
     for (pc = (uintptr_t)code; size > 0; pc += count, size -= count) {
         fprintf(out, "0x%08" PRIxPTR ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+        count = print_insn(pc, &s.info);
 	fprintf(out, "\n");
 	if (count < 0)
 	    break;
@@ -365,19 +374,20 @@ const char *lookup_symbol(target_ulong orig_addr)
 
 #if !defined(CONFIG_USER_ONLY)
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 static int monitor_disas_is_physical;
-static CPUArchState *monitor_disas_env;
 
 static int
 monitor_read_memory (bfd_vma memaddr, bfd_byte *myaddr, int length,
                      struct disassemble_info *info)
 {
+    CPUDebug *s = container_of(info, CPUDebug, info);
+
     if (monitor_disas_is_physical) {
         cpu_physical_memory_read(memaddr, myaddr, length);
     } else {
-        cpu_memory_rw_debug(monitor_disas_env, memaddr,myaddr, length, 0);
+        cpu_memory_rw_debug(s->env, memaddr,myaddr, length, 0);
     }
     return 0;
 }
@@ -396,30 +406,31 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
                    target_ulong pc, int nb_insn, int is_physical, int flags)
 {
     int count, i;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, (FILE *)mon, monitor_fprintf);
+    INIT_DISASSEMBLE_INFO(s.info, (FILE *)mon, monitor_fprintf);
 
-    monitor_disas_env = env;
+    s.env = env;
     monitor_disas_is_physical = is_physical;
-    disasm_info.read_memory_func = monitor_read_memory;
-    disasm_info.print_address_func = generic_print_target_address;
+    s.info.read_memory_func = monitor_read_memory;
+    s.info.print_address_func = generic_print_target_address;
 
-    disasm_info.buffer_vma = pc;
+    s.info.buffer_vma = pc;
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(TARGET_I386)
-    if (flags == 2)
-        disasm_info.mach = bfd_mach_x86_64;
-    else if (flags == 1)
-        disasm_info.mach = bfd_mach_i386_i8086;
-    else
-        disasm_info.mach = bfd_mach_i386_i386;
+    if (flags == 2) {
+        s.info.mach = bfd_mach_x86_64;
+    } else if (flags == 1) {
+        s.info.mach = bfd_mach_i386_i8086;
+    } else {
+        s.info.mach = bfd_mach_i386_i386;
+    }
     print_insn = print_insn_i386;
 #elif defined(TARGET_ARM)
     print_insn = print_insn_arm;
@@ -428,13 +439,13 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
 #elif defined(TARGET_SPARC)
     print_insn = print_insn_sparc;
 #ifdef TARGET_SPARC64
-    disasm_info.mach = bfd_mach_sparc_v9b;
+    s.info.mach = bfd_mach_sparc_v9b;
 #endif
 #elif defined(TARGET_PPC)
 #ifdef TARGET_PPC64
-    disasm_info.mach = bfd_mach_ppc64;
+    s.info.mach = bfd_mach_ppc64;
 #else
-    disasm_info.mach = bfd_mach_ppc;
+    s.info.mach = bfd_mach_ppc;
 #endif
     print_insn = print_insn_ppc;
 #elif defined(TARGET_M68K)
@@ -446,13 +457,13 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
     print_insn = print_insn_little_mips;
 #endif
 #elif defined(TARGET_SH4)
-    disasm_info.mach = bfd_mach_sh4;
+    s.info.mach = bfd_mach_sh4;
     print_insn = print_insn_sh;
 #elif defined(TARGET_S390X)
-    disasm_info.mach = bfd_mach_s390_64;
+    s.info.mach = bfd_mach_s390_64;
     print_insn = print_insn_s390;
 #elif defined(TARGET_LM32)
-    disasm_info.mach = bfd_mach_lm32;
+    s.info.mach = bfd_mach_lm32;
     print_insn = print_insn_lm32;
 #else
     monitor_printf(mon, "0x" TARGET_FMT_lx
@@ -462,7 +473,7 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
 
     for(i = 0; i < nb_insn; i++) {
 	monitor_printf(mon, "0x" TARGET_FMT_lx ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+        count = print_insn(pc, &s.info);
 	monitor_printf(mon, "\n");
 	if (count < 0)
 	    break;
diff --git a/disas/Makefile.objs b/disas/Makefile.objs
new file mode 100644
index 00000000000..ed75f9a99bd
--- /dev/null
+++ b/disas/Makefile.objs
@@ -0,0 +1,18 @@
+common-obj-$(CONFIG_ALPHA_DIS) += alpha.o
+common-obj-$(CONFIG_ARM_DIS) += arm.o
+common-obj-$(CONFIG_CRIS_DIS) += cris.o
+common-obj-$(CONFIG_HPPA_DIS) += hppa.o
+common-obj-$(CONFIG_I386_DIS) += i386.o
+common-obj-$(CONFIG_IA64_DIS) += ia64.o
+common-obj-$(CONFIG_M68K_DIS) += m68k.o
+common-obj-$(CONFIG_MICROBLAZE_DIS) += microblaze.o
+common-obj-$(CONFIG_MIPS_DIS) += mips.o
+common-obj-$(CONFIG_PPC_DIS) += ppc.o
+common-obj-$(CONFIG_S390_DIS) += s390.o
+common-obj-$(CONFIG_SH4_DIS) += sh4.o
+common-obj-$(CONFIG_SPARC_DIS) += sparc.o
+common-obj-$(CONFIG_LM32_DIS) += lm32.o
+
+# TODO: As long as the TCG interpreter and its generated code depend
+# on the QEMU target, we cannot compile the disassembler here.
+#common-obj-$(CONFIG_TCI_DIS) += tci.o
diff --git a/alpha-dis.c b/disas/alpha.c
index ae331b35b8f..a950b9cee0c 100644
--- a/alpha-dis.c
+++ b/disas/alpha.c
@@ -20,7 +20,7 @@ along with this file; see the file COPYING.  If not, see
 <http://www.gnu.org/licenses/>. */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* MAX is redefined below, so remove any previous definition. */
 #undef MAX
diff --git a/arm-dis.c b/disas/arm.c
index 6bc4d716981..4927d8ad7ca 100644
--- a/arm-dis.c
+++ b/disas/arm.c
@@ -22,7 +22,7 @@
 /* Start of qemu specific additions.  Mostly this is stub definitions
    for things we don't care about.  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #define ATTRIBUTE_UNUSED __attribute__((unused))
 #define ISSPACE(x) ((x) == ' ' || (x) == '\t' || (x) == '\n')
 
diff --git a/cris-dis.c b/disas/cris.c
index 1d174ba8ccf..9dfb4e38855 100644
--- a/cris-dis.c
+++ b/disas/cris.c
@@ -19,7 +19,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
 #include "qemu-common.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 //#include "sysdep.h"
 #include "target-cris/opcode-cris.h"
 //#include "libiberty.h"
diff --git a/hppa-dis.c b/disas/hppa.c
index 420a7d22d0f..c7c8be66a28 100644
--- a/hppa-dis.c
+++ b/disas/hppa.c
@@ -18,7 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* HP PA-RISC SOM object file format:  definitions internal to BFD.
    Copyright 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000,
diff --git a/i386-dis.c b/disas/i386.c
index c4a81c99ad5..3b006b13e09 100644
--- a/i386-dis.c
+++ b/disas/i386.c
@@ -32,7 +32,7 @@
    the Intel manual for details.  */
 
 #include <stdlib.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 /* include/opcode/i386.h r1.78 */
 
 /* opcode/i386.h -- Intel 80386 opcode macros
diff --git a/ia64-dis.c b/disas/ia64.c
index 2a103e6b5cb..a8fe26c4136 100644
--- a/ia64-dis.c
+++ b/disas/ia64.c
@@ -21,7 +21,7 @@
 #include <assert.h>
 #include <string.h>
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* ia64.h -- Header file for ia64 opcode table
    Copyright (C) 1998, 1999, 2000, 2002, 2005, 2006
diff --git a/lm32-dis.c b/disas/lm32.c
index 709ed3215cd..a8eefe05b18 100644
--- a/lm32-dis.c
+++ b/disas/lm32.c
@@ -19,7 +19,7 @@
  */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 typedef enum {
     LM32_OP_SRUI = 0, LM32_OP_NORI, LM32_OP_MULI, LM32_OP_SH, LM32_OP_LB,
diff --git a/m68k-dis.c b/disas/m68k.c
index 2b155de51c4..c950241f790 100644
--- a/m68k-dis.c
+++ b/disas/m68k.c
@@ -5,7 +5,7 @@
 #include <math.h>
 #include <stdio.h>
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* **** floatformat.h from sourceware.org CVS 2005-08-14.  */
 /* IEEE floating point support declarations, for GDB, the GNU Debugger.
diff --git a/microblaze-dis.c b/disas/microblaze.c
index 16c312f2f5d..ec91af386d6 100644
--- a/microblaze-dis.c
+++ b/disas/microblaze.c
@@ -582,7 +582,7 @@ char pvr_register_prefix[] = "rpvr";
 
 #endif /* MICROBLAZE_OPC */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include <strings.h>
 
 #define get_field_rd(instr) get_field(instr, RD_MASK, RD_LOW)
diff --git a/mips-dis.c b/disas/mips.c
index e3a6e0b49ec..2106b574cbd 100644
--- a/mips-dis.c
+++ b/disas/mips.c
@@ -19,7 +19,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* mips.h.  Mips opcode list for GDB, the GNU debugger.
    Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
diff --git a/ppc-dis.c b/disas/ppc.c
index bc98cbe655d..c149506fd88 100644
--- a/ppc-dis.c
+++ b/disas/ppc.c
@@ -18,7 +18,7 @@ the GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this file; see the file COPYING.  If not,
 see <http://www.gnu.org/licenses/>.  */
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #define BFD_DEFAULT_TARGET_SIZE 64
 
 /* ppc.h -- Header file for PowerPC opcode table
diff --git a/s390-dis.c b/disas/s390.c
index 8abcdf0128a..25499ba4199 100644
--- a/s390-dis.c
+++ b/disas/s390.c
@@ -21,7 +21,7 @@
    02110-1301, USA.  */
 
 #include "qemu-common.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* include/opcode/s390.h revision 1.9 */
 /* s390.h -- Header file for S390 opcode table
@@ -589,6 +589,16 @@ static const struct s390_operand s390_operands[] =
   { 4, 32, S390_OPERAND_CCODE },
 #define I8_32 46                  /* 8 bit signed value starting at 32 */
   { 8, 32, S390_OPERAND_SIGNED },
+#define U8_24 47                  /* 8 bit unsigned value starting at 24 */
+  { 8, 24, 0 },
+#define U8_32 48                  /* 8 bit unsigned value starting at 32 */
+  { 8, 32, 0 },
+#define I16_32 49
+  { 16, 32, S390_OPERAND_SIGNED },
+#define M4_16 50                  /* 4-bit condition-code starting at 12 */
+  { 4, 16, S390_OPERAND_CCODE },
+#define I8_16 51
+  { 8, 16, S390_OPERAND_SIGNED },
 /* QEMU-END */
 };
 
@@ -663,7 +673,9 @@ static const struct s390_operand s390_operands[] =
    This is just a workaround for existing code e.g. glibc.  */
 #define INSTR_RRE_RR_OPT 4, { R_24,RO_28,0,0,0,0 }             /* efpc, sfpc */
 #define INSTR_RRF_F0FF   4, { F_16,F_24,F_28,0,0,0 }           /* e.g. madbr */
-#define INSTR_RRF_F0FF2  4, { F_24,F_16,F_28,0,0,0 }           /* e.g. cpsdr */
+/* QEMU-MOD */
+#define INSTR_RRF_F0FF2  4, { F_24,F_28,F_16,0,0,0 }           /* e.g. cpsdr */
+/* QEMU-END */
 #define INSTR_RRF_F0FR   4, { F_24,F_16,R_28,0,0,0 }           /* e.g. iedtr */
 #define INSTR_RRF_FUFF   4, { F_24,F_16,F_28,U4_20,0,0 }       /* e.g. didbr */
 #define INSTR_RRF_RURR   4, { R_24,R_28,R_16,U4_20,0,0 }       /* e.g. .insn */
@@ -801,11 +813,35 @@ static const struct s390_operand s390_operands[] =
 #define MASK_SSF_RRDRD   { 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00 }
 
 /* QEMU-ADD: */
-#define INSTR_RIE_MRRP   6, { M4_32,R_8,R_12,J16_16,0,0 }	/* e.g. crj */
+#define INSTR_RIE_MRRP   6, { M4_32, R_8, R_12, J16_16, 0, 0 } /* e.g. crj */
 #define MASK_RIE_MRRP    { 0xff, 0x00, 0x00, 0x00, 0x0f, 0xff }
 
-#define INSTR_RIE_MRIP   6, { M4_12,R_8,I8_32,J16_16,0,0 }      /* e.g. cij */
+#define INSTR_RIE_MRIP   6, { M4_12, R_8, I8_32, J16_16, 0, 0 } /* e.g. cij */
 #define MASK_RIE_MRIP    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RIE_RRIII  6, { R_8, R_12, U8_16, U8_24, U8_32, 0 } /* risbg */
+#define MASK_RIE_RRIII   { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_MRI    6, { M4_32, R_8, I16_16, 0, 0, 0 }    /* e.g. cit */
+#define MASK_RIE_MRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_MRU    6, { M4_32, R_8, U16_16, 0, 0, 0 }    /* e.g. clfit */
+#define MASK_RIE_MRU     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_RRI    6, { R_8, R_12, I16_16, 0, 0, 0 }
+#define MASK_RIE_RRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RXY_URRD   6, { U8_8, D20_20, X_12, B_16, 0, 0 }
+#define MASK_RXY_URRD    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_SIL_DRI    6, { D_20, B_16, I16_32, 0, 0, 0 }
+#define MASK_SIL_DRI     { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }
+
+#define INSTR_RSY_MRRD   6, { M4_12, R_8, D20_20, B_16, 0, 0 }
+#define MASK_SRY_MRRD    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RRF_MRR    6, { M4_16, R_24, R_28, 0, 0, 0 }
+#define MASK_RRF_MRR     { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }
+
+#define INSTR_SIY_DRI    6, { D20_20, B_16, I8_16, 0, 0, 0 }
+#define MASK_SIY_DRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
 /* QEMU-END */
 
 /* The opcode formats table (blueprints for .insn pseudo mnemonic).  */
@@ -926,6 +962,30 @@ static const struct s390_opcode s390_opcodes[] =
   { "ldeb", OP48(0xed0000000004LL), MASK_RXE_FRRD, INSTR_RXE_FRRD, 3, 0},
   { "brxlg", OP48(0xec0000000045LL), MASK_RIE_RRP, INSTR_RIE_RRP, 2, 2},
   { "brxhg", OP48(0xec0000000044LL), MASK_RIE_RRP, INSTR_RIE_RRP, 2, 2},
+/* QEMU-ADD: */
+  { "crj",   OP48(0xec0000000076LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "cgrj",  OP48(0xec0000000064LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "clrj",  OP48(0xec0000000077LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "clgrj", OP48(0xec0000000065LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "cij",   OP48(0xec000000007eLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "cgij",  OP48(0xec000000007cLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "clij",  OP48(0xec000000007fLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "clgij", OP48(0xec000000007dLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "risbg", OP48(0xec0000000055LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "risbhg", OP48(0xec000000005dLL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "risblg", OP48(0xec0000000051LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rnsbg", OP48(0xec0000000054LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rosbg", OP48(0xec0000000056LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rxsbg", OP48(0xec0000000057LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "cit", OP48(0xec0000000072LL), MASK_RIE_MRI, INSTR_RIE_MRI, 3, 6},
+  { "cgit", OP48(0xec0000000070LL), MASK_RIE_MRI, INSTR_RIE_MRI, 3, 6},
+  { "clfit", OP48(0xec0000000073LL), MASK_RIE_MRU, INSTR_RIE_MRU, 3, 6},
+  { "clgit", OP48(0xec0000000071LL), MASK_RIE_MRU, INSTR_RIE_MRU, 3, 6},
+  { "ahik", OP48(0xec00000000d8LL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "aghik", OP48(0xec00000000d9LL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "alhsik", OP48(0xec00000000daLL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "alghsik", OP48(0xec00000000dbLL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+/* QEMU-END */
   { "tp", OP48(0xeb00000000c0LL), MASK_RSL_R0RD, INSTR_RSL_R0RD, 3, 0},
   { "stamy", OP48(0xeb000000009bLL), MASK_RSY_AARD, INSTR_RSY_AARD, 2, 3},
   { "lamy", OP48(0xeb000000009aLL), MASK_RSY_AARD, INSTR_RSY_AARD, 2, 3},
@@ -985,6 +1045,20 @@ static const struct s390_opcode s390_opcodes[] =
   { "srag", OP48(0xeb000000000aLL), MASK_RSE_RRRD, INSTR_RSE_RRRD, 2, 2},
   { "lmg", OP48(0xeb0000000004LL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 2, 3},
   { "lmg", OP48(0xeb0000000004LL), MASK_RSE_RRRD, INSTR_RSE_RRRD, 2, 2},
+/* QEMU-ADD: */
+  { "loc", OP48(0xeb00000000f2LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "locg", OP48(0xeb00000000e2LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "stoc", OP48(0xeb00000000f3LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "stocg", OP48(0xeb00000000e3LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "srak", OP48(0xeb00000000dcLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "slak", OP48(0xeb00000000ddLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "srlk", OP48(0xeb00000000deLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "sllk", OP48(0xeb00000000dfLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "asi", OP48(0xeb000000006aLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "alsi", OP48(0xeb000000006eLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "agsi", OP48(0xeb000000007aLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "algsi", OP48(0xeb000000007eLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+/* QEMU-END */
   { "unpka", OP8(0xeaLL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "pka", OP8(0xe9LL), MASK_SS_L2RDRD, INSTR_SS_L2RDRD, 3, 0},
   { "mvcin", OP8(0xe8LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
@@ -993,6 +1067,17 @@ static const struct s390_opcode s390_opcodes[] =
   { "tprot", OP16(0xe501LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 3, 0},
   { "strag", OP48(0xe50000000002LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 2, 2},
   { "lasp", OP16(0xe500LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 3, 0},
+/* QEMU-ADD: */
+  { "mvhhi", OP16(0xe544LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "mvghi", OP16(0xe548LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "mvhi", OP16(0xe54cLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "chhsi", OP16(0xe554LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clhhsi", OP16(0xe555LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "cghsi", OP16(0xe558LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clghsi", OP16(0xe559LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "chsi", OP16(0xe55cLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clfhsi", OP16(0xe55dLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+/* QEMU-END */
   { "slb", OP48(0xe30000000099LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 3, 3},
   { "slb", OP48(0xe30000000099LL), MASK_RXE_RRRD, INSTR_RXE_RRRD, 3, 2},
   { "alc", OP48(0xe30000000098LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 3, 3},
@@ -1116,6 +1201,9 @@ static const struct s390_opcode s390_opcodes[] =
   { "lrag", OP48(0xe30000000003LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 2, 3},
   { "lrag", OP48(0xe30000000003LL), MASK_RXE_RRRD, INSTR_RXE_RRRD, 2, 2},
   { "ltg", OP48(0xe30000000002LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 2, 4},
+/* QEMU-ADD: */
+  { "pfd", OP48(0xe30000000036LL), MASK_RXY_URRD, INSTR_RXY_URRD, 3, 6},
+/* QEMU-END */
   { "unpku", OP8(0xe2LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "pku", OP8(0xe1LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "edmk", OP8(0xdfLL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
@@ -1135,6 +1223,32 @@ static const struct s390_opcode s390_opcodes[] =
   { "csst", OP16(0xc802LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 5},
   { "ectg", OP16(0xc801LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 5},
   { "mvcos", OP16(0xc800LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 4},
+/* QEMU-ADD: */
+  { "exrl", OP16(0xc600ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "pfdrl", OP16(0xc602ll), MASK_RIL_UP, INSTR_RIL_UP, 3, 6},
+  { "cghrl", OP16(0xc604ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "chrl", OP16(0xc605ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clghrl", OP16(0xc606ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clhrl", OP16(0xc607ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "cgrl", OP16(0xc608ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clgrl", OP16(0xc60all), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "cgfrl", OP16(0xc60cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "crl", OP16(0xc60dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clgfrl", OP16(0xc60ell), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clrl", OP16(0xc60fll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+
+  { "llhrl", OP16(0xc400ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lghrl", OP16(0xc404ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lhrl", OP16(0xc405ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "llghrl", OP16(0xc406ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "sthrl", OP16(0xc407ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lgrl", OP16(0xc408ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "stgrl", OP16(0xc40bll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lgfrl", OP16(0xc40cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lrl", OP16(0xc40dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "llgfrl", OP16(0xc40ell), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "strl", OP16(0xc40fll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+/* QEMU-END */
   { "clfi", OP16(0xc20fLL), MASK_RIL_RU, INSTR_RIL_RU, 2, 4},
   { "clgfi", OP16(0xc20eLL), MASK_RIL_RU, INSTR_RIL_RU, 2, 4},
   { "cfi", OP16(0xc20dLL), MASK_RIL_RI, INSTR_RIL_RI, 2, 4},
@@ -1265,6 +1379,29 @@ static const struct s390_opcode s390_opcodes[] =
   { "ltgr", OP16(0xb902LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
   { "lngr", OP16(0xb901LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
   { "lpgr", OP16(0xb900LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
+/* QEMU-ADD: */
+  { "crt", OP16(0xb972LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "cgrt", OP16(0xb960LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "clrt", OP16(0xb973LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "clgrt", OP16(0xb961LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "locr", OP16(0xb9f2LL), MASK_RRF_MRR, INSTR_RRF_MRR, 3, 6},
+  { "locgr", OP16(0xb9e2LL), MASK_RRF_MRR, INSTR_RRF_MRR, 3, 6},
+  { "popcnt", OP16(0xb9e1LL), MASK_RRE_RR, INSTR_RRE_RR, 3, 6},
+  { "ngrk", OP16(0xb9e4LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ogrk", OP16(0xb9e6LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "xgrk", OP16(0xb9e7LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "agrk", OP16(0xb9e8LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "sgrk", OP16(0xb9e9LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "algrk", OP16(0xb9eaLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "slgrk", OP16(0xb9ebLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "nrk", OP16(0xb9f4LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ork", OP16(0xb9f6LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "xrk", OP16(0xb9f7LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ark", OP16(0xb9f8LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "srk", OP16(0xb9f9LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "alrk", OP16(0xb9faLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "slrk", OP16(0xb9fbLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+/* QEMU-END */
   { "lctl", OP8(0xb7LL), MASK_RS_CCRD, INSTR_RS_CCRD, 3, 0},
   { "stctl", OP8(0xb6LL), MASK_RS_CCRD, INSTR_RS_CCRD, 3, 0},
   { "rrxtr", OP16(0xb3ffLL), MASK_RRF_FFFU, INSTR_RRF_FFFU, 2, 5},
@@ -1426,6 +1563,20 @@ static const struct s390_opcode s390_opcodes[] =
   { "ltebr", OP16(0xb302LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
   { "lnebr", OP16(0xb301LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
   { "lpebr", OP16(0xb300LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
+/* QEMU-ADD: */
+  { "clfebr", OP16(0xb39cLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clfdbr", OP16(0xb39dLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clfxbr", OP16(0xb39eLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgebr", OP16(0xb3acLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgdbr", OP16(0xb3adLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgxbr", OP16(0xb3aeLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "celfbr", OP16(0xb390LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cdlfbr", OP16(0xb391LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cxlfbr", OP16(0xb392LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "celgbr", OP16(0xb3a0LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cdlgbr", OP16(0xb3a1LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cxlgbr", OP16(0xb3a2LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+/* QEMU-END */
   { "trap4", OP16(0xb2ffLL), MASK_S_RD, INSTR_S_RD, 3, 0},
   { "lfas", OP16(0xb2bdLL), MASK_S_RD, INSTR_S_RD, 2, 5},
   { "srnmt", OP16(0xb2b9LL), MASK_S_RD, INSTR_S_RD, 2, 5},
@@ -1774,22 +1925,6 @@ static const struct s390_opcode s390_opcodes[] =
   { "sckpf", OP16(0x0107LL), MASK_E, INSTR_E, 3, 0},
   { "upt", OP16(0x0102LL), MASK_E, INSTR_E, 3, 0},
   { "pr", OP16(0x0101LL), MASK_E, INSTR_E, 3, 0},
-
-/* QEMU-ADD: */
-  { "crj",   OP48(0xec0000000076LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "cgrj",  OP48(0xec0000000064LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "clrj",  OP48(0xec0000000077LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "clgrj", OP48(0xec0000000065LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-
-  { "cij",   OP48(0xec000000007eLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "cgij",  OP48(0xec000000007cLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "clij",  OP48(0xec000000007fLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "clgij", OP48(0xec000000007dLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-
-  { "lrl",   OP16(0xc40dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-  { "lgrl",  OP16(0xc408ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-  { "lgfrl", OP16(0xc40cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-/* QEMU-END */
 };
 
 static const int s390_num_opcodes =
diff --git a/sh4-dis.c b/disas/sh4.c
index 673bc783801..f6cadd55c00 100644
--- a/sh4-dis.c
+++ b/disas/sh4.c
@@ -16,7 +16,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 #define DEFINE_TABLE
 
diff --git a/sparc-dis.c b/disas/sparc.c
index cdd337a7386..8eb22e6fc3a 100644
--- a/sparc-dis.c
+++ b/disas/sparc.c
@@ -27,7 +27,7 @@
    see <http://www.gnu.org/licenses/>.  */
 
 #include <stdlib.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* The SPARC opcode table (and other related data) is defined in
    the opcodes library in sparc-opc.c.  If you change anything here, make
@@ -3270,6 +3270,6 @@ print_insn_sparc (bfd_vma memaddr, disassemble_info *info)
     }
 
   info->insn_type = dis_noninsn;        /* Mark as non-valid instruction.  */
-  (*info->fprintf_func) (stream, _("unknown"));
+  (*info->fprintf_func) (stream, ".long %#08lx", insn);
   return sizeof (buffer);
 }
diff --git a/tci-dis.c b/disas/tci.c
index 10c411be8c1..a606b63a2a8 100644
--- a/tci-dis.c
+++ b/disas/tci.c
@@ -17,7 +17,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include "tcg/tcg.h"
 
 /* Disassemble TCI bytecode. */
diff --git a/dma-helpers.c b/dma-helpers.c
index 433d8b21b34..272632f3671 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -7,14 +7,15 @@
  * (GNU GPL), version 2 or later.
  */
 
-#include "dma.h"
+#include "sysemu/dma.h"
 #include "trace.h"
-#include "range.h"
-#include "qemu-thread.h"
+#include "qemu/range.h"
+#include "qemu/thread.h"
 
 /* #define DEBUG_IOMMU */
 
-static void do_dma_memory_set(dma_addr_t addr, uint8_t c, dma_addr_t len)
+static void do_dma_memory_set(AddressSpace *as,
+                              dma_addr_t addr, uint8_t c, dma_addr_t len)
 {
 #define FILLBUF_SIZE 512
     uint8_t fillbuf[FILLBUF_SIZE];
@@ -23,7 +24,7 @@ static void do_dma_memory_set(dma_addr_t addr, uint8_t c, dma_addr_t len)
     memset(fillbuf, c, FILLBUF_SIZE);
     while (len > 0) {
         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
-        cpu_physical_memory_rw(addr, fillbuf, l, true);
+        address_space_rw(as, addr, fillbuf, l, true);
         len -= l;
         addr += l;
     }
@@ -36,7 +37,7 @@ int dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, dma_addr_t len)
     if (dma_has_iommu(dma)) {
         return iommu_dma_memory_set(dma, addr, c, len);
     }
-    do_dma_memory_set(addr, c, len);
+    do_dma_memory_set(dma->as, addr, c, len);
 
     return 0;
 }
@@ -194,7 +195,7 @@ static void dma_aio_cancel(BlockDriverAIOCB *acb)
     dma_complete(dbs, 0);
 }
 
-static AIOPool dma_aio_pool = {
+static const AIOCBInfo dma_aiocb_info = {
     .aiocb_size         = sizeof(DMAAIOCB),
     .cancel             = dma_aio_cancel,
 };
@@ -204,7 +205,7 @@ BlockDriverAIOCB *dma_bdrv_io(
     DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
     void *opaque, DMADirection dir)
 {
-    DMAAIOCB *dbs = qemu_aio_get(&dma_aio_pool, bs, cb, opaque);
+    DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque);
 
     trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
 
@@ -280,7 +281,7 @@ void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
 bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len,
                             DMADirection dir)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
 
 #ifdef DEBUG_IOMMU
     fprintf(stderr, "dma_memory_check context=%p addr=0x" DMA_ADDR_FMT
@@ -307,7 +308,7 @@ bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len,
 int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
                         void *buf, dma_addr_t len, DMADirection dir)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     int err;
 
 #ifdef DEBUG_IOMMU
@@ -332,8 +333,7 @@ int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
             plen = len;
         }
 
-        cpu_physical_memory_rw(paddr, buf, plen,
-                               dir == DMA_DIRECTION_FROM_DEVICE);
+        address_space_rw(dma->as, paddr, buf, plen, dir == DMA_DIRECTION_FROM_DEVICE);
 
         len -= plen;
         addr += plen;
@@ -346,7 +346,7 @@ int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
 int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
                          dma_addr_t len)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     int err;
 
 #ifdef DEBUG_IOMMU
@@ -366,7 +366,7 @@ int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
             plen = len;
         }
 
-        do_dma_memory_set(paddr, c, plen);
+        do_dma_memory_set(dma->as, paddr, c, plen);
 
         len -= plen;
         addr += plen;
@@ -375,13 +375,14 @@ int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
     return 0;
 }
 
-void dma_context_init(DMAContext *dma, DMATranslateFunc translate,
+void dma_context_init(DMAContext *dma, AddressSpace *as, DMATranslateFunc translate,
                       DMAMapFunc map, DMAUnmapFunc unmap)
 {
 #ifdef DEBUG_IOMMU
     fprintf(stderr, "dma_context_init(%p, %p, %p, %p)\n",
             dma, translate, map, unmap);
 #endif
+    dma->as = as;
     dma->translate = translate;
     dma->map = map;
     dma->unmap = unmap;
@@ -391,7 +392,7 @@ void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len,
                            DMADirection dir)
 {
     int err;
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     void *buf;
 
     if (dma->map) {
@@ -407,14 +408,13 @@ void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len,
     /*
      * If this is true, the virtual region is contiguous,
      * but the translated physical region isn't. We just
-     * clamp *len, much like cpu_physical_memory_map() does.
+     * clamp *len, much like address_space_map() does.
      */
     if (plen < *len) {
         *len = plen;
     }
 
-    buf = cpu_physical_memory_map(paddr, &plen,
-                                  dir == DMA_DIRECTION_FROM_DEVICE);
+    buf = address_space_map(dma->as, paddr, &plen, dir == DMA_DIRECTION_FROM_DEVICE);
     *len = plen;
 
     return buf;
@@ -428,8 +428,7 @@ void iommu_dma_memory_unmap(DMAContext *dma, void *buffer, dma_addr_t len,
         return;
     }
 
-    cpu_physical_memory_unmap(buffer, len,
-                              dir == DMA_DIRECTION_FROM_DEVICE,
-                              access_len);
+    address_space_unmap(dma->as, buffer, len, dir == DMA_DIRECTION_FROM_DEVICE,
+                        access_len);
 
 }
diff --git a/docs/q35-chipset.cfg b/docs/q35-chipset.cfg
new file mode 100644
index 00000000000..1b6efc0f2c1
--- /dev/null
+++ b/docs/q35-chipset.cfg
@@ -0,0 +1,129 @@
+################################################################
+#
+# qemu -M q35 creates a bare machine with just the very essential
+# chipset devices being present:
+#
+#     00.0 - Host bridge
+#     1f.0 - ISA bridge / LPC
+#     1f.2 - SATA (AHCI) controller
+#     1f.3 - SMBus controller
+#
+# This config file documents the other devices and how they are
+# created.  You can simply use "-readconfig $thisfile" to create
+# them all.  Here is a overview:
+#
+#     19.0 - Ethernet controller (not created, our e1000 emulation
+#                                 doesn't emulate the ich9 device).
+#     1a.* - USB Controller #2 (ehci + uhci companions)
+#     1b.0 - HD Audio Controller
+#     1c.* - PCI Express Ports
+#     1d.* - USB Controller #1 (ehci + uhci companions,
+#                               "qemu -M q35 -usb" creates these too)
+#     1e.0 - PCI Bridge
+#
+
+[device "ich9-ehci-2"]
+  driver = "ich9-usb-ehci2"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.7"
+
+[device "ich9-uhci-4"]
+  driver = "ich9-usb-uhci4"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.0"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "0"
+
+[device "ich9-uhci-5"]
+  driver = "ich9-usb-uhci5"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.1"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "2"
+
+[device "ich9-uhci-6"]
+  driver = "ich9-usb-uhci6"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.2"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "4"
+
+
+[device "ich9-hda-audio"]
+  driver = "ich9-intel-hda"
+  bus = "pcie.0"
+  addr = "1b.0"
+
+
+[device "ich9-pcie-port-1"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.0"
+  port = "1"
+  chassis = "1"
+
+[device "ich9-pcie-port-2"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.1"
+  port = "2"
+  chassis = "2"
+
+[device "ich9-pcie-port-3"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.2"
+  port = "3"
+  chassis = "3"
+
+[device "ich9-pcie-port-4"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.3"
+  port = "4"
+  chassis = "4"
+
+
+[device "ich9-ehci-1"]
+  driver = "ich9-usb-ehci1"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.7"
+
+[device "ich9-uhci-1"]
+  driver = "ich9-usb-uhci1"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.0"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "0"
+
+[device "ich9-uhci-2"]
+  driver = "ich9-usb-uhci2"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.1"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "2"
+
+[device "ich9-uhci-3"]
+  driver = "ich9-usb-uhci3"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.2"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "4"
+
+
+[device "ich9-pci-bridge"]
+  driver = "i82801b11-bridge"
+  bus = "pcie.0"
+  addr = "1e.0"
diff --git a/docs/qemupciserial.inf b/docs/qemupciserial.inf
new file mode 100644
index 00000000000..3474310a4b7
--- /dev/null
+++ b/docs/qemupciserial.inf
@@ -0,0 +1,109 @@
+; qemupciserial.inf for QEMU, based on MSPORTS.INF
+
+; The driver itself is shipped with Windows (serial.sys).  This is
+; just a inf file to tell windows which pci id the serial pci card
+; emulated by qemu has, and to apply a name tag to it which windows
+; will show in the device manager.
+
+; Installing the driver: Go to device manager.  You should find a "pci
+; serial card" tagged with a yellow question mark.  Open properties.
+; Pick "update driver".  Then "select driver manually".  Pick "Ports
+; (Com+Lpt)" from the list.  Click "Have a disk".  Select this file.
+; Procedure may vary a bit depending on the windows version.
+
+; FIXME: This file covers the single port version only.
+
+[Version]
+Signature="$CHICAGO$"
+Class=Ports
+ClassGuid={4D36E978-E325-11CE-BFC1-08002BE10318}
+Provider=%QEMU%
+DriverVer=09/24/2012,1.3.0
+
+[SourceDisksNames]
+3426=windows cd
+
+[SourceDisksFiles]
+serial.sys 		= 3426
+serenum.sys 		= 3426
+
+[DestinationDirs]
+DefaultDestDir  = 11        ;LDID_SYS
+ComPort.NT.Copy = 12        ;DIRID_DRIVERS
+SerialEnumerator.NT.Copy=12 ;DIRID_DRIVERS
+
+; Drivers
+;----------------------------------------------------------
+[Manufacturer]
+%QEMU%=QEMU,NTx86
+
+[QEMU.NTx86]
+%QEMU-PCI_SERIAL.DeviceDesc% = ComPort, "PCI\VEN_1b36&DEV_0002&CC_0700"
+
+; COM sections
+;----------------------------------------------------------
+[ComPort.AddReg]
+HKR,,PortSubClass,1,01
+
+[ComPort.NT]
+AddReg=ComPort.AddReg, ComPort.NT.AddReg
+LogConfig=caa
+SyssetupPnPFlags = 1
+
+[ComPort.NT.HW]
+AddReg=ComPort.NT.HW.AddReg
+
+[ComPort.NT.AddReg]
+HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider"
+
+[ComPort.NT.HW.AddReg]
+HKR,,"UpperFilters",0x00010000,"serenum"
+
+;-------------- Service installation
+; Port Driver (function driver for this device)
+[ComPort.NT.Services]
+AddService = Serial, 0x00000002, Serial_Service_Inst, Serial_EventLog_Inst
+AddService = Serenum,,Serenum_Service_Inst
+
+; -------------- Serial Port Driver install sections
+[Serial_Service_Inst]
+DisplayName    = %Serial.SVCDESC%
+ServiceType    = 1               ; SERVICE_KERNEL_DRIVER
+StartType      = 1               ; SERVICE_SYSTEM_START (this driver may do detection)
+ErrorControl   = 0               ; SERVICE_ERROR_IGNORE
+ServiceBinary  = %12%\serial.sys
+LoadOrderGroup = Extended base
+
+; -------------- Serenum Driver install section
+[Serenum_Service_Inst]
+DisplayName    = %Serenum.SVCDESC%
+ServiceType    = 1               ; SERVICE_KERNEL_DRIVER
+StartType      = 3               ; SERVICE_DEMAND_START
+ErrorControl   = 1               ; SERVICE_ERROR_NORMAL
+ServiceBinary  = %12%\serenum.sys
+LoadOrderGroup = PNP Filter
+
+[Serial_EventLog_Inst]
+AddReg = Serial_EventLog_AddReg
+
+[Serial_EventLog_AddReg]
+HKR,,EventMessageFile,0x00020000,"%%SystemRoot%%\System32\IoLogMsg.dll;%%SystemRoot%%\System32\drivers\serial.sys"
+HKR,,TypesSupported,0x00010001,7
+
+; The following sections are COM port resource configs.
+; Section name format means:
+; Char 1 = c (COM port)
+; Char 2 = I/O config: 1 (3f8), 2 (2f8), 3 (3e8), 4 (2e8), a (any)
+; Char 3 = IRQ config: #, a (any)
+
+[caa]                   ; Any base, any IRQ
+ConfigPriority=HARDRECONFIG
+IOConfig=8@100-ffff%fff8(3ff::)
+IRQConfig=S:3,4,5,7,9,10,11,12,14,15
+
+[Strings]
+QEMU="QEMU"
+QEMU-PCI_SERIAL.DeviceDesc="QEMU Serial PCI Card"
+
+Serial.SVCDESC   = "Serial port driver"
+Serenum.SVCDESC = "Serenum Filter Driver"
diff --git a/docs/specs/pci-ids.txt b/docs/specs/pci-ids.txt
new file mode 100644
index 00000000000..3c65e1a6ef5
--- /dev/null
+++ b/docs/specs/pci-ids.txt
@@ -0,0 +1,50 @@
+
+PCI IDs for qemu
+================
+
+Red Hat, Inc. donates a part of its device ID range to qemu, to be used for
+virtual devices.  The vendor IDs are 1af4 (formerly Qumranet ID) and 1b36.
+
+Contact Gerd Hoffmann <kraxel@redhat.com> to get a device ID assigned
+for your devices.
+
+1af4 vendor ID
+--------------
+
+The 1000 -> 10ff device ID range is used as follows for virtio-pci devices.
+Note that this allocation separate from the virtio device IDs, which are
+maintained as part of the virtio specification.
+
+1af4:1000  network device
+1af4:1001  block device
+1af4:1002  balloon device
+1af4:1003  console device
+1af4:1004  SCSI host bus adapter device
+1af4:1005  entropy generator device
+1af4:1009  9p filesystem device
+
+1af4:10f0  Available for experimental usage without registration.  Must get
+   to      official ID when the code leaves the test lab (i.e. when seeking
+1af4:10ff  upstream merge or shipping a distro/product) to avoid conflicts.
+
+1af4:1100  Used as PCI Subsystem ID for existing hardware devices emulated
+           by qemu.
+
+1af4:1110  ivshmem device (shared memory, docs/specs/ivshmem_device_spec.txt)
+
+All other device IDs are reserved.
+
+1b36 vendor ID
+--------------
+
+The 0000 -> 00ff device ID range is used as follows for QEMU-specific
+PCI devices (other than virtio):
+
+1b36:0001  PCI-PCI bridge
+1b36:0002  PCI serial port (16550A) adapter (docs/specs/pci-serial.txt)
+1b36:0003  PCI Dual-port 16550A adapter (docs/specs/pci-serial.txt)
+1b36:0004  PCI Quad-port 16550A adapter (docs/specs/pci-serial.txt)
+
+All these devices are documented in docs/specs.
+
+The 0100 device ID is used for the QXL video card device.
diff --git a/docs/specs/pci-serial.txt b/docs/specs/pci-serial.txt
new file mode 100644
index 00000000000..66c761f2b40
--- /dev/null
+++ b/docs/specs/pci-serial.txt
@@ -0,0 +1,34 @@
+
+QEMU pci serial devices
+=======================
+
+There is one single-port variant and two muliport-variants.  Linux
+guests out-of-the box with all cards.  There is a Windows inf file
+(docs/qemupciserial.inf) to setup the single-port card in Windows
+guests.
+
+
+single-port card
+----------------
+
+Name:   pci-serial
+PCI ID: 1b36:0002
+
+PCI Region 0:
+   IO bar, 8 bytes long, with the 16550 uart mapped to it.
+   Interrupt is wired to pin A.
+
+
+multiport cards
+---------------
+
+Name:   pci-serial-2x
+PCI ID: 1b36:0003
+
+Name:   pci-serial-4x
+PCI ID: 1b36:0004
+
+PCI Region 0:
+   IO bar, with two/four 16550 uart mapped after each other.
+   The first is at offset 0, second at offset 8, ...
+   Interrupt is wired to pin A.
diff --git a/docs/specs/ppc-spapr-hcalls.txt b/docs/specs/ppc-spapr-hcalls.txt
index 52ba8d42ab2..667b3fa00e2 100644
--- a/docs/specs/ppc-spapr-hcalls.txt
+++ b/docs/specs/ppc-spapr-hcalls.txt
@@ -31,7 +31,7 @@ Arguments:
 
 Returns:
 
-  H_SUCCESS   : Successully called the RTAS function (RTAS result
+  H_SUCCESS   : Successfully called the RTAS function (RTAS result
                 will have been stored in the parameter block)
   H_PARAMETER : Unknown token
 
diff --git a/docs/specs/standard-vga.txt b/docs/specs/standard-vga.txt
new file mode 100644
index 00000000000..8a4c1e93cd4
--- /dev/null
+++ b/docs/specs/standard-vga.txt
@@ -0,0 +1,65 @@
+
+QEMU Standard VGA
+=================
+
+Exists in two variants, for isa and pci.
+
+command line switches:
+    -vga std            [ picks isa for -M isapc, otherwise pci ]
+    -device VGA         [ pci variant ]
+    -device isa-vga     [ isa variant ]
+
+
+PCI spec
+--------
+
+Applies to the pci variant only for obvious reasons.
+
+PCI ID: 1234:1111
+
+PCI Region 0:
+   Framebuffer memory, 16 MB in size (by default).
+   Size is tunable via vga_mem_mb property.
+
+PCI Region 1:
+   Reserved (so we have the option to make the framebuffer bar 64bit).
+
+PCI Region 2:
+   MMIO bar, 4096 bytes in size (qemu 1.3+)
+
+PCI ROM Region:
+   Holds the vgabios (qemu 0.14+).
+
+
+IO ports used
+-------------
+
+03c0 - 03df : standard vga ports
+01ce        : bochs vbe interface index port
+01cf        : bochs vbe interface data port (x86 only)
+01d0        : bochs vbe interface data port
+
+
+Memory regions used
+-------------------
+
+0xe0000000 : Framebuffer memory, isa variant only.
+
+The pci variant used to mirror the framebuffer bar here, qemu 0.14+
+stops doing that (except when in -M pc-$old compat mode).
+
+
+MMIO area spec
+--------------
+
+Likewise applies to the pci variant only for obvious reasons.
+
+0000 - 03ff : reserved, for possible virtio extension.
+0400 - 041f : vga ioports (0x3c0 -> 0x3df), remapped 1:1.
+              word access is supported, bytes are written
+              in little endia order (aka index port first),
+              so indexed registers can be updated with a
+              single mmio write (and thus only one vmexit).
+0500 - 0515 : bochs dispi interface registers, mapped flat
+              without index/data ports.  Use (index << 1)
+              as offset for (16bit) register access.
diff --git a/docs/spice-port-fqdn.txt b/docs/spice-port-fqdn.txt
new file mode 100644
index 00000000000..50778952e52
--- /dev/null
+++ b/docs/spice-port-fqdn.txt
@@ -0,0 +1,19 @@
+A Spice port channel is an arbitrary communication between the Spice
+server host side and the client side.
+
+Thanks to the associated reverse fully qualified domain name (fqdn),
+a Spice client can handle the various ports appropriately.
+
+The following fqdn names are reserved by the QEMU project:
+
+org.qemu.monitor.hmp.0
+  QEMU human monitor
+
+org.qemu.monitor.qmp.0:
+  QEMU control monitor
+
+org.qemu.console.serial.0
+  QEMU virtual serial port
+
+org.qemu.console.debug.0
+  QEMU debug console
diff --git a/docs/tracing.txt b/docs/tracing.txt
index c5411333689..453cc4a63d9 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -139,6 +139,10 @@ having a common prefix in a batch. For example, virtio-blk trace events could
 be enabled using:
   trace-event virtio_blk_* on
 
+If a line in the "-trace events=<file>" file begins with a '-', the trace event
+will be disabled instead of enabled.  This is useful when a wildcard was used
+to enable an entire family of events but one noisy event needs to be disabled.
+
 == Trace backends ==
 
 The "tracetool" script automates tedious trace event code generation and also
@@ -185,15 +189,6 @@ records the char* pointer value instead of the string that is pointed to.
 
 ==== Monitor commands ====
 
-* info trace
-  Display the contents of trace buffer.  This command dumps the trace buffer
-  with simple formatting.  For full pretty-printing, use the simpletrace.py
-  script on a binary trace file.
-
-  The trace buffer is written into until full.  The full trace buffer is
-  flushed and emptied.  This means the 'info trace' will display few or no
-  entries if the buffer has just been flushed.
-
 * trace-file on|off|flush|set <path>
   Enable/disable/flush the trace file or set the trace file name.
 
diff --git a/docs/usb-storage.txt b/docs/usb-storage.txt
index e58e849d4d3..fa93111cf68 100644
--- a/docs/usb-storage.txt
+++ b/docs/usb-storage.txt
@@ -2,7 +2,7 @@
 qemu usb storage emulation
 --------------------------
 
-QEMU has two emulations for usb storage devices.
+QEMU has three devices for usb storage emulation.
 
 Number one emulates the classic bulk-only transport protocol which is
 used by 99% of the usb sticks on the marked today and is called
@@ -31,6 +31,15 @@ with tree logical units:
        -device scsi-cd,bus=uas.0,scsi-id=0,lun=5,drive=uas-cdrom
 
 
+Number three emulates the classic bulk-only transport protocol too.
+It's called "usb-bot".  It shares most code with "usb-storage", and
+the guest will not be able to see the difference.  The qemu command
+line interface is simliar to usb-uas though, i.e. no automatic scsi
+disk creation.  It also features support for up to 16 LUNs.  The LUN
+numbers must be continous, i.e. for three devices you must use 0+1+2.
+The 0+1+5 numbering from the "usb-uas" example isn't going to work
+with "usb-bot".
+
 enjoy,
   Gerd
 
diff --git a/docs/usb2.txt b/docs/usb2.txt
index d17e3c0044b..43dacdec283 100644
--- a/docs/usb2.txt
+++ b/docs/usb2.txt
@@ -58,11 +58,11 @@ try ...
 xhci controller support
 -----------------------
 
-There also is xhci host controller support available.  It got alot
+There is also xhci host controller support available.  It got a lot
 less testing than ehci and there are a bunch of known limitations, so
 ehci may work better for you.  On the other hand the xhci hardware
 design is much more virtualization-friendly, thus xhci emulation uses
-less ressources (especially cpu).  If you wanna give xhci a try
+less resources (especially cpu).  If you want to give xhci a try
 use this to add the host controller ...
 
     qemu -device nec-usb-xhci,id=xhci
diff --git a/docs/virtio-balloon-stats.txt b/docs/virtio-balloon-stats.txt
new file mode 100644
index 00000000000..f74612f468d
--- /dev/null
+++ b/docs/virtio-balloon-stats.txt
@@ -0,0 +1,104 @@
+virtio balloon memory statistics
+================================
+
+The virtio balloon driver supports guest memory statistics reporting. These
+statistics are available to QEMU users as QOM (QEMU Object Model) device
+properties via a polling mechanism.
+
+Before querying the available stats, clients first have to enable polling.
+This is done by writing a time interval value (in seconds) to the
+guest-stats-polling-interval property. This value can be:
+
+  > 0  enables polling in the specified interval. If polling is already
+       enabled, the polling time interval is changed to the new value
+
+  0    disables polling. Previous polled statistics are still valid and
+       can be queried.
+
+Once polling is enabled, the virtio-balloon device in QEMU will start
+polling the guest's balloon driver for new stats in the specified time
+interval.
+
+To retrieve those stats, clients have to query the guest-stats property,
+which will return a dictionary containing:
+
+  o A key named 'stats', containing all available stats. If the guest
+    doesn't support a particular stat, or if it couldn't be retrieved,
+    its value will be -1. Currently, the following stats are supported:
+
+      - stat-swap-in
+      - stat-swap-out
+      - stat-major-faults
+      - stat-minor-faults
+      - stat-free-memory
+      - stat-total-memory
+
+  o A key named last-update, which contains the last stats update
+    timestamp in seconds. Since this timestamp is generated by the host,
+    a buggy guest can't influence its value
+
+It's also important to note the following:
+
+ - Previously polled statistics remain available even if the polling is
+   later disabled
+
+ - As noted above, if a guest doesn't support a particular stat its value
+   will always be -1. However, it's also possible that a guest temporarily
+   couldn't update one or even all stats. If this happens, just wait for
+   the next update
+
+ - Polling can be enabled even if the guest doesn't have stats support
+   or the balloon driver wasn't loaded in the guest. If this is the case
+   and stats are queried, an error will be returned
+
+ - The polling timer is only re-armed when the guest responds to the
+   statistics request. This means that if a (buggy) guest doesn't ever
+   respond to the request the timer will never be re-armed, which has
+   the same effect as disabling polling
+
+Here are a few examples. QEMU is started with '-balloon virtio', which
+generates '/machine/peripheral-anon/device[1]' as the QOM path for the
+balloon device.
+
+Enable polling with 2 seconds interval:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "guest-stats-polling-interval", "value": 2 } }
+
+{ "return": {} }
+
+Change polling to 10 seconds:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "guest-stats-polling-interval", "value": 10 } }
+
+{ "return": {} }
+
+Get stats:
+
+{ "execute": "qom-get",
+  "arguments": { "path": "/machine/peripheral-anon/device[1]",
+  "property": "guest-stats" } }
+{
+    "return": {
+        "stats": {
+            "stat-swap-out": 0,
+            "stat-free-memory": 844943360,
+            "stat-minor-faults": 219028,
+            "stat-major-faults": 235,
+            "stat-total-memory": 1044406272,
+            "stat-swap-in": 0
+        },
+        "last-update": 1358529861
+    }
+}
+
+Disable polling:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "stats-polling-interval", "value": 0 } }
+
+{ "return": {} }
diff --git a/dump-stub.c b/dump-stub.c
index 56d4564f0fc..a9d0b3c67b1 100644
--- a/dump-stub.c
+++ b/dump-stub.c
@@ -12,8 +12,8 @@
  */
 
 #include "qemu-common.h"
-#include "dump.h"
-#include "qerror.h"
+#include "sysemu/dump.h"
+#include "qapi/qmp/qerror.h"
 #include "qmp-commands.h"
 
 /* we need this function in hmp.c */
diff --git a/dump.c b/dump.c
index 2bf8d8d9942..4ed1fa8622c 100644
--- a/dump.c
+++ b/dump.c
@@ -14,16 +14,16 @@
 #include "qemu-common.h"
 #include "elf.h"
 #include "cpu.h"
-#include "cpu-all.h"
-#include "targphys.h"
-#include "monitor.h"
-#include "kvm.h"
-#include "dump.h"
-#include "sysemu.h"
-#include "memory_mapping.h"
-#include "error.h"
+#include "exec/cpu-all.h"
+#include "exec/hwaddr.h"
+#include "monitor/monitor.h"
+#include "sysemu/kvm.h"
+#include "sysemu/dump.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/memory_mapping.h"
+#include "qapi/error.h"
 #include "qmp-commands.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 static uint16_t cpu_convert_to_target16(uint16_t val, int endian)
 {
@@ -66,7 +66,7 @@ typedef struct DumpState {
     bool have_section;
     bool resume;
     size_t note_size;
-    target_phys_addr_t memory_offset;
+    hwaddr memory_offset;
     int fd;
 
     RAMBlock *block;
@@ -100,18 +100,11 @@ static void dump_error(DumpState *s, const char *reason)
 static int fd_write_vmcore(void *buf, size_t size, void *opaque)
 {
     DumpState *s = opaque;
-    int fd = s->fd;
-    size_t writen_size;
+    size_t written_size;
 
-    /* The fd may be passed from user, and it can be non-blocked */
-    while (size) {
-        writen_size = qemu_write_full(fd, buf, size);
-        if (writen_size != size && errno != EAGAIN) {
-            return -1;
-        }
-
-        buf += writen_size;
-        size -= writen_size;
+    written_size = qemu_write_full(s->fd, buf, size);
+    if (written_size != size) {
+        return -1;
     }
 
     return 0;
@@ -194,7 +187,7 @@ static int write_elf32_header(DumpState *s)
 }
 
 static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, target_phys_addr_t offset)
+                            int phdr_index, hwaddr offset)
 {
     Elf64_Phdr phdr;
     int ret;
@@ -223,7 +216,7 @@ static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
 }
 
 static int write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, target_phys_addr_t offset)
+                            int phdr_index, hwaddr offset)
 {
     Elf32_Phdr phdr;
     int ret;
@@ -255,7 +248,7 @@ static int write_elf64_note(DumpState *s)
 {
     Elf64_Phdr phdr;
     int endian = s->dump_info.d_endian;
-    target_phys_addr_t begin = s->memory_offset - s->note_size;
+    hwaddr begin = s->memory_offset - s->note_size;
     int ret;
 
     memset(&phdr, 0, sizeof(Elf64_Phdr));
@@ -303,7 +296,7 @@ static int write_elf64_notes(DumpState *s)
 
 static int write_elf32_note(DumpState *s)
 {
-    target_phys_addr_t begin = s->memory_offset - s->note_size;
+    hwaddr begin = s->memory_offset - s->note_size;
     Elf32_Phdr phdr;
     int endian = s->dump_info.d_endian;
     int ret;
@@ -421,11 +414,11 @@ static int write_memory(DumpState *s, RAMBlock *block, ram_addr_t start,
 }
 
 /* get the memory's offset in the vmcore */
-static target_phys_addr_t get_offset(target_phys_addr_t phys_addr,
+static hwaddr get_offset(hwaddr phys_addr,
                                      DumpState *s)
 {
     RAMBlock *block;
-    target_phys_addr_t offset = s->memory_offset;
+    hwaddr offset = s->memory_offset;
     int64_t size_in_block, start;
 
     if (s->has_filter) {
@@ -434,7 +427,7 @@ static target_phys_addr_t get_offset(target_phys_addr_t phys_addr,
         }
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (s->has_filter) {
             if (block->offset >= s->begin + s->length ||
                 block->offset + block->length <= s->begin) {
@@ -470,7 +463,7 @@ static target_phys_addr_t get_offset(target_phys_addr_t phys_addr,
 
 static int write_elf_loads(DumpState *s)
 {
-    target_phys_addr_t offset;
+    hwaddr offset;
     MemoryMapping *memory_mapping;
     uint32_t phdr_index = 1;
     int ret;
@@ -601,7 +594,7 @@ static int dump_completed(DumpState *s)
 static int get_next_block(DumpState *s, RAMBlock *block)
 {
     while (1) {
-        block = QLIST_NEXT(block, next);
+        block = QTAILQ_NEXT(block, next);
         if (!block) {
             /* no more block */
             return 1;
@@ -677,11 +670,11 @@ static ram_addr_t get_start_block(DumpState *s)
     RAMBlock *block;
 
     if (!s->has_filter) {
-        s->block = QLIST_FIRST(&ram_list.blocks);
+        s->block = QTAILQ_FIRST(&ram_list.blocks);
         return 0;
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block->offset >= s->begin + s->length ||
             block->offset + block->length <= s->begin) {
             /* This block is out of the range */
@@ -836,9 +829,8 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
 
 #if !defined(WIN32)
     if (strstart(file, "fd:", &p)) {
-        fd = monitor_get_fd(cur_mon, p);
+        fd = monitor_get_fd(cur_mon, p, errp);
         if (fd == -1) {
-            error_set(errp, QERR_FD_NOT_FOUND, p);
             return;
         }
     }
diff --git a/dyngen-exec.h b/dyngen-exec.h
deleted file mode 100644
index 083e20bed78..00000000000
--- a/dyngen-exec.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- *  dyngen defines for micro operation code
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-#if !defined(__DYNGEN_EXEC_H__)
-#define __DYNGEN_EXEC_H__
-
-#if defined(CONFIG_TCG_INTERPRETER)
-/* The TCG interpreter does not need a special register AREG0,
- * but it is possible to use one by defining AREG0.
- * On i386, register edi seems to work. */
-/* Run without special register AREG0 or use a value defined elsewhere. */
-#elif defined(__i386__)
-#define AREG0 "ebp"
-#elif defined(__x86_64__)
-#define AREG0 "r14"
-#elif defined(_ARCH_PPC)
-#define AREG0 "r27"
-#elif defined(__arm__)
-#define AREG0 "r6"
-#elif defined(__hppa__)
-#define AREG0 "r17"
-#elif defined(__mips__)
-#define AREG0 "s0"
-#elif defined(__sparc__)
-#ifdef CONFIG_SOLARIS
-#define AREG0 "g2"
-#else
-#ifdef __sparc_v9__
-#define AREG0 "g5"
-#else
-#define AREG0 "g6"
-#endif
-#endif
-#elif defined(__s390__)
-#define AREG0 "r10"
-#elif defined(__alpha__)
-/* Note $15 is the frame pointer, so anything in op-i386.c that would
-   require a frame pointer, like alloca, would probably loose.  */
-#define AREG0 "$15"
-#elif defined(__mc68000)
-#define AREG0 "%a5"
-#elif defined(__ia64__)
-#define AREG0 "r7"
-#else
-#error unsupported CPU
-#endif
-
-#if defined(AREG0)
-register CPUArchState *env asm(AREG0);
-#else
-/* TODO: Try env = cpu_single_env. */
-extern CPUArchState *env;
-#endif
-
-#endif /* !defined(__DYNGEN_EXEC_H__) */
diff --git a/event_notifier.c b/event_notifier.c
deleted file mode 100644
index 2c207e1399b..00000000000
--- a/event_notifier.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * event notifier support
- *
- * Copyright Red Hat, Inc. 2010
- *
- * Authors:
- *  Michael S. Tsirkin <mst@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-#include "qemu-common.h"
-#include "event_notifier.h"
-#include "qemu-char.h"
-
-#ifdef CONFIG_EVENTFD
-#include <sys/eventfd.h>
-#endif
-
-void event_notifier_init_fd(EventNotifier *e, int fd)
-{
-    e->fd = fd;
-}
-
-int event_notifier_init(EventNotifier *e, int active)
-{
-#ifdef CONFIG_EVENTFD
-    int fd = eventfd(!!active, EFD_NONBLOCK | EFD_CLOEXEC);
-    if (fd < 0)
-        return -errno;
-    e->fd = fd;
-    return 0;
-#else
-    return -ENOSYS;
-#endif
-}
-
-void event_notifier_cleanup(EventNotifier *e)
-{
-    close(e->fd);
-}
-
-int event_notifier_get_fd(EventNotifier *e)
-{
-    return e->fd;
-}
-
-int event_notifier_set_handler(EventNotifier *e,
-                               EventNotifierHandler *handler)
-{
-    return qemu_set_fd_handler(e->fd, (IOHandler *)handler, NULL, e);
-}
-
-int event_notifier_set(EventNotifier *e)
-{
-    uint64_t value = 1;
-    int r = write(e->fd, &value, sizeof(value));
-    return r == sizeof(value);
-}
-
-int event_notifier_test_and_clear(EventNotifier *e)
-{
-    uint64_t value;
-    int r = read(e->fd, &value, sizeof(value));
-    return r == sizeof(value);
-}
diff --git a/exec.c b/exec.c
index 58347664b21..b85508ba30d 100644
--- a/exec.c
+++ b/exec.c
@@ -1,5 +1,5 @@
 /*
- *  virtual page mapping and translated block handling
+ *  Virtual page mapping
  *
  *  Copyright (c) 2003 Fabrice Bellard
  *
@@ -29,94 +29,43 @@
 #include "tcg.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
-#include "osdep.h"
-#include "kvm.h"
+#include "qemu/osdep.h"
+#include "sysemu/kvm.h"
 #include "hw/xen.h"
-#include "qemu-timer.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "qemu/config-file.h"
+#include "exec/memory.h"
+#include "sysemu/dma.h"
+#include "exec/address-spaces.h"
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-#include <sys/param.h>
-#if __FreeBSD_version >= 700104
-#define HAVE_KINFO_GETVMMAP
-#define sigqueue sigqueue_freebsd  /* avoid redefinition */
-#include <sys/time.h>
-#include <sys/proc.h>
-#include <machine/profile.h>
-#define _KERNEL
-#include <sys/user.h>
-#undef _KERNEL
-#undef sigqueue
-#include <libutil.h>
-#endif
-#endif
 #else /* !CONFIG_USER_ONLY */
-#include "xen-mapcache.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
 #endif
+#include "exec/cpu-all.h"
 
-#include "cputlb.h"
+#include "exec/cputlb.h"
+#include "translate-all.h"
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "exec/memory-internal.h"
 
-//#define DEBUG_TB_INVALIDATE
-//#define DEBUG_FLUSH
 //#define DEBUG_UNASSIGNED
-
-/* make various TB consistency checks */
-//#define DEBUG_TB_CHECK
-
-//#define DEBUG_IOPORT
 //#define DEBUG_SUBPAGE
 
 #if !defined(CONFIG_USER_ONLY)
-/* TB consistency checks only implemented for usermode emulation.  */
-#undef DEBUG_TB_CHECK
-#endif
-
-#define SMC_BITMAP_USE_THRESHOLD 10
-
-static TranslationBlock *tbs;
-static int code_gen_max_blocks;
-TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
-static int nb_tbs;
-/* any access to the tbs or the page table must use this lock */
-spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
-
-#if defined(__arm__) || defined(__sparc_v9__)
-/* The prologue must be reachable with a direct jump. ARM and Sparc64
- have limited branch ranges (possibly also PPC) so place it in a
- section close to code segment. */
-#define code_gen_section                                \
-    __attribute__((__section__(".gen_code")))           \
-    __attribute__((aligned (32)))
-#elif defined(_WIN32) && !defined(_WIN64)
-#define code_gen_section                                \
-    __attribute__((aligned (16)))
-#else
-#define code_gen_section                                \
-    __attribute__((aligned (32)))
-#endif
-
-uint8_t code_gen_prologue[1024] code_gen_section;
-static uint8_t *code_gen_buffer;
-static unsigned long code_gen_buffer_size;
-/* threshold to flush the translated code buffer */
-static unsigned long code_gen_buffer_max_size;
-static uint8_t *code_gen_ptr;
-
-#if !defined(CONFIG_USER_ONLY)
 int phys_ram_fd;
 static int in_migration;
 
-RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
+RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
 
+AddressSpace address_space_io;
+AddressSpace address_space_memory;
+DMAContext dma_context_memory;
+
 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
 static MemoryRegion io_mem_subpage_ram;
 
@@ -129,63 +78,9 @@ DEFINE_TLS(CPUArchState *,cpu_single_env);
 /* 0 = Do not count executed instructions.
    1 = Precise instruction counting.
    2 = Adaptive rate instruction counting.  */
-int use_icount = 0;
-
-typedef struct PageDesc {
-    /* list of TBs intersecting this ram page */
-    TranslationBlock *first_tb;
-    /* in order to optimize self modifying code, we count the number
-       of lookups we do to a given page to use a bitmap */
-    unsigned int code_write_count;
-    uint8_t *code_bitmap;
-#if defined(CONFIG_USER_ONLY)
-    unsigned long flags;
-#endif
-} PageDesc;
-
-/* In system mode we want L1_MAP to be based on ram offsets,
-   while in user mode we want it to be based on virtual addresses.  */
-#if !defined(CONFIG_USER_ONLY)
-#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
-# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
-#else
-# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
-#endif
-#else
-# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
-#endif
-
-/* Size of the L2 (and L3, etc) page tables.  */
-#define L2_BITS 10
-#define L2_SIZE (1 << L2_BITS)
-
-#define P_L2_LEVELS \
-    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
-
-/* The bits remaining after N lower levels of page tables.  */
-#define V_L1_BITS_REM \
-    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
-
-#if V_L1_BITS_REM < 4
-#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
-#else
-#define V_L1_BITS  V_L1_BITS_REM
-#endif
-
-#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
-
-#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
-
-uintptr_t qemu_real_host_page_size;
-uintptr_t qemu_host_page_size;
-uintptr_t qemu_host_page_mask;
-
-/* This is a multi-level map on the virtual address space.
-   The bottom level has pointers to PageDesc.  */
-static void *l1_map[V_L1_SIZE];
+int use_icount;
 
 #if !defined(CONFIG_USER_ONLY)
-typedef struct PhysPageEntry PhysPageEntry;
 
 static MemoryRegionSection *phys_sections;
 static unsigned phys_sections_nb, phys_sections_nb_alloc;
@@ -194,199 +89,19 @@ static uint16_t phys_section_notdirty;
 static uint16_t phys_section_rom;
 static uint16_t phys_section_watch;
 
-struct PhysPageEntry {
-    uint16_t is_leaf : 1;
-     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
-    uint16_t ptr : 15;
-};
-
 /* Simple allocator for PhysPageEntry nodes */
 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
 
 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 
-/* This is a multi-level map on the physical address space.
-   The bottom level has pointers to MemoryRegionSections.  */
-static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
-
 static void io_mem_init(void);
 static void memory_map_init(void);
+static void *qemu_safe_ram_ptr(ram_addr_t addr);
 
 static MemoryRegion io_mem_watch;
 #endif
 
-/* statistics */
-static int tb_flush_count;
-static int tb_phys_invalidate_count;
-
-#ifdef _WIN32
-static void map_exec(void *addr, long size)
-{
-    DWORD old_protect;
-    VirtualProtect(addr, size,
-                   PAGE_EXECUTE_READWRITE, &old_protect);
-    
-}
-#else
-static void map_exec(void *addr, long size)
-{
-    unsigned long start, end, page_size;
-    
-    page_size = getpagesize();
-    start = (unsigned long)addr;
-    start &= ~(page_size - 1);
-    
-    end = (unsigned long)addr + size;
-    end += page_size - 1;
-    end &= ~(page_size - 1);
-    
-    mprotect((void *)start, end - start,
-             PROT_READ | PROT_WRITE | PROT_EXEC);
-}
-#endif
-
-static void page_init(void)
-{
-    /* NOTE: we can always suppose that qemu_host_page_size >=
-       TARGET_PAGE_SIZE */
-#ifdef _WIN32
-    {
-        SYSTEM_INFO system_info;
-
-        GetSystemInfo(&system_info);
-        qemu_real_host_page_size = system_info.dwPageSize;
-    }
-#else
-    qemu_real_host_page_size = getpagesize();
-#endif
-    if (qemu_host_page_size == 0)
-        qemu_host_page_size = qemu_real_host_page_size;
-    if (qemu_host_page_size < TARGET_PAGE_SIZE)
-        qemu_host_page_size = TARGET_PAGE_SIZE;
-    qemu_host_page_mask = ~(qemu_host_page_size - 1);
-
-#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
-    {
-#ifdef HAVE_KINFO_GETVMMAP
-        struct kinfo_vmentry *freep;
-        int i, cnt;
-
-        freep = kinfo_getvmmap(getpid(), &cnt);
-        if (freep) {
-            mmap_lock();
-            for (i = 0; i < cnt; i++) {
-                unsigned long startaddr, endaddr;
-
-                startaddr = freep[i].kve_start;
-                endaddr = freep[i].kve_end;
-                if (h2g_valid(startaddr)) {
-                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
-
-                    if (h2g_valid(endaddr)) {
-                        endaddr = h2g(endaddr);
-                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-                    } else {
-#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
-                        endaddr = ~0ul;
-                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-#endif
-                    }
-                }
-            }
-            free(freep);
-            mmap_unlock();
-        }
-#else
-        FILE *f;
-
-        last_brk = (unsigned long)sbrk(0);
-
-        f = fopen("/compat/linux/proc/self/maps", "r");
-        if (f) {
-            mmap_lock();
-
-            do {
-                unsigned long startaddr, endaddr;
-                int n;
-
-                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
-
-                if (n == 2 && h2g_valid(startaddr)) {
-                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
-
-                    if (h2g_valid(endaddr)) {
-                        endaddr = h2g(endaddr);
-                    } else {
-                        endaddr = ~0ul;
-                    }
-                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-                }
-            } while (!feof(f));
-
-            fclose(f);
-            mmap_unlock();
-        }
-#endif
-    }
-#endif
-}
-
-static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
-{
-    PageDesc *pd;
-    void **lp;
-    int i;
-
-#if defined(CONFIG_USER_ONLY)
-    /* We can't use g_malloc because it may recurse into a locked mutex. */
-# define ALLOC(P, SIZE)                                 \
-    do {                                                \
-        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
-                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
-    } while (0)
-#else
-# define ALLOC(P, SIZE) \
-    do { P = g_malloc0(SIZE); } while (0)
-#endif
-
-    /* Level 1.  Always allocated.  */
-    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
-
-    /* Level 2..N-1.  */
-    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
-
-        if (p == NULL) {
-            if (!alloc) {
-                return NULL;
-            }
-            ALLOC(p, sizeof(void *) * L2_SIZE);
-            *lp = p;
-        }
-
-        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
-    }
-
-    pd = *lp;
-    if (pd == NULL) {
-        if (!alloc) {
-            return NULL;
-        }
-        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
-        *lp = pd;
-    }
-
-#undef ALLOC
-
-    return pd + (index & (L2_SIZE - 1));
-}
-
-static inline PageDesc *page_find(tb_page_addr_t index)
-{
-    return page_find_alloc(index, 0);
-}
-
 #if !defined(CONFIG_USER_ONLY)
 
 static void phys_map_node_reserve(unsigned nodes)
@@ -422,13 +137,13 @@ static void phys_map_nodes_reset(void)
 }
 
 
-static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
-                                target_phys_addr_t *nb, uint16_t leaf,
+static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
+                                hwaddr *nb, uint16_t leaf,
                                 int level)
 {
     PhysPageEntry *p;
     int i;
-    target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS);
+    hwaddr step = (hwaddr)1 << (level * L2_BITS);
 
     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
         lp->ptr = phys_map_node_alloc();
@@ -457,18 +172,19 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
     }
 }
 
-static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,
+static void phys_page_set(AddressSpaceDispatch *d,
+                          hwaddr index, hwaddr nb,
                           uint16_t leaf)
 {
     /* Wildly overreserve - it doesn't matter much. */
     phys_map_node_reserve(3 * P_L2_LEVELS);
 
-    phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
+    phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 }
 
-MemoryRegionSection *phys_page_find(target_phys_addr_t index)
+MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 {
-    PhysPageEntry lp = phys_map;
+    PhysPageEntry lp = d->phys_map;
     PhysPageEntry *p;
     int i;
     uint16_t s_index = phys_section_unassigned;
@@ -492,149 +208,12 @@ bool memory_region_is_unassigned(MemoryRegion *mr)
         && mr != &io_mem_notdirty && !mr->rom_device
         && mr != &io_mem_watch;
 }
-
-#define mmap_lock() do { } while(0)
-#define mmap_unlock() do { } while(0)
-#endif
-
-#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
-
-#if defined(CONFIG_USER_ONLY)
-/* Currently it is not recommended to allocate big chunks of data in
-   user mode. It will change when a dedicated libc will be used */
-#define USE_STATIC_CODE_GEN_BUFFER
 #endif
 
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
-               __attribute__((aligned (CODE_GEN_ALIGN)));
-#endif
-
-static void code_gen_alloc(unsigned long tb_size)
-{
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-    code_gen_buffer = static_code_gen_buffer;
-    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
-    map_exec(code_gen_buffer, code_gen_buffer_size);
-#else
-    code_gen_buffer_size = tb_size;
-    if (code_gen_buffer_size == 0) {
-#if defined(CONFIG_USER_ONLY)
-        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
-#else
-        /* XXX: needs adjustments */
-        code_gen_buffer_size = (unsigned long)(ram_size / 4);
-#endif
-    }
-    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
-        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
-    /* The code gen buffer location may have constraints depending on
-       the host cpu and OS */
-#if defined(__linux__) 
-    {
-        int flags;
-        void *start = NULL;
-
-        flags = MAP_PRIVATE | MAP_ANONYMOUS;
-#if defined(__x86_64__)
-        flags |= MAP_32BIT;
-        /* Cannot map more than that */
-        if (code_gen_buffer_size > (800 * 1024 * 1024))
-            code_gen_buffer_size = (800 * 1024 * 1024);
-#elif defined(__sparc_v9__)
-        // Map the buffer below 2G, so we can use direct calls and branches
-        flags |= MAP_FIXED;
-        start = (void *) 0x60000000UL;
-        if (code_gen_buffer_size > (512 * 1024 * 1024))
-            code_gen_buffer_size = (512 * 1024 * 1024);
-#elif defined(__arm__)
-        /* Keep the buffer no bigger than 16MB to branch between blocks */
-        if (code_gen_buffer_size > 16 * 1024 * 1024)
-            code_gen_buffer_size = 16 * 1024 * 1024;
-#elif defined(__s390x__)
-        /* Map the buffer so that we can use direct calls and branches.  */
-        /* We have a +- 4GB range on the branches; leave some slop.  */
-        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
-            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
-        }
-        start = (void *)0x90000000UL;
-#endif
-        code_gen_buffer = mmap(start, code_gen_buffer_size,
-                               PROT_WRITE | PROT_READ | PROT_EXEC,
-                               flags, -1, 0);
-        if (code_gen_buffer == MAP_FAILED) {
-            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-            exit(1);
-        }
-    }
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__)
-    {
-        int flags;
-        void *addr = NULL;
-        flags = MAP_PRIVATE | MAP_ANONYMOUS;
-#if defined(__x86_64__)
-        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
-         * 0x40000000 is free */
-        flags |= MAP_FIXED;
-        addr = (void *)0x40000000;
-        /* Cannot map more than that */
-        if (code_gen_buffer_size > (800 * 1024 * 1024))
-            code_gen_buffer_size = (800 * 1024 * 1024);
-#elif defined(__sparc_v9__)
-        // Map the buffer below 2G, so we can use direct calls and branches
-        flags |= MAP_FIXED;
-        addr = (void *) 0x60000000UL;
-        if (code_gen_buffer_size > (512 * 1024 * 1024)) {
-            code_gen_buffer_size = (512 * 1024 * 1024);
-        }
-#endif
-        code_gen_buffer = mmap(addr, code_gen_buffer_size,
-                               PROT_WRITE | PROT_READ | PROT_EXEC, 
-                               flags, -1, 0);
-        if (code_gen_buffer == MAP_FAILED) {
-            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-            exit(1);
-        }
-    }
-#else
-    code_gen_buffer = g_malloc(code_gen_buffer_size);
-    map_exec(code_gen_buffer, code_gen_buffer_size);
-#endif
-#endif /* !USE_STATIC_CODE_GEN_BUFFER */
-    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
-    code_gen_buffer_max_size = code_gen_buffer_size -
-        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
-    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
-    tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
-}
-
-/* Must be called before using the QEMU cpus. 'tb_size' is the size
-   (in bytes) allocated to the translation buffer. Zero means default
-   size. */
-void tcg_exec_init(unsigned long tb_size)
-{
-    cpu_gen_init();
-    code_gen_alloc(tb_size);
-    code_gen_ptr = code_gen_buffer;
-    tcg_register_jit(code_gen_buffer, code_gen_buffer_size);
-    page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
-    /* There's no guest base to take into account, so go ahead and
-       initialize the prologue now.  */
-    tcg_prologue_init(&tcg_ctx);
-#endif
-}
-
-bool tcg_enabled(void)
-{
-    return code_gen_buffer != NULL;
-}
-
 void cpu_exec_init_all(void)
 {
 #if !defined(CONFIG_USER_ONLY)
+    qemu_mutex_init(&ram_list.mutex);
     memory_map_init();
     io_mem_init();
 #endif
@@ -668,21 +247,25 @@ static const VMStateDescription vmstate_cpu_common = {
 };
 #endif
 
-CPUArchState *qemu_get_cpu(int cpu)
+CPUState *qemu_get_cpu(int index)
 {
     CPUArchState *env = first_cpu;
+    CPUState *cpu = NULL;
 
     while (env) {
-        if (env->cpu_index == cpu)
+        cpu = ENV_GET_CPU(env);
+        if (cpu->cpu_index == index) {
             break;
+        }
         env = env->next_cpu;
     }
 
-    return env;
+    return cpu;
 }
 
 void cpu_exec_init(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     CPUArchState **penv;
     int cpu_index;
 
@@ -696,12 +279,12 @@ void cpu_exec_init(CPUArchState *env)
         penv = &(*penv)->next_cpu;
         cpu_index++;
     }
-    env->cpu_index = cpu_index;
-    env->numa_node = 0;
+    cpu->cpu_index = cpu_index;
+    cpu->numa_node = 0;
     QTAILQ_INIT(&env->breakpoints);
     QTAILQ_INIT(&env->watchpoints);
 #ifndef CONFIG_USER_ONLY
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
 #endif
     *penv = env;
 #if defined(CONFIG_USER_ONLY)
@@ -714,752 +297,6 @@ void cpu_exec_init(CPUArchState *env)
 #endif
 }
 
-/* Allocate a new translation block. Flush the translation buffer if
-   too many translation blocks or too much generated code. */
-static TranslationBlock *tb_alloc(target_ulong pc)
-{
-    TranslationBlock *tb;
-
-    if (nb_tbs >= code_gen_max_blocks ||
-        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
-        return NULL;
-    tb = &tbs[nb_tbs++];
-    tb->pc = pc;
-    tb->cflags = 0;
-    return tb;
-}
-
-void tb_free(TranslationBlock *tb)
-{
-    /* In practice this is mostly used for single use temporary TB
-       Ignore the hard cases and just back up if this TB happens to
-       be the last one generated.  */
-    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
-        code_gen_ptr = tb->tc_ptr;
-        nb_tbs--;
-    }
-}
-
-static inline void invalidate_page_bitmap(PageDesc *p)
-{
-    if (p->code_bitmap) {
-        g_free(p->code_bitmap);
-        p->code_bitmap = NULL;
-    }
-    p->code_write_count = 0;
-}
-
-/* Set to NULL all the 'first_tb' fields in all PageDescs. */
-
-static void page_flush_tb_1 (int level, void **lp)
-{
-    int i;
-
-    if (*lp == NULL) {
-        return;
-    }
-    if (level == 0) {
-        PageDesc *pd = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            pd[i].first_tb = NULL;
-            invalidate_page_bitmap(pd + i);
-        }
-    } else {
-        void **pp = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            page_flush_tb_1 (level - 1, pp + i);
-        }
-    }
-}
-
-static void page_flush_tb(void)
-{
-    int i;
-    for (i = 0; i < V_L1_SIZE; i++) {
-        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
-    }
-}
-
-/* flush all the translation blocks */
-/* XXX: tb_flush is currently not thread safe */
-void tb_flush(CPUArchState *env1)
-{
-    CPUArchState *env;
-#if defined(DEBUG_FLUSH)
-    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
-           (unsigned long)(code_gen_ptr - code_gen_buffer),
-           nb_tbs, nb_tbs > 0 ?
-           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
-#endif
-    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
-        cpu_abort(env1, "Internal error: code buffer overflow\n");
-
-    nb_tbs = 0;
-
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
-    }
-
-    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
-    page_flush_tb();
-
-    code_gen_ptr = code_gen_buffer;
-    /* XXX: flush processor icache at this point if cache flush is
-       expensive */
-    tb_flush_count++;
-}
-
-#ifdef DEBUG_TB_CHECK
-
-static void tb_invalidate_check(target_ulong address)
-{
-    TranslationBlock *tb;
-    int i;
-    address &= TARGET_PAGE_MASK;
-    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
-            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
-                  address >= tb->pc + tb->size)) {
-                printf("ERROR invalidate: address=" TARGET_FMT_lx
-                       " PC=%08lx size=%04x\n",
-                       address, (long)tb->pc, tb->size);
-            }
-        }
-    }
-}
-
-/* verify that all the pages have correct rights for code */
-static void tb_page_check(void)
-{
-    TranslationBlock *tb;
-    int i, flags1, flags2;
-
-    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
-            flags1 = page_get_flags(tb->pc);
-            flags2 = page_get_flags(tb->pc + tb->size - 1);
-            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
-                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
-                       (long)tb->pc, tb->size, flags1, flags2);
-            }
-        }
-    }
-}
-
-#endif
-
-/* invalidate one TB */
-static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
-                             int next_offset)
-{
-    TranslationBlock *tb1;
-    for(;;) {
-        tb1 = *ptb;
-        if (tb1 == tb) {
-            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
-            break;
-        }
-        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
-    }
-}
-
-static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
-{
-    TranslationBlock *tb1;
-    unsigned int n1;
-
-    for(;;) {
-        tb1 = *ptb;
-        n1 = (uintptr_t)tb1 & 3;
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        if (tb1 == tb) {
-            *ptb = tb1->page_next[n1];
-            break;
-        }
-        ptb = &tb1->page_next[n1];
-    }
-}
-
-static inline void tb_jmp_remove(TranslationBlock *tb, int n)
-{
-    TranslationBlock *tb1, **ptb;
-    unsigned int n1;
-
-    ptb = &tb->jmp_next[n];
-    tb1 = *ptb;
-    if (tb1) {
-        /* find tb(n) in circular list */
-        for(;;) {
-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == n && tb1 == tb)
-                break;
-            if (n1 == 2) {
-                ptb = &tb1->jmp_first;
-            } else {
-                ptb = &tb1->jmp_next[n1];
-            }
-        }
-        /* now we can suppress tb(n) from the list */
-        *ptb = tb->jmp_next[n];
-
-        tb->jmp_next[n] = NULL;
-    }
-}
-
-/* reset the jump entry 'n' of a TB so that it is not chained to
-   another TB */
-static inline void tb_reset_jump(TranslationBlock *tb, int n)
-{
-    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
-}
-
-void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
-{
-    CPUArchState *env;
-    PageDesc *p;
-    unsigned int h, n1;
-    tb_page_addr_t phys_pc;
-    TranslationBlock *tb1, *tb2;
-
-    /* remove the TB from the hash list */
-    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-    h = tb_phys_hash_func(phys_pc);
-    tb_remove(&tb_phys_hash[h], tb,
-              offsetof(TranslationBlock, phys_hash_next));
-
-    /* remove the TB from the page list */
-    if (tb->page_addr[0] != page_addr) {
-        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
-        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-
-    tb_invalidated_flag = 1;
-
-    /* remove the TB from the hash list */
-    h = tb_jmp_cache_hash_func(tb->pc);
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->tb_jmp_cache[h] == tb)
-            env->tb_jmp_cache[h] = NULL;
-    }
-
-    /* suppress this TB from the two jump lists */
-    tb_jmp_remove(tb, 0);
-    tb_jmp_remove(tb, 1);
-
-    /* suppress any remaining jumps to this TB */
-    tb1 = tb->jmp_first;
-    for(;;) {
-        n1 = (uintptr_t)tb1 & 3;
-        if (n1 == 2)
-            break;
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        tb2 = tb1->jmp_next[n1];
-        tb_reset_jump(tb1, n1);
-        tb1->jmp_next[n1] = NULL;
-        tb1 = tb2;
-    }
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
-
-    tb_phys_invalidate_count++;
-}
-
-static inline void set_bits(uint8_t *tab, int start, int len)
-{
-    int end, mask, end1;
-
-    end = start + len;
-    tab += start >> 3;
-    mask = 0xff << (start & 7);
-    if ((start & ~7) == (end & ~7)) {
-        if (start < end) {
-            mask &= ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    } else {
-        *tab++ |= mask;
-        start = (start + 8) & ~7;
-        end1 = end & ~7;
-        while (start < end1) {
-            *tab++ = 0xff;
-            start += 8;
-        }
-        if (start < end) {
-            mask = ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    }
-}
-
-static void build_page_bitmap(PageDesc *p)
-{
-    int n, tb_start, tb_end;
-    TranslationBlock *tb;
-
-    p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
-
-    tb = p->first_tb;
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-        /* NOTE: this is subtle as a TB may span two physical pages */
-        if (n == 0) {
-            /* NOTE: tb_end may be after the end of the page, but
-               it is not a problem */
-            tb_start = tb->pc & ~TARGET_PAGE_MASK;
-            tb_end = tb_start + tb->size;
-            if (tb_end > TARGET_PAGE_SIZE)
-                tb_end = TARGET_PAGE_SIZE;
-        } else {
-            tb_start = 0;
-            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
-        }
-        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
-        tb = tb->page_next[n];
-    }
-}
-
-TranslationBlock *tb_gen_code(CPUArchState *env,
-                              target_ulong pc, target_ulong cs_base,
-                              int flags, int cflags)
-{
-    TranslationBlock *tb;
-    uint8_t *tc_ptr;
-    tb_page_addr_t phys_pc, phys_page2;
-    target_ulong virt_page2;
-    int code_gen_size;
-
-    phys_pc = get_page_addr_code(env, pc);
-    tb = tb_alloc(pc);
-    if (!tb) {
-        /* flush must be done */
-        tb_flush(env);
-        /* cannot fail at this point */
-        tb = tb_alloc(pc);
-        /* Don't forget to invalidate previous TB info.  */
-        tb_invalidated_flag = 1;
-    }
-    tc_ptr = code_gen_ptr;
-    tb->tc_ptr = tc_ptr;
-    tb->cs_base = cs_base;
-    tb->flags = flags;
-    tb->cflags = cflags;
-    cpu_gen_code(env, tb, &code_gen_size);
-    code_gen_ptr = (void *)(((uintptr_t)code_gen_ptr + code_gen_size +
-                             CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
-
-    /* check next page if needed */
-    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
-    phys_page2 = -1;
-    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
-        phys_page2 = get_page_addr_code(env, virt_page2);
-    }
-    tb_link_page(tb, phys_pc, phys_page2);
-    return tb;
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end may refer to *different* physical pages.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- */
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
-                              int is_cpu_write_access)
-{
-    while (start < end) {
-        tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
-        start &= TARGET_PAGE_MASK;
-        start += TARGET_PAGE_SIZE;
-    }
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end must refer to the *same* physical page.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- */
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
-                                   int is_cpu_write_access)
-{
-    TranslationBlock *tb, *tb_next, *saved_tb;
-    CPUArchState *env = cpu_single_env;
-    tb_page_addr_t tb_start, tb_end;
-    PageDesc *p;
-    int n;
-#ifdef TARGET_HAS_PRECISE_SMC
-    int current_tb_not_found = is_cpu_write_access;
-    TranslationBlock *current_tb = NULL;
-    int current_tb_modified = 0;
-    target_ulong current_pc = 0;
-    target_ulong current_cs_base = 0;
-    int current_flags = 0;
-#endif /* TARGET_HAS_PRECISE_SMC */
-
-    p = page_find(start >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    if (!p->code_bitmap &&
-        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
-        is_cpu_write_access) {
-        /* build code bitmap */
-        build_page_bitmap(p);
-    }
-
-    /* we remove all the TBs in the range [start, end[ */
-    /* XXX: see if in some cases it could be faster to invalidate all the code */
-    tb = p->first_tb;
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-        tb_next = tb->page_next[n];
-        /* NOTE: this is subtle as a TB may span two physical pages */
-        if (n == 0) {
-            /* NOTE: tb_end may be after the end of the page, but
-               it is not a problem */
-            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-            tb_end = tb_start + tb->size;
-        } else {
-            tb_start = tb->page_addr[1];
-            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
-        }
-        if (!(tb_end <= start || tb_start >= end)) {
-#ifdef TARGET_HAS_PRECISE_SMC
-            if (current_tb_not_found) {
-                current_tb_not_found = 0;
-                current_tb = NULL;
-                if (env->mem_io_pc) {
-                    /* now we have a real cpu fault */
-                    current_tb = tb_find_pc(env->mem_io_pc);
-                }
-            }
-            if (current_tb == tb &&
-                (current_tb->cflags & CF_COUNT_MASK) != 1) {
-                /* If we are modifying the current TB, we must stop
-                its execution. We could be more precise by checking
-                that the modification is after the current PC, but it
-                would require a specialized function to partially
-                restore the CPU state */
-
-                current_tb_modified = 1;
-                cpu_restore_state(current_tb, env, env->mem_io_pc);
-                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
-                                     &current_flags);
-            }
-#endif /* TARGET_HAS_PRECISE_SMC */
-            /* we need to do that to handle the case where a signal
-               occurs while doing tb_phys_invalidate() */
-            saved_tb = NULL;
-            if (env) {
-                saved_tb = env->current_tb;
-                env->current_tb = NULL;
-            }
-            tb_phys_invalidate(tb, -1);
-            if (env) {
-                env->current_tb = saved_tb;
-                if (env->interrupt_request && env->current_tb)
-                    cpu_interrupt(env, env->interrupt_request);
-            }
-        }
-        tb = tb_next;
-    }
-#if !defined(CONFIG_USER_ONLY)
-    /* if no code remaining, no need to continue to use slow writes */
-    if (!p->first_tb) {
-        invalidate_page_bitmap(p);
-        if (is_cpu_write_access) {
-            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
-        }
-    }
-#endif
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (current_tb_modified) {
-        /* we generate a block containing just the instruction
-           modifying the memory. It will ensure that it cannot modify
-           itself */
-        env->current_tb = NULL;
-        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(env, NULL);
-    }
-#endif
-}
-
-/* len must be <= 8 and start must be a multiple of len */
-static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
-{
-    PageDesc *p;
-    int offset, b;
-#if 0
-    if (1) {
-        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
-                  cpu_single_env->mem_io_vaddr, len,
-                  cpu_single_env->eip,
-                  cpu_single_env->eip +
-                  (intptr_t)cpu_single_env->segs[R_CS].base);
-    }
-#endif
-    p = page_find(start >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    if (p->code_bitmap) {
-        offset = start & ~TARGET_PAGE_MASK;
-        b = p->code_bitmap[offset >> 3] >> (offset & 7);
-        if (b & ((1 << len) - 1))
-            goto do_invalidate;
-    } else {
-    do_invalidate:
-        tb_invalidate_phys_page_range(start, start + len, 1);
-    }
-}
-
-#if !defined(CONFIG_SOFTMMU)
-static void tb_invalidate_phys_page(tb_page_addr_t addr,
-                                    uintptr_t pc, void *puc)
-{
-    TranslationBlock *tb;
-    PageDesc *p;
-    int n;
-#ifdef TARGET_HAS_PRECISE_SMC
-    TranslationBlock *current_tb = NULL;
-    CPUArchState *env = cpu_single_env;
-    int current_tb_modified = 0;
-    target_ulong current_pc = 0;
-    target_ulong current_cs_base = 0;
-    int current_flags = 0;
-#endif
-
-    addr &= TARGET_PAGE_MASK;
-    p = page_find(addr >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    tb = p->first_tb;
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (tb && pc != 0) {
-        current_tb = tb_find_pc(pc);
-    }
-#endif
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-#ifdef TARGET_HAS_PRECISE_SMC
-        if (current_tb == tb &&
-            (current_tb->cflags & CF_COUNT_MASK) != 1) {
-                /* If we are modifying the current TB, we must stop
-                   its execution. We could be more precise by checking
-                   that the modification is after the current PC, but it
-                   would require a specialized function to partially
-                   restore the CPU state */
-
-            current_tb_modified = 1;
-            cpu_restore_state(current_tb, env, pc);
-            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
-                                 &current_flags);
-        }
-#endif /* TARGET_HAS_PRECISE_SMC */
-        tb_phys_invalidate(tb, addr);
-        tb = tb->page_next[n];
-    }
-    p->first_tb = NULL;
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (current_tb_modified) {
-        /* we generate a block containing just the instruction
-           modifying the memory. It will ensure that it cannot modify
-           itself */
-        env->current_tb = NULL;
-        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(env, puc);
-    }
-#endif
-}
-#endif
-
-/* add the tb in the target page and protect it if necessary */
-static inline void tb_alloc_page(TranslationBlock *tb,
-                                 unsigned int n, tb_page_addr_t page_addr)
-{
-    PageDesc *p;
-#ifndef CONFIG_USER_ONLY
-    bool page_already_protected;
-#endif
-
-    tb->page_addr[n] = page_addr;
-    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
-    tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
-    page_already_protected = p->first_tb != NULL;
-#endif
-    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
-    invalidate_page_bitmap(p);
-
-#if defined(TARGET_HAS_SMC) || 1
-
-#if defined(CONFIG_USER_ONLY)
-    if (p->flags & PAGE_WRITE) {
-        target_ulong addr;
-        PageDesc *p2;
-        int prot;
-
-        /* force the host page as non writable (writes will have a
-           page fault + mprotect overhead) */
-        page_addr &= qemu_host_page_mask;
-        prot = 0;
-        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
-            addr += TARGET_PAGE_SIZE) {
-
-            p2 = page_find (addr >> TARGET_PAGE_BITS);
-            if (!p2)
-                continue;
-            prot |= p2->flags;
-            p2->flags &= ~PAGE_WRITE;
-          }
-        mprotect(g2h(page_addr), qemu_host_page_size,
-                 (prot & PAGE_BITS) & ~PAGE_WRITE);
-#ifdef DEBUG_TB_INVALIDATE
-        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
-               page_addr);
-#endif
-    }
-#else
-    /* if some code is already present, then the pages are already
-       protected. So we handle the case where only the first TB is
-       allocated in a physical page */
-    if (!page_already_protected) {
-        tlb_protect_code(page_addr);
-    }
-#endif
-
-#endif /* TARGET_HAS_SMC */
-}
-
-/* add a new TB and link it to the physical page tables. phys_page2 is
-   (-1) to indicate that only one page contains the TB. */
-void tb_link_page(TranslationBlock *tb,
-                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
-{
-    unsigned int h;
-    TranslationBlock **ptb;
-
-    /* Grab the mmap lock to stop another thread invalidating this TB
-       before we are done.  */
-    mmap_lock();
-    /* add in the physical hash table */
-    h = tb_phys_hash_func(phys_pc);
-    ptb = &tb_phys_hash[h];
-    tb->phys_hash_next = *ptb;
-    *ptb = tb;
-
-    /* add in the page list */
-    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
-    if (phys_page2 != -1)
-        tb_alloc_page(tb, 1, phys_page2);
-    else
-        tb->page_addr[1] = -1;
-
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
-    tb->jmp_next[0] = NULL;
-    tb->jmp_next[1] = NULL;
-
-    /* init original jump addresses */
-    if (tb->tb_next_offset[0] != 0xffff)
-        tb_reset_jump(tb, 0);
-    if (tb->tb_next_offset[1] != 0xffff)
-        tb_reset_jump(tb, 1);
-
-#ifdef DEBUG_TB_CHECK
-    tb_page_check();
-#endif
-    mmap_unlock();
-}
-
-/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
-   tb[1].tc_ptr. Return NULL if not found */
-TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
-{
-    int m_min, m_max, m;
-    uintptr_t v;
-    TranslationBlock *tb;
-
-    if (nb_tbs <= 0)
-        return NULL;
-    if (tc_ptr < (uintptr_t)code_gen_buffer ||
-        tc_ptr >= (uintptr_t)code_gen_ptr) {
-        return NULL;
-    }
-    /* binary search (cf Knuth) */
-    m_min = 0;
-    m_max = nb_tbs - 1;
-    while (m_min <= m_max) {
-        m = (m_min + m_max) >> 1;
-        tb = &tbs[m];
-        v = (uintptr_t)tb->tc_ptr;
-        if (v == tc_ptr)
-            return tb;
-        else if (tc_ptr < v) {
-            m_max = m - 1;
-        } else {
-            m_min = m + 1;
-        }
-    }
-    return &tbs[m_max];
-}
-
-static void tb_reset_jump_recursive(TranslationBlock *tb);
-
-static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
-{
-    TranslationBlock *tb1, *tb_next, **ptb;
-    unsigned int n1;
-
-    tb1 = tb->jmp_next[n];
-    if (tb1 != NULL) {
-        /* find head of list */
-        for(;;) {
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == 2)
-                break;
-            tb1 = tb1->jmp_next[n1];
-        }
-        /* we are now sure now that tb jumps to tb1 */
-        tb_next = tb1;
-
-        /* remove tb from the jmp_first list */
-        ptb = &tb_next->jmp_first;
-        for(;;) {
-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == n && tb1 == tb)
-                break;
-            ptb = &tb1->jmp_next[n1];
-        }
-        *ptb = tb->jmp_next[n];
-        tb->jmp_next[n] = NULL;
-
-        /* suppress the jump to next tb in generated code */
-        tb_reset_jump(tb, n);
-
-        /* suppress jumps in the tb on which we could have jumped */
-        tb_reset_jump_recursive(tb_next);
-    }
-}
-
-static void tb_reset_jump_recursive(TranslationBlock *tb)
-{
-    tb_reset_jump_recursive2(tb, 0);
-    tb_reset_jump_recursive2(tb, 1);
-}
-
 #if defined(TARGET_HAS_ICE)
 #if defined(CONFIG_USER_ONLY)
 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
@@ -1467,21 +304,6 @@ static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 }
 #else
-void tb_invalidate_phys_addr(target_phys_addr_t addr)
-{
-    ram_addr_t ram_addr;
-    MemoryRegionSection *section;
-
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
-    if (!(memory_region_is_ram(section->mr)
-          || (section->mr->rom_device && section->mr->readable))) {
-        return;
-    }
-    ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
-        + memory_region_section_addr(section, addr);
-    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
-}
-
 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 {
     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
@@ -1663,66 +485,6 @@ void cpu_single_step(CPUArchState *env, int enabled)
 #endif
 }
 
-static void cpu_unlink_tb(CPUArchState *env)
-{
-    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
-       problem and hope the cpu will stop of its own accord.  For userspace
-       emulation this often isn't actually as bad as it sounds.  Often
-       signals are used primarily to interrupt blocking syscalls.  */
-    TranslationBlock *tb;
-    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
-
-    spin_lock(&interrupt_lock);
-    tb = env->current_tb;
-    /* if the cpu is currently executing code, we must unlink it and
-       all the potentially executing TB */
-    if (tb) {
-        env->current_tb = NULL;
-        tb_reset_jump_recursive(tb);
-    }
-    spin_unlock(&interrupt_lock);
-}
-
-#ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUArchState *env, int mask)
-{
-    int old_mask;
-
-    old_mask = env->interrupt_request;
-    env->interrupt_request |= mask;
-
-    /*
-     * If called from iothread context, wake the target cpu in
-     * case its halted.
-     */
-    if (!qemu_cpu_is_self(env)) {
-        qemu_cpu_kick(env);
-        return;
-    }
-
-    if (use_icount) {
-        env->icount_decr.u16.high = 0xffff;
-        if (!can_do_io(env)
-            && (mask & ~old_mask) != 0) {
-            cpu_abort(env, "Raised interrupt while not in I/O function");
-        }
-    } else {
-        cpu_unlink_tb(env);
-    }
-}
-
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
-#else /* CONFIG_USER_ONLY */
-
-void cpu_interrupt(CPUArchState *env, int mask)
-{
-    env->interrupt_request |= mask;
-    cpu_unlink_tb(env);
-}
-#endif /* CONFIG_USER_ONLY */
-
 void cpu_reset_interrupt(CPUArchState *env, int mask)
 {
     env->interrupt_request &= ~mask;
@@ -1744,20 +506,12 @@ void cpu_abort(CPUArchState *env, const char *fmt, ...)
     fprintf(stderr, "qemu: fatal: ");
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
-#ifdef TARGET_I386
-    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
-#else
-    cpu_dump_state(env, stderr, fprintf, 0);
-#endif
+    cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
     if (qemu_log_enabled()) {
         qemu_log("qemu: fatal: ");
         qemu_log_vprintf(fmt, ap2);
         qemu_log("\n");
-#ifdef TARGET_I386
-        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
-#else
-        log_cpu_state(env, 0);
-#endif
+        log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
         qemu_log_flush();
         qemu_log_close();
     }
@@ -1778,7 +532,6 @@ CPUArchState *cpu_copy(CPUArchState *env)
 {
     CPUArchState *new_env = cpu_init(env->cpu_model_str);
     CPUArchState *next_cpu = new_env->next_cpu;
-    int cpu_index = new_env->cpu_index;
 #if defined(TARGET_HAS_ICE)
     CPUBreakpoint *bp;
     CPUWatchpoint *wp;
@@ -1786,9 +539,8 @@ CPUArchState *cpu_copy(CPUArchState *env)
 
     memcpy(new_env, env, sizeof(CPUArchState));
 
-    /* Preserve chaining and index. */
+    /* Preserve chaining. */
     new_env->next_cpu = next_cpu;
-    new_env->cpu_index = cpu_index;
 
     /* Clone all break/watchpoints.
        Note: Once we support ptrace with hw-debug register access, make sure
@@ -1809,21 +561,6 @@ CPUArchState *cpu_copy(CPUArchState *env)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
-{
-    unsigned int i;
-
-    /* Discard jump cache entries for any tb which might potentially
-       overlap the flushed page.  */
-    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
-    memset (&env->tb_jmp_cache[i], 0, 
-            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
-
-    i = tb_jmp_cache_hash_page(addr);
-    memset (&env->tb_jmp_cache[i], 0, 
-            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
-}
-
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
                                       uintptr_t length)
 {
@@ -1861,21 +598,21 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
     }
 }
 
-int cpu_physical_memory_set_dirty_tracking(int enable)
+static int cpu_physical_memory_set_dirty_tracking(int enable)
 {
     int ret = 0;
     in_migration = enable;
     return ret;
 }
 
-target_phys_addr_t memory_region_section_get_iotlb(CPUArchState *env,
+hwaddr memory_region_section_get_iotlb(CPUArchState *env,
                                                    MemoryRegionSection *section,
                                                    target_ulong vaddr,
-                                                   target_phys_addr_t paddr,
+                                                   hwaddr paddr,
                                                    int prot,
                                                    target_ulong *address)
 {
-    target_phys_addr_t iotlb;
+    hwaddr iotlb;
     CPUWatchpoint *wp;
 
     if (memory_region_is_ram(section->mr)) {
@@ -1913,264 +650,6 @@ target_phys_addr_t memory_region_section_get_iotlb(CPUArchState *env,
 
     return iotlb;
 }
-
-#else
-/*
- * Walks guest process memory "regions" one by one
- * and calls callback function 'fn' for each region.
- */
-
-struct walk_memory_regions_data
-{
-    walk_memory_regions_fn fn;
-    void *priv;
-    uintptr_t start;
-    int prot;
-};
-
-static int walk_memory_regions_end(struct walk_memory_regions_data *data,
-                                   abi_ulong end, int new_prot)
-{
-    if (data->start != -1ul) {
-        int rc = data->fn(data->priv, data->start, end, data->prot);
-        if (rc != 0) {
-            return rc;
-        }
-    }
-
-    data->start = (new_prot ? end : -1ul);
-    data->prot = new_prot;
-
-    return 0;
-}
-
-static int walk_memory_regions_1(struct walk_memory_regions_data *data,
-                                 abi_ulong base, int level, void **lp)
-{
-    abi_ulong pa;
-    int i, rc;
-
-    if (*lp == NULL) {
-        return walk_memory_regions_end(data, base, 0);
-    }
-
-    if (level == 0) {
-        PageDesc *pd = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            int prot = pd[i].flags;
-
-            pa = base | (i << TARGET_PAGE_BITS);
-            if (prot != data->prot) {
-                rc = walk_memory_regions_end(data, pa, prot);
-                if (rc != 0) {
-                    return rc;
-                }
-            }
-        }
-    } else {
-        void **pp = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            pa = base | ((abi_ulong)i <<
-                (TARGET_PAGE_BITS + L2_BITS * level));
-            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
-            if (rc != 0) {
-                return rc;
-            }
-        }
-    }
-
-    return 0;
-}
-
-int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
-{
-    struct walk_memory_regions_data data;
-    uintptr_t i;
-
-    data.fn = fn;
-    data.priv = priv;
-    data.start = -1ul;
-    data.prot = 0;
-
-    for (i = 0; i < V_L1_SIZE; i++) {
-        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
-                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
-        if (rc != 0) {
-            return rc;
-        }
-    }
-
-    return walk_memory_regions_end(&data, 0, 0);
-}
-
-static int dump_region(void *priv, abi_ulong start,
-    abi_ulong end, unsigned long prot)
-{
-    FILE *f = (FILE *)priv;
-
-    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
-        " "TARGET_ABI_FMT_lx" %c%c%c\n",
-        start, end, end - start,
-        ((prot & PAGE_READ) ? 'r' : '-'),
-        ((prot & PAGE_WRITE) ? 'w' : '-'),
-        ((prot & PAGE_EXEC) ? 'x' : '-'));
-
-    return (0);
-}
-
-/* dump memory mappings */
-void page_dump(FILE *f)
-{
-    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
-            "start", "end", "size", "prot");
-    walk_memory_regions(f, dump_region);
-}
-
-int page_get_flags(target_ulong address)
-{
-    PageDesc *p;
-
-    p = page_find(address >> TARGET_PAGE_BITS);
-    if (!p)
-        return 0;
-    return p->flags;
-}
-
-/* Modify the flags of a page and invalidate the code if necessary.
-   The flag PAGE_WRITE_ORG is positioned automatically depending
-   on PAGE_WRITE.  The mmap_lock should already be held.  */
-void page_set_flags(target_ulong start, target_ulong end, int flags)
-{
-    target_ulong addr, len;
-
-    /* This function should never be called with addresses outside the
-       guest address space.  If this assert fires, it probably indicates
-       a missing call to h2g_valid.  */
-#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
-#endif
-    assert(start < end);
-
-    start = start & TARGET_PAGE_MASK;
-    end = TARGET_PAGE_ALIGN(end);
-
-    if (flags & PAGE_WRITE) {
-        flags |= PAGE_WRITE_ORG;
-    }
-
-    for (addr = start, len = end - start;
-         len != 0;
-         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-
-        /* If the write protection bit is set, then we invalidate
-           the code inside.  */
-        if (!(p->flags & PAGE_WRITE) &&
-            (flags & PAGE_WRITE) &&
-            p->first_tb) {
-            tb_invalidate_phys_page(addr, 0, NULL);
-        }
-        p->flags = flags;
-    }
-}
-
-int page_check_range(target_ulong start, target_ulong len, int flags)
-{
-    PageDesc *p;
-    target_ulong end;
-    target_ulong addr;
-
-    /* This function should never be called with addresses outside the
-       guest address space.  If this assert fires, it probably indicates
-       a missing call to h2g_valid.  */
-#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
-#endif
-
-    if (len == 0) {
-        return 0;
-    }
-    if (start + len - 1 < start) {
-        /* We've wrapped around.  */
-        return -1;
-    }
-
-    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
-    start = start & TARGET_PAGE_MASK;
-
-    for (addr = start, len = end - start;
-         len != 0;
-         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-        p = page_find(addr >> TARGET_PAGE_BITS);
-        if( !p )
-            return -1;
-        if( !(p->flags & PAGE_VALID) )
-            return -1;
-
-        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
-            return -1;
-        if (flags & PAGE_WRITE) {
-            if (!(p->flags & PAGE_WRITE_ORG))
-                return -1;
-            /* unprotect the page if it was put read-only because it
-               contains translated code */
-            if (!(p->flags & PAGE_WRITE)) {
-                if (!page_unprotect(addr, 0, NULL))
-                    return -1;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-
-/* called from signal handler: invalidate the code and unprotect the
-   page. Return TRUE if the fault was successfully handled. */
-int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
-{
-    unsigned int prot;
-    PageDesc *p;
-    target_ulong host_start, host_end, addr;
-
-    /* Technically this isn't safe inside a signal handler.  However we
-       know this only ever happens in a synchronous SEGV handler, so in
-       practice it seems to be ok.  */
-    mmap_lock();
-
-    p = page_find(address >> TARGET_PAGE_BITS);
-    if (!p) {
-        mmap_unlock();
-        return 0;
-    }
-
-    /* if the page was really writable, then we change its
-       protection back to writable */
-    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
-        host_start = address & qemu_host_page_mask;
-        host_end = host_start + qemu_host_page_size;
-
-        prot = 0;
-        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
-            p = page_find(addr >> TARGET_PAGE_BITS);
-            p->flags |= PAGE_WRITE;
-            prot |= p->flags;
-
-            /* and since the content will be modified, we must invalidate
-               the corresponding translated code. */
-            tb_invalidate_phys_page(addr, pc, puc);
-#ifdef DEBUG_TB_CHECK
-            tb_invalidate_check(addr);
-#endif
-        }
-        mprotect((void *)g2h(host_start), qemu_host_page_size,
-                 prot & PAGE_BITS);
-
-        mmap_unlock();
-        return 1;
-    }
-    mmap_unlock();
-    return 0;
-}
 #endif /* defined(CONFIG_USER_ONLY) */
 
 #if !defined(CONFIG_USER_ONLY)
@@ -2178,13 +657,13 @@ int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 typedef struct subpage_t {
     MemoryRegion iomem;
-    target_phys_addr_t base;
+    hwaddr base;
     uint16_t sub_section[TARGET_PAGE_SIZE];
 } subpage_t;
 
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
-static subpage_t *subpage_init(target_phys_addr_t base);
+static subpage_t *subpage_init(hwaddr base);
 static void destroy_page_desc(uint16_t section_index)
 {
     MemoryRegionSection *section = &phys_sections[section_index];
@@ -2218,9 +697,9 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
     lp->ptr = PHYS_MAP_NODE_NIL;
 }
 
-static void destroy_all_mappings(void)
+static void destroy_all_mappings(AddressSpaceDispatch *d)
 {
-    destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);
+    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
     phys_map_nodes_reset();
 }
 
@@ -2240,24 +719,24 @@ static void phys_sections_clear(void)
     phys_sections_nb = 0;
 }
 
-static void register_subpage(MemoryRegionSection *section)
+static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 {
     subpage_t *subpage;
-    target_phys_addr_t base = section->offset_within_address_space
+    hwaddr base = section->offset_within_address_space
         & TARGET_PAGE_MASK;
-    MemoryRegionSection *existing = phys_page_find(base >> TARGET_PAGE_BITS);
+    MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
     MemoryRegionSection subsection = {
         .offset_within_address_space = base,
         .size = TARGET_PAGE_SIZE,
     };
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 
     if (!(existing->mr->subpage)) {
         subpage = subpage_init(base);
         subsection.mr = &subpage->iomem;
-        phys_page_set(base >> TARGET_PAGE_BITS, 1,
+        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
                       phys_section_add(&subsection));
     } else {
         subpage = container_of(existing->mr, subpage_t, iomem);
@@ -2268,23 +747,23 @@ static void register_subpage(MemoryRegionSection *section)
 }
 
 
-static void register_multipage(MemoryRegionSection *section)
+static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 {
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint16_t section_index = phys_section_add(section);
 
     assert(size);
 
     addr = start_addr;
-    phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
+    phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
                   section_index);
 }
 
-void cpu_register_physical_memory_log(MemoryRegionSection *section,
-                                      bool readonly)
+static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 {
+    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
     MemoryRegionSection now = *section, remain = *section;
 
     if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
@@ -2292,7 +771,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
         now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
                        - now.offset_within_address_space,
                        now.size);
-        register_subpage(&now);
+        register_subpage(d, &now);
         remain.size -= now.size;
         remain.offset_within_address_space += now.size;
         remain.offset_within_region += now.size;
@@ -2301,10 +780,10 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
         now = remain;
         if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
             now.size = TARGET_PAGE_SIZE;
-            register_subpage(&now);
+            register_subpage(d, &now);
         } else {
             now.size &= TARGET_PAGE_MASK;
-            register_multipage(&now);
+            register_multipage(d, &now);
         }
         remain.size -= now.size;
         remain.offset_within_address_space += now.size;
@@ -2312,27 +791,24 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
     }
     now = remain;
     if (now.size) {
-        register_subpage(&now);
+        register_subpage(d, &now);
     }
 }
 
-
-void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
+void qemu_flush_coalesced_mmio_buffer(void)
 {
     if (kvm_enabled())
-        kvm_coalesce_mmio_region(addr, size);
+        kvm_flush_coalesced_mmio_buffer();
 }
 
-void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
+void qemu_mutex_lock_ramlist(void)
 {
-    if (kvm_enabled())
-        kvm_uncoalesce_mmio_region(addr, size);
+    qemu_mutex_lock(&ram_list.mutex);
 }
 
-void qemu_flush_coalesced_mmio_buffer(void)
+void qemu_mutex_unlock_ramlist(void)
 {
-    if (kvm_enabled())
-        kvm_flush_coalesced_mmio_buffer();
+    qemu_mutex_unlock(&ram_list.mutex);
 }
 
 #if defined(__linux__) && !defined(TARGET_S390X)
@@ -2387,18 +863,16 @@ static void *file_ram_alloc(RAMBlock *block,
         return NULL;
     }
 
-    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
-        return NULL;
-    }
+    filename = g_strdup_printf("%s/qemu_back_mem.XXXXXX", path);
 
     fd = mkstemp(filename);
     if (fd < 0) {
         perror("unable to create backing store for hugepages");
-        free(filename);
+        g_free(filename);
         return NULL;
     }
     unlink(filename);
-    free(filename);
+    g_free(filename);
 
     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 
@@ -2436,15 +910,15 @@ static ram_addr_t find_ram_offset(ram_addr_t size)
     RAMBlock *block, *next_block;
     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 
-    if (QLIST_EMPTY(&ram_list.blocks))
+    if (QTAILQ_EMPTY(&ram_list.blocks))
         return 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         ram_addr_t end, next = RAM_ADDR_MAX;
 
         end = block->offset + block->length;
 
-        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
             if (next_block->offset >= end) {
                 next = MIN(next, next_block->offset);
             }
@@ -2464,12 +938,12 @@ static ram_addr_t find_ram_offset(ram_addr_t size)
     return offset;
 }
 
-static ram_addr_t last_ram_offset(void)
+ram_addr_t last_ram_offset(void)
 {
     RAMBlock *block;
     ram_addr_t last = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next)
+    QTAILQ_FOREACH(block, &ram_list.blocks, next)
         last = MAX(last, block->offset + block->length);
 
     return last;
@@ -2498,7 +972,7 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     RAMBlock *new_block, *block;
 
     new_block = NULL;
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block->offset == addr) {
             new_block = block;
             break;
@@ -2516,23 +990,41 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     }
     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                     new_block->idstr);
             abort();
         }
     }
+    qemu_mutex_unlock_ramlist();
+}
+
+static int memory_try_enable_merging(void *addr, size_t len)
+{
+    QemuOpts *opts;
+
+    opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
+        /* disabled by the user */
+        return 0;
+    }
+
+    return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
 }
 
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr)
 {
-    RAMBlock *new_block;
+    RAMBlock *block, *new_block;
 
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
 
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
     new_block->mr = mr;
     new_block->offset = find_ram_offset(size);
     if (host) {
@@ -2544,7 +1036,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
             new_block->host = file_ram_alloc(new_block, size, mem_path);
             if (!new_block->host) {
                 new_block->host = qemu_vmalloc(size);
-                qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
+                memory_try_enable_merging(new_block->host, size);
             }
 #else
             fprintf(stderr, "-mem-path option unsupported\n");
@@ -2559,12 +1051,26 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
             } else {
                 new_block->host = qemu_vmalloc(size);
             }
-            qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
+            memory_try_enable_merging(new_block->host, size);
         }
     }
     new_block->length = size;
 
-    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
+    /* Keep the list sorted from biggest to smallest block.  */
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        if (block->length < new_block->length) {
+            break;
+        }
+    }
+    if (block) {
+        QTAILQ_INSERT_BEFORE(block, new_block, next);
+    } else {
+        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
+    }
+    ram_list.mru_block = NULL;
+
+    ram_list.version++;
+    qemu_mutex_unlock_ramlist();
 
     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
                                        last_ram_offset() >> TARGET_PAGE_BITS);
@@ -2573,6 +1079,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
 
     qemu_ram_setup_dump(new_block->host, size);
+    qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
 
     if (kvm_enabled())
         kvm_setup_guest_memory(new_block->host, size);
@@ -2589,22 +1096,31 @@ void qemu_ram_free_from_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr == block->offset) {
-            QLIST_REMOVE(block, next);
+            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            ram_list.mru_block = NULL;
+            ram_list.version++;
             g_free(block);
-            return;
+            break;
         }
     }
+    qemu_mutex_unlock_ramlist();
 }
 
 void qemu_ram_free(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr == block->offset) {
-            QLIST_REMOVE(block, next);
+            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            ram_list.mru_block = NULL;
+            ram_list.version++;
             if (block->flags & RAM_PREALLOC_MASK) {
                 ;
             } else if (mem_path) {
@@ -2630,9 +1146,10 @@ void qemu_ram_free(ram_addr_t addr)
 #endif
             }
             g_free(block);
-            return;
+            break;
         }
     }
+    qemu_mutex_unlock_ramlist();
 
 }
 
@@ -2644,7 +1161,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
     int flags;
     void *area, *vaddr;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         offset = addr - block->offset;
         if (offset < block->length) {
             vaddr = block->host + offset;
@@ -2689,7 +1206,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
                             length, addr);
                     exit(1);
                 }
-                qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
+                memory_try_enable_merging(vaddr, length);
                 qemu_ram_setup_dump(vaddr, length);
             }
             return;
@@ -2710,43 +1227,48 @@ void *qemu_get_ram_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* The list is protected by the iothread lock here.  */
+    block = ram_list.mru_block;
+    if (block && addr - block->offset < block->length) {
+        goto found;
+    }
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->length) {
-            /* Move this entry to to start of the list.  */
-            if (block != QLIST_FIRST(&ram_list.blocks)) {
-                QLIST_REMOVE(block, next);
-                QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
-            }
-            if (xen_enabled()) {
-                /* We need to check if the requested address is in the RAM
-                 * because we don't want to map the entire memory in QEMU.
-                 * In that case just map until the end of the page.
-                 */
-                if (block->offset == 0) {
-                    return xen_map_cache(addr, 0, 0);
-                } else if (block->host == NULL) {
-                    block->host =
-                        xen_map_cache(block->offset, block->length, 1);
-                }
-            }
-            return block->host + (addr - block->offset);
+            goto found;
         }
     }
 
     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
     abort();
 
-    return NULL;
+found:
+    ram_list.mru_block = block;
+    if (xen_enabled()) {
+        /* We need to check if the requested address is in the RAM
+         * because we don't want to map the entire memory in QEMU.
+         * In that case just map until the end of the page.
+         */
+        if (block->offset == 0) {
+            return xen_map_cache(addr, 0, 0);
+        } else if (block->host == NULL) {
+            block->host =
+                xen_map_cache(block->offset, block->length, 1);
+        }
+    }
+    return block->host + (addr - block->offset);
 }
 
-/* Return a host pointer to ram allocated with qemu_ram_alloc.
- * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
+/* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
+ * qemu_get_ram_ptr but do not touch ram_list.mru_block.
+ *
+ * ??? Is this still necessary?
  */
-void *qemu_safe_ram_ptr(ram_addr_t addr)
+static void *qemu_safe_ram_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* The list is protected by the iothread lock here.  */
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->length) {
             if (xen_enabled()) {
                 /* We need to check if the requested address is in the RAM
@@ -2772,7 +1294,7 @@ void *qemu_safe_ram_ptr(ram_addr_t addr)
 
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
  * but takes a size argument */
-void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
+static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
 {
     if (*size == 0) {
         return NULL;
@@ -2782,7 +1304,7 @@ void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
     } else {
         RAMBlock *block;
 
-        QLIST_FOREACH(block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
             if (addr - block->offset < block->length) {
                 if (addr - block->offset + *size > block->length)
                     *size = block->length - addr + block->offset;
@@ -2810,7 +1332,7 @@ int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
         return 0;
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         /* This case append when the block is not mapped. */
         if (block->host == NULL) {
             continue;
@@ -2837,7 +1359,7 @@ ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
     return ram_addr;
 }
 
-static uint64_t unassigned_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2849,7 +1371,7 @@ static uint64_t unassigned_mem_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void unassigned_mem_write(void *opaque, target_phys_addr_t addr,
+static void unassigned_mem_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2866,13 +1388,13 @@ static const MemoryRegionOps unassigned_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t error_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t error_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     abort();
 }
 
-static void error_mem_write(void *opaque, target_phys_addr_t addr,
+static void error_mem_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     abort();
@@ -2890,7 +1412,7 @@ static const MemoryRegionOps rom_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void notdirty_mem_write(void *opaque, target_phys_addr_t ram_addr,
+static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
                                uint64_t val, unsigned size)
 {
     int dirty_flags;
@@ -2933,7 +1455,6 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 {
     CPUArchState *env = cpu_single_env;
     target_ulong pc, cs_base;
-    TranslationBlock *tb;
     target_ulong vaddr;
     CPUWatchpoint *wp;
     int cpu_flags;
@@ -2952,13 +1473,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
             wp->flags |= BP_WATCHPOINT_HIT;
             if (!env->watchpoint_hit) {
                 env->watchpoint_hit = wp;
-                tb = tb_find_pc(env->mem_io_pc);
-                if (!tb) {
-                    cpu_abort(env, "check_watchpoint: could not find TB for "
-                              "pc=%p", (void *)env->mem_io_pc);
-                }
-                cpu_restore_state(tb, env, env->mem_io_pc);
-                tb_phys_invalidate(tb, -1);
+                tb_check_watchpoint(env);
                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                     env->exception_index = EXCP_DEBUG;
                     cpu_loop_exit(env);
@@ -2977,7 +1492,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
    so these check for a hit then pass through to the normal out-of-line
    phys routines.  */
-static uint64_t watch_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t watch_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
@@ -2989,7 +1504,7 @@ static uint64_t watch_mem_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void watch_mem_write(void *opaque, target_phys_addr_t addr,
+static void watch_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
@@ -3013,7 +1528,7 @@ static const MemoryRegionOps watch_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
+static uint64_t subpage_read(void *opaque, hwaddr addr,
                              unsigned len)
 {
     subpage_t *mmio = opaque;
@@ -3031,7 +1546,7 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
     return io_mem_read(section->mr, addr, len);
 }
 
-static void subpage_write(void *opaque, target_phys_addr_t addr,
+static void subpage_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned len)
 {
     subpage_t *mmio = opaque;
@@ -3056,7 +1571,7 @@ static const MemoryRegionOps subpage_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t subpage_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     ram_addr_t raddr = addr;
@@ -3069,7 +1584,7 @@ static uint64_t subpage_ram_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void subpage_ram_write(void *opaque, target_phys_addr_t addr,
+static void subpage_ram_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     ram_addr_t raddr = addr;
@@ -3113,7 +1628,7 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
     return 0;
 }
 
-static subpage_t *subpage_init(target_phys_addr_t base)
+static subpage_t *subpage_init(hwaddr base)
 {
     subpage_t *mmio;
 
@@ -3144,7 +1659,7 @@ static uint16_t dummy_section(MemoryRegion *mr)
     return phys_section_add(&section);
 }
 
-MemoryRegion *iotlb_to_region(target_phys_addr_t index)
+MemoryRegion *iotlb_to_region(hwaddr index)
 {
     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
 }
@@ -3163,18 +1678,24 @@ static void io_mem_init(void)
                           "watch", UINT64_MAX);
 }
 
+static void mem_begin(MemoryListener *listener)
+{
+    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
+
+    destroy_all_mappings(d);
+    d->phys_map.ptr = PHYS_MAP_NODE_NIL;
+}
+
 static void core_begin(MemoryListener *listener)
 {
-    destroy_all_mappings();
     phys_sections_clear();
-    phys_map.ptr = PHYS_MAP_NODE_NIL;
     phys_section_unassigned = dummy_section(&io_mem_unassigned);
     phys_section_notdirty = dummy_section(&io_mem_notdirty);
     phys_section_rom = dummy_section(&io_mem_rom);
     phys_section_watch = dummy_section(&io_mem_watch);
 }
 
-static void core_commit(MemoryListener *listener)
+static void tcg_commit(MemoryListener *listener)
 {
     CPUArchState *env;
 
@@ -3186,38 +1707,6 @@ static void core_commit(MemoryListener *listener)
     }
 }
 
-static void core_region_add(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-    cpu_register_physical_memory_log(section, section->readonly);
-}
-
-static void core_region_del(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-}
-
-static void core_region_nop(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-    cpu_register_physical_memory_log(section, section->readonly);
-}
-
-static void core_log_start(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
-static void core_log_stop(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
-static void core_log_sync(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
 static void core_log_global_start(MemoryListener *listener)
 {
     cpu_physical_memory_set_dirty_tracking(1);
@@ -3228,26 +1717,6 @@ static void core_log_global_stop(MemoryListener *listener)
     cpu_physical_memory_set_dirty_tracking(0);
 }
 
-static void core_eventfd_add(MemoryListener *listener,
-                             MemoryRegionSection *section,
-                             bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void core_eventfd_del(MemoryListener *listener,
-                             MemoryRegionSection *section,
-                             bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void io_begin(MemoryListener *listener)
-{
-}
-
-static void io_commit(MemoryListener *listener)
-{
-}
-
 static void io_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
 {
@@ -3266,90 +1735,66 @@ static void io_region_del(MemoryListener *listener,
     isa_unassign_ioport(section->offset_within_address_space, section->size);
 }
 
-static void io_region_nop(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
-static void io_log_start(MemoryListener *listener,
-                         MemoryRegionSection *section)
-{
-}
-
-static void io_log_stop(MemoryListener *listener,
-                        MemoryRegionSection *section)
-{
-}
-
-static void io_log_sync(MemoryListener *listener,
-                        MemoryRegionSection *section)
-{
-}
-
-static void io_log_global_start(MemoryListener *listener)
-{
-}
-
-static void io_log_global_stop(MemoryListener *listener)
-{
-}
-
-static void io_eventfd_add(MemoryListener *listener,
-                           MemoryRegionSection *section,
-                           bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void io_eventfd_del(MemoryListener *listener,
-                           MemoryRegionSection *section,
-                           bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
 static MemoryListener core_memory_listener = {
     .begin = core_begin,
-    .commit = core_commit,
-    .region_add = core_region_add,
-    .region_del = core_region_del,
-    .region_nop = core_region_nop,
-    .log_start = core_log_start,
-    .log_stop = core_log_stop,
-    .log_sync = core_log_sync,
     .log_global_start = core_log_global_start,
     .log_global_stop = core_log_global_stop,
-    .eventfd_add = core_eventfd_add,
-    .eventfd_del = core_eventfd_del,
-    .priority = 0,
+    .priority = 1,
 };
 
 static MemoryListener io_memory_listener = {
-    .begin = io_begin,
-    .commit = io_commit,
     .region_add = io_region_add,
     .region_del = io_region_del,
-    .region_nop = io_region_nop,
-    .log_start = io_log_start,
-    .log_stop = io_log_stop,
-    .log_sync = io_log_sync,
-    .log_global_start = io_log_global_start,
-    .log_global_stop = io_log_global_stop,
-    .eventfd_add = io_eventfd_add,
-    .eventfd_del = io_eventfd_del,
     .priority = 0,
 };
 
+static MemoryListener tcg_memory_listener = {
+    .commit = tcg_commit,
+};
+
+void address_space_init_dispatch(AddressSpace *as)
+{
+    AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
+
+    d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
+    d->listener = (MemoryListener) {
+        .begin = mem_begin,
+        .region_add = mem_add,
+        .region_nop = mem_add,
+        .priority = 0,
+    };
+    as->dispatch = d;
+    memory_listener_register(&d->listener, as);
+}
+
+void address_space_destroy_dispatch(AddressSpace *as)
+{
+    AddressSpaceDispatch *d = as->dispatch;
+
+    memory_listener_unregister(&d->listener);
+    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
+    g_free(d);
+    as->dispatch = NULL;
+}
+
 static void memory_map_init(void)
 {
     system_memory = g_malloc(sizeof(*system_memory));
     memory_region_init(system_memory, "system", INT64_MAX);
-    set_system_memory_map(system_memory);
+    address_space_init(&address_space_memory, system_memory);
+    address_space_memory.name = "memory";
 
     system_io = g_malloc(sizeof(*system_io));
     memory_region_init(system_io, "io", 65536);
-    set_system_io_map(system_io);
+    address_space_init(&address_space_io, system_io);
+    address_space_io.name = "I/O";
+
+    memory_listener_register(&core_memory_listener, &address_space_memory);
+    memory_listener_register(&io_memory_listener, &address_space_io);
+    memory_listener_register(&tcg_memory_listener, &address_space_memory);
 
-    memory_listener_register(&core_memory_listener, system_memory);
-    memory_listener_register(&io_memory_listener, system_io);
+    dma_context_init(&dma_context_memory, &address_space_memory,
+                     NULL, NULL, NULL);
 }
 
 MemoryRegion *get_system_memory(void)
@@ -3406,13 +1851,27 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
 }
 
 #else
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
-                            int len, int is_write)
+
+static void invalidate_and_set_dirty(hwaddr addr,
+                                     hwaddr length)
+{
+    if (!cpu_physical_memory_is_dirty(addr)) {
+        /* invalidate code */
+        tb_invalidate_phys_page_range(addr, addr + length, 0);
+        /* set dirty bit */
+        cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
+    }
+    xen_modified_memory(addr, length);
+}
+
+void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
+                      int len, bool is_write)
 {
+    AddressSpaceDispatch *d = as->dispatch;
     int l;
     uint8_t *ptr;
     uint32_t val;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
 
     while (len > 0) {
@@ -3420,11 +1879,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (is_write) {
             if (!memory_region_is_ram(section->mr)) {
-                target_phys_addr_t addr1;
+                hwaddr addr1;
                 addr1 = memory_region_section_addr(section, addr);
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
@@ -3451,19 +1910,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                 /* RAM case */
                 ptr = qemu_get_ram_ptr(addr1);
                 memcpy(ptr, buf, l);
-                if (!cpu_physical_memory_is_dirty(addr1)) {
-                    /* invalidate code */
-                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
-                    /* set dirty bit */
-                    cpu_physical_memory_set_dirty_flags(
-                        addr1, (0xff & ~CODE_DIRTY_FLAG));
-                }
+                invalidate_and_set_dirty(addr1, l);
                 qemu_put_ram_ptr(ptr);
             }
         } else {
             if (!(memory_region_is_ram(section->mr) ||
                   memory_region_is_romd(section->mr))) {
-                target_phys_addr_t addr1;
+                hwaddr addr1;
                 /* I/O case */
                 addr1 = memory_region_section_addr(section, addr);
                 if (l >= 4 && ((addr1 & 3) == 0)) {
@@ -3497,13 +1950,39 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
     }
 }
 
+void address_space_write(AddressSpace *as, hwaddr addr,
+                         const uint8_t *buf, int len)
+{
+    address_space_rw(as, addr, (uint8_t *)buf, len, true);
+}
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
+{
+    address_space_rw(as, addr, buf, len, false);
+}
+
+
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
+                            int len, int is_write)
+{
+    return address_space_rw(&address_space_memory, addr, buf, len, is_write);
+}
+
 /* used for ROM loading : can write in RAM and ROM */
-void cpu_physical_memory_write_rom(target_phys_addr_t addr,
+void cpu_physical_memory_write_rom(hwaddr addr,
                                    const uint8_t *buf, int len)
 {
+    AddressSpaceDispatch *d = address_space_memory.dispatch;
     int l;
     uint8_t *ptr;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
 
     while (len > 0) {
@@ -3511,7 +1990,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (!(memory_region_is_ram(section->mr) ||
               memory_region_is_romd(section->mr))) {
@@ -3523,6 +2002,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
             /* ROM/RAM case */
             ptr = qemu_get_ram_ptr(addr1);
             memcpy(ptr, buf, l);
+            invalidate_and_set_dirty(addr1, l);
             qemu_put_ram_ptr(ptr);
         }
         len -= l;
@@ -3533,8 +2013,8 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
 
 typedef struct {
     void *buffer;
-    target_phys_addr_t addr;
-    target_phys_addr_t len;
+    hwaddr addr;
+    hwaddr len;
 } BounceBuffer;
 
 static BounceBuffer bounce;
@@ -3558,7 +2038,7 @@ void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
     return client;
 }
 
-void cpu_unregister_map_client(void *_client)
+static void cpu_unregister_map_client(void *_client)
 {
     MapClient *client = (MapClient *)_client;
 
@@ -3584,14 +2064,16 @@ static void cpu_notify_map_clients(void)
  * Use cpu_register_map_client() to know when retrying the map operation is
  * likely to succeed.
  */
-void *cpu_physical_memory_map(target_phys_addr_t addr,
-                              target_phys_addr_t *plen,
-                              int is_write)
-{
-    target_phys_addr_t len = *plen;
-    target_phys_addr_t todo = 0;
+void *address_space_map(AddressSpace *as,
+                        hwaddr addr,
+                        hwaddr *plen,
+                        bool is_write)
+{
+    AddressSpaceDispatch *d = as->dispatch;
+    hwaddr len = *plen;
+    hwaddr todo = 0;
     int l;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
     ram_addr_t raddr = RAM_ADDR_MAX;
     ram_addr_t rlen;
@@ -3602,7 +2084,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
             if (todo || bounce.buffer) {
@@ -3612,7 +2094,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
             bounce.addr = addr;
             bounce.len = l;
             if (!is_write) {
-                cpu_physical_memory_read(addr, bounce.buffer, l);
+                address_space_read(as, addr, bounce.buffer, l);
             }
 
             *plen = l;
@@ -3633,12 +2115,12 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
     return ret;
 }
 
-/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
+/* Unmaps a memory region previously mapped by address_space_map().
  * Will also mark the memory as dirty if is_write == 1.  access_len gives
  * the amount of memory that was actually read or written by the caller.
  */
-void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
-                               int is_write, target_phys_addr_t access_len)
+void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
+                         int is_write, hwaddr access_len)
 {
     if (buffer != bounce.buffer) {
         if (is_write) {
@@ -3648,13 +2130,7 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                 l = TARGET_PAGE_SIZE;
                 if (l > access_len)
                     l = access_len;
-                if (!cpu_physical_memory_is_dirty(addr1)) {
-                    /* invalidate code */
-                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
-                    /* set dirty bit */
-                    cpu_physical_memory_set_dirty_flags(
-                        addr1, (0xff & ~CODE_DIRTY_FLAG));
-                }
+                invalidate_and_set_dirty(addr1, l);
                 addr1 += l;
                 access_len -= l;
             }
@@ -3665,22 +2141,35 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
         return;
     }
     if (is_write) {
-        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
+        address_space_write(as, bounce.addr, bounce.buffer, access_len);
     }
     qemu_vfree(bounce.buffer);
     bounce.buffer = NULL;
     cpu_notify_map_clients();
 }
 
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
+                              int is_write)
+{
+    return address_space_map(&address_space_memory, addr, plen, is_write);
+}
+
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len)
+{
+    return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
+}
+
 /* warning: addr must be aligned */
-static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
+static inline uint32_t ldl_phys_internal(hwaddr addr,
                                          enum device_endian endian)
 {
     uint8_t *ptr;
     uint32_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3716,30 +2205,30 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint32_t ldl_phys(target_phys_addr_t addr)
+uint32_t ldl_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint32_t ldl_le_phys(target_phys_addr_t addr)
+uint32_t ldl_le_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint32_t ldl_be_phys(target_phys_addr_t addr)
+uint32_t ldl_be_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
 
 /* warning: addr must be aligned */
-static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
+static inline uint64_t ldq_phys_internal(hwaddr addr,
                                          enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3775,23 +2264,23 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint64_t ldq_phys(target_phys_addr_t addr)
+uint64_t ldq_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint64_t ldq_le_phys(target_phys_addr_t addr)
+uint64_t ldq_le_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint64_t ldq_be_phys(target_phys_addr_t addr)
+uint64_t ldq_be_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-uint32_t ldub_phys(target_phys_addr_t addr)
+uint32_t ldub_phys(hwaddr addr)
 {
     uint8_t val;
     cpu_physical_memory_read(addr, &val, 1);
@@ -3799,14 +2288,14 @@ uint32_t ldub_phys(target_phys_addr_t addr)
 }
 
 /* warning: addr must be aligned */
-static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
+static inline uint32_t lduw_phys_internal(hwaddr addr,
                                           enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3842,17 +2331,17 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint32_t lduw_phys(target_phys_addr_t addr)
+uint32_t lduw_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint32_t lduw_le_phys(target_phys_addr_t addr)
+uint32_t lduw_le_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint32_t lduw_be_phys(target_phys_addr_t addr)
+uint32_t lduw_be_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
@@ -3860,12 +2349,12 @@ uint32_t lduw_be_phys(target_phys_addr_t addr)
 /* warning: addr must be aligned. The ram page is not masked as dirty
    and the code inside is not invalidated. It is useful if the dirty
    bits are used to track modified PTEs */
-void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
+void stl_phys_notdirty(hwaddr addr, uint32_t val)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3892,12 +2381,12 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
     }
 }
 
-void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
+void stq_phys_notdirty(hwaddr addr, uint64_t val)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3920,13 +2409,13 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
 }
 
 /* warning: addr must be aligned */
-static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
+static inline void stl_phys_internal(hwaddr addr, uint32_t val,
                                      enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3960,46 +2449,40 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
             stl_p(ptr, val);
             break;
         }
-        if (!cpu_physical_memory_is_dirty(addr1)) {
-            /* invalidate code */
-            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
-            /* set dirty bit */
-            cpu_physical_memory_set_dirty_flags(addr1,
-                (0xff & ~CODE_DIRTY_FLAG));
-        }
+        invalidate_and_set_dirty(addr1, 4);
     }
 }
 
-void stl_phys(target_phys_addr_t addr, uint32_t val)
+void stl_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
 }
 
-void stl_le_phys(target_phys_addr_t addr, uint32_t val)
+void stl_le_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
 }
 
-void stl_be_phys(target_phys_addr_t addr, uint32_t val)
+void stl_be_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-void stb_phys(target_phys_addr_t addr, uint32_t val)
+void stb_phys(hwaddr addr, uint32_t val)
 {
     uint8_t v = val;
     cpu_physical_memory_write(addr, &v, 1);
 }
 
 /* warning: addr must be aligned */
-static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
+static inline void stw_phys_internal(hwaddr addr, uint32_t val,
                                      enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -4033,45 +2516,39 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
             stw_p(ptr, val);
             break;
         }
-        if (!cpu_physical_memory_is_dirty(addr1)) {
-            /* invalidate code */
-            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
-            /* set dirty bit */
-            cpu_physical_memory_set_dirty_flags(addr1,
-                (0xff & ~CODE_DIRTY_FLAG));
-        }
+        invalidate_and_set_dirty(addr1, 2);
     }
 }
 
-void stw_phys(target_phys_addr_t addr, uint32_t val)
+void stw_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
 }
 
-void stw_le_phys(target_phys_addr_t addr, uint32_t val)
+void stw_le_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
 }
 
-void stw_be_phys(target_phys_addr_t addr, uint32_t val)
+void stw_be_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-void stq_phys(target_phys_addr_t addr, uint64_t val)
+void stq_phys(hwaddr addr, uint64_t val)
 {
     val = tswap64(val);
     cpu_physical_memory_write(addr, &val, 8);
 }
 
-void stq_le_phys(target_phys_addr_t addr, uint64_t val)
+void stq_le_phys(hwaddr addr, uint64_t val)
 {
     val = cpu_to_le64(val);
     cpu_physical_memory_write(addr, &val, 8);
 }
 
-void stq_be_phys(target_phys_addr_t addr, uint64_t val)
+void stq_be_phys(hwaddr addr, uint64_t val)
 {
     val = cpu_to_be64(val);
     cpu_physical_memory_write(addr, &val, 8);
@@ -4082,7 +2559,7 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
                         uint8_t *buf, int len, int is_write)
 {
     int l;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page;
 
     while (len > 0) {
@@ -4107,119 +2584,8 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
 }
 #endif
 
-/* in deterministic execution mode, instructions doing device I/Os
-   must be at the end of the TB */
-void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-    uint32_t n, cflags;
-    target_ulong pc, cs_base;
-    uint64_t flags;
-
-    tb = tb_find_pc(retaddr);
-    if (!tb) {
-        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
-                  (void *)retaddr);
-    }
-    n = env->icount_decr.u16.low + tb->icount;
-    cpu_restore_state(tb, env, retaddr);
-    /* Calculate how many instructions had been executed before the fault
-       occurred.  */
-    n = n - env->icount_decr.u16.low;
-    /* Generate a new TB ending on the I/O insn.  */
-    n++;
-    /* On MIPS and SH, delay slot instructions can only be restarted if
-       they were already the first instruction in the TB.  If this is not
-       the first instruction in a TB then re-execute the preceding
-       branch.  */
-#if defined(TARGET_MIPS)
-    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
-        env->active_tc.PC -= 4;
-        env->icount_decr.u16.low++;
-        env->hflags &= ~MIPS_HFLAG_BMASK;
-    }
-#elif defined(TARGET_SH4)
-    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
-            && n > 1) {
-        env->pc -= 2;
-        env->icount_decr.u16.low++;
-        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
-    }
-#endif
-    /* This should never happen.  */
-    if (n > CF_COUNT_MASK)
-        cpu_abort(env, "TB too big during recompile");
-
-    cflags = n | CF_LAST_IO;
-    pc = tb->pc;
-    cs_base = tb->cs_base;
-    flags = tb->flags;
-    tb_phys_invalidate(tb, -1);
-    /* FIXME: In theory this could raise an exception.  In practice
-       we have already translated the block once so it's probably ok.  */
-    tb_gen_code(env, pc, cs_base, flags, cflags);
-    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
-       the first in the TB) then we end up generating a whole new TB and
-       repeating the fault, which is horribly inefficient.
-       Better would be to execute just this insn uncached, or generate a
-       second new TB.  */
-    cpu_resume_from_signal(env, NULL);
-}
-
 #if !defined(CONFIG_USER_ONLY)
 
-void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
-{
-    int i, target_code_size, max_target_code_size;
-    int direct_jmp_count, direct_jmp2_count, cross_page;
-    TranslationBlock *tb;
-
-    target_code_size = 0;
-    max_target_code_size = 0;
-    cross_page = 0;
-    direct_jmp_count = 0;
-    direct_jmp2_count = 0;
-    for(i = 0; i < nb_tbs; i++) {
-        tb = &tbs[i];
-        target_code_size += tb->size;
-        if (tb->size > max_target_code_size)
-            max_target_code_size = tb->size;
-        if (tb->page_addr[1] != -1)
-            cross_page++;
-        if (tb->tb_next_offset[0] != 0xffff) {
-            direct_jmp_count++;
-            if (tb->tb_next_offset[1] != 0xffff) {
-                direct_jmp2_count++;
-            }
-        }
-    }
-    /* XXX: avoid using doubles ? */
-    cpu_fprintf(f, "Translation buffer state:\n");
-    cpu_fprintf(f, "gen code size       %td/%ld\n",
-                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
-    cpu_fprintf(f, "TB count            %d/%d\n", 
-                nb_tbs, code_gen_max_blocks);
-    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
-                nb_tbs ? target_code_size / nb_tbs : 0,
-                max_target_code_size);
-    cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
-                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
-                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
-    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
-            cross_page,
-            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
-    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
-                direct_jmp_count,
-                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
-                direct_jmp2_count,
-                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
-    cpu_fprintf(f, "\nStatistics:\n");
-    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
-    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
-    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
-    tcg_dump_info(f, cpu_fprintf);
-}
-
 /*
  * A helper function for the _utterly broken_ virtio device model to find out if
  * it's running on a big endian machine. Don't do this at home kids!
@@ -4237,11 +2603,12 @@ bool virtio_is_big_endian(void)
 #endif
 
 #ifndef CONFIG_USER_ONLY
-bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr)
+bool cpu_physical_memory_is_io(hwaddr phys_addr)
 {
     MemoryRegionSection *section;
 
-    section = phys_page_find(phys_addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch,
+                             phys_addr >> TARGET_PAGE_BITS);
 
     return !(memory_region_is_ram(section->mr) ||
              memory_region_is_romd(section->mr));
diff --git a/fpu/softfloat-specialize.h b/fpu/softfloat-specialize.h
index 490245004fa..518f694a680 100644
--- a/fpu/softfloat-specialize.h
+++ b/fpu/softfloat-specialize.h
@@ -41,6 +41,13 @@ these four paragraphs for those parts of this code that are retained.
 #define SNAN_BIT_IS_ONE		0
 #endif
 
+#if defined(TARGET_XTENSA)
+/* Define for architectures which deviate from IEEE in not supporting
+ * signaling NaNs (so all NaNs are treated as quiet).
+ */
+#define NO_SIGNALING_NANS 1
+#endif
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
@@ -57,7 +64,8 @@ const float16 float16_default_nan = const_float16(0xFE00);
 *----------------------------------------------------------------------------*/
 #if defined(TARGET_SPARC)
 const float32 float32_default_nan = const_float32(0x7FFFFFFF);
-#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA)
+#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA) || \
+      defined(TARGET_XTENSA)
 const float32 float32_default_nan = const_float32(0x7FC00000);
 #elif SNAN_BIT_IS_ONE
 const float32 float32_default_nan = const_float32(0x7FBFFFFF);
@@ -127,6 +135,17 @@ typedef struct {
     uint64_t high, low;
 } commonNaNT;
 
+#ifdef NO_SIGNALING_NANS
+int float16_is_quiet_nan(float16 a_)
+{
+    return float16_is_any_nan(a_);
+}
+
+int float16_is_signaling_nan(float16 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the half-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -156,6 +175,7 @@ int float16_is_signaling_nan(float16 a_)
     return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the half-precision floating point value `a' is a
@@ -217,6 +237,17 @@ static float16 commonNaNToFloat16(commonNaNT a STATUS_PARAM)
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float32_is_quiet_nan(float32 a_)
+{
+    return float32_is_any_nan(a_);
+}
+
+int float32_is_signaling_nan(float32 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the single-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -246,6 +277,7 @@ int float32_is_signaling_nan( float32 a_ )
     return ( ( ( a>>22 ) & 0x1FF ) == 0x1FE ) && ( a & 0x003FFFFF );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the single-precision floating point value `a' is a
@@ -372,7 +404,7 @@ static int pickNaN(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
         return 1;
     }
 }
-#elif defined(TARGET_PPC)
+#elif defined(TARGET_PPC) || defined(TARGET_XTENSA)
 static int pickNaN(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
                    flag aIsLargerSignificand)
 {
@@ -454,6 +486,33 @@ static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
         return 1;
     }
 }
+#elif defined(TARGET_MIPS)
+static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
+                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+{
+    /* For MIPS, the (inf,zero,qnan) case sets InvalidOp and returns
+     * the default NaN
+     */
+    if (infzero) {
+        float_raise(float_flag_invalid STATUS_VAR);
+        return 3;
+    }
+
+    /* Prefer sNaN over qNaN, in the a, b, c order. */
+    if (aIsSNaN) {
+        return 0;
+    } else if (bIsSNaN) {
+        return 1;
+    } else if (cIsSNaN) {
+        return 2;
+    } else if (aIsQNaN) {
+        return 0;
+    } else if (bIsQNaN) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
 #elif defined(TARGET_PPC)
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
                          flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
@@ -586,6 +645,17 @@ static float32 propagateFloat32MulAddNaN(float32 a, float32 b,
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float64_is_quiet_nan(float64 a_)
+{
+    return float64_is_any_nan(a_);
+}
+
+int float64_is_signaling_nan(float64 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the double-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -619,6 +689,7 @@ int float64_is_signaling_nan( float64 a_ )
         && ( a & LIT64( 0x0007FFFFFFFFFFFF ) );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the double-precision floating point value `a' is a
@@ -773,6 +844,17 @@ static float64 propagateFloat64MulAddNaN(float64 a, float64 b,
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int floatx80_is_quiet_nan(floatx80 a_)
+{
+    return floatx80_is_any_nan(a_);
+}
+
+int floatx80_is_signaling_nan(floatx80 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the extended double-precision floating-point value `a' is a
 | quiet NaN; otherwise returns 0. This slightly differs from the same
@@ -816,6 +898,7 @@ int floatx80_is_signaling_nan( floatx80 a )
         && ( a.low == aLow );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the extended double-precision floating point value
@@ -929,6 +1012,17 @@ static floatx80 propagateFloatx80NaN( floatx80 a, floatx80 b STATUS_PARAM)
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float128_is_quiet_nan(float128 a_)
+{
+    return float128_is_any_nan(a_);
+}
+
+int float128_is_signaling_nan(float128 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the quadruple-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -964,6 +1058,7 @@ int float128_is_signaling_nan( float128 a )
         && ( a.low || ( a.high & LIT64( 0x00007FFFFFFFFFFF ) ) );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the quadruple-precision floating point value `a' is
diff --git a/fpu/softfloat.c b/fpu/softfloat.c
index b29256a8ebe..83ccc4b8cd3 100644
--- a/fpu/softfloat.c
+++ b/fpu/softfloat.c
@@ -40,7 +40,7 @@ these four paragraphs for those parts of this code that are retained.
  */
 #include "config.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 /*----------------------------------------------------------------------------
 | Primitive arithmetic functions, including multi-word arithmetic, and
@@ -1238,7 +1238,7 @@ float32 uint64_to_float32( uint64 a STATUS_PARAM )
     if ( a == 0 ) return float32_zero;
     shiftCount = countLeadingZeros64( a ) - 40;
     if ( 0 <= shiftCount ) {
-        return packFloat32( 1 > 0, 0x95 - shiftCount, a<<shiftCount );
+        return packFloat32(0, 0x95 - shiftCount, a<<shiftCount);
     }
     else {
         shiftCount += 7;
@@ -1248,7 +1248,7 @@ float32 uint64_to_float32( uint64 a STATUS_PARAM )
         else {
             a <<= shiftCount;
         }
-        return roundAndPackFloat32( 1 > 0, 0x9C - shiftCount, a STATUS_VAR );
+        return roundAndPackFloat32(0, 0x9C - shiftCount, a STATUS_VAR);
     }
 }
 
@@ -1271,11 +1271,18 @@ float64 int64_to_float64( int64 a STATUS_PARAM )
 
 }
 
-float64 uint64_to_float64( uint64 a STATUS_PARAM )
+float64 uint64_to_float64(uint64 a STATUS_PARAM)
 {
-    if ( a == 0 ) return float64_zero;
-    return normalizeRoundAndPackFloat64( 0, 0x43C, a STATUS_VAR );
+    int exp =  0x43C;
 
+    if (a == 0) {
+        return float64_zero;
+    }
+    if ((int64_t)a < 0) {
+        shift64RightJamming(a, 1, &a);
+        exp += 1;
+    }
+    return normalizeRoundAndPackFloat64(0, exp, a STATUS_VAR);
 }
 
 /*----------------------------------------------------------------------------
@@ -1332,6 +1339,14 @@ float128 int64_to_float128( int64 a STATUS_PARAM )
 
 }
 
+float128 uint64_to_float128(uint64 a STATUS_PARAM)
+{
+    if (a == 0) {
+        return float128_zero;
+    }
+    return normalizeRoundAndPackFloat128(0, 0x406E, a, 0 STATUS_VAR);
+}
+
 /*----------------------------------------------------------------------------
 | Returns the result of converting the single-precision floating-point value
 | `a' to the 32-bit two's complement integer format.  The conversion is
@@ -2219,7 +2234,7 @@ float32 float32_muladd(float32 a, float32 b, float32 c, int flags STATUS_PARAM)
             }
         }
         /* Zero plus something non-zero : just return the something */
-        return make_float32(float32_val(c) ^ (signflip << 31));
+        return packFloat32(cSign ^ signflip, cExp, cSig);
     }
 
     if (aExp == 0) {
@@ -3007,7 +3022,7 @@ float32 float16_to_float32(float16 a, flag ieee STATUS_PARAM)
         if (aSig) {
             return commonNaNToFloat32(float16ToCommonNaN(a STATUS_VAR) STATUS_VAR);
         }
-        return packFloat32(aSign, 0xff, aSig << 13);
+        return packFloat32(aSign, 0xff, 0);
     }
     if (aExp == 0) {
         int8 shiftCount;
@@ -3772,7 +3787,7 @@ float64 float64_muladd(float64 a, float64 b, float64 c, int flags STATUS_PARAM)
             }
         }
         /* Zero plus something non-zero : just return the something */
-        return make_float64(float64_val(c) ^ ((uint64_t)signflip << 63));
+        return packFloat64(cSign ^ signflip, cExp, cSig);
     }
 
     if (aExp == 0) {
diff --git a/fsdev/Makefile.objs b/fsdev/Makefile.objs
index cb1e2500b99..206289c49f1 100644
--- a/fsdev/Makefile.objs
+++ b/fsdev/Makefile.objs
@@ -1,9 +1,10 @@
 ifeq ($(CONFIG_REALLY_VIRTFS),y)
 common-obj-y = qemu-fsdev.o virtio-9p-marshal.o
-
-# Toplevel always builds this; targets without virtio will put it in
-# common-obj-y
-extra-obj-y = qemu-fsdev-dummy.o
 else
 common-obj-y = qemu-fsdev-dummy.o
 endif
+common-obj-y += qemu-fsdev-opts.o
+
+# Toplevel always builds this; targets without virtio will put it in
+# common-obj-y
+common-obj-$(CONFIG_ALL) += qemu-fsdev-dummy.o
diff --git a/fsdev/qemu-fsdev-dummy.c b/fsdev/qemu-fsdev-dummy.c
index 4e700dd4e4f..7dc2630a782 100644
--- a/fsdev/qemu-fsdev-dummy.c
+++ b/fsdev/qemu-fsdev-dummy.c
@@ -13,16 +13,10 @@
 #include <stdio.h>
 #include <string.h>
 #include "qemu-fsdev.h"
-#include "qemu-config.h"
+#include "qemu/config-file.h"
+#include "qemu/module.h"
 
 int qemu_fsdev_add(QemuOpts *opts)
 {
     return 0;
 }
-
-static void fsdev_register_config(void)
-{
-    qemu_add_opts(&qemu_fsdev_opts);
-    qemu_add_opts(&qemu_virtfs_opts);
-}
-machine_init(fsdev_register_config);
diff --git a/fsdev/qemu-fsdev-opts.c b/fsdev/qemu-fsdev-opts.c
new file mode 100644
index 00000000000..6311c7a7e56
--- /dev/null
+++ b/fsdev/qemu-fsdev-opts.c
@@ -0,0 +1,85 @@
+/*
+ * Virtio 9p
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/config-file.h"
+#include "qemu/option.h"
+#include "qemu/module.h"
+
+static QemuOptsList qemu_fsdev_opts = {
+    .name = "fsdev",
+    .implied_opt_name = "fsdriver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_fsdev_opts.head),
+    .desc = {
+        {
+            .name = "fsdriver",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "security_model",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "writeout",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+
+        }, {
+            .name = "socket",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "sock_fd",
+            .type = QEMU_OPT_NUMBER,
+        },
+
+        { /*End of list */ }
+    },
+};
+
+static QemuOptsList qemu_virtfs_opts = {
+    .name = "virtfs",
+    .implied_opt_name = "fsdriver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_virtfs_opts.head),
+    .desc = {
+        {
+            .name = "fsdriver",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "mount_tag",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "security_model",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "writeout",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "socket",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "sock_fd",
+            .type = QEMU_OPT_NUMBER,
+        },
+
+        { /*End of list */ }
+    },
+};
+
+static void fsdev_register_config(void)
+{
+    qemu_add_opts(&qemu_fsdev_opts);
+    qemu_add_opts(&qemu_virtfs_opts);
+}
+machine_init(fsdev_register_config);
diff --git a/fsdev/qemu-fsdev.c b/fsdev/qemu-fsdev.c
index e20202a4bfb..6eaf36dbfa2 100644
--- a/fsdev/qemu-fsdev.c
+++ b/fsdev/qemu-fsdev.c
@@ -13,10 +13,10 @@
 #include <stdio.h>
 #include <string.h>
 #include "qemu-fsdev.h"
-#include "qemu-queue.h"
-#include "osdep.h"
+#include "qemu/queue.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "qemu-config.h"
+#include "qemu/config-file.h"
 
 static QTAILQ_HEAD(FsDriverEntry_head, FsDriverListEntry) fsdriver_entries =
     QTAILQ_HEAD_INITIALIZER(fsdriver_entries);
@@ -97,11 +97,3 @@ FsDriverEntry *get_fsdev_fsentry(char *id)
     }
     return NULL;
 }
-
-static void fsdev_register_config(void)
-{
-    qemu_add_opts(&qemu_fsdev_opts);
-    qemu_add_opts(&qemu_virtfs_opts);
-}
-machine_init(fsdev_register_config);
-
diff --git a/fsdev/qemu-fsdev.h b/fsdev/qemu-fsdev.h
index 1af1f545d87..9fa45bf5107 100644
--- a/fsdev/qemu-fsdev.h
+++ b/fsdev/qemu-fsdev.h
@@ -12,7 +12,7 @@
  */
 #ifndef QEMU_FSDEV_H
 #define QEMU_FSDEV_H
-#include "qemu-option.h"
+#include "qemu/option.h"
 #include "file-op-9p.h"
 
 
diff --git a/fsdev/virtfs-proxy-helper.c b/fsdev/virtfs-proxy-helper.c
index f9a8270ee96..6b9afd32d59 100644
--- a/fsdev/virtfs-proxy-helper.c
+++ b/fsdev/virtfs-proxy-helper.c
@@ -21,8 +21,8 @@
 #include <linux/magic.h>
 #endif
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "qemu-xattr.h"
+#include "qemu/sockets.h"
+#include "qemu/xattr.h"
 #include "virtio-9p-marshal.h"
 #include "hw/9pfs/virtio-9p-proxy.h"
 #include "fsdev/virtio-9p-marshal.h"
@@ -272,31 +272,76 @@ static int send_status(int sockfd, struct iovec *iovec, int status)
 /*
  * from man 7 capabilities, section
  * Effect of User ID Changes on Capabilities:
- * 4. If the file system user ID is changed from 0 to nonzero (see setfsuid(2))
- * then the following capabilities are cleared from the effective set:
- * CAP_CHOWN, CAP_DAC_OVERRIDE, CAP_DAC_READ_SEARCH,  CAP_FOWNER, CAP_FSETID,
- * CAP_LINUX_IMMUTABLE  (since  Linux 2.2.30), CAP_MAC_OVERRIDE, and CAP_MKNOD
- * (since Linux 2.2.30). If the file system UID is changed from nonzero to 0,
- * then any of these capabilities that are enabled in the permitted set
- * are enabled in the effective set.
+ * If the effective user ID is changed from nonzero to 0, then the permitted
+ * set is copied to the effective set.  If the effective user ID is changed
+ * from 0 to nonzero, then all capabilities are are cleared from the effective
+ * set.
+ *
+ * The setfsuid/setfsgid man pages warn that changing the effective user ID may
+ * expose the program to unwanted signals, but this is not true anymore: for an
+ * unprivileged (without CAP_KILL) program to send a signal, the real or
+ * effective user ID of the sending process must equal the real or saved user
+ * ID of the target process.  Even when dropping privileges, it is enough to
+ * keep the saved UID to a "privileged" value and virtfs-proxy-helper won't
+ * be exposed to signals.  So just use setresuid/setresgid.
  */
-static int setfsugid(int uid, int gid)
+static int setugid(int uid, int gid, int *suid, int *sgid)
 {
+    int retval;
+
     /*
-     * We still need DAC_OVERRIDE because  we don't change
+     * We still need DAC_OVERRIDE because we don't change
      * supplementary group ids, and hence may be subjected DAC rules
      */
     cap_value_t cap_list[] = {
         CAP_DAC_OVERRIDE,
     };
 
-    setfsgid(gid);
-    setfsuid(uid);
+    *suid = geteuid();
+    *sgid = getegid();
+
+    if (setresgid(-1, gid, *sgid) == -1) {
+        retval = -errno;
+        goto err_out;
+    }
+
+    if (setresuid(-1, uid, *suid) == -1) {
+        retval = -errno;
+        goto err_sgid;
+    }
 
     if (uid != 0 || gid != 0) {
-        return do_cap_set(cap_list, ARRAY_SIZE(cap_list), 0);
+        if (do_cap_set(cap_list, ARRAY_SIZE(cap_list), 0) < 0) {
+            retval = -errno;
+            goto err_suid;
+        }
     }
     return 0;
+
+err_suid:
+    if (setresuid(-1, *suid, *suid) == -1) {
+        abort();
+    }
+err_sgid:
+    if (setresgid(-1, *sgid, *sgid) == -1) {
+        abort();
+    }
+err_out:
+    return retval;
+}
+
+/*
+ * This is used to reset the ugid back with the saved values
+ * There is nothing much we can do checking error values here.
+ */
+static void resetugid(int suid, int sgid)
+{
+    if (setresgid(-1, sgid, sgid) == -1) {
+        abort();
+    }
+    if (setresuid(-1, suid, suid) == -1) {
+        abort();
+    }
 }
 
 /*
@@ -578,18 +623,15 @@ static int do_create_others(int type, struct iovec *iovec)
 
     v9fs_string_init(&path);
     v9fs_string_init(&oldpath);
-    cur_uid = geteuid();
-    cur_gid = getegid();
 
     retval = proxy_unmarshal(iovec, offset, "dd", &uid, &gid);
     if (retval < 0) {
         return retval;
     }
     offset += retval;
-    retval = setfsugid(uid, gid);
+    retval = setugid(uid, gid, &cur_uid, &cur_gid);
     if (retval < 0) {
-        retval = -errno;
-        goto err_out;
+        goto unmarshal_err_out;
     }
     switch (type) {
     case T_MKNOD:
@@ -619,9 +661,10 @@ static int do_create_others(int type, struct iovec *iovec)
     }
 
 err_out:
+    resetugid(cur_uid, cur_gid);
+unmarshal_err_out:
     v9fs_string_free(&path);
     v9fs_string_free(&oldpath);
-    setfsugid(cur_uid, cur_gid);
     return retval;
 }
 
@@ -641,24 +684,16 @@ static int do_create(struct iovec *iovec)
     if (ret < 0) {
         goto unmarshal_err_out;
     }
-    cur_uid = geteuid();
-    cur_gid = getegid();
-    ret = setfsugid(uid, gid);
+    ret = setugid(uid, gid, &cur_uid, &cur_gid);
     if (ret < 0) {
-        /*
-         * On failure reset back to the
-         * old uid/gid
-         */
-        ret = -errno;
-        goto err_out;
+        goto unmarshal_err_out;
     }
     ret = open(path.data, flags, mode);
     if (ret < 0) {
         ret = -errno;
     }
 
-err_out:
-    setfsugid(cur_uid, cur_gid);
+    resetugid(cur_uid, cur_gid);
 unmarshal_err_out:
     v9fs_string_free(&path);
     return ret;
diff --git a/fsdev/virtio-9p-marshal.c b/fsdev/virtio-9p-marshal.c
index bf980bfa38f..20f308b7602 100644
--- a/fsdev/virtio-9p-marshal.c
+++ b/fsdev/virtio-9p-marshal.c
@@ -22,9 +22,9 @@
 #include <stdint.h>
 #include <errno.h>
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "virtio-9p-marshal.h"
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 void v9fs_string_free(V9fsString *str)
 {
diff --git a/gdbstub.c b/gdbstub.c
index 5d37dd98f49..6cd26f16193 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -29,17 +29,18 @@
 
 #include "qemu.h"
 #else
-#include "monitor.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "gdbstub.h"
+#include "monitor/monitor.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
 #endif
 
 #define MAX_PACKET_LENGTH 4096
 
 #include "cpu.h"
-#include "qemu_socket.h"
-#include "kvm.h"
+#include "qemu/sockets.h"
+#include "sysemu/kvm.h"
+#include "qemu/bitops.h"
 
 #ifndef TARGET_CPU_MEMORY_RW_DEBUG
 static inline int target_memory_rw_debug(CPUArchState *env, target_ulong addr,
@@ -1226,33 +1227,48 @@ static int cpu_gdb_write_register(CPUOpenRISCState *env,
 
 static int cpu_gdb_read_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 {
-    if (n < 8) {
+    switch (n) {
+    case 0 ... 7:
         if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
             GET_REGL(env->gregs[n + 16]);
         } else {
             GET_REGL(env->gregs[n]);
         }
-    } else if (n < 16) {
+    case 8 ... 15:
         GET_REGL(env->gregs[n]);
-    } else if (n >= 25 && n < 41) {
-	GET_REGL(env->fregs[(n - 25) + ((env->fpscr & FPSCR_FR) ? 16 : 0)]);
-    } else if (n >= 43 && n < 51) {
-	GET_REGL(env->gregs[n - 43]);
-    } else if (n >= 51 && n < 59) {
-	GET_REGL(env->gregs[n - (51 - 16)]);
-    }
-    switch (n) {
-    case 16: GET_REGL(env->pc);
-    case 17: GET_REGL(env->pr);
-    case 18: GET_REGL(env->gbr);
-    case 19: GET_REGL(env->vbr);
-    case 20: GET_REGL(env->mach);
-    case 21: GET_REGL(env->macl);
-    case 22: GET_REGL(env->sr);
-    case 23: GET_REGL(env->fpul);
-    case 24: GET_REGL(env->fpscr);
-    case 41: GET_REGL(env->ssr);
-    case 42: GET_REGL(env->spc);
+    case 16:
+        GET_REGL(env->pc);
+    case 17:
+        GET_REGL(env->pr);
+    case 18:
+        GET_REGL(env->gbr);
+    case 19:
+        GET_REGL(env->vbr);
+    case 20:
+        GET_REGL(env->mach);
+    case 21:
+        GET_REGL(env->macl);
+    case 22:
+        GET_REGL(env->sr);
+    case 23:
+        GET_REGL(env->fpul);
+    case 24:
+        GET_REGL(env->fpscr);
+    case 25 ... 40:
+        if (env->fpscr & FPSCR_FR) {
+            stfl_p(mem_buf, env->fregs[n - 9]);
+        } else {
+            stfl_p(mem_buf, env->fregs[n - 25]);
+        }
+        return 4;
+    case 41:
+        GET_REGL(env->ssr);
+    case 42:
+        GET_REGL(env->spc);
+    case 43 ... 50:
+        GET_REGL(env->gregs[n - 43]);
+    case 51 ... 58:
+        GET_REGL(env->gregs[n - (51 - 16)]);
     }
 
     return 0;
@@ -1260,42 +1276,63 @@ static int cpu_gdb_read_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 
 static int cpu_gdb_write_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 {
-    uint32_t tmp;
-
-    tmp = ldl_p(mem_buf);
-
-    if (n < 8) {
+    switch (n) {
+    case 0 ... 7:
         if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
-            env->gregs[n + 16] = tmp;
+            env->gregs[n + 16] = ldl_p(mem_buf);
         } else {
-            env->gregs[n] = tmp;
+            env->gregs[n] = ldl_p(mem_buf);
         }
-	return 4;
-    } else if (n < 16) {
-        env->gregs[n] = tmp;
-	return 4;
-    } else if (n >= 25 && n < 41) {
-	env->fregs[(n - 25) + ((env->fpscr & FPSCR_FR) ? 16 : 0)] = tmp;
-	return 4;
-    } else if (n >= 43 && n < 51) {
-	env->gregs[n - 43] = tmp;
-	return 4;
-    } else if (n >= 51 && n < 59) {
-	env->gregs[n - (51 - 16)] = tmp;
-	return 4;
-    }
-    switch (n) {
-    case 16: env->pc = tmp; break;
-    case 17: env->pr = tmp; break;
-    case 18: env->gbr = tmp; break;
-    case 19: env->vbr = tmp; break;
-    case 20: env->mach = tmp; break;
-    case 21: env->macl = tmp; break;
-    case 22: env->sr = tmp; break;
-    case 23: env->fpul = tmp; break;
-    case 24: env->fpscr = tmp; break;
-    case 41: env->ssr = tmp; break;
-    case 42: env->spc = tmp; break;
+        break;
+    case 8 ... 15:
+        env->gregs[n] = ldl_p(mem_buf);
+        break;
+    case 16:
+        env->pc = ldl_p(mem_buf);
+        break;
+    case 17:
+        env->pr = ldl_p(mem_buf);
+        break;
+    case 18:
+        env->gbr = ldl_p(mem_buf);
+        break;
+    case 19:
+        env->vbr = ldl_p(mem_buf);
+        break;
+    case 20:
+        env->mach = ldl_p(mem_buf);
+        break;
+    case 21:
+        env->macl = ldl_p(mem_buf);
+        break;
+    case 22:
+        env->sr = ldl_p(mem_buf);
+        break;
+    case 23:
+        env->fpul = ldl_p(mem_buf);
+        break;
+    case 24:
+        env->fpscr = ldl_p(mem_buf);
+        break;
+    case 25 ... 40:
+        if (env->fpscr & FPSCR_FR) {
+            env->fregs[n - 9] = ldfl_p(mem_buf);
+        } else {
+            env->fregs[n - 25] = ldfl_p(mem_buf);
+        }
+        break;
+    case 41:
+        env->ssr = ldl_p(mem_buf);
+        break;
+    case 42:
+        env->spc = ldl_p(mem_buf);
+        break;
+    case 43 ... 50:
+        env->gregs[n - 43] = ldl_p(mem_buf);
+        break;
+    case 51 ... 58:
+        env->gregs[n - (51 - 16)] = ldl_p(mem_buf);
+        break;
     default: return 0;
     }
 
@@ -1499,27 +1536,34 @@ static int cpu_gdb_write_register(CPUAlphaState *env, uint8_t *mem_buf, int n)
 }
 #elif defined (TARGET_S390X)
 
-#define NUM_CORE_REGS S390_NUM_TOTAL_REGS
+#define NUM_CORE_REGS  S390_NUM_REGS
 
 static int cpu_gdb_read_register(CPUS390XState *env, uint8_t *mem_buf, int n)
 {
+    uint64_t val;
+    int cc_op;
+
     switch (n) {
-        case S390_PSWM_REGNUM: GET_REGL(env->psw.mask); break;
-        case S390_PSWA_REGNUM: GET_REGL(env->psw.addr); break;
-        case S390_R0_REGNUM ... S390_R15_REGNUM:
-            GET_REGL(env->regs[n-S390_R0_REGNUM]); break;
-        case S390_A0_REGNUM ... S390_A15_REGNUM:
-            GET_REG32(env->aregs[n-S390_A0_REGNUM]); break;
-        case S390_FPC_REGNUM: GET_REG32(env->fpc); break;
-        case S390_F0_REGNUM ... S390_F15_REGNUM:
-            /* XXX */
-            break;
-        case S390_PC_REGNUM: GET_REGL(env->psw.addr); break;
-        case S390_CC_REGNUM:
-            env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst,
-                                 env->cc_vr);
-            GET_REG32(env->cc_op);
-            break;
+    case S390_PSWM_REGNUM:
+        cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
+        val = deposit64(env->psw.mask, 44, 2, cc_op);
+        GET_REGL(val);
+        break;
+    case S390_PSWA_REGNUM:
+        GET_REGL(env->psw.addr);
+        break;
+    case S390_R0_REGNUM ... S390_R15_REGNUM:
+        GET_REGL(env->regs[n-S390_R0_REGNUM]);
+        break;
+    case S390_A0_REGNUM ... S390_A15_REGNUM:
+        GET_REG32(env->aregs[n-S390_A0_REGNUM]);
+        break;
+    case S390_FPC_REGNUM:
+        GET_REG32(env->fpc);
+        break;
+    case S390_F0_REGNUM ... S390_F15_REGNUM:
+        GET_REG64(env->fregs[n-S390_F0_REGNUM].ll);
+        break;
     }
 
     return 0;
@@ -1534,20 +1578,30 @@ static int cpu_gdb_write_register(CPUS390XState *env, uint8_t *mem_buf, int n)
     tmp32 = ldl_p(mem_buf);
 
     switch (n) {
-        case S390_PSWM_REGNUM: env->psw.mask = tmpl; break;
-        case S390_PSWA_REGNUM: env->psw.addr = tmpl; break;
-        case S390_R0_REGNUM ... S390_R15_REGNUM:
-            env->regs[n-S390_R0_REGNUM] = tmpl; break;
-        case S390_A0_REGNUM ... S390_A15_REGNUM:
-            env->aregs[n-S390_A0_REGNUM] = tmp32; r=4; break;
-        case S390_FPC_REGNUM: env->fpc = tmp32; r=4; break;
-        case S390_F0_REGNUM ... S390_F15_REGNUM:
-            /* XXX */
-            break;
-        case S390_PC_REGNUM: env->psw.addr = tmpl; break;
-        case S390_CC_REGNUM: env->cc_op = tmp32; r=4; break;
+    case S390_PSWM_REGNUM:
+        env->psw.mask = tmpl;
+        env->cc_op = extract64(tmpl, 44, 2);
+        break;
+    case S390_PSWA_REGNUM:
+        env->psw.addr = tmpl;
+        break;
+    case S390_R0_REGNUM ... S390_R15_REGNUM:
+        env->regs[n-S390_R0_REGNUM] = tmpl;
+        break;
+    case S390_A0_REGNUM ... S390_A15_REGNUM:
+        env->aregs[n-S390_A0_REGNUM] = tmp32;
+        r = 4;
+        break;
+    case S390_FPC_REGNUM:
+        env->fpc = tmp32;
+        r = 4;
+        break;
+    case S390_F0_REGNUM ... S390_F15_REGNUM:
+        env->fregs[n-S390_F0_REGNUM].ll = tmpl;
+        break;
+    default:
+        return 0;
     }
-
     return r;
 }
 #elif defined (TARGET_LM32)
@@ -1660,6 +1714,10 @@ static int cpu_gdb_read_register(CPUXtensaState *env, uint8_t *mem_buf, int n)
         GET_REG32(env->uregs[reg->targno & 0xff]);
         break;
 
+    case 4: /*f*/
+        GET_REG32(float32_val(env->fregs[reg->targno & 0x0f]));
+        break;
+
     case 8: /*a*/
         GET_REG32(env->regs[reg->targno & 0x0f]);
         break;
@@ -1700,6 +1758,10 @@ static int cpu_gdb_write_register(CPUXtensaState *env, uint8_t *mem_buf, int n)
         env->uregs[reg->targno & 0xff] = tmp;
         break;
 
+    case 4: /*f*/
+        env->fregs[reg->targno & 0x0f] = make_float32(tmp);
+        break;
+
     case 8: /*a*/
         env->regs[reg->targno & 0x0f] = tmp;
         break;
@@ -2339,9 +2401,10 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
             thread = strtoull(p+16, (char **)&p, 16);
             env = find_cpu(thread);
             if (env != NULL) {
+                CPUState *cpu = ENV_GET_CPU(env);
                 cpu_synchronize_state(env);
                 len = snprintf((char *)mem_buf, sizeof(mem_buf),
-                               "CPU#%d [%s]", env->cpu_index,
+                               "CPU#%d [%s]", cpu->cpu_index,
                                env->halted ? "halted " : "running");
                 memtohex(buf, mem_buf, len);
                 put_packet(s, buf);
diff --git a/hmp-commands.hx b/hmp-commands.hx
index f6104b099f8..bdd48f3469e 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -99,16 +99,60 @@ ETEXI
 
     {
         .name       = "block_job_cancel",
-        .args_type  = "device:B",
-        .params     = "device",
-        .help       = "stop an active background block operation",
+        .args_type  = "force:-f,device:B",
+        .params     = "[-f] device",
+        .help       = "stop an active background block operation (use -f"
+                      "\n\t\t\t if the operation is currently paused)",
         .mhandler.cmd = hmp_block_job_cancel,
     },
 
 STEXI
 @item block_job_cancel
 @findex block_job_cancel
-Stop an active block streaming operation.
+Stop an active background block operation (streaming, mirroring).
+ETEXI
+
+    {
+        .name       = "block_job_complete",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "stop an active background block operation",
+        .mhandler.cmd = hmp_block_job_complete,
+    },
+
+STEXI
+@item block_job_complete
+@findex block_job_complete
+Manually trigger completion of an active background block operation.
+For mirroring, this will switch the device to the destination path.
+ETEXI
+
+    {
+        .name       = "block_job_pause",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "pause an active background block operation",
+        .mhandler.cmd = hmp_block_job_pause,
+    },
+
+STEXI
+@item block_job_pause
+@findex block_job_pause
+Pause an active block streaming operation.
+ETEXI
+
+    {
+        .name       = "block_job_resume",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "resume a paused background block operation",
+        .mhandler.cmd = hmp_block_job_resume,
+    },
+
+STEXI
+@item block_job_resume
+@findex block_job_resume
+Resume a paused block streaming operation.
 ETEXI
 
     {
@@ -194,8 +238,7 @@ ETEXI
         .args_type  = "filename:F",
         .params     = "filename",
         .help       = "save screen into PPM image 'filename'",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_screen_dump,
+        .mhandler.cmd = hmp_screen_dump,
     },
 
 STEXI
@@ -502,19 +545,19 @@ ETEXI
 
     {
         .name       = "sendkey",
-        .args_type  = "string:s,hold_time:i?",
+        .args_type  = "keys:s,hold-time:i?",
         .params     = "keys [hold_ms]",
         .help       = "send keys to the VM (e.g. 'sendkey ctrl-alt-f1', default hold time=100 ms)",
-        .mhandler.cmd = do_sendkey,
+        .mhandler.cmd = hmp_send_key,
     },
 
 STEXI
 @item sendkey @var{keys}
 @findex sendkey
 
-Send @var{keys} to the emulator. @var{keys} could be the name of the
-key or @code{#} followed by the raw value in either decimal or hexadecimal
-format. Use @code{-} to press several keys simultaneously. Example:
+Send @var{keys} to the guest. @var{keys} could be the name of the
+key or the raw value in hexadecimal format. Use @code{-} to press
+several keys simultaneously. Example:
 @example
 sendkey ctrl-alt-f1
 @end example
@@ -794,6 +837,45 @@ STEXI
 @item nmi @var{cpu}
 @findex nmi
 Inject an NMI on the given CPU (x86 only).
+
+ETEXI
+
+    {
+        .name       = "memchar_write",
+        .args_type  = "device:s,data:s",
+        .params     = "device data",
+        .help       = "Provide writing interface for CirMemCharDriver. Write"
+                      "'data' to it.",
+        .mhandler.cmd = hmp_memchar_write,
+    },
+
+STEXI
+@item memchar_write @var{device} @var{data}
+@findex memchar_write
+Provide writing interface for CirMemCharDriver. Write @var{data}
+to char device 'memory'.
+
+ETEXI
+
+    {
+        .name       = "memchar_read",
+        .args_type  = "device:s,size:i",
+        .params     = "device size",
+        .help       = "Provide read interface for CirMemCharDriver. Read from"
+                      "it and return the data with size.",
+        .mhandler.cmd = hmp_memchar_read,
+    },
+
+STEXI
+@item memchar_read @var{device}
+@findex memchar_read
+Provide read interface for CirMemCharDriver. Read from char device
+'memory' and return the data.
+
+@var{size} is the size of data want to read from. Refer to unencoded
+size of the raw data, would adjust to the init size of the memchar
+if the requested size is larger than it.
+
 ETEXI
 
     {
@@ -915,12 +997,11 @@ ETEXI
 #if defined(CONFIG_HAVE_CORE_DUMP)
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:-p,protocol:s,begin:i?,length:i?",
-        .params     = "[-p] protocol [begin] [length]",
+        .args_type  = "paging:-p,filename:F,begin:i?,length:i?",
+        .params     = "[-p] filename [begin] [length]",
         .help       = "dump guest memory to file"
                       "\n\t\t\t begin(optional): the starting physical address"
                       "\n\t\t\t length(optional): the memory size, in bytes",
-        .user_print = monitor_user_noop,
         .mhandler.cmd = hmp_dump_guest_memory,
     },
 
@@ -930,8 +1011,7 @@ STEXI
 @findex dump-guest-memory
 Dump guest memory to @var{protocol}. The file can be processed with crash or
 gdb.
-  protocol: destination file(started with "file:") or destination file
-            descriptor (started with "fd:")
+  filename: dump file name
     paging: do paging to get guest's memory mapping
      begin: the starting physical address. It's optional, and should be
             specified with length together.
@@ -963,6 +1043,27 @@ Snapshot device, using snapshot file as target if provided
 ETEXI
 
     {
+        .name       = "drive_mirror",
+        .args_type  = "reuse:-n,full:-f,device:B,target:s,format:s?",
+        .params     = "[-n] [-f] device target [format]",
+        .help       = "initiates live storage\n\t\t\t"
+                      "migration for a device. The device's contents are\n\t\t\t"
+                      "copied to the new image file, including data that\n\t\t\t"
+                      "is written after the command is started.\n\t\t\t"
+                      "The -n flag requests QEMU to reuse the image found\n\t\t\t"
+                      "in new-image-file, instead of recreating it from scratch.\n\t\t\t"
+                      "The -f flag requests QEMU to copy the whole disk,\n\t\t\t"
+                      "so that the result does not need a backing file.\n\t\t\t",
+        .mhandler.cmd = hmp_drive_mirror,
+    },
+STEXI
+@item drive_mirror
+@findex drive_mirror
+Start mirroring a block device's writes to a new destination,
+using the specified target.
+ETEXI
+
+    {
         .name       = "drive_add",
         .args_type  = "pci_addr:s,opts:s",
         .params     = "[[<domain>:]<bus>:]<slot>\n"
@@ -1248,6 +1349,51 @@ Remove all matches from the access control list, and set the default
 policy back to @code{deny}.
 ETEXI
 
+    {
+        .name       = "nbd_server_start",
+        .args_type  = "all:-a,writable:-w,uri:s",
+        .params     = "nbd_server_start [-a] [-w] host:port",
+        .help       = "serve block devices on the given host and port",
+        .mhandler.cmd = hmp_nbd_server_start,
+    },
+STEXI
+@item nbd_server_start @var{host}:@var{port}
+@findex nbd_server_start
+Start an NBD server on the given host and/or port.  If the @option{-a}
+option is included, all of the virtual machine's block devices that
+have an inserted media on them are automatically exported; in this case,
+the @option{-w} option makes the devices writable too.
+ETEXI
+
+    {
+        .name       = "nbd_server_add",
+        .args_type  = "writable:-w,device:B",
+        .params     = "nbd_server_add [-w] device",
+        .help       = "export a block device via NBD",
+        .mhandler.cmd = hmp_nbd_server_add,
+    },
+STEXI
+@item nbd_server_add @var{device}
+@findex nbd_server_add
+Export a block device through QEMU's NBD server, which must be started
+beforehand with @command{nbd_server_start}.  The @option{-w} option makes the
+exported device writable too.
+ETEXI
+
+    {
+        .name       = "nbd_server_stop",
+        .args_type  = "",
+        .params     = "nbd_server_stop",
+        .help       = "stop serving block devices using the NBD protocol",
+        .mhandler.cmd = hmp_nbd_server_stop,
+    },
+STEXI
+@item nbd_server_stop
+@findex nbd_server_stop
+Stop the QEMU embedded NBD server.
+ETEXI
+
+
 #if defined(TARGET_I386)
 
     {
@@ -1378,11 +1524,44 @@ passed since 1970, i.e. unix epoch.
 ETEXI
 
     {
+        .name       = "chardev-add",
+        .args_type  = "args:s",
+        .params     = "args",
+        .help       = "add chardev",
+        .mhandler.cmd = hmp_chardev_add,
+    },
+
+STEXI
+@item chardev_add args
+@findex chardev_add
+
+chardev_add accepts the same parameters as the -chardev command line switch.
+
+ETEXI
+
+    {
+        .name       = "chardev-remove",
+        .args_type  = "id:s",
+        .params     = "id",
+        .help       = "remove chardev",
+        .mhandler.cmd = hmp_chardev_remove,
+    },
+
+STEXI
+@item chardev_remove id
+@findex chardev_remove
+
+Removes the chardev @var{id}.
+
+ETEXI
+
+    {
         .name       = "info",
         .args_type  = "item:s?",
         .params     = "[subcommand]",
         .help       = "show various information about the system state",
-        .mhandler.cmd = do_info,
+        .mhandler.cmd = do_info_help,
+        .sub_table = info_cmds,
     },
 
 STEXI
@@ -1466,13 +1645,6 @@ show roms
 @end table
 ETEXI
 
-#ifdef CONFIG_TRACE_SIMPLE
-STEXI
-@item info trace
-show contents of trace buffer
-ETEXI
-#endif
-
 STEXI
 @item info trace-events
 show available trace events and their state
diff --git a/hmp.c b/hmp.c
index 81c8acb67aa..249b89b7e37 100644
--- a/hmp.c
+++ b/hmp.c
@@ -14,11 +14,14 @@
  */
 
 #include "hmp.h"
-#include "net.h"
-#include "qemu-option.h"
-#include "qemu-timer.h"
+#include "net/net.h"
+#include "char/char.h"
+#include "qemu/option.h"
+#include "qemu/timer.h"
 #include "qmp-commands.h"
-#include "monitor.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
 
 static void hmp_handle_error(Monitor *mon, Error **errp)
 {
@@ -28,7 +31,7 @@ static void hmp_handle_error(Monitor *mon, Error **errp)
     }
 }
 
-void hmp_info_name(Monitor *mon)
+void hmp_info_name(Monitor *mon, const QDict *qdict)
 {
     NameInfo *info;
 
@@ -39,7 +42,7 @@ void hmp_info_name(Monitor *mon)
     qapi_free_NameInfo(info);
 }
 
-void hmp_info_version(Monitor *mon)
+void hmp_info_version(Monitor *mon, const QDict *qdict)
 {
     VersionInfo *info;
 
@@ -52,7 +55,7 @@ void hmp_info_version(Monitor *mon)
     qapi_free_VersionInfo(info);
 }
 
-void hmp_info_kvm(Monitor *mon)
+void hmp_info_kvm(Monitor *mon, const QDict *qdict)
 {
     KvmInfo *info;
 
@@ -67,7 +70,7 @@ void hmp_info_kvm(Monitor *mon)
     qapi_free_KvmInfo(info);
 }
 
-void hmp_info_status(Monitor *mon)
+void hmp_info_status(Monitor *mon, const QDict *qdict)
 {
     StatusInfo *info;
 
@@ -86,7 +89,7 @@ void hmp_info_status(Monitor *mon)
     qapi_free_StatusInfo(info);
 }
 
-void hmp_info_uuid(Monitor *mon)
+void hmp_info_uuid(Monitor *mon, const QDict *qdict)
 {
     UuidInfo *info;
 
@@ -95,7 +98,7 @@ void hmp_info_uuid(Monitor *mon)
     qapi_free_UuidInfo(info);
 }
 
-void hmp_info_chardev(Monitor *mon)
+void hmp_info_chardev(Monitor *mon, const QDict *qdict)
 {
     ChardevInfoList *char_info, *info;
 
@@ -108,7 +111,7 @@ void hmp_info_chardev(Monitor *mon)
     qapi_free_ChardevInfoList(char_info);
 }
 
-void hmp_info_mice(Monitor *mon)
+void hmp_info_mice(Monitor *mon, const QDict *qdict)
 {
     MouseInfoList *mice_list, *mouse;
 
@@ -128,7 +131,7 @@ void hmp_info_mice(Monitor *mon)
     qapi_free_MouseInfoList(mice_list);
 }
 
-void hmp_info_migrate(Monitor *mon)
+void hmp_info_migrate(Monitor *mon, const QDict *qdict)
 {
     MigrationInfo *info;
     MigrationCapabilityStatusList *caps, *cap;
@@ -151,6 +154,14 @@ void hmp_info_migrate(Monitor *mon)
         monitor_printf(mon, "Migration status: %s\n", info->status);
         monitor_printf(mon, "total time: %" PRIu64 " milliseconds\n",
                        info->total_time);
+        if (info->has_expected_downtime) {
+            monitor_printf(mon, "expected downtime: %" PRIu64 " milliseconds\n",
+                           info->expected_downtime);
+        }
+        if (info->has_downtime) {
+            monitor_printf(mon, "downtime: %" PRIu64 " milliseconds\n",
+                           info->downtime);
+        }
     }
 
     if (info->has_ram) {
@@ -166,6 +177,10 @@ void hmp_info_migrate(Monitor *mon)
                        info->ram->normal);
         monitor_printf(mon, "normal bytes: %" PRIu64 " kbytes\n",
                        info->ram->normal_bytes >> 10);
+        if (info->ram->dirty_pages_rate) {
+            monitor_printf(mon, "dirty pages rate: %" PRIu64 " pages\n",
+                           info->ram->dirty_pages_rate);
+        }
     }
 
     if (info->has_disk) {
@@ -194,7 +209,7 @@ void hmp_info_migrate(Monitor *mon)
     qapi_free_MigrationCapabilityStatusList(caps);
 }
 
-void hmp_info_migrate_capabilities(Monitor *mon)
+void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict)
 {
     MigrationCapabilityStatusList *caps, *cap;
 
@@ -213,13 +228,13 @@ void hmp_info_migrate_capabilities(Monitor *mon)
     qapi_free_MigrationCapabilityStatusList(caps);
 }
 
-void hmp_info_migrate_cache_size(Monitor *mon)
+void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "xbzrel cache size: %" PRId64 " kbytes\n",
                    qmp_query_migrate_cache_size(NULL) >> 10);
 }
 
-void hmp_info_cpus(Monitor *mon)
+void hmp_info_cpus(Monitor *mon, const QDict *qdict)
 {
     CpuInfoList *cpu_list, *cpu;
 
@@ -232,20 +247,19 @@ void hmp_info_cpus(Monitor *mon)
             active = '*';
         }
 
-        monitor_printf(mon, "%c CPU #%" PRId64 ": ", active, cpu->value->CPU);
+        monitor_printf(mon, "%c CPU #%" PRId64 ":", active, cpu->value->CPU);
 
         if (cpu->value->has_pc) {
-            monitor_printf(mon, "pc=0x%016" PRIx64, cpu->value->pc);
+            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->pc);
         }
         if (cpu->value->has_nip) {
-            monitor_printf(mon, "nip=0x%016" PRIx64, cpu->value->nip);
+            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->nip);
         }
         if (cpu->value->has_npc) {
-            monitor_printf(mon, "pc=0x%016" PRIx64, cpu->value->pc);
-            monitor_printf(mon, "npc=0x%016" PRIx64, cpu->value->npc);
+            monitor_printf(mon, " npc=0x%016" PRIx64, cpu->value->npc);
         }
         if (cpu->value->has_PC) {
-            monitor_printf(mon, "PC=0x%016" PRIx64, cpu->value->PC);
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->PC);
         }
 
         if (cpu->value->halted) {
@@ -258,7 +272,7 @@ void hmp_info_cpus(Monitor *mon)
     qapi_free_CpuInfoList(cpu_list);
 }
 
-void hmp_info_block(Monitor *mon)
+void hmp_info_block(Monitor *mon, const QDict *qdict)
 {
     BlockInfoList *block_list, *info;
 
@@ -312,7 +326,7 @@ void hmp_info_block(Monitor *mon)
     qapi_free_BlockInfoList(block_list);
 }
 
-void hmp_info_blockstats(Monitor *mon)
+void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
 {
     BlockStatsList *stats_list, *stats;
 
@@ -346,7 +360,7 @@ void hmp_info_blockstats(Monitor *mon)
     qapi_free_BlockStatsList(stats_list);
 }
 
-void hmp_info_vnc(Monitor *mon)
+void hmp_info_vnc(Monitor *mon, const QDict *qdict)
 {
     VncInfo *info;
     Error *err = NULL;
@@ -392,7 +406,7 @@ out:
     qapi_free_VncInfo(info);
 }
 
-void hmp_info_spice(Monitor *mon)
+void hmp_info_spice(Monitor *mon, const QDict *qdict)
 {
     SpiceChannelList *chan;
     SpiceInfo *info;
@@ -413,6 +427,8 @@ void hmp_info_spice(Monitor *mon)
         monitor_printf(mon, "     address: %s:%" PRId64 " [tls]\n",
                        info->host, info->tls_port);
     }
+    monitor_printf(mon, "    migrated: %s\n",
+                   info->migrated ? "true" : "false");
     monitor_printf(mon, "        auth: %s\n", info->auth);
     monitor_printf(mon, "    compiled: %s\n", info->compiled_version);
     monitor_printf(mon, "  mouse-mode: %s\n",
@@ -437,7 +453,7 @@ out:
     qapi_free_SpiceInfo(info);
 }
 
-void hmp_info_balloon(Monitor *mon)
+void hmp_info_balloon(Monitor *mon, const QDict *qdict)
 {
     BalloonInfo *info;
     Error *err = NULL;
@@ -449,29 +465,7 @@ void hmp_info_balloon(Monitor *mon)
         return;
     }
 
-    monitor_printf(mon, "balloon: actual=%" PRId64, info->actual >> 20);
-    if (info->has_mem_swapped_in) {
-        monitor_printf(mon, " mem_swapped_in=%" PRId64, info->mem_swapped_in);
-    }
-    if (info->has_mem_swapped_out) {
-        monitor_printf(mon, " mem_swapped_out=%" PRId64, info->mem_swapped_out);
-    }
-    if (info->has_major_page_faults) {
-        monitor_printf(mon, " major_page_faults=%" PRId64,
-                       info->major_page_faults);
-    }
-    if (info->has_minor_page_faults) {
-        monitor_printf(mon, " minor_page_faults=%" PRId64,
-                       info->minor_page_faults);
-    }
-    if (info->has_free_mem) {
-        monitor_printf(mon, " free_mem=%" PRId64, info->free_mem);
-    }
-    if (info->has_total_mem) {
-        monitor_printf(mon, " total_mem=%" PRId64, info->total_mem);
-    }
-
-    monitor_printf(mon, "\n");
+    monitor_printf(mon, "balloon: actual=%" PRId64 "\n", info->actual >> 20);
 
     qapi_free_BalloonInfo(info);
 }
@@ -554,7 +548,7 @@ static void hmp_info_pci_device(Monitor *mon, const PciDeviceInfo *dev)
     }
 }
 
-void hmp_info_pci(Monitor *mon)
+void hmp_info_pci(Monitor *mon, const QDict *qdict)
 {
     PciInfoList *info_list, *info;
     Error *err = NULL;
@@ -577,7 +571,7 @@ void hmp_info_pci(Monitor *mon)
     qapi_free_PciInfoList(info_list);
 }
 
-void hmp_info_block_jobs(Monitor *mon)
+void hmp_info_block_jobs(Monitor *mon, const QDict *qdict)
 {
     BlockJobInfoList *list;
     Error *err = NULL;
@@ -668,6 +662,40 @@ void hmp_pmemsave(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &errp);
 }
 
+void hmp_memchar_write(Monitor *mon, const QDict *qdict)
+{
+    uint32_t size;
+    const char *chardev = qdict_get_str(qdict, "device");
+    const char *data = qdict_get_str(qdict, "data");
+    Error *errp = NULL;
+
+    size = strlen(data);
+    qmp_memchar_write(chardev, size, data, false, 0, &errp);
+
+    hmp_handle_error(mon, &errp);
+}
+
+void hmp_memchar_read(Monitor *mon, const QDict *qdict)
+{
+    uint32_t size = qdict_get_int(qdict, "size");
+    const char *chardev = qdict_get_str(qdict, "device");
+    MemCharRead *meminfo;
+    Error *errp = NULL;
+
+    meminfo = qmp_memchar_read(chardev, size, false, 0, &errp);
+    if (errp) {
+        monitor_printf(mon, "%s\n", error_get_pretty(errp));
+        error_free(errp);
+        return;
+    }
+
+    if (meminfo->count > 0) {
+        monitor_printf(mon, "%s\n", meminfo->data);
+    }
+
+    qapi_free_MemCharRead(meminfo);
+}
+
 static void hmp_cont_cb(void *opaque, int err)
 {
     if (!err) {
@@ -756,6 +784,35 @@ void hmp_block_resize(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &errp);
 }
 
+void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    const char *filename = qdict_get_str(qdict, "target");
+    const char *format = qdict_get_try_str(qdict, "format");
+    int reuse = qdict_get_try_bool(qdict, "reuse", 0);
+    int full = qdict_get_try_bool(qdict, "full", 0);
+    enum NewImageMode mode;
+    Error *errp = NULL;
+
+    if (!filename) {
+        error_set(&errp, QERR_MISSING_PARAMETER, "target");
+        hmp_handle_error(mon, &errp);
+        return;
+    }
+
+    if (reuse) {
+        mode = NEW_IMAGE_MODE_EXISTING;
+    } else {
+        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+    }
+
+    qmp_drive_mirror(device, filename, !!format, format,
+                     full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
+                     true, mode, false, 0, false, 0, false, 0,
+                     false, 0, false, 0, &errp);
+    hmp_handle_error(mon, &errp);
+}
+
 void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_str(qdict, "device");
@@ -927,7 +984,8 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
     int64_t speed = qdict_get_try_int(qdict, "speed", 0);
 
     qmp_block_stream(device, base != NULL, base,
-                     qdict_haskey(qdict, "speed"), speed, &error);
+                     qdict_haskey(qdict, "speed"), speed,
+                     BLOCKDEV_ON_ERROR_REPORT, true, &error);
 
     hmp_handle_error(mon, &error);
 }
@@ -947,8 +1005,39 @@ void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
 {
     Error *error = NULL;
     const char *device = qdict_get_str(qdict, "device");
+    bool force = qdict_get_try_bool(qdict, "force", 0);
+
+    qmp_block_job_cancel(device, true, force, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_pause(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
 
-    qmp_block_job_cancel(device, &error);
+    qmp_block_job_pause(device, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_resume(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
+
+    qmp_block_job_resume(device, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_complete(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
+
+    qmp_block_job_complete(device, &error);
 
     hmp_handle_error(mon, &error);
 }
@@ -1039,11 +1128,12 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *errp = NULL;
     int paging = qdict_get_try_bool(qdict, "paging", 0);
-    const char *file = qdict_get_str(qdict, "protocol");
+    const char *file = qdict_get_str(qdict, "filename");
     bool has_begin = qdict_haskey(qdict, "begin");
     bool has_length = qdict_haskey(qdict, "length");
     int64_t begin = 0;
     int64_t length = 0;
+    char *prot;
 
     if (has_begin) {
         begin = qdict_get_int(qdict, "begin");
@@ -1052,9 +1142,12 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
         length = qdict_get_int(qdict, "length");
     }
 
-    qmp_dump_guest_memory(paging, file, has_begin, begin, has_length, length,
+    prot = g_strconcat("file:", file, NULL);
+
+    qmp_dump_guest_memory(paging, prot, has_begin, begin, has_length, length,
                           &errp);
     hmp_handle_error(mon, &errp);
+    g_free(prot);
 }
 
 void hmp_netdev_add(Monitor *mon, const QDict *qdict)
@@ -1102,3 +1195,179 @@ void hmp_closefd(Monitor *mon, const QDict *qdict)
     qmp_closefd(fdname, &errp);
     hmp_handle_error(mon, &errp);
 }
+
+void hmp_send_key(Monitor *mon, const QDict *qdict)
+{
+    const char *keys = qdict_get_str(qdict, "keys");
+    KeyValueList *keylist, *head = NULL, *tmp = NULL;
+    int has_hold_time = qdict_haskey(qdict, "hold-time");
+    int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
+    Error *err = NULL;
+    char keyname_buf[16];
+    char *separator;
+    int keyname_len;
+
+    while (1) {
+        separator = strchr(keys, '-');
+        keyname_len = separator ? separator - keys : strlen(keys);
+        pstrcpy(keyname_buf, sizeof(keyname_buf), keys);
+
+        /* Be compatible with old interface, convert user inputted "<" */
+        if (!strncmp(keyname_buf, "<", 1) && keyname_len == 1) {
+            pstrcpy(keyname_buf, sizeof(keyname_buf), "less");
+            keyname_len = 4;
+        }
+        keyname_buf[keyname_len] = 0;
+
+        keylist = g_malloc0(sizeof(*keylist));
+        keylist->value = g_malloc0(sizeof(*keylist->value));
+
+        if (!head) {
+            head = keylist;
+        }
+        if (tmp) {
+            tmp->next = keylist;
+        }
+        tmp = keylist;
+
+        if (strstart(keyname_buf, "0x", NULL)) {
+            char *endp;
+            int value = strtoul(keyname_buf, &endp, 0);
+            if (*endp != '\0') {
+                goto err_out;
+            }
+            keylist->value->kind = KEY_VALUE_KIND_NUMBER;
+            keylist->value->number = value;
+        } else {
+            int idx = index_from_key(keyname_buf);
+            if (idx == Q_KEY_CODE_MAX) {
+                goto err_out;
+            }
+            keylist->value->kind = KEY_VALUE_KIND_QCODE;
+            keylist->value->qcode = idx;
+        }
+
+        if (!separator) {
+            break;
+        }
+        keys = separator + 1;
+    }
+
+    qmp_send_key(head, has_hold_time, hold_time, &err);
+    hmp_handle_error(mon, &err);
+
+out:
+    qapi_free_KeyValueList(head);
+    return;
+
+err_out:
+    monitor_printf(mon, "invalid parameter: %s\n", keyname_buf);
+    goto out;
+}
+
+void hmp_screen_dump(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    Error *err = NULL;
+
+    qmp_screendump(filename, &err);
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_nbd_server_start(Monitor *mon, const QDict *qdict)
+{
+    const char *uri = qdict_get_str(qdict, "uri");
+    int writable = qdict_get_try_bool(qdict, "writable", 0);
+    int all = qdict_get_try_bool(qdict, "all", 0);
+    Error *local_err = NULL;
+    BlockInfoList *block_list, *info;
+    SocketAddress *addr;
+
+    if (writable && !all) {
+        error_setg(&local_err, "-w only valid together with -a");
+        goto exit;
+    }
+
+    /* First check if the address is valid and start the server.  */
+    addr = socket_parse(uri, &local_err);
+    if (local_err != NULL) {
+        goto exit;
+    }
+
+    qmp_nbd_server_start(addr, &local_err);
+    qapi_free_SocketAddress(addr);
+    if (local_err != NULL) {
+        goto exit;
+    }
+
+    if (!all) {
+        return;
+    }
+
+    /* Then try adding all block devices.  If one fails, close all and
+     * exit.
+     */
+    block_list = qmp_query_block(NULL);
+
+    for (info = block_list; info; info = info->next) {
+        if (!info->value->has_inserted) {
+            continue;
+        }
+
+        qmp_nbd_server_add(info->value->device, true, writable, &local_err);
+
+        if (local_err != NULL) {
+            qmp_nbd_server_stop(NULL);
+            break;
+        }
+    }
+
+    qapi_free_BlockInfoList(block_list);
+
+exit:
+    hmp_handle_error(mon, &local_err);
+}
+
+void hmp_nbd_server_add(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    int writable = qdict_get_try_bool(qdict, "writable", 0);
+    Error *local_err = NULL;
+
+    qmp_nbd_server_add(device, true, writable, &local_err);
+
+    if (local_err != NULL) {
+        hmp_handle_error(mon, &local_err);
+    }
+}
+
+void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict)
+{
+    Error *errp = NULL;
+
+    qmp_nbd_server_stop(&errp);
+    hmp_handle_error(mon, &errp);
+}
+
+void hmp_chardev_add(Monitor *mon, const QDict *qdict)
+{
+    const char *args = qdict_get_str(qdict, "args");
+    Error *err = NULL;
+    QemuOpts *opts;
+
+    opts = qemu_opts_parse(qemu_find_opts("chardev"), args, 1);
+    if (opts == NULL) {
+        error_setg(&err, "Parsing chardev args failed\n");
+    } else {
+        qemu_chr_new_from_opts(opts, NULL, &err);
+    }
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_chardev_remove(Monitor *mon, const QDict *qdict)
+{
+    Error *local_err = NULL;
+
+    qmp_chardev_remove(qdict_get_str(qdict, "id"), &local_err);
+    hmp_handle_error(mon, &local_err);
+}
diff --git a/hmp.h b/hmp.h
index 7dd93bf0f45..076d8cf378f 100644
--- a/hmp.h
+++ b/hmp.h
@@ -16,26 +16,26 @@
 
 #include "qemu-common.h"
 #include "qapi-types.h"
-#include "qdict.h"
+#include "qapi/qmp/qdict.h"
 
-void hmp_info_name(Monitor *mon);
-void hmp_info_version(Monitor *mon);
-void hmp_info_kvm(Monitor *mon);
-void hmp_info_status(Monitor *mon);
-void hmp_info_uuid(Monitor *mon);
-void hmp_info_chardev(Monitor *mon);
-void hmp_info_mice(Monitor *mon);
-void hmp_info_migrate(Monitor *mon);
-void hmp_info_migrate_capabilities(Monitor *mon);
-void hmp_info_migrate_cache_size(Monitor *mon);
-void hmp_info_cpus(Monitor *mon);
-void hmp_info_block(Monitor *mon);
-void hmp_info_blockstats(Monitor *mon);
-void hmp_info_vnc(Monitor *mon);
-void hmp_info_spice(Monitor *mon);
-void hmp_info_balloon(Monitor *mon);
-void hmp_info_pci(Monitor *mon);
-void hmp_info_block_jobs(Monitor *mon);
+void hmp_info_name(Monitor *mon, const QDict *qdict);
+void hmp_info_version(Monitor *mon, const QDict *qdict);
+void hmp_info_kvm(Monitor *mon, const QDict *qdict);
+void hmp_info_status(Monitor *mon, const QDict *qdict);
+void hmp_info_uuid(Monitor *mon, const QDict *qdict);
+void hmp_info_chardev(Monitor *mon, const QDict *qdict);
+void hmp_info_mice(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict);
+void hmp_info_cpus(Monitor *mon, const QDict *qdict);
+void hmp_info_block(Monitor *mon, const QDict *qdict);
+void hmp_info_blockstats(Monitor *mon, const QDict *qdict);
+void hmp_info_vnc(Monitor *mon, const QDict *qdict);
+void hmp_info_spice(Monitor *mon, const QDict *qdict);
+void hmp_info_balloon(Monitor *mon, const QDict *qdict);
+void hmp_info_pci(Monitor *mon, const QDict *qdict);
+void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
@@ -43,6 +43,8 @@ void hmp_system_powerdown(Monitor *mon, const QDict *qdict);
 void hmp_cpu(Monitor *mon, const QDict *qdict);
 void hmp_memsave(Monitor *mon, const QDict *qdict);
 void hmp_pmemsave(Monitor *mon, const QDict *qdict);
+void hmp_memchar_write(Monitor *mon, const QDict *qdict);
+void hmp_memchar_read(Monitor *mon, const QDict *qdict);
 void hmp_cont(Monitor *mon, const QDict *qdict);
 void hmp_system_wakeup(Monitor *mon, const QDict *qdict);
 void hmp_inject_nmi(Monitor *mon, const QDict *qdict);
@@ -51,6 +53,7 @@ void hmp_block_passwd(Monitor *mon, const QDict *qdict);
 void hmp_balloon(Monitor *mon, const QDict *qdict);
 void hmp_block_resize(Monitor *mon, const QDict *qdict);
 void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
+void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
 void hmp_migrate_cancel(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_downtime(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_speed(Monitor *mon, const QDict *qdict);
@@ -64,6 +67,9 @@ void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict);
 void hmp_block_stream(Monitor *mon, const QDict *qdict);
 void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
 void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
+void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
+void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
+void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
 void hmp_migrate(Monitor *mon, const QDict *qdict);
 void hmp_device_del(Monitor *mon, const QDict *qdict);
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict);
@@ -71,5 +77,12 @@ void hmp_netdev_add(Monitor *mon, const QDict *qdict);
 void hmp_netdev_del(Monitor *mon, const QDict *qdict);
 void hmp_getfd(Monitor *mon, const QDict *qdict);
 void hmp_closefd(Monitor *mon, const QDict *qdict);
+void hmp_send_key(Monitor *mon, const QDict *qdict);
+void hmp_screen_dump(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_start(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_add(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict);
+void hmp_chardev_add(Monitor *mon, const QDict *qdict);
+void hmp_chardev_remove(Monitor *mon, const QDict *qdict);
 
 #endif
diff --git a/hw/9pfs/Makefile.objs b/hw/9pfs/Makefile.objs
index 972df240504..1e9b595cb4d 100644
--- a/hw/9pfs/Makefile.objs
+++ b/hw/9pfs/Makefile.objs
@@ -1,9 +1,9 @@
-hw-obj-y  = virtio-9p.o
-hw-obj-y += virtio-9p-local.o virtio-9p-xattr.o
-hw-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
-hw-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
-hw-obj-y += coxattr.o virtio-9p-synth.o
-hw-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
-hw-obj-y += virtio-9p-proxy.o
+common-obj-y  = virtio-9p.o
+common-obj-y += virtio-9p-local.o virtio-9p-xattr.o
+common-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
+common-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
+common-obj-y += coxattr.o virtio-9p-synth.o
+common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
+common-obj-y += virtio-9p-proxy.o
 
 obj-y += virtio-9p-device.o
diff --git a/hw/9pfs/codir.c b/hw/9pfs/codir.c
index 3d188284ba3..65ad3298be9 100644
--- a/hw/9pfs/codir.c
+++ b/hw/9pfs/codir.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_readdir_r(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent *dent,
diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c
index 9345aaeb2ef..2efebf35710 100644
--- a/hw/9pfs/cofile.c
+++ b/hw/9pfs/cofile.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode,
diff --git a/hw/9pfs/cofs.c b/hw/9pfs/cofs.c
index 83f125bd47c..38910507485 100644
--- a/hw/9pfs/cofs.c
+++ b/hw/9pfs/cofs.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_readlink(V9fsPDU *pdu, V9fsPath *path, V9fsString *buf)
diff --git a/hw/9pfs/coxattr.c b/hw/9pfs/coxattr.c
index 8a482287023..18ee08df0f5 100644
--- a/hw/9pfs/coxattr.c
+++ b/hw/9pfs/coxattr.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_llistxattr(V9fsPDU *pdu, V9fsPath *path, void *value, size_t size)
diff --git a/hw/9pfs/virtio-9p-coth.c b/hw/9pfs/virtio-9p-coth.c
index 25556cc6a74..ae6cde80053 100644
--- a/hw/9pfs/virtio-9p-coth.c
+++ b/hw/9pfs/virtio-9p-coth.c
@@ -12,10 +12,9 @@
  *
  */
 
-#include "qemu-char.h"
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 /* v9fs glib thread pool */
diff --git a/hw/9pfs/virtio-9p-coth.h b/hw/9pfs/virtio-9p-coth.h
index c31c96578bd..86d5ed41699 100644
--- a/hw/9pfs/virtio-9p-coth.h
+++ b/hw/9pfs/virtio-9p-coth.h
@@ -15,8 +15,8 @@
 #ifndef _QEMU_VIRTIO_9P_COTH_H
 #define _QEMU_VIRTIO_9P_COTH_H
 
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p.h"
 #include <glib.h>
 
diff --git a/hw/9pfs/virtio-9p-device.c b/hw/9pfs/virtio-9p-device.c
index b8220abae79..6f427dfc5d6 100644
--- a/hw/9pfs/virtio-9p-device.c
+++ b/hw/9pfs/virtio-9p-device.c
@@ -13,7 +13,7 @@
 
 #include "hw/virtio.h"
 #include "hw/pc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "hw/virtio-pci.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
@@ -170,14 +170,14 @@ static void virtio_9p_class_init(ObjectClass *klass, void *data)
 
     k->init = virtio_9p_init_pci;
     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
-    k->device_id = 0x1009;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_9P;
     k->revision = VIRTIO_PCI_ABI_VERSION;
     k->class_id = 0x2;
     dc->props = virtio_9p_properties;
     dc->reset = virtio_pci_reset;
 }
 
-static TypeInfo virtio_9p_info = {
+static const TypeInfo virtio_9p_info = {
     .name          = "virtio-9p-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
diff --git a/hw/9pfs/virtio-9p-handle.c b/hw/9pfs/virtio-9p-handle.c
index f96d17a9749..e30fdb6730e 100644
--- a/hw/9pfs/virtio-9p-handle.c
+++ b/hw/9pfs/virtio-9p-handle.c
@@ -19,7 +19,7 @@
 #include <grp.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include <unistd.h>
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
diff --git a/hw/9pfs/virtio-9p-local.c b/hw/9pfs/virtio-9p-local.c
index 33a41d2e18b..113602144cd 100644
--- a/hw/9pfs/virtio-9p-local.c
+++ b/hw/9pfs/virtio-9p-local.c
@@ -19,7 +19,7 @@
 #include <grp.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include <libgen.h>
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
diff --git a/hw/9pfs/virtio-9p-posix-acl.c b/hw/9pfs/virtio-9p-posix-acl.c
index a1948e3affc..08bb0e8bcaf 100644
--- a/hw/9pfs/virtio-9p-posix-acl.c
+++ b/hw/9pfs/virtio-9p-posix-acl.c
@@ -12,7 +12,7 @@
  */
 
 #include <sys/types.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include "hw/virtio.h"
 #include "virtio-9p.h"
 #include "fsdev/file-op-9p.h"
@@ -44,7 +44,8 @@ static ssize_t mp_pacl_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, ACL_ACCESS, len);
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_ACCESS, len);
     return 0;
 }
 
@@ -95,7 +96,8 @@ static ssize_t mp_dacl_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, ACL_DEFAULT, len);
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_ACCESS, len);
     return 0;
 }
 
diff --git a/hw/9pfs/virtio-9p-synth.c b/hw/9pfs/virtio-9p-synth.c
index 92e0b09d38f..e95a856d25e 100644
--- a/hw/9pfs/virtio-9p-synth.c
+++ b/hw/9pfs/virtio-9p-synth.c
@@ -58,7 +58,7 @@ static V9fsSynthNode *v9fs_add_dir_node(V9fsSynthNode *parent, int mode,
         node->attr->read  = NULL;
     }
     node->private = node;
-    strncpy(node->name, name, sizeof(node->name));
+    pstrcpy(node->name, sizeof(node->name), name);
     QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
     return node;
 }
@@ -132,7 +132,7 @@ int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
     node->attr->write  = write;
     node->attr->mode   = mode;
     node->private      = arg;
-    strncpy(node->name, name, sizeof(node->name));
+    pstrcpy(node->name, sizeof(node->name), name);
     QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
     ret = 0;
 err_out:
diff --git a/hw/9pfs/virtio-9p-synth.h b/hw/9pfs/virtio-9p-synth.h
index e03f4346336..ab05a8e78c6 100644
--- a/hw/9pfs/virtio-9p-synth.h
+++ b/hw/9pfs/virtio-9p-synth.h
@@ -10,6 +10,8 @@
  * the COPYING file in the top-level directory.
  *
  */
+#ifndef HW_9PFS_VIRTIO9P_SYNTH_H
+#define HW_9PFS_VIRTIO9P_SYNTH_H 1
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -48,3 +50,5 @@ extern int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
 extern int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
                                     const char *name, v9fs_synth_read read,
                                     v9fs_synth_write write, void *arg);
+
+#endif
diff --git a/hw/9pfs/virtio-9p-xattr-user.c b/hw/9pfs/virtio-9p-xattr-user.c
index 5044a3e5ab4..5bb60200709 100644
--- a/hw/9pfs/virtio-9p-xattr-user.c
+++ b/hw/9pfs/virtio-9p-xattr-user.c
@@ -61,7 +61,8 @@ static ssize_t mp_user_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, name, name_size);
+    /* name_size includes the trailing NUL. */
+    memcpy(value, name, name_size);
     return name_size;
 }
 
diff --git a/hw/9pfs/virtio-9p-xattr.c b/hw/9pfs/virtio-9p-xattr.c
index 7f08f6e1763..a83960676d1 100644
--- a/hw/9pfs/virtio-9p-xattr.c
+++ b/hw/9pfs/virtio-9p-xattr.c
@@ -53,7 +53,8 @@ ssize_t pt_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, name, name_size);
+    /* no need for strncpy: name_size is strlen(name)+1 */
+    memcpy(value, name, name_size);
     return name_size;
 }
 
diff --git a/hw/9pfs/virtio-9p-xattr.h b/hw/9pfs/virtio-9p-xattr.h
index 9437280c99e..41cc6cbc7ba 100644
--- a/hw/9pfs/virtio-9p-xattr.h
+++ b/hw/9pfs/virtio-9p-xattr.h
@@ -13,7 +13,7 @@
 #ifndef _QEMU_VIRTIO_9P_XATTR_H
 #define _QEMU_VIRTIO_9P_XATTR_H
 
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 
 typedef struct xattr_operations
 {
diff --git a/hw/9pfs/virtio-9p.c b/hw/9pfs/virtio-9p.c
index 4b52540116b..0aaf0d2de07 100644
--- a/hw/9pfs/virtio-9p.c
+++ b/hw/9pfs/virtio-9p.c
@@ -13,14 +13,14 @@
 
 #include "hw/virtio.h"
 #include "hw/pc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "hw/virtio-pci.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
 #include "virtio-9p-xattr.h"
 #include "virtio-9p-coth.h"
 #include "trace.h"
-#include "migration.h"
+#include "migration/migration.h"
 
 int open_fd_hw;
 int total_open_fd;
@@ -505,7 +505,6 @@ static void virtfs_reset(V9fsPDU *pdu)
         error_report("9pfs:%s: One or more uncluncked fids "
                      "found during reset", __func__);
     }
-    return;
 }
 
 #define P9_QID_TYPE_DIR         0x80
@@ -934,7 +933,6 @@ static void v9fs_version(void *opaque)
 out:
     complete_pdu(s, pdu, offset);
     v9fs_string_free(&version);
-    return;
 }
 
 static void v9fs_attach(void *opaque)
@@ -1314,7 +1312,6 @@ out_nofid:
         g_free(wnames);
         g_free(qids);
     }
-    return;
 }
 
 static int32_t get_iounit(V9fsPDU *pdu, V9fsPath *path)
@@ -2257,7 +2254,6 @@ static void v9fs_flush(void *opaque)
         free_pdu(pdu->s, cancel_pdu);
     }
     complete_pdu(s, pdu, 7);
-    return;
 }
 
 static void v9fs_link(void *opaque)
@@ -2763,7 +2759,6 @@ out:
     put_fid(pdu, fidp);
 out_nofid:
     complete_pdu(s, pdu, retval);
-    return;
 }
 
 static void v9fs_mknod(void *opaque)
diff --git a/hw/9pfs/virtio-9p.h b/hw/9pfs/virtio-9p.h
index 579794404b7..406fe522db8 100644
--- a/hw/9pfs/virtio-9p.h
+++ b/hw/9pfs/virtio-9p.h
@@ -9,8 +9,8 @@
 #include "hw/virtio.h"
 #include "fsdev/file-op-9p.h"
 #include "fsdev/virtio-9p-marshal.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 
 
 /* The feature bitmap for virtio 9P */
diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index 6dfebd2978c..c7556cfa277 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -1,140 +1,150 @@
-hw-obj-y = usb/ ide/
-hw-obj-y += loader.o
-hw-obj-$(CONFIG_VIRTIO) += virtio-console.o
-hw-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
-hw-obj-y += fw_cfg.o
-hw-obj-$(CONFIG_PCI) += pci.o pci_bridge.o pci_bridge_dev.o
-hw-obj-$(CONFIG_PCI) += msix.o msi.o
-hw-obj-$(CONFIG_PCI) += shpc.o
-hw-obj-$(CONFIG_PCI) += slotid_cap.o
-hw-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
-hw-obj-$(CONFIG_PCI) += ioh3420.o xio3130_upstream.o xio3130_downstream.o
-hw-obj-y += watchdog.o
-hw-obj-$(CONFIG_ISA_MMIO) += isa_mmio.o
-hw-obj-$(CONFIG_ECC) += ecc.o
-hw-obj-$(CONFIG_NAND) += nand.o
-hw-obj-$(CONFIG_PFLASH_CFI01) += pflash_cfi01.o
-hw-obj-$(CONFIG_PFLASH_CFI02) += pflash_cfi02.o
-
-hw-obj-$(CONFIG_M48T59) += m48t59.o
-hw-obj-$(CONFIG_ESCC) += escc.o
-hw-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
-
-hw-obj-$(CONFIG_SERIAL) += serial.o
-hw-obj-$(CONFIG_PARALLEL) += parallel.o
-hw-obj-$(CONFIG_I8254) += i8254_common.o i8254.o
-hw-obj-$(CONFIG_PCSPK) += pcspk.o
-hw-obj-$(CONFIG_PCKBD) += pckbd.o
-hw-obj-$(CONFIG_FDC) += fdc.o
-hw-obj-$(CONFIG_ACPI) += acpi.o acpi_piix4.o
-hw-obj-$(CONFIG_APM) += pm_smbus.o apm.o
-hw-obj-$(CONFIG_DMA) += dma.o
-hw-obj-$(CONFIG_I82374) += i82374.o
-hw-obj-$(CONFIG_HPET) += hpet.o
-hw-obj-$(CONFIG_APPLESMC) += applesmc.o
-hw-obj-$(CONFIG_SMARTCARD) += ccid-card-passthru.o
-hw-obj-$(CONFIG_SMARTCARD_NSS) += ccid-card-emulated.o
-hw-obj-$(CONFIG_I8259) += i8259_common.o i8259.o
+# core qdev-related obj files, also used by *-user:
+common-obj-y += qdev.o qdev-properties.o
+# irq.o needed for qdev GPIO handling:
+common-obj-y += irq.o
+
+ifeq ($(CONFIG_SOFTMMU),y)
+common-obj-y += usb/ ide/ pci/
+common-obj-y += loader.o
+common-obj-$(CONFIG_VIRTIO) += virtio-console.o
+common-obj-$(CONFIG_VIRTIO) += virtio-rng.o
+common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
+common-obj-$(CONFIG_VIRTIO) += virtio-bus.o
+common-obj-y += fw_cfg.o
+common-obj-$(CONFIG_PCI) += pci_bridge_dev.o
+common-obj-$(CONFIG_PCI) += ioh3420.o xio3130_upstream.o xio3130_downstream.o
+common-obj-$(CONFIG_PCI) += i82801b11.o
+common-obj-y += watchdog.o
+common-obj-$(CONFIG_ISA_MMIO) += isa_mmio.o
+common-obj-$(CONFIG_ECC) += ecc.o
+common-obj-$(CONFIG_NAND) += nand.o
+common-obj-$(CONFIG_PFLASH_CFI01) += pflash_cfi01.o
+common-obj-$(CONFIG_PFLASH_CFI02) += pflash_cfi02.o
+
+common-obj-$(CONFIG_M48T59) += m48t59.o
+common-obj-$(CONFIG_ESCC) += escc.o
+common-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
+
+common-obj-$(CONFIG_SERIAL) += serial.o serial-isa.o
+common-obj-$(CONFIG_SERIAL_PCI) += serial-pci.o
+common-obj-$(CONFIG_PARALLEL) += parallel.o
+common-obj-$(CONFIG_I8254) += i8254_common.o i8254.o
+common-obj-$(CONFIG_PCSPK) += pcspk.o
+common-obj-$(CONFIG_PCKBD) += pckbd.o
+common-obj-$(CONFIG_FDC) += fdc.o
+common-obj-$(CONFIG_ACPI) += acpi.o acpi_piix4.o acpi_ich9.o smbus_ich9.o
+common-obj-$(CONFIG_APM) += pm_smbus.o apm.o
+common-obj-$(CONFIG_DMA) += dma.o
+common-obj-$(CONFIG_I82374) += i82374.o
+common-obj-$(CONFIG_HPET) += hpet.o
+common-obj-$(CONFIG_APPLESMC) += applesmc.o
+common-obj-y += ccid-card-passthru.o
+common-obj-$(CONFIG_SMARTCARD_NSS) += ccid-card-emulated.o
+common-obj-$(CONFIG_I8259) += i8259_common.o i8259.o
+common-obj-y += fifo.o
+common-obj-y += pam.o
 
 # PPC devices
-hw-obj-$(CONFIG_PREP_PCI) += prep_pci.o
-hw-obj-$(CONFIG_I82378) += i82378.o
+common-obj-$(CONFIG_PREP_PCI) += prep_pci.o
+common-obj-$(CONFIG_I82378) += i82378.o
+common-obj-$(CONFIG_PC87312) += pc87312.o
 # Mac shared devices
-hw-obj-$(CONFIG_MACIO) += macio.o
-hw-obj-$(CONFIG_CUDA) += cuda.o
-hw-obj-$(CONFIG_ADB) += adb.o
-hw-obj-$(CONFIG_MAC_NVRAM) += mac_nvram.o
-hw-obj-$(CONFIG_MAC_DBDMA) += mac_dbdma.o
+common-obj-$(CONFIG_MACIO) += macio.o
+common-obj-$(CONFIG_CUDA) += cuda.o
+common-obj-$(CONFIG_ADB) += adb.o
+common-obj-$(CONFIG_MAC_NVRAM) += mac_nvram.o
+common-obj-$(CONFIG_MAC_DBDMA) += mac_dbdma.o
 # OldWorld PowerMac
-hw-obj-$(CONFIG_HEATHROW_PIC) += heathrow_pic.o
-hw-obj-$(CONFIG_GRACKLE_PCI) += grackle_pci.o
+common-obj-$(CONFIG_HEATHROW_PIC) += heathrow_pic.o
+common-obj-$(CONFIG_GRACKLE_PCI) += grackle_pci.o
 # NewWorld PowerMac
-hw-obj-$(CONFIG_UNIN_PCI) += unin_pci.o
-hw-obj-$(CONFIG_DEC_PCI) += dec_pci.o
+common-obj-$(CONFIG_UNIN_PCI) += unin_pci.o
+common-obj-$(CONFIG_DEC_PCI) += dec_pci.o
 # PowerPC E500 boards
-hw-obj-$(CONFIG_PPCE500_PCI) += ppce500_pci.o
+common-obj-$(CONFIG_PPCE500_PCI) += ppce500_pci.o
 
 # MIPS devices
-hw-obj-$(CONFIG_PIIX4) += piix4.o
-hw-obj-$(CONFIG_G364FB) += g364fb.o
-hw-obj-$(CONFIG_JAZZ_LED) += jazz_led.o
+common-obj-$(CONFIG_PIIX4) += piix4.o
+common-obj-$(CONFIG_G364FB) += g364fb.o
+common-obj-$(CONFIG_JAZZ_LED) += jazz_led.o
 
 # Xilinx devices
-hw-obj-$(CONFIG_XILINX) += xilinx_intc.o
-hw-obj-$(CONFIG_XILINX) += xilinx_timer.o
-hw-obj-$(CONFIG_XILINX) += xilinx_uartlite.o
-hw-obj-$(CONFIG_XILINX_AXI) += xilinx_axidma.o
-hw-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
-hw-obj-$(CONFIG_XILINX_AXI) += stream.o
+common-obj-$(CONFIG_XILINX) += xilinx_intc.o
+common-obj-$(CONFIG_XILINX) += xilinx_timer.o
+common-obj-$(CONFIG_XILINX) += xilinx_uartlite.o
+common-obj-$(CONFIG_XILINX_AXI) += xilinx_axidma.o
+common-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
+common-obj-$(CONFIG_XILINX_AXI) += stream.o
 
 # PKUnity SoC devices
-hw-obj-$(CONFIG_PUV3) += puv3_intc.o
-hw-obj-$(CONFIG_PUV3) += puv3_ost.o
-hw-obj-$(CONFIG_PUV3) += puv3_gpio.o
-hw-obj-$(CONFIG_PUV3) += puv3_pm.o
-hw-obj-$(CONFIG_PUV3) += puv3_dma.o
+common-obj-$(CONFIG_PUV3) += puv3_intc.o
+common-obj-$(CONFIG_PUV3) += puv3_ost.o
+common-obj-$(CONFIG_PUV3) += puv3_gpio.o
+common-obj-$(CONFIG_PUV3) += puv3_pm.o
+common-obj-$(CONFIG_PUV3) += puv3_dma.o
 
 # ARM devices
-hw-obj-$(CONFIG_ARM_TIMER) += arm_timer.o
-hw-obj-$(CONFIG_PL011) += pl011.o
-hw-obj-$(CONFIG_PL022) += pl022.o
-hw-obj-$(CONFIG_PL031) += pl031.o
-hw-obj-$(CONFIG_PL041) += pl041.o lm4549.o
-hw-obj-$(CONFIG_PL050) += pl050.o
-hw-obj-$(CONFIG_PL061) += pl061.o
-hw-obj-$(CONFIG_PL080) += pl080.o
-hw-obj-$(CONFIG_PL110) += pl110.o
-hw-obj-$(CONFIG_PL181) += pl181.o
-hw-obj-$(CONFIG_PL190) += pl190.o
-hw-obj-$(CONFIG_PL310) += arm_l2x0.o
-hw-obj-$(CONFIG_VERSATILE_PCI) += versatile_pci.o
-hw-obj-$(CONFIG_VERSATILE_I2C) += versatile_i2c.o
-hw-obj-$(CONFIG_CADENCE) += cadence_uart.o
-hw-obj-$(CONFIG_CADENCE) += cadence_ttc.o
-hw-obj-$(CONFIG_CADENCE) += cadence_gem.o
-hw-obj-$(CONFIG_XGMAC) += xgmac.o
+common-obj-$(CONFIG_ARM_TIMER) += arm_timer.o
+common-obj-$(CONFIG_PL011) += pl011.o
+common-obj-$(CONFIG_PL022) += pl022.o
+common-obj-$(CONFIG_PL031) += pl031.o
+common-obj-$(CONFIG_PL041) += pl041.o lm4549.o
+common-obj-$(CONFIG_PL050) += pl050.o
+common-obj-$(CONFIG_PL061) += pl061.o
+common-obj-$(CONFIG_PL080) += pl080.o
+common-obj-$(CONFIG_PL110) += pl110.o
+common-obj-$(CONFIG_PL181) += pl181.o
+common-obj-$(CONFIG_PL190) += pl190.o
+common-obj-$(CONFIG_PL310) += arm_l2x0.o
+common-obj-$(CONFIG_VERSATILE_PCI) += versatile_pci.o
+common-obj-$(CONFIG_VERSATILE_I2C) += versatile_i2c.o
+common-obj-$(CONFIG_CADENCE) += cadence_uart.o
+common-obj-$(CONFIG_CADENCE) += cadence_ttc.o
+common-obj-$(CONFIG_CADENCE) += cadence_gem.o
+common-obj-$(CONFIG_XGMAC) += xgmac.o
 
 # PCI watchdog devices
-hw-obj-$(CONFIG_PCI) += wdt_i6300esb.o
+common-obj-$(CONFIG_PCI) += wdt_i6300esb.o
 
-hw-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
+# IndustryPack
+common-obj-$(CONFIG_IPACK) += tpci200.o ipoctal232.o ipack.o
 
 # PCI network cards
-hw-obj-$(CONFIG_NE2000_PCI) += ne2000.o
-hw-obj-$(CONFIG_EEPRO100_PCI) += eepro100.o
-hw-obj-$(CONFIG_PCNET_PCI) += pcnet-pci.o
-hw-obj-$(CONFIG_PCNET_COMMON) += pcnet.o
-hw-obj-$(CONFIG_E1000_PCI) += e1000.o
-hw-obj-$(CONFIG_RTL8139_PCI) += rtl8139.o
-
-hw-obj-$(CONFIG_SMC91C111) += smc91c111.o
-hw-obj-$(CONFIG_LAN9118) += lan9118.o
-hw-obj-$(CONFIG_NE2000_ISA) += ne2000-isa.o
-hw-obj-$(CONFIG_OPENCORES_ETH) += opencores_eth.o
+common-obj-$(CONFIG_NE2000_PCI) += ne2000.o
+common-obj-$(CONFIG_EEPRO100_PCI) += eepro100.o
+common-obj-$(CONFIG_PCNET_PCI) += pcnet-pci.o
+common-obj-$(CONFIG_PCNET_COMMON) += pcnet.o
+common-obj-$(CONFIG_E1000_PCI) += e1000.o
+common-obj-$(CONFIG_RTL8139_PCI) += rtl8139.o
+
+common-obj-$(CONFIG_SMC91C111) += smc91c111.o
+common-obj-$(CONFIG_LAN9118) += lan9118.o
+common-obj-$(CONFIG_NE2000_ISA) += ne2000-isa.o
+common-obj-$(CONFIG_OPENCORES_ETH) += opencores_eth.o
 
 # SCSI layer
-hw-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
-hw-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
-hw-obj-$(CONFIG_ESP) += esp.o
-hw-obj-$(CONFIG_ESP_PCI) += esp-pci.o
+common-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
+common-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
+common-obj-$(CONFIG_ESP) += esp.o
+common-obj-$(CONFIG_ESP_PCI) += esp-pci.o
 
-hw-obj-y += sysbus.o isa-bus.o
-hw-obj-y += qdev-addr.o
+common-obj-y += sysbus.o isa-bus.o
+common-obj-y += qdev-addr.o
 
 # VGA
-hw-obj-$(CONFIG_VGA_PCI) += vga-pci.o
-hw-obj-$(CONFIG_VGA_ISA) += vga-isa.o
-hw-obj-$(CONFIG_VGA_ISA_MM) += vga-isa-mm.o
-hw-obj-$(CONFIG_VMWARE_VGA) += vmware_vga.o
-hw-obj-$(CONFIG_VMMOUSE) += vmmouse.o
-hw-obj-$(CONFIG_VGA_CIRRUS) += cirrus_vga.o
+common-obj-$(CONFIG_VGA_PCI) += vga-pci.o
+common-obj-$(CONFIG_VGA_ISA) += vga-isa.o
+common-obj-$(CONFIG_VGA_ISA_MM) += vga-isa-mm.o
+common-obj-$(CONFIG_VMWARE_VGA) += vmware_vga.o
+common-obj-$(CONFIG_VMMOUSE) += vmmouse.o
+common-obj-$(CONFIG_VGA_CIRRUS) += cirrus_vga.o
 
-hw-obj-$(CONFIG_RC4030) += rc4030.o
-hw-obj-$(CONFIG_DP8393X) += dp8393x.o
-hw-obj-$(CONFIG_DS1225Y) += ds1225y.o
-hw-obj-$(CONFIG_MIPSNET) += mipsnet.o
+common-obj-$(CONFIG_RC4030) += rc4030.o
+common-obj-$(CONFIG_DP8393X) += dp8393x.o
+common-obj-$(CONFIG_DS1225Y) += ds1225y.o
+common-obj-$(CONFIG_MIPSNET) += mipsnet.o
 
-hw-obj-y += null-machine.o
+common-obj-y += null-machine.o
 
 # Sound
 sound-obj-y =
@@ -148,12 +158,11 @@ sound-obj-$(CONFIG_HDA) += intel-hda.o hda-audio.o
 
 $(obj)/adlib.o $(obj)/fmopl.o: QEMU_CFLAGS += -DBUILD_Y8950=0
 
-hw-obj-$(CONFIG_SOUND) += $(sound-obj-y)
+common-obj-$(CONFIG_SOUND) += $(sound-obj-y)
 
-hw-obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
+common-obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
 
 common-obj-y += usb/
-common-obj-y += irq.o
 common-obj-$(CONFIG_PTIMER) += ptimer.o
 common-obj-$(CONFIG_MAX7310) += max7310.o
 common-obj-$(CONFIG_WM8750) += wm8750.o
@@ -173,12 +182,14 @@ common-obj-y += scsi-disk.o cdrom.o hd-geometry.o block-common.o
 common-obj-y += scsi-generic.o scsi-bus.o
 common-obj-y += hid.o
 common-obj-$(CONFIG_SSI) += ssi.o
+common-obj-$(CONFIG_SSI_M25P80) += m25p80.o
 common-obj-$(CONFIG_SSI_SD) += ssi-sd.o
 common-obj-$(CONFIG_SD) += sd.o
 common-obj-y += bt.o bt-l2cap.o bt-sdp.o bt-hci.o bt-hid.o
 common-obj-y += bt-hci-csr.o
 common-obj-y += msmouse.o ps2.o
-common-obj-y += qdev.o qdev-properties.o qdev-monitor.o
+common-obj-y += qdev-monitor.o
+common-obj-y += qdev-properties-system.o
 common-obj-$(CONFIG_BRLAPI) += baum.o
 
 # xen backend driver support
@@ -188,17 +199,24 @@ common-obj-$(CONFIG_XEN_BACKEND) += xen_console.o xenfb.o xen_disk.o xen_nic.o
 # Per-target files
 # virtio has to be here due to weird dependency between PCI and virtio-net.
 # need to fix this properly
+obj-$(CONFIG_VIRTIO) += dataplane/
 obj-$(CONFIG_VIRTIO) += virtio.o virtio-blk.o virtio-balloon.o virtio-net.o
 obj-$(CONFIG_VIRTIO) += virtio-serial-bus.o virtio-scsi.o
+ifeq ($(CONFIG_VIRTIO), y)
+obj-$(CONFIG_LINUX) += vhost-scsi.o
+endif
 obj-$(CONFIG_SOFTMMU) += vhost_net.o
-obj-$(CONFIG_VHOST_NET) += vhost.o
+obj-$(CONFIG_LINUX) += vhost.o
 obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
-obj-$(CONFIG_NO_PCI) += pci-stub.o
 obj-$(CONFIG_VGA) += vga.o
 obj-$(CONFIG_SOFTMMU) += device-hotplug.o
 obj-$(CONFIG_XEN) += xen_domainbuild.o xen_machine_pv.o
 
-# Inter-VM PCI shared memory
+# Inter-VM PCI shared memory & VFIO PCI device assignment
 ifeq ($(CONFIG_PCI), y)
 obj-$(CONFIG_KVM) += ivshmem.o
+obj-$(CONFIG_LINUX) += vfio_pci.o
+endif
+
+$(obj)/baum.o: QEMU_CFLAGS += $(SDL_CFLAGS) 
 endif
diff --git a/hw/a15mpcore.c b/hw/a15mpcore.c
index fc0a02ae86b..fe6c34ca533 100644
--- a/hw/a15mpcore.c
+++ b/hw/a15mpcore.c
@@ -46,7 +46,7 @@ static int a15mp_priv_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-irq", s->num_irq);
     qdev_prop_set_uint32(s->gic, "revision", 2);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -93,7 +93,7 @@ static void a15mp_priv_class_init(ObjectClass *klass, void *data)
     /* We currently have no savable state */
 }
 
-static TypeInfo a15mp_priv_info = {
+static const TypeInfo a15mp_priv_info = {
     .name  = "a15mpcore_priv",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size  = sizeof(A15MPPrivState),
diff --git a/hw/a9mpcore.c b/hw/a9mpcore.c
index ebd5b291734..673bbd8c427 100644
--- a/hw/a9mpcore.c
+++ b/hw/a9mpcore.c
@@ -19,14 +19,13 @@ typedef struct a9mp_priv_state {
     uint32_t old_timer_status[8];
     uint32_t num_cpu;
     MemoryRegion scu_iomem;
-    MemoryRegion ptimer_iomem;
     MemoryRegion container;
     DeviceState *mptimer;
     DeviceState *gic;
     uint32_t num_irq;
 } a9mp_priv_state;
 
-static uint64_t a9_scu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t a9_scu_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     a9mp_priv_state *s = (a9mp_priv_state *)opaque;
@@ -57,7 +56,7 @@ static uint64_t a9_scu_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void a9_scu_write(void *opaque, target_phys_addr_t offset,
+static void a9_scu_write(void *opaque, hwaddr offset,
                          uint64_t value, unsigned size)
 {
     a9mp_priv_state *s = (a9mp_priv_state *)opaque;
@@ -113,7 +112,7 @@ static const MemoryRegionOps a9_scu_ops = {
 
 static void a9mp_priv_reset(DeviceState *dev)
 {
-    a9mp_priv_state *s = FROM_SYSBUS(a9mp_priv_state, sysbus_from_qdev(dev));
+    a9mp_priv_state *s = FROM_SYSBUS(a9mp_priv_state, SYS_BUS_DEVICE(dev));
     int i;
     s->scu_control = 0;
     for (i = 0; i < ARRAY_SIZE(s->old_timer_status); i++) {
@@ -137,7 +136,7 @@ static int a9mp_priv_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(s->gic, "num-irq", s->num_irq);
     qdev_init_nofail(s->gic);
-    gicbusdev = sysbus_from_qdev(s->gic);
+    gicbusdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, gicbusdev);
@@ -148,7 +147,7 @@ static int a9mp_priv_init(SysBusDevice *dev)
     s->mptimer = qdev_create(NULL, "arm_mptimer");
     qdev_prop_set_uint32(s->mptimer, "num-cpu", s->num_cpu);
     qdev_init_nofail(s->mptimer);
-    busdev = sysbus_from_qdev(s->mptimer);
+    busdev = SYS_BUS_DEVICE(s->mptimer);
 
     /* Memory map (addresses are offsets from PERIPHBASE):
      *  0x0000-0x00ff -- Snoop Control Unit
@@ -227,7 +226,7 @@ static void a9mp_priv_class_init(ObjectClass *klass, void *data)
     dc->reset = a9mp_priv_reset;
 }
 
-static TypeInfo a9mp_priv_info = {
+static const TypeInfo a9mp_priv_info = {
     .name          = "a9mpcore_priv",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(a9mp_priv_state),
diff --git a/hw/ac97.c b/hw/ac97.c
index 0f561fa5c1c..6c565e755c0 100644
--- a/hw/ac97.c
+++ b/hw/ac97.c
@@ -20,8 +20,8 @@
 #include "hw.h"
 #include "audiodev.h"
 #include "audio/audio.h"
-#include "pci.h"
-#include "dma.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
 
 enum {
     AC97_Reset                     = 0x00,
@@ -1226,32 +1226,101 @@ static const VMStateDescription vmstate_ac97 = {
     }
 };
 
-static const MemoryRegionPortio nam_portio[] = {
-    { 0, 256 * 1, 1, .read = nam_readb, },
-    { 0, 256 * 2, 2, .read = nam_readw, },
-    { 0, 256 * 4, 4, .read = nam_readl, },
-    { 0, 256 * 1, 1, .write = nam_writeb, },
-    { 0, 256 * 2, 2, .write = nam_writew, },
-    { 0, 256 * 4, 4, .write = nam_writel, },
-    PORTIO_END_OF_LIST (),
-};
+static uint64_t nam_read(void *opaque, hwaddr addr, unsigned size)
+{
+    if ((addr / size) > 256) {
+        return -1;
+    }
+
+    switch (size) {
+    case 1:
+        return nam_readb(opaque, addr);
+    case 2:
+        return nam_readw(opaque, addr);
+    case 4:
+        return nam_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void nam_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    if ((addr / size) > 256) {
+        return;
+    }
+
+    switch (size) {
+    case 1:
+        nam_writeb(opaque, addr, val);
+        break;
+    case 2:
+        nam_writew(opaque, addr, val);
+        break;
+    case 4:
+        nam_writel(opaque, addr, val);
+        break;
+    }
+}
 
 static const MemoryRegionOps ac97_io_nam_ops = {
-    .old_portio = nam_portio,
+    .read = nam_read,
+    .write = nam_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static const MemoryRegionPortio nabm_portio[] = {
-    { 0, 64 * 1, 1, .read = nabm_readb, },
-    { 0, 64 * 2, 2, .read = nabm_readw, },
-    { 0, 64 * 4, 4, .read = nabm_readl, },
-    { 0, 64 * 1, 1, .write = nabm_writeb, },
-    { 0, 64 * 2, 2, .write = nabm_writew, },
-    { 0, 64 * 4, 4, .write = nabm_writel, },
-    PORTIO_END_OF_LIST ()
-};
+static uint64_t nabm_read(void *opaque, hwaddr addr, unsigned size)
+{
+    if ((addr / size) > 64) {
+        return -1;
+    }
+
+    switch (size) {
+    case 1:
+        return nabm_readb(opaque, addr);
+    case 2:
+        return nabm_readw(opaque, addr);
+    case 4:
+        return nabm_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void nabm_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    if ((addr / size) > 64) {
+        return;
+    }
+
+    switch (size) {
+    case 1:
+        nabm_writeb(opaque, addr, val);
+        break;
+    case 2:
+        nabm_writew(opaque, addr, val);
+        break;
+    case 4:
+        nabm_writel(opaque, addr, val);
+        break;
+    }
+}
+
 
 static const MemoryRegionOps ac97_io_nabm_ops = {
-    .old_portio = nabm_portio,
+    .read = nabm_read,
+    .write = nabm_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void ac97_on_reset (void *opaque)
@@ -1354,7 +1423,7 @@ static void ac97_class_init (ObjectClass *klass, void *data)
     dc->props = ac97_properties;
 }
 
-static TypeInfo ac97_info = {
+static const TypeInfo ac97_info = {
     .name          = "AC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (AC97LinkState),
diff --git a/hw/acpi.c b/hw/acpi.c
index f7950be2675..8c9dcc51c4b 100644
--- a/hw/acpi.c
+++ b/hw/acpi.c
@@ -18,11 +18,11 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "hw.h"
 #include "pc.h"
 #include "acpi.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 struct acpi_table_header {
     uint16_t _length;         /* our length, not actual part of the hdr */
@@ -61,18 +61,6 @@ static int acpi_checksum(const uint8_t *data, int len)
     return (-sum) & 0xff;
 }
 
-/* like strncpy() but zero-fills the tail of destination */
-static void strzcpy(char *dst, const char *src, size_t size)
-{
-    size_t len = strlen(src);
-    if (len >= size) {
-        len = size;
-    } else {
-      memset(dst + len, 0, size - len);
-    }
-    memcpy(dst, src, len);
-}
-
 /* XXX fixme: this function uses obsolete argument parsing interface */
 int acpi_table_add(const char *t)
 {
@@ -116,7 +104,7 @@ int acpi_table_add(const char *t)
     /* now read in the data files, reallocating buffer as needed */
 
     for (f = strtok(buf, ":"); f; f = strtok(NULL, ":")) {
-        int fd = open(f, O_RDONLY);
+        int fd = open(f, O_RDONLY | O_BINARY);
 
         if (fd < 0) {
             fprintf(stderr, "can't open file %s: %s\n", f, strerror(errno));
@@ -157,7 +145,8 @@ int acpi_table_add(const char *t)
     hdr._length = cpu_to_le16(len);
 
     if (get_param_value(buf, sizeof(buf), "sig", t)) {
-        strzcpy(hdr.sig, buf, sizeof(hdr.sig));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.sig, buf, sizeof(hdr.sig));
         ++changed;
     }
 
@@ -187,12 +176,14 @@ int acpi_table_add(const char *t)
     }
 
     if (get_param_value(buf, sizeof(buf), "oem_id", t)) {
-        strzcpy(hdr.oem_id, buf, sizeof(hdr.oem_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.oem_id, buf, sizeof(hdr.oem_id));
         ++changed;
     }
 
     if (get_param_value(buf, sizeof(buf), "oem_table_id", t)) {
-        strzcpy(hdr.oem_table_id, buf, sizeof(hdr.oem_table_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.oem_table_id, buf, sizeof(hdr.oem_table_id));
         ++changed;
     }
 
@@ -207,7 +198,8 @@ int acpi_table_add(const char *t)
     }
 
     if (get_param_value(buf, sizeof(buf), "asl_compiler_id", t)) {
-        strzcpy(hdr.asl_compiler_id, buf, sizeof(hdr.asl_compiler_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.asl_compiler_id, buf, sizeof(hdr.asl_compiler_id));
         ++changed;
     }
 
@@ -283,7 +275,7 @@ uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar)
     return ar->pm1.evt.sts;
 }
 
-void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
+static void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
 {
     uint16_t pm1_sts = acpi_pm1_evt_get_sts(ar);
     if (pm1_sts & val & ACPI_BITMASK_TIMER_STATUS) {
@@ -293,7 +285,7 @@ void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
     ar->pm1.evt.sts &= ~val;
 }
 
-void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val)
+static void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val)
 {
     ar->pm1.evt.en = val;
     qemu_system_wakeup_enable(QEMU_WAKEUP_REASON_RTC,
@@ -318,6 +310,51 @@ void acpi_pm1_evt_reset(ACPIREGS *ar)
     qemu_system_wakeup_enable(QEMU_WAKEUP_REASON_PMTIMER, 0);
 }
 
+static uint64_t acpi_pm_evt_read(void *opaque, hwaddr addr, unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    switch (addr) {
+    case 0:
+        return acpi_pm1_evt_get_sts(ar);
+    case 2:
+        return ar->pm1.evt.en;
+    default:
+        return 0;
+    }
+}
+
+static void acpi_pm_evt_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    switch (addr) {
+    case 0:
+        acpi_pm1_evt_write_sts(ar, val);
+        ar->pm1.evt.update_sci(ar);
+        break;
+    case 2:
+        acpi_pm1_evt_write_en(ar, val);
+        ar->pm1.evt.update_sci(ar);
+        break;
+    }
+}
+
+static const MemoryRegionOps acpi_pm_evt_ops = {
+    .read = acpi_pm_evt_read,
+    .write = acpi_pm_evt_write,
+    .valid.min_access_size = 2,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm1_evt_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                       MemoryRegion *parent)
+{
+    ar->pm1.evt.update_sci = update_sci;
+    memory_region_init_io(&ar->pm1.evt.io, &acpi_pm_evt_ops, ar, "acpi-evt", 4);
+    memory_region_add_subregion(parent, 0, &ar->pm1.evt.io);
+}
+
 /* ACPI PM_TMR */
 void acpi_pm_tmr_update(ACPIREGS *ar, bool enable)
 {
@@ -339,7 +376,7 @@ void acpi_pm_tmr_calc_overflow_time(ACPIREGS *ar)
     ar->tmr.overflow_time = (d + 0x800000LL) & ~0x7fffffLL;
 }
 
-uint32_t acpi_pm_tmr_get(ACPIREGS *ar)
+static uint32_t acpi_pm_tmr_get(ACPIREGS *ar)
 {
     uint32_t d = acpi_pm_tmr_get_clock();
     return d & 0xffffff;
@@ -352,10 +389,25 @@ static void acpi_pm_tmr_timer(void *opaque)
     ar->tmr.update_sci(ar);
 }
 
-void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci)
+static uint64_t acpi_pm_tmr_read(void *opaque, hwaddr addr, unsigned width)
+{
+    return acpi_pm_tmr_get(opaque);
+}
+
+static const MemoryRegionOps acpi_pm_tmr_ops = {
+    .read = acpi_pm_tmr_read,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                      MemoryRegion *parent)
 {
     ar->tmr.update_sci = update_sci;
     ar->tmr.timer = qemu_new_timer_ns(vm_clock, acpi_pm_tmr_timer, ar);
+    memory_region_init_io(&ar->tmr.io, &acpi_pm_tmr_ops, ar, "acpi-tmr", 4);
+    memory_region_add_subregion(parent, 8, &ar->tmr.io);
 }
 
 void acpi_pm_tmr_reset(ACPIREGS *ar)
@@ -365,13 +417,7 @@ void acpi_pm_tmr_reset(ACPIREGS *ar)
 }
 
 /* ACPI PM1aCNT */
-void acpi_pm1_cnt_init(ACPIREGS *ar)
-{
-    ar->wakeup.notify = acpi_notify_wakeup;
-    qemu_register_wakeup_notifier(&ar->wakeup);
-}
-
-void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4)
+static void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val)
 {
     ar->pm1.cnt.cnt = val & ~(ACPI_BITMASK_SLEEP_ENABLE);
 
@@ -386,7 +432,7 @@ void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4)
             qemu_system_suspend_request();
             break;
         default:
-            if (sus_typ == s4) { /* S4 request */
+            if (sus_typ == ar->pm1.cnt.s4_val) { /* S4 request */
                 monitor_protocol_event(QEVENT_SUSPEND_DISK, NULL);
                 qemu_system_shutdown_request();
             }
@@ -406,6 +452,34 @@ void acpi_pm1_cnt_update(ACPIREGS *ar,
     }
 }
 
+static uint64_t acpi_pm_cnt_read(void *opaque, hwaddr addr, unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    return ar->pm1.cnt.cnt;
+}
+
+static void acpi_pm_cnt_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
+{
+    acpi_pm1_cnt_write(opaque, val);
+}
+
+static const MemoryRegionOps acpi_pm_cnt_ops = {
+    .read = acpi_pm_cnt_read,
+    .write = acpi_pm_cnt_write,
+    .valid.min_access_size = 2,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent)
+{
+    ar->wakeup.notify = acpi_notify_wakeup;
+    qemu_register_wakeup_notifier(&ar->wakeup);
+    memory_region_init_io(&ar->pm1.cnt.io, &acpi_pm_cnt_ops, ar, "acpi-cnt", 2);
+    memory_region_add_subregion(parent, 4, &ar->pm1.cnt.io);
+}
+
 void acpi_pm1_cnt_reset(ACPIREGS *ar)
 {
     ar->pm1.cnt.cnt = 0;
@@ -419,11 +493,6 @@ void acpi_gpe_init(ACPIREGS *ar, uint8_t len)
     ar->gpe.en = g_malloc0(len / 2);
 }
 
-void acpi_gpe_blk(ACPIREGS *ar, uint32_t blk)
-{
-    ar->gpe.blk = blk;
-}
-
 void acpi_gpe_reset(ACPIREGS *ar)
 {
     memset(ar->gpe.sts, 0, ar->gpe.len / 2);
@@ -449,7 +518,6 @@ void acpi_gpe_ioport_writeb(ACPIREGS *ar, uint32_t addr, uint32_t val)
 {
     uint8_t *cur;
 
-    addr -= ar->gpe.blk;
     cur = acpi_gpe_ioport_get_ptr(ar, addr);
     if (addr < ar->gpe.len / 2) {
         /* GPE_STS */
@@ -467,7 +535,6 @@ uint32_t acpi_gpe_ioport_readb(ACPIREGS *ar, uint32_t addr)
     uint8_t *cur;
     uint32_t val;
 
-    addr -= ar->gpe.blk;
     cur = acpi_gpe_ioport_get_ptr(ar, addr);
     val = 0;
     if (cur != NULL) {
diff --git a/hw/acpi.h b/hw/acpi.h
index 7337f418577..c3628d070db 100644
--- a/hw/acpi.h
+++ b/hw/acpi.h
@@ -84,22 +84,26 @@ typedef void (*acpi_update_sci_fn)(ACPIREGS *ar);
 
 struct ACPIPMTimer {
     QEMUTimer *timer;
+    MemoryRegion io;
     int64_t overflow_time;
 
     acpi_update_sci_fn update_sci;
 };
 
 struct ACPIPM1EVT {
+    MemoryRegion io;
     uint16_t sts;
     uint16_t en;
+    acpi_update_sci_fn update_sci;
 };
 
 struct ACPIPM1CNT {
+    MemoryRegion io;
     uint16_t cnt;
+    uint8_t s4_val;
 };
 
 struct ACPIGPE {
-    uint32_t blk;
     uint8_t len;
 
     uint8_t *sts;
@@ -119,11 +123,11 @@ struct ACPIREGS {
 /* PM_TMR */
 void acpi_pm_tmr_update(ACPIREGS *ar, bool enable);
 void acpi_pm_tmr_calc_overflow_time(ACPIREGS *ar);
-uint32_t acpi_pm_tmr_get(ACPIREGS *ar);
-void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci);
+void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                      MemoryRegion *parent);
 void acpi_pm_tmr_reset(ACPIREGS *ar);
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 static inline int64_t acpi_pm_tmr_get_clock(void)
 {
     return muldiv64(qemu_get_clock_ns(vm_clock), PM_TIMER_FREQUENCY,
@@ -132,21 +136,19 @@ static inline int64_t acpi_pm_tmr_get_clock(void)
 
 /* PM1a_EVT: piix and ich9 don't implement PM1b. */
 uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar);
-void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val);
-void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val);
 void acpi_pm1_evt_power_down(ACPIREGS *ar);
 void acpi_pm1_evt_reset(ACPIREGS *ar);
+void acpi_pm1_evt_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                       MemoryRegion *parent);
 
 /* PM1a_CNT: piix and ich9 don't implement PM1b CNT. */
-void acpi_pm1_cnt_init(ACPIREGS *ar);
-void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4);
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent);
 void acpi_pm1_cnt_update(ACPIREGS *ar,
                          bool sci_enable, bool sci_disable);
 void acpi_pm1_cnt_reset(ACPIREGS *ar);
 
 /* GPE0 */
 void acpi_gpe_init(ACPIREGS *ar, uint8_t len);
-void acpi_gpe_blk(ACPIREGS *ar, uint32_t blk);
 void acpi_gpe_reset(ACPIREGS *ar);
 
 void acpi_gpe_ioport_writeb(ACPIREGS *ar, uint32_t addr, uint32_t val);
diff --git a/hw/acpi_ich9.c b/hw/acpi_ich9.c
new file mode 100644
index 00000000000..d2f9808242b
--- /dev/null
+++ b/hw/acpi_ich9.c
@@ -0,0 +1,230 @@
+/*
+ * ACPI implementation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on acpi.c.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include "hw.h"
+#include "pc.h"
+#include "pci/pci.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "acpi.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
+
+#include "ich9.h"
+
+//#define DEBUG
+
+#ifdef DEBUG
+#define ICH9_DEBUG(fmt, ...) \
+do { printf("%s "fmt, __func__, ## __VA_ARGS__); } while (0)
+#else
+#define ICH9_DEBUG(fmt, ...)    do { } while (0)
+#endif
+
+static void pm_update_sci(ICH9LPCPMRegs *pm)
+{
+    int sci_level, pm1a_sts;
+
+    pm1a_sts = acpi_pm1_evt_get_sts(&pm->acpi_regs);
+
+    sci_level = (((pm1a_sts & pm->acpi_regs.pm1.evt.en) &
+                  (ACPI_BITMASK_RT_CLOCK_ENABLE |
+                   ACPI_BITMASK_POWER_BUTTON_ENABLE |
+                   ACPI_BITMASK_GLOBAL_LOCK_ENABLE |
+                   ACPI_BITMASK_TIMER_ENABLE)) != 0);
+    qemu_set_irq(pm->irq, sci_level);
+
+    /* schedule a timer interruption if needed */
+    acpi_pm_tmr_update(&pm->acpi_regs,
+                       (pm->acpi_regs.pm1.evt.en & ACPI_BITMASK_TIMER_ENABLE) &&
+                       !(pm1a_sts & ACPI_BITMASK_TIMER_STATUS));
+}
+
+static void ich9_pm_update_sci_fn(ACPIREGS *regs)
+{
+    ICH9LPCPMRegs *pm = container_of(regs, ICH9LPCPMRegs, acpi_regs);
+    pm_update_sci(pm);
+}
+
+static uint64_t ich9_gpe_readb(void *opaque, hwaddr addr, unsigned width)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    return acpi_gpe_ioport_readb(&pm->acpi_regs, addr);
+}
+
+static void ich9_gpe_writeb(void *opaque, hwaddr addr, uint64_t val,
+                            unsigned width)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    acpi_gpe_ioport_writeb(&pm->acpi_regs, addr, val);
+}
+
+static const MemoryRegionOps ich9_gpe_ops = {
+    .read = ich9_gpe_readb,
+    .write = ich9_gpe_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t ich9_smi_readl(void *opaque, hwaddr addr, unsigned width)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    switch (addr) {
+    case 0:
+        return pm->smi_en;
+    case 4:
+        return pm->smi_sts;
+    default:
+        return 0;
+    }
+}
+
+static void ich9_smi_writel(void *opaque, hwaddr addr, uint64_t val,
+                            unsigned width)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    switch (addr) {
+    case 0:
+        pm->smi_en = val;
+        break;
+    }
+}
+
+static const MemoryRegionOps ich9_smi_ops = {
+    .read = ich9_smi_readl,
+    .write = ich9_smi_writel,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base)
+{
+    ICH9_DEBUG("to 0x%x\n", pm_io_base);
+
+    assert((pm_io_base & ICH9_PMIO_MASK) == 0);
+
+    pm->pm_io_base = pm_io_base;
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&pm->io, pm->pm_io_base != 0);
+    memory_region_set_address(&pm->io, pm->pm_io_base);
+    memory_region_transaction_commit();
+}
+
+static int ich9_pm_post_load(void *opaque, int version_id)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint32_t pm_io_base = pm->pm_io_base;
+    pm->pm_io_base = 0;
+    ich9_pm_iospace_update(pm, pm_io_base);
+    return 0;
+}
+
+#define VMSTATE_GPE_ARRAY(_field, _state)                            \
+ {                                                                   \
+     .name       = (stringify(_field)),                              \
+     .version_id = 0,                                                \
+     .num        = ICH9_PMIO_GPE0_LEN,                               \
+     .info       = &vmstate_info_uint8,                              \
+     .size       = sizeof(uint8_t),                                  \
+     .flags      = VMS_ARRAY | VMS_POINTER,                          \
+     .offset     = vmstate_offset_pointer(_state, _field, uint8_t),  \
+ }
+
+const VMStateDescription vmstate_ich9_pm = {
+    .name = "ich9_pm",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = ich9_pm_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(acpi_regs.pm1.evt.sts, ICH9LPCPMRegs),
+        VMSTATE_UINT16(acpi_regs.pm1.evt.en, ICH9LPCPMRegs),
+        VMSTATE_UINT16(acpi_regs.pm1.cnt.cnt, ICH9LPCPMRegs),
+        VMSTATE_TIMER(acpi_regs.tmr.timer, ICH9LPCPMRegs),
+        VMSTATE_INT64(acpi_regs.tmr.overflow_time, ICH9LPCPMRegs),
+        VMSTATE_GPE_ARRAY(acpi_regs.gpe.sts, ICH9LPCPMRegs),
+        VMSTATE_GPE_ARRAY(acpi_regs.gpe.en, ICH9LPCPMRegs),
+        VMSTATE_UINT32(smi_en, ICH9LPCPMRegs),
+        VMSTATE_UINT32(smi_sts, ICH9LPCPMRegs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void pm_reset(void *opaque)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    ich9_pm_iospace_update(pm, 0);
+
+    acpi_pm1_evt_reset(&pm->acpi_regs);
+    acpi_pm1_cnt_reset(&pm->acpi_regs);
+    acpi_pm_tmr_reset(&pm->acpi_regs);
+    acpi_gpe_reset(&pm->acpi_regs);
+
+    if (kvm_enabled()) {
+        /* Mark SMM as already inited to prevent SMM from running. KVM does not
+         * support SMM mode. */
+        pm->smi_en |= ICH9_PMIO_SMI_EN_APMC_EN;
+    }
+
+    pm_update_sci(pm);
+}
+
+static void pm_powerdown_req(Notifier *n, void *opaque)
+{
+    ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, powerdown_notifier);
+
+    acpi_pm1_evt_power_down(&pm->acpi_regs);
+}
+
+void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
+                  qemu_irq sci_irq, qemu_irq cmos_s3)
+{
+    memory_region_init(&pm->io, "ich9-pm", ICH9_PMIO_SIZE);
+    memory_region_set_enabled(&pm->io, false);
+    memory_region_add_subregion(pci_address_space_io(lpc_pci),
+                                0, &pm->io);
+
+    acpi_pm_tmr_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
+    acpi_pm1_evt_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
+    acpi_pm1_cnt_init(&pm->acpi_regs, &pm->io);
+
+    acpi_gpe_init(&pm->acpi_regs, ICH9_PMIO_GPE0_LEN);
+    memory_region_init_io(&pm->io_gpe, &ich9_gpe_ops, pm, "apci-gpe0",
+                          ICH9_PMIO_GPE0_LEN);
+    memory_region_add_subregion(&pm->io, ICH9_PMIO_GPE0_STS, &pm->io_gpe);
+
+    memory_region_init_io(&pm->io_smi, &ich9_smi_ops, pm, "apci-smi",
+                          8);
+    memory_region_add_subregion(&pm->io, ICH9_PMIO_SMI_EN, &pm->io_smi);
+
+    pm->irq = sci_irq;
+    qemu_register_reset(pm_reset, pm);
+    pm->powerdown_notifier.notify = pm_powerdown_req;
+    qemu_register_powerdown_notifier(&pm->powerdown_notifier);
+}
diff --git a/hw/acpi_ich9.h b/hw/acpi_ich9.h
new file mode 100644
index 00000000000..ecb82abc65b
--- /dev/null
+++ b/hw/acpi_ich9.h
@@ -0,0 +1,52 @@
+/*
+ * QEMU GMCH/ICH9 LPC PM Emulation
+ *
+ *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                     VA Linux Systems Japan K.K.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef HW_ACPI_ICH9_H
+#define HW_ACPI_ICH9_H
+
+#include "acpi.h"
+
+typedef struct ICH9LPCPMRegs {
+    /*
+     * In ich9 spec says that pm1_cnt register is 32bit width and
+     * that the upper 16bits are reserved and unused.
+     * PM1a_CNT_BLK = 2 in FADT so it is defined as uint16_t.
+     */
+    ACPIREGS acpi_regs;
+
+    MemoryRegion io;
+    MemoryRegion io_gpe;
+    MemoryRegion io_smi;
+
+    uint32_t smi_en;
+    uint32_t smi_sts;
+
+    qemu_irq irq;      /* SCI */
+
+    uint32_t pm_io_base;
+    Notifier powerdown_notifier;
+} ICH9LPCPMRegs;
+
+void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
+                  qemu_irq sci_irq, qemu_irq cmos_s3_resume);
+void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base);
+extern const VMStateDescription vmstate_ich9_pm;
+
+#endif /* HW_ACPI_ICH9_H */
diff --git a/hw/acpi_piix4.c b/hw/acpi_piix4.c
index c56220b3686..0d33849e953 100644
--- a/hw/acpi_piix4.c
+++ b/hw/acpi_piix4.c
@@ -22,12 +22,13 @@
 #include "pc.h"
 #include "apm.h"
 #include "pm_smbus.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "acpi.h"
-#include "sysemu.h"
-#include "range.h"
-#include "ioport.h"
+#include "sysemu/sysemu.h"
+#include "qemu/range.h"
+#include "exec/ioport.h"
 #include "fw_cfg.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG
 
@@ -37,10 +38,11 @@
 # define PIIX4_DPRINTF(format, ...)     do { } while (0)
 #endif
 
-#define ACPI_DBG_IO_ADDR  0xb044
-
 #define GPE_BASE 0xafe0
 #define GPE_LEN 4
+
+#define PCI_HOTPLUG_ADDR 0xae00
+#define PCI_HOTPLUG_SIZE 0x000f
 #define PCI_UP_BASE 0xae00
 #define PCI_DOWN_BASE 0xae04
 #define PCI_EJ_BASE 0xae08
@@ -55,7 +57,10 @@ struct pci_status {
 
 typedef struct PIIX4PMState {
     PCIDevice dev;
-    IORange ioport;
+
+    MemoryRegion io;
+    MemoryRegion io_gpe;
+    MemoryRegion io_pci;
     ACPIREGS ar;
 
     APMState apm;
@@ -67,6 +72,7 @@ typedef struct PIIX4PMState {
     qemu_irq smi_irq;
     int kvm_enabled;
     Notifier machine_ready;
+    Notifier powerdown_notifier;
 
     /* for pci hotplug */
     struct pci_status pci0_status;
@@ -78,7 +84,8 @@ typedef struct PIIX4PMState {
     uint8_t s4_val;
 } PIIX4PMState;
 
-static void piix4_acpi_system_hot_add_init(PCIBus *bus, PIIX4PMState *s);
+static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+                                           PCIBus *bus, PIIX4PMState *s);
 
 #define ACPI_ENABLE 0xf1
 #define ACPI_DISABLE 0xf0
@@ -108,67 +115,6 @@ static void pm_tmr_timer(ACPIREGS *ar)
     pm_update_sci(s);
 }
 
-static void pm_ioport_write(IORange *ioport, uint64_t addr, unsigned width,
-                            uint64_t val)
-{
-    PIIX4PMState *s = container_of(ioport, PIIX4PMState, ioport);
-
-    if (width != 2) {
-        PIIX4_DPRINTF("PM write port=0x%04x width=%d val=0x%08x\n",
-                      (unsigned)addr, width, (unsigned)val);
-    }
-
-    switch(addr) {
-    case 0x00:
-        acpi_pm1_evt_write_sts(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x02:
-        acpi_pm1_evt_write_en(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x04:
-        acpi_pm1_cnt_write(&s->ar, val, s->s4_val);
-        break;
-    default:
-        break;
-    }
-    PIIX4_DPRINTF("PM writew port=0x%04x val=0x%04x\n", (unsigned int)addr,
-                  (unsigned int)val);
-}
-
-static void pm_ioport_read(IORange *ioport, uint64_t addr, unsigned width,
-                            uint64_t *data)
-{
-    PIIX4PMState *s = container_of(ioport, PIIX4PMState, ioport);
-    uint32_t val;
-
-    switch(addr) {
-    case 0x00:
-        val = acpi_pm1_evt_get_sts(&s->ar);
-        break;
-    case 0x02:
-        val = s->ar.pm1.evt.en;
-        break;
-    case 0x04:
-        val = s->ar.pm1.cnt.cnt;
-        break;
-    case 0x08:
-        val = acpi_pm_tmr_get(&s->ar);
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    PIIX4_DPRINTF("PM readw port=0x%04x val=0x%04x\n", (unsigned int)addr, val);
-    *data = val;
-}
-
-static const IORangeOps pm_iorange_ops = {
-    .read = pm_ioport_read,
-    .write = pm_ioport_write,
-};
-
 static void apm_ctrl_changed(uint32_t val, void *arg)
 {
     PIIX4PMState *s = arg;
@@ -183,32 +129,42 @@ static void apm_ctrl_changed(uint32_t val, void *arg)
     }
 }
 
-static void acpi_dbg_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    PIIX4_DPRINTF("ACPI: DBG: 0x%08x\n", val);
-}
-
 static void pm_io_space_update(PIIX4PMState *s)
 {
     uint32_t pm_io_base;
 
-    if (s->dev.config[0x80] & 1) {
-        pm_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x40));
-        pm_io_base &= 0xffc0;
+    pm_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x40));
+    pm_io_base &= 0xffc0;
 
-        /* XXX: need to improve memory and ioport allocation */
-        PIIX4_DPRINTF("PM: mapping to 0x%x\n", pm_io_base);
-        iorange_init(&s->ioport, &pm_iorange_ops, pm_io_base, 64);
-        ioport_register(&s->ioport);
-    }
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->io, s->dev.config[0x80] & 1);
+    memory_region_set_address(&s->io, pm_io_base);
+    memory_region_transaction_commit();
+}
+
+static void smbus_io_space_update(PIIX4PMState *s)
+{
+    s->smb_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x90));
+    s->smb_io_base &= 0xffc0;
+
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->smb.io, s->dev.config[0xd2] & 1);
+    memory_region_set_address(&s->smb.io, s->smb_io_base);
+    memory_region_transaction_commit();
 }
 
 static void pm_write_config(PCIDevice *d,
                             uint32_t address, uint32_t val, int len)
 {
     pci_default_write_config(d, address, val, len);
-    if (range_covers_byte(address, len, 0x80))
+    if (range_covers_byte(address, len, 0x80) ||
+        ranges_overlap(address, len, 0x40, 4)) {
         pm_io_space_update((PIIX4PMState *)d);
+    }
+    if (range_covers_byte(address, len, 0xd2) ||
+        ranges_overlap(address, len, 0x90, 4)) {
+        smbus_io_space_update((PIIX4PMState *)d);
+    }
 }
 
 static void vmstate_pci_status_pre_save(void *opaque)
@@ -234,10 +190,9 @@ static int vmstate_acpi_post_load(void *opaque, int version_id)
  {                                                                   \
      .name       = (stringify(_field)),                              \
      .version_id = 0,                                                \
-     .num        = GPE_LEN,                                          \
      .info       = &vmstate_info_uint16,                             \
      .size       = sizeof(uint16_t),                                 \
-     .flags      = VMS_ARRAY | VMS_POINTER,                          \
+     .flags      = VMS_SINGLE | VMS_POINTER,                         \
      .offset     = vmstate_offset_pointer(_state, _field, uint8_t),  \
  }
 
@@ -266,11 +221,54 @@ static const VMStateDescription vmstate_pci_status = {
     }
 };
 
+static int acpi_load_old(QEMUFile *f, void *opaque, int version_id)
+{
+    PIIX4PMState *s = opaque;
+    int ret, i;
+    uint16_t temp;
+
+    ret = pci_device_load(&s->dev, f);
+    if (ret < 0) {
+        return ret;
+    }
+    qemu_get_be16s(f, &s->ar.pm1.evt.sts);
+    qemu_get_be16s(f, &s->ar.pm1.evt.en);
+    qemu_get_be16s(f, &s->ar.pm1.cnt.cnt);
+
+    ret = vmstate_load_state(f, &vmstate_apm, opaque, 1);
+    if (ret) {
+        return ret;
+    }
+
+    qemu_get_timer(f, s->ar.tmr.timer);
+    qemu_get_sbe64s(f, &s->ar.tmr.overflow_time);
+
+    qemu_get_be16s(f, (uint16_t *)s->ar.gpe.sts);
+    for (i = 0; i < 3; i++) {
+        qemu_get_be16s(f, &temp);
+    }
+
+    qemu_get_be16s(f, (uint16_t *)s->ar.gpe.en);
+    for (i = 0; i < 3; i++) {
+        qemu_get_be16s(f, &temp);
+    }
+
+    ret = vmstate_load_state(f, &vmstate_pci_status, opaque, 1);
+    return ret;
+}
+
+/* qemu-kvm 1.2 uses version 3 but advertised as 2
+ * To support incoming qemu-kvm 1.2 migration, change version_id
+ * and minimum_version_id to 2 below (which breaks migration from
+ * qemu 1.2).
+ *
+ */
 static const VMStateDescription vmstate_acpi = {
     .name = "piix4_pm",
-    .version_id = 2,
-    .minimum_version_id = 1,
+    .version_id = 3,
+    .minimum_version_id = 3,
     .minimum_version_id_old = 1,
+    .load_state_old = acpi_load_old,
     .post_load = vmstate_acpi_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_PCI_DEVICE(dev, PIIX4PMState),
@@ -362,9 +360,9 @@ static void piix4_reset(void *opaque)
     piix4_update_hotplug(s);
 }
 
-static void piix4_powerdown(void *opaque, int irq, int power_failing)
+static void piix4_pm_powerdown_req(Notifier *n, void *opaque)
 {
-    PIIX4PMState *s = opaque;
+    PIIX4PMState *s = container_of(n, PIIX4PMState, powerdown_notifier);
 
     assert(s != NULL);
     acpi_pm1_evt_power_down(&s->ar);
@@ -395,9 +393,7 @@ static int piix4_pm_initfn(PCIDevice *dev)
     pci_conf[0x3d] = 0x01; // interrupt pin 1
 
     /* APM */
-    apm_init(&s->apm, apm_ctrl_changed, s);
-
-    register_ioport_write(ACPI_DBG_IO_ADDR, 4, 4, acpi_dbg_writel, s);
+    apm_init(dev, &s->apm, apm_ctrl_changed, s);
 
     if (s->kvm_enabled) {
         /* Mark SMM as already inited to prevent SMM from running.  KVM does not
@@ -410,19 +406,29 @@ static int piix4_pm_initfn(PCIDevice *dev)
     pci_conf[0x90] = s->smb_io_base | 1;
     pci_conf[0x91] = s->smb_io_base >> 8;
     pci_conf[0xd2] = 0x09;
-    register_ioport_write(s->smb_io_base, 64, 1, smb_ioport_writeb, &s->smb);
-    register_ioport_read(s->smb_io_base, 64, 1, smb_ioport_readb, &s->smb);
-
-    acpi_pm_tmr_init(&s->ar, pm_tmr_timer);
+    pm_smbus_init(&s->dev.qdev, &s->smb);
+    memory_region_set_enabled(&s->smb.io, pci_conf[0xd2] & 1);
+    memory_region_add_subregion(pci_address_space_io(dev),
+                                s->smb_io_base, &s->smb.io);
+
+    memory_region_init(&s->io, "piix4-pm", 64);
+    memory_region_set_enabled(&s->io, false);
+    memory_region_add_subregion(pci_address_space_io(dev),
+                                0, &s->io);
+
+    acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_cnt_init(&s->ar, &s->io);
     acpi_gpe_init(&s->ar, GPE_LEN);
 
-    qemu_system_powerdown = *qemu_allocate_irqs(piix4_powerdown, s, 1);
+    s->powerdown_notifier.notify = piix4_pm_powerdown_req;
+    qemu_register_powerdown_notifier(&s->powerdown_notifier);
 
-    pm_smbus_init(&s->dev.qdev, &s->smb);
     s->machine_ready.notify = piix4_pm_machine_ready;
     qemu_add_machine_init_done_notifier(&s->machine_ready);
     qemu_register_reset(piix4_reset, s);
-    piix4_acpi_system_hot_add_init(dev->bus, s);
+
+    piix4_acpi_system_hot_add_init(pci_address_space_io(dev), dev->bus, s);
 
     return 0;
 }
@@ -439,7 +445,6 @@ i2c_bus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
 
     s = DO_UPCAST(PIIX4PMState, dev, dev);
     s->irq = sci_irq;
-    acpi_pm1_cnt_init(&s->ar);
     s->smi_irq = smi_irq;
     s->kvm_enabled = kvm_enabled;
 
@@ -482,7 +487,7 @@ static void piix4_pm_class_init(ObjectClass *klass, void *data)
     dc->props = piix4_pm_properties;
 }
 
-static TypeInfo piix4_pm_info = {
+static const TypeInfo piix4_pm_info = {
     .name          = "PIIX4_PM",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PIIX4PMState),
@@ -496,7 +501,7 @@ static void piix4_pm_register_types(void)
 
 type_init(piix4_pm_register_types)
 
-static uint32_t gpe_readb(void *opaque, uint32_t addr)
+static uint64_t gpe_readb(void *opaque, hwaddr addr, unsigned width)
 {
     PIIX4PMState *s = opaque;
     uint32_t val = acpi_gpe_ioport_readb(&s->ar, addr);
@@ -505,7 +510,8 @@ static uint32_t gpe_readb(void *opaque, uint32_t addr)
     return val;
 }
 
-static void gpe_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void gpe_writeb(void *opaque, hwaddr addr, uint64_t val,
+                       unsigned width)
 {
     PIIX4PMState *s = opaque;
 
@@ -515,67 +521,84 @@ static void gpe_writeb(void *opaque, uint32_t addr, uint32_t val)
     PIIX4_DPRINTF("gpe write %x <== %d\n", addr, val);
 }
 
-static uint32_t pci_up_read(void *opaque, uint32_t addr)
-{
-    PIIX4PMState *s = opaque;
-    uint32_t val;
-
-    /* Manufacture an "up" value to cause a device check on any hotplug
-     * slot with a device.  Extra device checks are harmless. */
-    val = s->pci0_slot_device_present & s->pci0_hotplug_enable;
-
-    PIIX4_DPRINTF("pci_up_read %x\n", val);
-    return val;
-}
+static const MemoryRegionOps piix4_gpe_ops = {
+    .read = gpe_readb,
+    .write = gpe_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static uint32_t pci_down_read(void *opaque, uint32_t addr)
+static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size)
 {
     PIIX4PMState *s = opaque;
-    uint32_t val = s->pci0_status.down;
+    uint32_t val = 0;
+
+    switch (addr) {
+    case PCI_UP_BASE - PCI_HOTPLUG_ADDR:
+        /* Manufacture an "up" value to cause a device check on any hotplug
+         * slot with a device.  Extra device checks are harmless. */
+        val = s->pci0_slot_device_present & s->pci0_hotplug_enable;
+        PIIX4_DPRINTF("pci_up_read %x\n", val);
+        break;
+    case PCI_DOWN_BASE - PCI_HOTPLUG_ADDR:
+        val = s->pci0_status.down;
+        PIIX4_DPRINTF("pci_down_read %x\n", val);
+        break;
+    case PCI_EJ_BASE - PCI_HOTPLUG_ADDR:
+        /* No feature defined yet */
+        PIIX4_DPRINTF("pci_features_read %x\n", val);
+        break;
+    case PCI_RMV_BASE - PCI_HOTPLUG_ADDR:
+        val = s->pci0_hotplug_enable;
+        break;
+    default:
+        break;
+    }
 
-    PIIX4_DPRINTF("pci_down_read %x\n", val);
     return val;
 }
 
-static uint32_t pci_features_read(void *opaque, uint32_t addr)
-{
-    /* No feature defined yet */
-    PIIX4_DPRINTF("pci_features_read %x\n", 0);
-    return 0;
-}
-
-static void pciej_write(void *opaque, uint32_t addr, uint32_t val)
+static void pci_write(void *opaque, hwaddr addr, uint64_t data,
+                      unsigned int size)
 {
-    acpi_piix_eject_slot(opaque, val);
-
-    PIIX4_DPRINTF("pciej write %x <== %d\n", addr, val);
+    switch (addr) {
+    case PCI_EJ_BASE - PCI_HOTPLUG_ADDR:
+        acpi_piix_eject_slot(opaque, (uint32_t)data);
+        PIIX4_DPRINTF("pciej write %" HWADDR_PRIx " <== % " PRIu64 "\n",
+                      addr, data);
+        break;
+    default:
+        break;
+    }
 }
 
-static uint32_t pcirmv_read(void *opaque, uint32_t addr)
-{
-    PIIX4PMState *s = opaque;
-
-    return s->pci0_hotplug_enable;
-}
+static const MemoryRegionOps piix4_pci_ops = {
+    .read = pci_read,
+    .write = pci_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
 static int piix4_device_hotplug(DeviceState *qdev, PCIDevice *dev,
                                 PCIHotplugState state);
 
-static void piix4_acpi_system_hot_add_init(PCIBus *bus, PIIX4PMState *s)
+static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+                                           PCIBus *bus, PIIX4PMState *s)
 {
-
-    register_ioport_write(GPE_BASE, GPE_LEN, 1, gpe_writeb, s);
-    register_ioport_read(GPE_BASE, GPE_LEN, 1,  gpe_readb, s);
-    acpi_gpe_blk(&s->ar, GPE_BASE);
-
-    register_ioport_read(PCI_UP_BASE, 4, 4, pci_up_read, s);
-    register_ioport_read(PCI_DOWN_BASE, 4, 4, pci_down_read, s);
-
-    register_ioport_write(PCI_EJ_BASE, 4, 4, pciej_write, s);
-    register_ioport_read(PCI_EJ_BASE, 4, 4,  pci_features_read, s);
-
-    register_ioport_read(PCI_RMV_BASE, 4, 4,  pcirmv_read, s);
-
+    memory_region_init_io(&s->io_gpe, &piix4_gpe_ops, s, "apci-gpe0",
+                          GPE_LEN);
+    memory_region_add_subregion(parent, GPE_BASE, &s->io_gpe);
+
+    memory_region_init_io(&s->io_pci, &piix4_pci_ops, s, "apci-pci-hotplug",
+                          PCI_HOTPLUG_SIZE);
+    memory_region_add_subregion(parent, PCI_HOTPLUG_ADDR,
+                                &s->io_pci);
     pci_bus_hotplug(bus, piix4_device_hotplug, &s->dev.qdev);
 }
 
diff --git a/hw/adb.c b/hw/adb.c
index aa15f55dc9a..6cf54650c8a 100644
--- a/hw/adb.c
+++ b/hw/adb.c
@@ -23,7 +23,7 @@
  */
 #include "hw.h"
 #include "adb.h"
-#include "console.h"
+#include "ui/console.h"
 
 /* debug ADB */
 //#define DEBUG_ADB
@@ -48,16 +48,21 @@ do { printf("ADB: " fmt , ## __VA_ARGS__); } while (0)
 #define ADB_CMD_CHANGE_ID_AND_ENABLE	0x00
 
 /* ADB default device IDs (upper 4 bits of ADB command byte) */
-#define ADB_DONGLE	1
-#define ADB_KEYBOARD	2
-#define ADB_MOUSE	3
-#define ADB_TABLET	4
-#define ADB_MODEM	5
-#define ADB_MISC	7
+#define ADB_DEVID_DONGLE   1
+#define ADB_DEVID_KEYBOARD 2
+#define ADB_DEVID_MOUSE    3
+#define ADB_DEVID_TABLET   4
+#define ADB_DEVID_MODEM    5
+#define ADB_DEVID_MISC     7
 
 /* error codes */
 #define ADB_RET_NOTPRESENT (-2)
 
+static void adb_device_reset(ADBDevice *d)
+{
+    qdev_reset_all(DEVICE(d));
+}
+
 int adb_request(ADBBusState *s, uint8_t *obuf, const uint8_t *buf, int len)
 {
     ADBDevice *d;
@@ -66,18 +71,17 @@ int adb_request(ADBBusState *s, uint8_t *obuf, const uint8_t *buf, int len)
     cmd = buf[0] & 0xf;
     if (cmd == ADB_BUSRESET) {
         for(i = 0; i < s->nb_devices; i++) {
-            d = &s->devices[i];
-            if (d->devreset) {
-                d->devreset(d);
-            }
+            d = s->devices[i];
+            adb_device_reset(d);
         }
         return 0;
     }
     devaddr = buf[0] >> 4;
     for(i = 0; i < s->nb_devices; i++) {
-        d = &s->devices[i];
+        d = s->devices[i];
         if (d->devaddr == devaddr) {
-            return d->devreq(d, obuf, buf, len);
+            ADBDeviceClass *adc = ADB_DEVICE_GET_CLASS(d);
+            return adc->devreq(d, obuf, buf, len);
         }
     }
     return ADB_RET_NOTPRESENT;
@@ -94,7 +98,7 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
     for(i = 0; i < s->nb_devices; i++) {
         if (s->poll_index >= s->nb_devices)
             s->poll_index = 0;
-        d = &s->devices[s->poll_index];
+        d = s->devices[s->poll_index];
         buf[0] = ADB_READREG | (d->devaddr << 4);
         olen = adb_request(s, obuf + 1, buf, 1);
         /* if there is data, we poll again the same device */
@@ -108,32 +112,67 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
     return olen;
 }
 
-ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
-                               ADBDeviceRequest *devreq,
-                               ADBDeviceReset *devreset,
-                               void *opaque)
+static const TypeInfo adb_bus_type_info = {
+    .name = TYPE_ADB_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(ADBBusState),
+};
+
+static void adb_device_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    if (s->nb_devices >= MAX_ADB_DEVICES)
-        return NULL;
-    d = &s->devices[s->nb_devices++];
-    d->bus = s;
-    d->devaddr = devaddr;
-    d->devreq = devreq;
-    d->devreset = devreset;
-    d->opaque = opaque;
-    qemu_register_reset((QEMUResetHandler *)devreset, d);
-    return d;
+    ADBDevice *d = ADB_DEVICE(dev);
+    ADBBusState *bus = ADB_BUS(qdev_get_parent_bus(dev));
+
+    if (bus->nb_devices >= MAX_ADB_DEVICES) {
+        return;
+    }
+
+    bus->devices[bus->nb_devices++] = d;
 }
 
+static void adb_device_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = adb_device_realizefn;
+    dc->bus_type = TYPE_ADB_BUS;
+}
+
+static const TypeInfo adb_device_type_info = {
+    .name = TYPE_ADB_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(ADBDevice),
+    .abstract = true,
+    .class_init = adb_device_class_init,
+};
+
 /***************************************************************/
 /* Keyboard ADB device */
 
+#define ADB_KEYBOARD(obj) OBJECT_CHECK(KBDState, (obj), TYPE_ADB_KEYBOARD)
+
 typedef struct KBDState {
+    /*< private >*/
+    ADBDevice parent_obj;
+    /*< public >*/
+
     uint8_t data[128];
     int rptr, wptr, count;
 } KBDState;
 
+#define ADB_KEYBOARD_CLASS(class) \
+    OBJECT_CLASS_CHECK(ADBKeyboardClass, (class), TYPE_ADB_KEYBOARD)
+#define ADB_KEYBOARD_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBKeyboardClass, (obj), TYPE_ADB_KEYBOARD)
+
+typedef struct ADBKeyboardClass {
+    /*< private >*/
+    ADBDeviceClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+} ADBKeyboardClass;
+
 static const uint8_t pc_to_adb_keycode[256] = {
   0, 53, 18, 19, 20, 21, 23, 22, 26, 28, 25, 29, 27, 24, 51, 48,
  12, 13, 14, 15, 17, 16, 32, 34, 31, 35, 33, 30, 36, 54,  0,  1,
@@ -155,8 +194,7 @@ static const uint8_t pc_to_adb_keycode[256] = {
 
 static void adb_kbd_put_keycode(void *opaque, int keycode)
 {
-    ADBDevice *d = opaque;
-    KBDState *s = d->opaque;
+    KBDState *s = opaque;
 
     if (s->count < sizeof(s->data)) {
         s->data[s->wptr] = keycode;
@@ -169,7 +207,7 @@ static void adb_kbd_put_keycode(void *opaque, int keycode)
 static int adb_kbd_poll(ADBDevice *d, uint8_t *obuf)
 {
     static int ext_keycode;
-    KBDState *s = d->opaque;
+    KBDState *s = ADB_KEYBOARD(d);
     int adb_keycode, keycode;
     int olen;
 
@@ -203,7 +241,7 @@ static int adb_kbd_poll(ADBDevice *d, uint8_t *obuf)
 static int adb_kbd_request(ADBDevice *d, uint8_t *obuf,
                            const uint8_t *buf, int len)
 {
-    KBDState *s = d->opaque;
+    KBDState *s = ADB_KEYBOARD(d);
     int cmd, reg, olen;
 
     if ((buf[0] & 0x0f) == ADB_FLUSH) {
@@ -275,41 +313,90 @@ static const VMStateDescription vmstate_adb_kbd = {
     }
 };
 
-static int adb_kbd_reset(ADBDevice *d)
+static void adb_kbd_reset(DeviceState *dev)
 {
-    KBDState *s = d->opaque;
+    ADBDevice *d = ADB_DEVICE(dev);
+    KBDState *s = ADB_KEYBOARD(dev);
 
     d->handler = 1;
-    d->devaddr = ADB_KEYBOARD;
-    memset(s, 0, sizeof(KBDState));
-
-    return 0;
+    d->devaddr = ADB_DEVID_KEYBOARD;
+    memset(s->data, 0, sizeof(s->data));
+    s->rptr = 0;
+    s->wptr = 0;
+    s->count = 0;
 }
 
-void adb_kbd_init(ADBBusState *bus)
+static void adb_kbd_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    KBDState *s;
-    s = g_malloc0(sizeof(KBDState));
-    d = adb_register_device(bus, ADB_KEYBOARD, adb_kbd_request,
-                            adb_kbd_reset, s);
+    ADBDevice *d = ADB_DEVICE(dev);
+    ADBKeyboardClass *akc = ADB_KEYBOARD_GET_CLASS(dev);
+
+    akc->parent_realize(dev, errp);
+
     qemu_add_kbd_event_handler(adb_kbd_put_keycode, d);
-    vmstate_register(NULL, -1, &vmstate_adb_kbd, s);
 }
 
+static void adb_kbd_initfn(Object *obj)
+{
+    ADBDevice *d = ADB_DEVICE(obj);
+
+    d->devaddr = ADB_DEVID_KEYBOARD;
+}
+
+static void adb_kbd_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    ADBDeviceClass *adc = ADB_DEVICE_CLASS(oc);
+    ADBKeyboardClass *akc = ADB_KEYBOARD_CLASS(oc);
+
+    akc->parent_realize = dc->realize;
+    dc->realize = adb_kbd_realizefn;
+
+    adc->devreq = adb_kbd_request;
+    dc->reset = adb_kbd_reset;
+    dc->vmsd = &vmstate_adb_kbd;
+}
+
+static const TypeInfo adb_kbd_type_info = {
+    .name = TYPE_ADB_KEYBOARD,
+    .parent = TYPE_ADB_DEVICE,
+    .instance_size = sizeof(KBDState),
+    .instance_init = adb_kbd_initfn,
+    .class_init = adb_kbd_class_init,
+    .class_size = sizeof(ADBKeyboardClass),
+};
+
 /***************************************************************/
 /* Mouse ADB device */
 
+#define ADB_MOUSE(obj) OBJECT_CHECK(MouseState, (obj), TYPE_ADB_MOUSE)
+
 typedef struct MouseState {
+    /*< public >*/
+    ADBDevice parent_obj;
+    /*< private >*/
+
     int buttons_state, last_buttons_state;
     int dx, dy, dz;
 } MouseState;
 
+#define ADB_MOUSE_CLASS(class) \
+    OBJECT_CLASS_CHECK(ADBMouseClass, (class), TYPE_ADB_MOUSE)
+#define ADB_MOUSE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBMouseClass, (obj), TYPE_ADB_MOUSE)
+
+typedef struct ADBMouseClass {
+    /*< public >*/
+    ADBDeviceClass parent_class;
+    /*< private >*/
+
+    DeviceRealize parent_realize;
+} ADBMouseClass;
+
 static void adb_mouse_event(void *opaque,
                             int dx1, int dy1, int dz1, int buttons_state)
 {
-    ADBDevice *d = opaque;
-    MouseState *s = d->opaque;
+    MouseState *s = opaque;
 
     s->dx += dx1;
     s->dy += dy1;
@@ -320,7 +407,7 @@ static void adb_mouse_event(void *opaque,
 
 static int adb_mouse_poll(ADBDevice *d, uint8_t *obuf)
 {
-    MouseState *s = d->opaque;
+    MouseState *s = ADB_MOUSE(d);
     int dx, dy;
 
     if (s->last_buttons_state == s->buttons_state &&
@@ -359,7 +446,7 @@ static int adb_mouse_poll(ADBDevice *d, uint8_t *obuf)
 static int adb_mouse_request(ADBDevice *d, uint8_t *obuf,
                              const uint8_t *buf, int len)
 {
-    MouseState *s = d->opaque;
+    MouseState *s = ADB_MOUSE(d);
     int cmd, reg, olen;
 
     if ((buf[0] & 0x0f) == ADB_FLUSH) {
@@ -416,15 +503,15 @@ static int adb_mouse_request(ADBDevice *d, uint8_t *obuf,
     return olen;
 }
 
-static int adb_mouse_reset(ADBDevice *d)
+static void adb_mouse_reset(DeviceState *dev)
 {
-    MouseState *s = d->opaque;
+    ADBDevice *d = ADB_DEVICE(dev);
+    MouseState *s = ADB_MOUSE(dev);
 
     d->handler = 2;
-    d->devaddr = ADB_MOUSE;
-    memset(s, 0, sizeof(MouseState));
-
-    return 0;
+    d->devaddr = ADB_DEVID_MOUSE;
+    s->last_buttons_state = s->buttons_state = 0;
+    s->dx = s->dy = s->dz = 0;
 }
 
 static const VMStateDescription vmstate_adb_mouse = {
@@ -442,14 +529,53 @@ static const VMStateDescription vmstate_adb_mouse = {
     }
 };
 
-void adb_mouse_init(ADBBusState *bus)
+static void adb_mouse_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    MouseState *s;
+    MouseState *s = ADB_MOUSE(dev);
+    ADBMouseClass *amc = ADB_MOUSE_GET_CLASS(dev);
+
+    amc->parent_realize(dev, errp);
+
+    qemu_add_mouse_event_handler(adb_mouse_event, s, 0, "QEMU ADB Mouse");
+}
+
+static void adb_mouse_initfn(Object *obj)
+{
+    ADBDevice *d = ADB_DEVICE(obj);
+
+    d->devaddr = ADB_DEVID_MOUSE;
+}
+
+static void adb_mouse_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    ADBDeviceClass *adc = ADB_DEVICE_CLASS(oc);
+    ADBMouseClass *amc = ADB_MOUSE_CLASS(oc);
+
+    amc->parent_realize = dc->realize;
+    dc->realize = adb_mouse_realizefn;
 
-    s = g_malloc0(sizeof(MouseState));
-    d = adb_register_device(bus, ADB_MOUSE, adb_mouse_request,
-                            adb_mouse_reset, s);
-    qemu_add_mouse_event_handler(adb_mouse_event, d, 0, "QEMU ADB Mouse");
-    vmstate_register(NULL, -1, &vmstate_adb_mouse, s);
+    adc->devreq = adb_mouse_request;
+    dc->reset = adb_mouse_reset;
+    dc->vmsd = &vmstate_adb_mouse;
 }
+
+static const TypeInfo adb_mouse_type_info = {
+    .name = TYPE_ADB_MOUSE,
+    .parent = TYPE_ADB_DEVICE,
+    .instance_size = sizeof(MouseState),
+    .instance_init = adb_mouse_initfn,
+    .class_init = adb_mouse_class_init,
+    .class_size = sizeof(ADBMouseClass),
+};
+
+
+static void adb_register_types(void)
+{
+    type_register_static(&adb_bus_type_info);
+    type_register_static(&adb_device_type_info);
+    type_register_static(&adb_kbd_type_info);
+    type_register_static(&adb_mouse_type_info);
+}
+
+type_init(adb_register_types)
diff --git a/hw/adb.h b/hw/adb.h
index b2a591c546b..721f1ac43ef 100644
--- a/hw/adb.h
+++ b/hw/adb.h
@@ -26,42 +26,62 @@
 #if !defined(__ADB_H__)
 #define __ADB_H__
 
+#include "qdev.h"
+
 #define MAX_ADB_DEVICES 16
 
 #define ADB_MAX_OUT_LEN 16
 
+typedef struct ADBBusState ADBBusState;
 typedef struct ADBDevice ADBDevice;
 
 /* buf = NULL means polling */
 typedef int ADBDeviceRequest(ADBDevice *d, uint8_t *buf_out,
                               const uint8_t *buf, int len);
-typedef int ADBDeviceReset(ADBDevice *d);
+
+#define TYPE_ADB_DEVICE "adb-device"
+#define ADB_DEVICE(obj) OBJECT_CHECK(ADBDevice, (obj), TYPE_ADB_DEVICE)
 
 struct ADBDevice {
-    struct ADBBusState *bus;
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
+
     int devaddr;
     int handler;
-    ADBDeviceRequest *devreq;
-    ADBDeviceReset *devreset;
-    void *opaque;
 };
 
-typedef struct ADBBusState {
-    ADBDevice devices[MAX_ADB_DEVICES];
+#define ADB_DEVICE_CLASS(cls) \
+    OBJECT_CLASS_CHECK(ADBDeviceClass, (cls), TYPE_ADB_DEVICE)
+#define ADB_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBDeviceClass, (obj), TYPE_ADB_DEVICE)
+
+typedef struct ADBDeviceClass {
+    /*< private >*/
+    DeviceClass parent_class;
+    /*< public >*/
+
+    ADBDeviceRequest *devreq;
+} ADBDeviceClass;
+
+#define TYPE_ADB_BUS "apple-desktop-bus"
+#define ADB_BUS(obj) OBJECT_CHECK(ADBBusState, (obj), TYPE_ADB_BUS)
+
+struct ADBBusState {
+    /*< private >*/
+    BusState parent_obj;
+    /*< public >*/
+
+    ADBDevice *devices[MAX_ADB_DEVICES];
     int nb_devices;
     int poll_index;
-} ADBBusState;
+};
 
 int adb_request(ADBBusState *s, uint8_t *buf_out,
                 const uint8_t *buf, int len);
 int adb_poll(ADBBusState *s, uint8_t *buf_out);
 
-ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
-                               ADBDeviceRequest *devreq,
-                               ADBDeviceReset *devreset,
-                               void *opaque);
-void adb_kbd_init(ADBBusState *bus);
-void adb_mouse_init(ADBBusState *bus);
+#define TYPE_ADB_KEYBOARD "adb-keyboard"
+#define TYPE_ADB_MOUSE "adb-mouse"
 
-extern ADBBusState adb_bus;
 #endif /* !defined(__ADB_H__) */
diff --git a/hw/adlib.c b/hw/adlib.c
index d39cd97384d..07c69fc9678 100644
--- a/hw/adlib.c
+++ b/hw/adlib.c
@@ -32,7 +32,7 @@
 #define ADLIB_KILL_TIMERS 1
 
 #ifdef DEBUG
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #endif
 
 #define dolog(...) AUD_log ("adlib", __VA_ARGS__)
diff --git a/hw/ads7846.c b/hw/ads7846.c
index 41c7f101c4c..29e5585d915 100644
--- a/hw/ads7846.c
+++ b/hw/ads7846.c
@@ -11,7 +11,7 @@
  */
 
 #include "ssi.h"
-#include "console.h"
+#include "ui/console.h"
 
 typedef struct {
     SSISlave ssidev;
@@ -119,11 +119,12 @@ static int ads7856_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_ads7846 = {
     .name = "ads7846",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .minimum_version_id_old = 0,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
     .post_load = ads7856_post_load,
     .fields      = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, ADS7846State),
         VMSTATE_INT32_ARRAY(input, ADS7846State, 8),
         VMSTATE_INT32(noise, ADS7846State),
         VMSTATE_INT32(cycle, ADS7846State),
@@ -161,7 +162,7 @@ static void ads7846_class_init(ObjectClass *klass, void *data)
     k->transfer = ads7846_transfer;
 }
 
-static TypeInfo ads7846_info = {
+static const TypeInfo ads7846_info = {
     .name          = "ads7846",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ADS7846State),
diff --git a/hw/alpha_dp264.c b/hw/alpha_dp264.c
index 9eb939f3833..1cd549c69fa 100644
--- a/hw/alpha_dp264.c
+++ b/hw/alpha_dp264.c
@@ -11,10 +11,11 @@
 #include "loader.h"
 #include "boards.h"
 #include "alpha_sys.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "mc146818rtc.h"
 #include "ide.h"
 #include "i8254.h"
+#include "serial.h"
 
 #define MAX_IDE_BUS 2
 
@@ -42,14 +43,14 @@ static int clipper_pci_map_irq(PCIDevice *d, int irq_num)
     return (slot + 1) * 4 + irq_num;
 }
 
-static void clipper_init(ram_addr_t ram_size,
-                         const char *boot_device,
-                         const char *kernel_filename,
-                         const char *kernel_cmdline,
-                         const char *initrd_filename,
-                         const char *cpu_model)
+static void clipper_init(QEMUMachineInitArgs *args)
 {
-    CPUAlphaState *cpus[4];
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    AlphaCPU *cpus[4];
     PCIBus *pci_bus;
     ISABus *isa_bus;
     qemu_irq rtc_irq;
@@ -61,12 +62,12 @@ static void clipper_init(ram_addr_t ram_size,
     /* Create up to 4 cpus.  */
     memset(cpus, 0, sizeof(cpus));
     for (i = 0; i < smp_cpus; ++i) {
-        cpus[i] = cpu_init(cpu_model ? cpu_model : "ev67");
+        cpus[i] = cpu_alpha_init(cpu_model ? cpu_model : "ev67");
     }
 
-    cpus[0]->trap_arg0 = ram_size;
-    cpus[0]->trap_arg1 = 0;
-    cpus[0]->trap_arg2 = smp_cpus;
+    cpus[0]->env.trap_arg0 = ram_size;
+    cpus[0]->env.trap_arg1 = 0;
+    cpus[0]->env.trap_arg2 = smp_cpus;
 
     /* Init the chipset.  */
     pci_bus = typhoon_init(ram_size, &isa_bus, &rtc_irq, cpus,
@@ -77,7 +78,7 @@ static void clipper_init(ram_addr_t ram_size,
     isa_create_simple(isa_bus, "i8042");
 
     /* VGA setup.  Don't bother loading the bios.  */
-    alpha_pci_vga_setup(pci_bus);
+    pci_vga_init(pci_bus);
 
     /* Serial code setup.  */
     for (i = 0; i < MAX_SERIAL_PORTS; ++i) {
@@ -118,9 +119,9 @@ static void clipper_init(ram_addr_t ram_size,
 
     /* Start all cpus at the PALcode RESET entry point.  */
     for (i = 0; i < smp_cpus; ++i) {
-        cpus[i]->pal_mode = 1;
-        cpus[i]->pc = palcode_entry;
-        cpus[i]->palbr = palcode_entry;
+        cpus[i]->env.pal_mode = 1;
+        cpus[i]->env.pc = palcode_entry;
+        cpus[i]->env.palbr = palcode_entry;
     }
 
     /* Load a kernel.  */
@@ -135,7 +136,7 @@ static void clipper_init(ram_addr_t ram_size,
             exit(1);
         }
 
-        cpus[0]->trap_arg1 = kernel_entry;
+        cpus[0]->env.trap_arg1 = kernel_entry;
 
         param_offset = kernel_low - 0x6000;
 
@@ -170,6 +171,7 @@ static QEMUMachine clipper_machine = {
     .init = clipper_init,
     .max_cpus = 4,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void clipper_machine_init(void)
diff --git a/hw/alpha_pci.c b/hw/alpha_pci.c
index ea546f83f18..7327d488fdb 100644
--- a/hw/alpha_pci.c
+++ b/hw/alpha_pci.c
@@ -8,16 +8,14 @@
 
 #include "config.h"
 #include "alpha_sys.h"
-#include "qemu-log.h"
-#include "sysemu.h"
-#include "vmware_vga.h"
-#include "vga-pci.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 
 
 /* PCI IO reads/writes, to byte-word addressable memory.  */
 /* ??? Doesn't handle multiple PCI busses.  */
 
-static uint64_t bw_io_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t bw_io_read(void *opaque, hwaddr addr, unsigned size)
 {
     switch (size) {
     case 1:
@@ -30,7 +28,7 @@ static uint64_t bw_io_read(void *opaque, target_phys_addr_t addr, unsigned size)
     abort();
 }
 
-static void bw_io_write(void *opaque, target_phys_addr_t addr,
+static void bw_io_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     switch (size) {
@@ -59,14 +57,14 @@ const MemoryRegionOps alpha_pci_bw_io_ops = {
 };
 
 /* PCI config space reads/writes, to byte-word addressable memory.  */
-static uint64_t bw_conf1_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bw_conf1_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PCIBus *b = opaque;
     return pci_data_read(b, addr, size);
 }
 
-static void bw_conf1_write(void *opaque, target_phys_addr_t addr,
+static void bw_conf1_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     PCIBus *b = opaque;
@@ -85,12 +83,12 @@ const MemoryRegionOps alpha_pci_conf1_ops = {
 
 /* PCI/EISA Interrupt Acknowledge Cycle.  */
 
-static uint64_t iack_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t iack_read(void *opaque, hwaddr addr, unsigned size)
 {
     return pic_read_irq(isa_pic);
 }
 
-static void special_write(void *opaque, target_phys_addr_t addr,
+static void special_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     qemu_log("pci: special write cycle");
@@ -109,25 +107,3 @@ const MemoryRegionOps alpha_pci_iack_ops = {
         .max_access_size = 4,
     },
 };
-
-void alpha_pci_vga_setup(PCIBus *pci_bus)
-{
-    switch (vga_interface_type) {
-#ifdef CONFIG_SPICE
-    case VGA_QXL:
-        pci_create_simple(pci_bus, -1, "qxl-vga");
-        return;
-#endif
-    case VGA_CIRRUS:
-        pci_cirrus_vga_init(pci_bus);
-        return;
-    case VGA_VMWARE:
-        pci_vmsvga_init(pci_bus);
-        return;
-    }
-    /* If VGA is enabled at all, and one of the above didn't work, then
-       fallback to Standard VGA.  */
-    if (vga_interface_type != VGA_NONE) {
-        pci_vga_init(pci_bus);
-    }
-}
diff --git a/hw/alpha_sys.h b/hw/alpha_sys.h
index de40f8b613b..233a71ecdbd 100644
--- a/hw/alpha_sys.h
+++ b/hw/alpha_sys.h
@@ -3,15 +3,14 @@
 #ifndef HW_ALPHA_H
 #define HW_ALPHA_H 1
 
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "ide.h"
-#include "net.h"
 #include "pc.h"
 #include "irq.h"
 
 
-PCIBus *typhoon_init(ram_addr_t, ISABus **, qemu_irq *, CPUAlphaState *[4],
+PCIBus *typhoon_init(ram_addr_t, ISABus **, qemu_irq *, AlphaCPU *[4],
                      pci_map_irq_fn);
 
 /* alpha_pci.c.  */
@@ -19,6 +18,4 @@ extern const MemoryRegionOps alpha_pci_bw_io_ops;
 extern const MemoryRegionOps alpha_pci_conf1_ops;
 extern const MemoryRegionOps alpha_pci_iack_ops;
 
-void alpha_pci_vga_setup(PCIBus *pci_bus);
-
 #endif
diff --git a/hw/alpha_typhoon.c b/hw/alpha_typhoon.c
index b7cf4e29001..bf9aabfc084 100644
--- a/hw/alpha_typhoon.c
+++ b/hw/alpha_typhoon.c
@@ -7,12 +7,12 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "hw.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "alpha_sys.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 
 #define TYPE_TYPHOON_PCI_HOST_BRIDGE "typhoon-pcihost"
@@ -23,7 +23,7 @@ typedef struct TyphoonCchip {
     uint64_t drir;
     uint64_t dim[4];
     uint32_t iic[4];
-    CPUAlphaState *cpu[4];
+    AlphaCPU *cpu[4];
 } TyphoonCchip;
 
 typedef struct TyphoonWindow {
@@ -58,10 +58,11 @@ typedef struct TyphoonState {
 } TyphoonState;
 
 /* Called when one of DRIR or DIM changes.  */
-static void cpu_irq_change(CPUAlphaState *env, uint64_t req)
+static void cpu_irq_change(AlphaCPU *cpu, uint64_t req)
 {
     /* If there are any non-masked interrupts, tell the cpu.  */
-    if (env) {
+    if (cpu != NULL) {
+        CPUAlphaState *env = &cpu->env;
         if (req) {
             cpu_interrupt(env, CPU_INTERRUPT_HARD);
         } else {
@@ -70,10 +71,11 @@ static void cpu_irq_change(CPUAlphaState *env, uint64_t req)
     }
 }
 
-static uint64_t cchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t cchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     CPUAlphaState *env = cpu_single_env;
     TyphoonState *s = opaque;
+    CPUState *cpu;
     uint64_t ret = 0;
 
     if (addr & 4) {
@@ -94,7 +96,8 @@ static uint64_t cchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
 
     case 0x0080:
         /* MISC: Miscellaneous Register.  */
-        ret = s->cchip.misc | (env->cpu_index & 3);
+        cpu = ENV_GET_CPU(env);
+        ret = s->cchip.misc | (cpu->cpu_index & 3);
         break;
 
     case 0x00c0:
@@ -203,13 +206,13 @@ static uint64_t cchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static uint64_t dchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t dchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     /* Skip this.  It's all related to DRAM timing and setup.  */
     return 0;
 }
 
-static uint64_t pchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t pchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     TyphoonState *s = opaque;
     uint64_t ret = 0;
@@ -306,7 +309,7 @@ static uint64_t pchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static void cchip_write(void *opaque, target_phys_addr_t addr,
+static void cchip_write(void *opaque, hwaddr addr,
                         uint64_t v32, unsigned size)
 {
     TyphoonState *s = opaque;
@@ -353,8 +356,9 @@ static void cchip_write(void *opaque, target_phys_addr_t addr,
         if ((newval ^ oldval) & 0xff0) {
             int i;
             for (i = 0; i < 4; ++i) {
-                CPUAlphaState *env = s->cchip.cpu[i];
-                if (env) {
+                AlphaCPU *cpu = s->cchip.cpu[i];
+                if (cpu != NULL) {
+                    CPUAlphaState *env = &cpu->env;
                     /* IPI can be either cleared or set by the write.  */
                     if (newval & (1 << (i + 8))) {
                         cpu_interrupt(env, CPU_INTERRUPT_SMP);
@@ -463,13 +467,13 @@ static void cchip_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void dchip_write(void *opaque, target_phys_addr_t addr,
+static void dchip_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     /* Skip this.  It's all related to DRAM timing and setup.  */
 }
 
-static void pchip_write(void *opaque, target_phys_addr_t addr,
+static void pchip_write(void *opaque, hwaddr addr,
                         uint64_t v32, unsigned size)
 {
     TyphoonState *s = opaque;
@@ -661,8 +665,8 @@ static void typhoon_set_timer_irq(void *opaque, int irq, int level)
 
     /* Deliver the interrupt to each CPU, considering each CPU's IIC.  */
     for (i = 0; i < 4; ++i) {
-        CPUAlphaState *env = s->cchip.cpu[i];
-        if (env) {
+        AlphaCPU *cpu = s->cchip.cpu[i];
+        if (cpu != NULL) {
             uint32_t iic = s->cchip.iic[i];
 
             /* ??? The verbage in Section 10.2.2.10 isn't 100% clear.
@@ -681,7 +685,7 @@ static void typhoon_set_timer_irq(void *opaque, int irq, int level)
                 /* Set the ITI bit for this cpu.  */
                 s->cchip.misc |= 1 << (i + 4);
                 /* And signal the interrupt.  */
-                cpu_interrupt(env, CPU_INTERRUPT_TIMER);
+                cpu_interrupt(&cpu->env, CPU_INTERRUPT_TIMER);
             }
         }
     }
@@ -694,12 +698,12 @@ static void typhoon_alarm_timer(void *opaque)
 
     /* Set the ITI bit for this cpu.  */
     s->cchip.misc |= 1 << (cpu + 4);
-    cpu_interrupt(s->cchip.cpu[cpu], CPU_INTERRUPT_TIMER);
+    cpu_interrupt(&s->cchip.cpu[cpu]->env, CPU_INTERRUPT_TIMER);
 }
 
 PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
                      qemu_irq *p_rtc_irq,
-                     CPUAlphaState *cpus[4], pci_map_irq_fn sys_map_irq)
+                     AlphaCPU *cpus[4], pci_map_irq_fn sys_map_irq)
 {
     const uint64_t MB = 1024 * 1024;
     const uint64_t GB = 1024 * MB;
@@ -719,10 +723,10 @@ PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
 
     /* Remember the CPUs so that we can deliver interrupts to them.  */
     for (i = 0; i < 4; i++) {
-        CPUAlphaState *env = cpus[i];
-        s->cchip.cpu[i] = env;
-        if (env) {
-            env->alarm_timer = qemu_new_timer_ns(rtc_clock,
+        AlphaCPU *cpu = cpus[i];
+        s->cchip.cpu[i] = cpu;
+        if (cpu != NULL) {
+            cpu->alarm_timer = qemu_new_timer_ns(rtc_clock,
                                                  typhoon_alarm_timer,
                                                  (void *)((uintptr_t)s + i));
         }
diff --git a/hw/an5206.c b/hw/an5206.c
index 25407c0f506..750115a3aaf 100644
--- a/hw/an5206.c
+++ b/hw/an5206.c
@@ -11,7 +11,7 @@
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
 #define AN5206_MBAR_ADDR 0x10000000
@@ -19,15 +19,15 @@
 
 /* Board init.  */
 
-static void an5206_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void an5206_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *sram = g_new(MemoryRegion, 1);
@@ -86,6 +86,7 @@ static QEMUMachine an5206_machine = {
     .name = "an5206",
     .desc = "Arnewsh 5206",
     .init = an5206_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void an5206_machine_init(void)
diff --git a/hw/apb_pci.c b/hw/apb_pci.c
index c28411a4605..7eb0c2bbcb7 100644
--- a/hw/apb_pci.c
+++ b/hw/apb_pci.c
@@ -27,13 +27,13 @@
    the secondary PCI bridge.  */
 
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "pci/pci_bridge.h"
+#include "pci/pci_bus.h"
 #include "apb_pci.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 /* debug APB */
 //#define DEBUG_APB
@@ -87,7 +87,7 @@ typedef struct APBState {
 
 static void pci_apb_set_irq(void *opaque, int irq_num, int level);
 
-static void apb_config_writel (void *opaque, target_phys_addr_t addr,
+static void apb_config_writel (void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     APBState *s = opaque;
@@ -152,7 +152,7 @@ static void apb_config_writel (void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t apb_config_readl (void *opaque,
-                                  target_phys_addr_t addr, unsigned size)
+                                  hwaddr addr, unsigned size)
 {
     APBState *s = opaque;
     uint32_t val;
@@ -212,7 +212,7 @@ static const MemoryRegionOps apb_config_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void apb_pci_config_write(void *opaque, target_phys_addr_t addr,
+static void apb_pci_config_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
     APBState *s = opaque;
@@ -222,7 +222,7 @@ static void apb_pci_config_write(void *opaque, target_phys_addr_t addr,
     pci_data_write(s->bus, addr, val, size);
 }
 
-static uint64_t apb_pci_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t apb_pci_config_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     uint32_t ret;
@@ -234,25 +234,25 @@ static uint64_t apb_pci_config_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void pci_apb_iowriteb (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowriteb (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outb(addr & IOPORTS_MASK, val);
 }
 
-static void pci_apb_iowritew (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowritew (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outw(addr & IOPORTS_MASK, bswap16(val));
 }
 
-static void pci_apb_iowritel (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowritel (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     cpu_outl(addr & IOPORTS_MASK, bswap32(val));
 }
 
-static uint32_t pci_apb_ioreadb (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadb (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -260,7 +260,7 @@ static uint32_t pci_apb_ioreadb (void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t pci_apb_ioreadw (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadw (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -268,7 +268,7 @@ static uint32_t pci_apb_ioreadw (void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t pci_apb_ioreadl (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadl (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -351,8 +351,8 @@ static int apb_pci_bridge_initfn(PCIDevice *dev)
     return 0;
 }
 
-PCIBus *pci_apb_init(target_phys_addr_t special_base,
-                     target_phys_addr_t mem_base,
+PCIBus *pci_apb_init(hwaddr special_base,
+                     hwaddr mem_base,
                      qemu_irq *ivec_irqs, PCIBus **bus2, PCIBus **bus3,
                      qemu_irq **pbm_irqs)
 {
@@ -365,7 +365,7 @@ PCIBus *pci_apb_init(target_phys_addr_t special_base,
     /* Ultrasparc PBM main bus */
     dev = qdev_create(NULL, "pbm");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* apb_config */
     sysbus_mmio_map(s, 0, special_base);
     /* PCI configuration space */
@@ -486,7 +486,7 @@ static void pbm_pci_host_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_BRIDGE_HOST;
 }
 
-static TypeInfo pbm_pci_host_info = {
+static const TypeInfo pbm_pci_host_info = {
     .name          = "pbm-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -502,7 +502,7 @@ static void pbm_host_class_init(ObjectClass *klass, void *data)
     dc->reset = pci_pbm_reset;
 }
 
-static TypeInfo pbm_host_info = {
+static const TypeInfo pbm_host_info = {
     .name          = "pbm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(APBState),
@@ -525,7 +525,7 @@ static void pbm_pci_bridge_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pci_device;
 }
 
-static TypeInfo pbm_pci_bridge_info = {
+static const TypeInfo pbm_pci_bridge_info = {
     .name          = "pbm-bridge",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBridge),
diff --git a/hw/apb_pci.h b/hw/apb_pci.h
index 55f7c4c3b21..736db6118e1 100644
--- a/hw/apb_pci.h
+++ b/hw/apb_pci.h
@@ -3,8 +3,8 @@
 
 #include "qemu-common.h"
 
-PCIBus *pci_apb_init(target_phys_addr_t special_base,
-                     target_phys_addr_t mem_base,
+PCIBus *pci_apb_init(hwaddr special_base,
+                     hwaddr mem_base,
                      qemu_irq *ivec_irqs, PCIBus **bus2, PCIBus **bus3,
                      qemu_irq **pbm_irqs);
 #endif
diff --git a/hw/apic.c b/hw/apic.c
index 385555eb43c..fd14b73023d 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -16,12 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 #include "apic_internal.h"
 #include "apic.h"
 #include "ioapic.h"
-#include "msi.h"
-#include "host-utils.h"
+#include "pci/msi.h"
+#include "qemu/host-utils.h"
 #include "trace.h"
 #include "pc.h"
 #include "apic-msidef.h"
@@ -107,7 +107,7 @@ static void apic_sync_vapic(APICCommonState *s, int sync_type)
         length = offsetof(VAPICState, enabled) - offsetof(VAPICState, isr);
 
         if (sync_type & SYNC_TO_VAPIC) {
-            assert(qemu_cpu_is_self(s->cpu_env));
+            assert(qemu_cpu_is_self(CPU(s->cpu)));
 
             vapic_state.tpr = s->tpr;
             vapic_state.enabled = 1;
@@ -151,15 +151,15 @@ static void apic_local_deliver(APICCommonState *s, int vector)
 
     switch ((lvt >> 8) & 7) {
     case APIC_DM_SMI:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_SMI);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_SMI);
         break;
 
     case APIC_DM_NMI:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_NMI);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_NMI);
         break;
 
     case APIC_DM_EXTINT:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
         break;
 
     case APIC_DM_FIXED:
@@ -187,7 +187,7 @@ void apic_deliver_pic_intr(DeviceState *d, int level)
             reset_bit(s->irr, lvt & 0xff);
             /* fall through */
         case APIC_DM_EXTINT:
-            cpu_reset_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+            cpu_reset_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
             break;
         }
     }
@@ -248,18 +248,22 @@ static void apic_bus_deliver(const uint32_t *deliver_bitmask,
 
         case APIC_DM_SMI:
             foreach_apic(apic_iter, deliver_bitmask,
-                cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_SMI) );
+                cpu_interrupt(&apic_iter->cpu->env, CPU_INTERRUPT_SMI)
+            );
             return;
 
         case APIC_DM_NMI:
             foreach_apic(apic_iter, deliver_bitmask,
-                cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_NMI) );
+                cpu_interrupt(&apic_iter->cpu->env, CPU_INTERRUPT_NMI)
+            );
             return;
 
         case APIC_DM_INIT:
             /* normal INIT IPI sent to processors */
             foreach_apic(apic_iter, deliver_bitmask,
-                         cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_INIT) );
+                         cpu_interrupt(&apic_iter->cpu->env,
+                                       CPU_INTERRUPT_INIT)
+            );
             return;
 
         case APIC_DM_EXTINT:
@@ -293,7 +297,7 @@ static void apic_set_base(APICCommonState *s, uint64_t val)
     /* if disabled, cannot be enabled again */
     if (!(val & MSR_IA32_APICBASE_ENABLE)) {
         s->apicbase &= ~MSR_IA32_APICBASE_ENABLE;
-        cpu_clear_apic_feature(s->cpu_env);
+        cpu_clear_apic_feature(&s->cpu->env);
         s->spurious_vec &= ~APIC_SV_ENABLE;
     }
 }
@@ -359,13 +363,15 @@ static int apic_irq_pending(APICCommonState *s)
 /* signal the CPU if an irq is pending */
 static void apic_update_irq(APICCommonState *s)
 {
+    CPUState *cpu = CPU(s->cpu);
+
     if (!(s->spurious_vec & APIC_SV_ENABLE)) {
         return;
     }
-    if (!qemu_cpu_is_self(s->cpu_env)) {
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_POLL);
+    if (!qemu_cpu_is_self(cpu)) {
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_POLL);
     } else if (apic_irq_pending(s) > 0) {
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
     }
 }
 
@@ -472,18 +478,18 @@ static void apic_get_delivery_bitmask(uint32_t *deliver_bitmask,
 static void apic_startup(APICCommonState *s, int vector_num)
 {
     s->sipi_vector = vector_num;
-    cpu_interrupt(s->cpu_env, CPU_INTERRUPT_SIPI);
+    cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_SIPI);
 }
 
 void apic_sipi(DeviceState *d)
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
 
-    cpu_reset_interrupt(s->cpu_env, CPU_INTERRUPT_SIPI);
+    cpu_reset_interrupt(&s->cpu->env, CPU_INTERRUPT_SIPI);
 
     if (!s->wait_for_sipi)
         return;
-    cpu_x86_load_seg_cache_sipi(s->cpu_env, s->sipi_vector);
+    cpu_x86_load_seg_cache_sipi(s->cpu, s->sipi_vector);
     s->wait_for_sipi = 0;
 }
 
@@ -630,25 +636,25 @@ static void apic_timer(void *opaque)
     apic_timer_update(s, s->next_time);
 }
 
-static uint32_t apic_mem_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readb(void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static uint32_t apic_mem_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readw(void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static void apic_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
 }
 
-static void apic_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writew(void *opaque, hwaddr addr, uint32_t val)
 {
 }
 
-static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readl(void *opaque, hwaddr addr)
 {
     DeviceState *d;
     APICCommonState *s;
@@ -672,7 +678,7 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
     case 0x08:
         apic_sync_vapic(s, SYNC_FROM_VAPIC);
         if (apic_report_tpr_access) {
-            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_READ);
+            cpu_report_tpr_access(&s->cpu->env, TPR_ACCESS_READ);
         }
         val = s->tpr;
         break;
@@ -732,7 +738,7 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void apic_send_msi(target_phys_addr_t addr, uint32_t data)
+static void apic_send_msi(hwaddr addr, uint32_t data)
 {
     uint8_t dest = (addr & MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT;
     uint8_t vector = (data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT;
@@ -743,7 +749,7 @@ static void apic_send_msi(target_phys_addr_t addr, uint32_t data)
     apic_deliver_irq(dest, dest_mode, delivery, vector, trigger_mode);
 }
 
-static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     DeviceState *d;
     APICCommonState *s;
@@ -774,7 +780,7 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
         break;
     case 0x08:
         if (apic_report_tpr_access) {
-            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_WRITE);
+            cpu_report_tpr_access(&s->cpu->env, TPR_ACCESS_WRITE);
         }
         s->tpr = val;
         apic_sync_vapic(s, SYNC_TO_VAPIC);
@@ -889,7 +895,7 @@ static void apic_class_init(ObjectClass *klass, void *data)
     k->post_load = apic_post_load;
 }
 
-static TypeInfo apic_info = {
+static const TypeInfo apic_info = {
     .name          = "apic",
     .instance_size = sizeof(APICCommonState),
     .parent        = TYPE_APIC_COMMON,
diff --git a/hw/apic_common.c b/hw/apic_common.c
index 371f95d909f..6e1b1e0320e 100644
--- a/hw/apic_common.c
+++ b/hw/apic_common.c
@@ -20,7 +20,7 @@
 #include "apic.h"
 #include "apic_internal.h"
 #include "trace.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static int apic_irq_delivered;
 bool apic_report_tpr_access;
@@ -89,7 +89,7 @@ void apic_enable_tpr_access_reporting(DeviceState *d, bool enable)
     }
 }
 
-void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr)
+void apic_enable_vapic(DeviceState *d, hwaddr paddr)
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
     APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
@@ -103,7 +103,7 @@ void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
 
-    vapic_report_tpr_access(s->vapic, s->cpu_env, ip, access);
+    vapic_report_tpr_access(s->vapic, &s->cpu->env, ip, access);
 }
 
 void apic_report_irq_delivered(int delivered)
@@ -217,7 +217,7 @@ static void apic_reset_common(DeviceState *d)
     APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
     bool bsp;
 
-    bsp = cpu_is_bsp(x86_env_get_cpu(s->cpu_env));
+    bsp = cpu_is_bsp(s->cpu);
     s->apicbase = 0xfee00000 |
         (bsp ? MSR_IA32_APICBASE_BSP : 0) | MSR_IA32_APICBASE_ENABLE;
 
@@ -368,7 +368,6 @@ static const VMStateDescription vmstate_apic_common = {
 
 static Property apic_properties_common[] = {
     DEFINE_PROP_UINT8("id", APICCommonState, id, -1),
-    DEFINE_PROP_PTR("cpu_env", APICCommonState, cpu_env),
     DEFINE_PROP_BIT("vapic", APICCommonState, vapic_control, VAPIC_ENABLE_BIT,
                     true),
     DEFINE_PROP_END_OF_LIST(),
@@ -386,7 +385,7 @@ static void apic_common_class_init(ObjectClass *klass, void *data)
     sc->init = apic_init_common;
 }
 
-static TypeInfo apic_common_type = {
+static const TypeInfo apic_common_type = {
     .name = TYPE_APIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/apic_internal.h b/hw/apic_internal.h
index 4d8ff490ce3..dcbbfd41cb9 100644
--- a/hw/apic_internal.h
+++ b/hw/apic_internal.h
@@ -20,9 +20,9 @@
 #ifndef QEMU_APIC_INTERNAL_H
 #define QEMU_APIC_INTERNAL_H
 
-#include "memory.h"
+#include "exec/memory.h"
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* APIC Local Vector Table */
 #define APIC_LVT_TIMER                  0
@@ -95,8 +95,9 @@ typedef struct APICCommonClass
 
 struct APICCommonState {
     SysBusDevice busdev;
+
     MemoryRegion io_memory;
-    void *cpu_env;
+    X86CPU *cpu;
     uint32_t apicbase;
     uint8_t id;
     uint8_t arb_id;
@@ -124,7 +125,7 @@ struct APICCommonState {
 
     uint32_t vapic_control;
     DeviceState *vapic;
-    target_phys_addr_t vapic_paddr; /* note: persistence via kvmvapic */
+    hwaddr vapic_paddr; /* note: persistence via kvmvapic */
 };
 
 typedef struct VAPICState {
@@ -140,7 +141,7 @@ extern bool apic_report_tpr_access;
 void apic_report_irq_delivered(int delivered);
 bool apic_next_timer(APICCommonState *s, int64_t current_time);
 void apic_enable_tpr_access_reporting(DeviceState *d, bool enable);
-void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr);
+void apic_enable_vapic(DeviceState *d, hwaddr paddr);
 
 void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
                              TPRAccess access);
diff --git a/hw/apm.c b/hw/apm.c
index 2aead52a747..2e1b1372d24 100644
--- a/hw/apm.c
+++ b/hw/apm.c
@@ -22,6 +22,7 @@
 
 #include "apm.h"
 #include "hw.h"
+#include "pci/pci.h"
 
 //#define DEBUG
 
@@ -35,7 +36,8 @@
 #define APM_CNT_IOPORT  0xb2
 #define APM_STS_IOPORT  0xb3
 
-static void apm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void apm_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
 {
     APMState *apm = opaque;
     addr &= 1;
@@ -51,7 +53,7 @@ static void apm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t apm_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t apm_ioport_readb(void *opaque, hwaddr addr, unsigned size)
 {
     APMState *apm = opaque;
     uint32_t val;
@@ -78,12 +80,23 @@ const VMStateDescription vmstate_apm = {
     }
 };
 
-void apm_init(APMState *apm, apm_ctrl_changed_t callback, void *arg)
+static const MemoryRegionOps apm_ops = {
+    .read = apm_ioport_readb,
+    .write = apm_ioport_writeb,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+void apm_init(PCIDevice *dev, APMState *apm, apm_ctrl_changed_t callback,
+              void *arg)
 {
     apm->callback = callback;
     apm->arg = arg;
 
     /* ioport 0xb2, 0xb3 */
-    register_ioport_write(APM_CNT_IOPORT, 2, 1, apm_ioport_writeb, apm);
-    register_ioport_read(APM_CNT_IOPORT, 2, 1, apm_ioport_readb, apm);
+    memory_region_init_io(&apm->io, &apm_ops, apm, "apm-io", 2);
+    memory_region_add_subregion(pci_address_space_io(dev), APM_CNT_IOPORT,
+                                &apm->io);
 }
diff --git a/hw/apm.h b/hw/apm.h
index f7c741e3276..9abb47f99ff 100644
--- a/hw/apm.h
+++ b/hw/apm.h
@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include "qemu-common.h"
 #include "hw.h"
+#include "exec/memory.h"
 
 typedef void (*apm_ctrl_changed_t)(uint32_t val, void *arg);
 
@@ -13,9 +14,11 @@ typedef struct APMState {
 
     apm_ctrl_changed_t callback;
     void *arg;
+    MemoryRegion io;
 } APMState;
 
-void apm_init(APMState *s, apm_ctrl_changed_t callback, void *arg);
+void apm_init(PCIDevice *dev, APMState *s, apm_ctrl_changed_t callback,
+              void *arg);
 
 extern const VMStateDescription vmstate_apm;
 
diff --git a/hw/applesmc.c b/hw/applesmc.c
index 8bedaad3108..5a8c4ff2d26 100644
--- a/hw/applesmc.c
+++ b/hw/applesmc.c
@@ -32,8 +32,8 @@
 
 #include "hw.h"
 #include "isa.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 
 /* #define DEBUG_SMC */
 
@@ -236,7 +236,7 @@ static void qdev_applesmc_class_init(ObjectClass *klass, void *data)
     dc->props = applesmc_isa_properties;
 }
 
-static TypeInfo applesmc_isa_info = {
+static const TypeInfo applesmc_isa_info = {
     .name          = "isa-applesmc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(struct AppleSMCStatus),
diff --git a/hw/arm-misc.h b/hw/arm-misc.h
index bdd8fecc99e..cba7553039b 100644
--- a/hw/arm-misc.h
+++ b/hw/arm-misc.h
@@ -11,7 +11,8 @@
 #ifndef ARM_MISC_H
 #define ARM_MISC_H 1
 
-#include "memory.h"
+#include "exec/memory.h"
+#include "hw/irq.h"
 
 /* The CPU is also modeled as an interrupt controller.  */
 #define ARM_PIC_CPU_IRQ 0
@@ -30,15 +31,15 @@ struct arm_boot_info {
     const char *kernel_cmdline;
     const char *initrd_filename;
     const char *dtb_filename;
-    target_phys_addr_t loader_start;
+    hwaddr loader_start;
     /* multicore boards that use the default secondary core boot functions
      * need to put the address of the secondary boot code, the boot reg,
      * and the GIC address in the next 3 values, respectively. boards that
      * have their own boot functions can use these values as they want.
      */
-    target_phys_addr_t smp_loader_start;
-    target_phys_addr_t smp_bootreg_addr;
-    target_phys_addr_t gic_cpu_if_addr;
+    hwaddr smp_loader_start;
+    hwaddr smp_bootreg_addr;
+    hwaddr gic_cpu_if_addr;
     int nb_cpus;
     int board_id;
     int (*atag_board)(const struct arm_boot_info *info, void *p);
@@ -56,8 +57,9 @@ struct arm_boot_info {
                                      const struct arm_boot_info *info);
     /* Used internally by arm_boot.c */
     int is_linux;
-    target_phys_addr_t initrd_size;
-    target_phys_addr_t entry;
+    hwaddr initrd_start;
+    hwaddr initrd_size;
+    hwaddr entry;
 };
 void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info);
 
diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs
index 2b39fb3c855..6d049e7de6a 100644
--- a/hw/arm/Makefile.objs
+++ b/hw/arm/Makefile.objs
@@ -1,6 +1,7 @@
 obj-y = integratorcp.o versatilepb.o arm_pic.o
 obj-y += arm_boot.o
 obj-y += xilinx_zynq.o zynq_slcr.o
+obj-y += xilinx_spips.o
 obj-y += arm_gic.o arm_gic_common.o
 obj-y += realview_gic.o realview.o arm_sysctl.o arm11mpcore.o a9mpcore.o
 obj-y += exynos4210_gic.o exynos4210_combiner.o exynos4210.o
diff --git a/hw/arm11mpcore.c b/hw/arm11mpcore.c
index 1bff3d32821..324e503dd9e 100644
--- a/hw/arm11mpcore.c
+++ b/hw/arm11mpcore.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* MPCore private memory region.  */
 
@@ -27,7 +27,7 @@ typedef struct mpcore_priv_state {
 
 /* Per-CPU private memory mapped IO.  */
 
-static uint64_t mpcore_scu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mpcore_scu_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     mpcore_priv_state *s = (mpcore_priv_state *)opaque;
@@ -44,11 +44,13 @@ static uint64_t mpcore_scu_read(void *opaque, target_phys_addr_t offset,
     case 0x0c: /* Invalidate all.  */
         return 0;
     default:
-        hw_error("mpcore_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "mpcore_priv_read: Bad offset %x\n", (int)offset);
+        return 0;
     }
 }
 
-static void mpcore_scu_write(void *opaque, target_phys_addr_t offset,
+static void mpcore_scu_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     mpcore_priv_state *s = (mpcore_priv_state *)opaque;
@@ -61,7 +63,8 @@ static void mpcore_scu_write(void *opaque, target_phys_addr_t offset,
         /* This is a no-op as cache is not emulated.  */
         break;
     default:
-        hw_error("mpcore_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "mpcore_priv_read: Bad offset %x\n", (int)offset);
     }
 }
 
@@ -80,8 +83,8 @@ static void mpcore_priv_set_irq(void *opaque, int irq, int level)
 static void mpcore_priv_map_setup(mpcore_priv_state *s)
 {
     int i;
-    SysBusDevice *gicbusdev = sysbus_from_qdev(s->gic);
-    SysBusDevice *busdev = sysbus_from_qdev(s->mptimer);
+    SysBusDevice *gicbusdev = SYS_BUS_DEVICE(s->gic);
+    SysBusDevice *busdev = SYS_BUS_DEVICE(s->mptimer);
     memory_region_init(&s->container, "mpcode-priv-container", 0x2000);
     memory_region_init_io(&s->iomem, &mpcore_scu_ops, s, "mpcore-scu", 0x100);
     memory_region_add_subregion(&s->container, 0, &s->iomem);
@@ -89,7 +92,7 @@ static void mpcore_priv_map_setup(mpcore_priv_state *s)
      * at 0x200, 0x300...
      */
     for (i = 0; i < (s->num_cpu + 1); i++) {
-        target_phys_addr_t offset = 0x100 + (i * 0x100);
+        hwaddr offset = 0x100 + (i * 0x100);
         memory_region_add_subregion(&s->container, offset,
                                     sysbus_mmio_get_region(gicbusdev, i + 1));
     }
@@ -98,7 +101,7 @@ static void mpcore_priv_map_setup(mpcore_priv_state *s)
      */
     for (i = 0; i < (s->num_cpu + 1) * 2; i++) {
         /* Timers at 0x600, 0x700, ...; watchdogs at 0x620, 0x720, ... */
-        target_phys_addr_t offset = 0x600 + (i >> 1) * 0x100 + (i & 1) * 0x20;
+        hwaddr offset = 0x600 + (i >> 1) * 0x100 + (i & 1) * 0x20;
         memory_region_add_subregion(&s->container, offset,
                                     sysbus_mmio_get_region(busdev, i));
     }
@@ -128,7 +131,7 @@ static int mpcore_priv_init(SysBusDevice *dev)
     qdev_init_nofail(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
-    sysbus_pass_irq(dev, sysbus_from_qdev(s->gic));
+    sysbus_pass_irq(dev, SYS_BUS_DEVICE(s->gic));
 
     /* Pass through inbound GPIO lines to the GIC */
     qdev_init_gpio_in(&s->busdev.qdev, mpcore_priv_set_irq, s->num_irq - 32);
@@ -187,7 +190,7 @@ static int realview_mpcore_init(SysBusDevice *dev)
     priv = qdev_create(NULL, "arm11mpcore_priv");
     qdev_prop_set_uint32(priv, "num-cpu", s->num_cpu);
     qdev_init_nofail(priv);
-    s->priv = sysbus_from_qdev(priv);
+    s->priv = SYS_BUS_DEVICE(priv);
     sysbus_pass_irq(dev, s->priv);
     for (i = 0; i < 32; i++) {
         s->cpuic[i] = qdev_get_gpio_in(priv, i);
@@ -219,7 +222,7 @@ static void mpcore_rirq_class_init(ObjectClass *klass, void *data)
     dc->props = mpcore_rirq_properties;
 }
 
-static TypeInfo mpcore_rirq_info = {
+static const TypeInfo mpcore_rirq_info = {
     .name          = "realview_mpcore",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mpcore_rirq_state),
@@ -249,7 +252,7 @@ static void mpcore_priv_class_init(ObjectClass *klass, void *data)
     dc->props = mpcore_priv_properties;
 }
 
-static TypeInfo mpcore_priv_info = {
+static const TypeInfo mpcore_priv_info = {
     .name          = "arm11mpcore_priv",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mpcore_priv_state),
diff --git a/hw/arm_boot.c b/hw/arm_boot.c
index a6e91436625..4065424d60b 100644
--- a/hw/arm_boot.c
+++ b/hw/arm_boot.c
@@ -10,15 +10,15 @@
 #include "config.h"
 #include "hw.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "qemu/config-file.h"
 
 #define KERNEL_ARGS_ADDR 0x100
 #define KERNEL_LOAD_ADDR 0x00010000
-#define INITRD_LOAD_ADDR 0x00d00000
 
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
 static uint32_t bootloader[] = {
@@ -45,11 +45,17 @@ static uint32_t bootloader[] = {
  * for an interprocessor interrupt and polling a configurable
  * location for the kernel secondary CPU entry point.
  */
+#define DSB_INSN 0xf57ff04f
+#define CP15_DSB_INSN 0xee070f9a /* mcr cp15, 0, r0, c7, c10, 4 */
+
 static uint32_t smpboot[] = {
-  0xe59f201c, /* ldr r2, gic_cpu_if */
-  0xe59f001c, /* ldr r0, startaddr */
+  0xe59f2028, /* ldr r2, gic_cpu_if */
+  0xe59f0028, /* ldr r0, startaddr */
   0xe3a01001, /* mov r1, #1 */
-  0xe5821000, /* str r1, [r2] */
+  0xe5821000, /* str r1, [r2] - set GICC_CTLR.Enable */
+  0xe3a010ff, /* mov r1, #0xff */
+  0xe5821004, /* str r1, [r2, 4] - set GIC_PMR.Priority to 0xff */
+  DSB_INSN,   /* dsb */
   0xe320f003, /* wfi */
   0xe5901000, /* ldr     r1, [r0] */
   0xe1110001, /* tst     r1, r1 */
@@ -66,6 +72,11 @@ static void default_write_secondary(ARMCPU *cpu,
     smpboot[ARRAY_SIZE(smpboot) - 1] = info->smp_bootreg_addr;
     smpboot[ARRAY_SIZE(smpboot) - 2] = info->gic_cpu_if_addr;
     for (n = 0; n < ARRAY_SIZE(smpboot); n++) {
+        /* Replace DSB with the pre-v7 DSB if necessary. */
+        if (!arm_feature(&cpu->env, ARM_FEATURE_V7) &&
+            smpboot[n] == DSB_INSN) {
+            smpboot[n] = CP15_DSB_INSN;
+        }
         smpboot[n] = tswap32(smpboot[n]);
     }
     rom_add_blob_fixed("smpboot", smpboot, sizeof(smpboot),
@@ -89,8 +100,8 @@ static void default_reset_secondary(ARMCPU *cpu,
 static void set_kernel_args(const struct arm_boot_info *info)
 {
     int initrd_size = info->initrd_size;
-    target_phys_addr_t base = info->loader_start;
-    target_phys_addr_t p;
+    hwaddr base = info->loader_start;
+    hwaddr p;
 
     p = base + KERNEL_ARGS_ADDR;
     /* ATAG_CORE */
@@ -109,7 +120,7 @@ static void set_kernel_args(const struct arm_boot_info *info)
         /* ATAG_INITRD2 */
         WRITE_WORD(p, 4);
         WRITE_WORD(p, 0x54420005);
-        WRITE_WORD(p, info->loader_start + INITRD_LOAD_ADDR);
+        WRITE_WORD(p, info->initrd_start);
         WRITE_WORD(p, initrd_size);
     }
     if (info->kernel_cmdline && *info->kernel_cmdline) {
@@ -142,10 +153,10 @@ static void set_kernel_args(const struct arm_boot_info *info)
 
 static void set_kernel_args_old(const struct arm_boot_info *info)
 {
-    target_phys_addr_t p;
+    hwaddr p;
     const char *s;
     int initrd_size = info->initrd_size;
-    target_phys_addr_t base = info->loader_start;
+    hwaddr base = info->loader_start;
 
     /* see linux/include/asm-arm/setup.h */
     p = base + KERNEL_ARGS_ADDR;
@@ -185,10 +196,11 @@ static void set_kernel_args_old(const struct arm_boot_info *info)
     /* pages_in_vram */
     WRITE_WORD(p, 0);
     /* initrd_start */
-    if (initrd_size)
-        WRITE_WORD(p, info->loader_start + INITRD_LOAD_ADDR);
-    else
+    if (initrd_size) {
+        WRITE_WORD(p, info->initrd_start);
+    } else {
         WRITE_WORD(p, 0);
+    }
     /* initrd_size */
     WRITE_WORD(p, initrd_size);
     /* rd_start */
@@ -213,7 +225,7 @@ static void set_kernel_args_old(const struct arm_boot_info *info)
     }
 }
 
-static int load_dtb(target_phys_addr_t addr, const struct arm_boot_info *binfo)
+static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo)
 {
 #ifdef CONFIG_FDT
     uint32_t *mem_reg_property;
@@ -281,14 +293,13 @@ static int load_dtb(target_phys_addr_t addr, const struct arm_boot_info *binfo)
 
     if (binfo->initrd_size) {
         rc = qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-start",
-                binfo->loader_start + INITRD_LOAD_ADDR);
+                binfo->initrd_start);
         if (rc < 0) {
             fprintf(stderr, "couldn't set /chosen/linux,initrd-start\n");
         }
 
         rc = qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-end",
-                    binfo->loader_start + INITRD_LOAD_ADDR +
-                    binfo->initrd_size);
+                    binfo->initrd_start + binfo->initrd_size);
         if (rc < 0) {
             fprintf(stderr, "couldn't set /chosen/linux,initrd-end\n");
         }
@@ -342,7 +353,7 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     int n;
     int is_linux = 0;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     int big_endian;
     QemuOpts *machine_opts;
 
@@ -375,6 +386,19 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     big_endian = 0;
 #endif
 
+    /* We want to put the initrd far enough into RAM that when the
+     * kernel is uncompressed it will not clobber the initrd. However
+     * on boards without much RAM we must ensure that we still leave
+     * enough room for a decent sized initrd, and on boards with large
+     * amounts of RAM we must avoid the initrd being so far up in RAM
+     * that it is outside lowmem and inaccessible to the kernel.
+     * So for boards with less  than 256MB of RAM we put the initrd
+     * halfway into RAM, and for boards with 256MB of RAM or more we put
+     * the initrd at 128MB.
+     */
+    info->initrd_start = info->loader_start +
+        MIN(info->ram_size / 2, 128 * 1024 * 1024);
+
     /* Assume that raw images are linux kernels, and ELF images are not.  */
     kernel_size = load_elf(info->kernel_filename, NULL, NULL, &elf_entry,
                            NULL, NULL, big_endian, ELF_MACHINE, 1);
@@ -398,10 +422,9 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     if (is_linux) {
         if (info->initrd_filename) {
             initrd_size = load_image_targphys(info->initrd_filename,
-                                              info->loader_start
-                                              + INITRD_LOAD_ADDR,
-                                              info->ram_size
-                                              - INITRD_LOAD_ADDR);
+                                              info->initrd_start,
+                                              info->ram_size -
+                                              info->initrd_start);
             if (initrd_size < 0) {
                 fprintf(stderr, "qemu: could not load initrd '%s'\n",
                         info->initrd_filename);
@@ -418,10 +441,12 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
          * we point to the kernel args.
          */
         if (info->dtb_filename) {
-            /* Place the DTB after the initrd in memory */
-            target_phys_addr_t dtb_start = TARGET_PAGE_ALIGN(info->loader_start
-                                                             + INITRD_LOAD_ADDR
-                                                             + initrd_size);
+            /* Place the DTB after the initrd in memory. Note that some
+             * kernels will trash anything in the 4K page the initrd
+             * ends in, so make sure the DTB isn't caught up in that.
+             */
+            hwaddr dtb_start = QEMU_ALIGN_UP(info->initrd_start + initrd_size,
+                                             4096);
             if (load_dtb(dtb_start, info)) {
                 exit(1);
             }
diff --git a/hw/arm_gic.c b/hw/arm_gic.c
index 55871fad19a..90e43d07287 100644
--- a/hw/arm_gic.c
+++ b/hw/arm_gic.c
@@ -36,17 +36,18 @@ static const uint8_t gic_id[] = {
 
 #define NUM_CPU(s) ((s)->num_cpu)
 
-static inline int gic_get_current_cpu(gic_state *s)
+static inline int gic_get_current_cpu(GICState *s)
 {
     if (s->num_cpu > 1) {
-        return cpu_single_env->cpu_index;
+        CPUState *cpu = ENV_GET_CPU(cpu_single_env);
+        return cpu->cpu_index;
     }
     return 0;
 }
 
 /* TODO: Many places that call this routine could be optimized.  */
 /* Update interrupt status after enabled or pending bits have been changed.  */
-void gic_update(gic_state *s)
+void gic_update(GICState *s)
 {
     int best_irq;
     int best_prio;
@@ -73,10 +74,10 @@ void gic_update(gic_state *s)
             }
         }
         level = 0;
-        if (best_prio <= s->priority_mask[cpu]) {
+        if (best_prio < s->priority_mask[cpu]) {
             s->current_pending[cpu] = best_irq;
             if (best_prio < s->running_priority[cpu]) {
-                DPRINTF("Raised pending IRQ %d\n", best_irq);
+                DPRINTF("Raised pending IRQ %d (cpu %d)\n", best_irq, cpu);
                 level = 1;
             }
         }
@@ -84,7 +85,7 @@ void gic_update(gic_state *s)
     }
 }
 
-void gic_set_pending_private(gic_state *s, int cpu, int irq)
+void gic_set_pending_private(GICState *s, int cpu, int irq)
 {
     int cm = 1 << cpu;
 
@@ -105,7 +106,7 @@ static void gic_set_irq(void *opaque, int irq, int level)
      *  [N+32..N+63] : PPI (internal interrupts for CPU 1
      *  ...
      */
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int cm, target;
     if (irq < (s->num_irq - GIC_INTERNAL)) {
         /* The first external input line is internal interrupt 32.  */
@@ -137,7 +138,7 @@ static void gic_set_irq(void *opaque, int irq, int level)
     gic_update(s);
 }
 
-static void gic_set_running_irq(gic_state *s, int cpu, int irq)
+static void gic_set_running_irq(GICState *s, int cpu, int irq)
 {
     s->running_irq[cpu] = irq;
     if (irq == 1023) {
@@ -148,7 +149,7 @@ static void gic_set_running_irq(gic_state *s, int cpu, int irq)
     gic_update(s);
 }
 
-uint32_t gic_acknowledge_irq(gic_state *s, int cpu)
+uint32_t gic_acknowledge_irq(GICState *s, int cpu)
 {
     int new_irq;
     int cm = 1 << cpu;
@@ -167,7 +168,7 @@ uint32_t gic_acknowledge_irq(gic_state *s, int cpu)
     return new_irq;
 }
 
-void gic_complete_irq(gic_state *s, int cpu, int irq)
+void gic_complete_irq(GICState *s, int cpu, int irq)
 {
     int update = 0;
     int cm = 1 << cpu;
@@ -212,9 +213,9 @@ void gic_complete_irq(gic_state *s, int cpu, int irq)
     }
 }
 
-static uint32_t gic_dist_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readb(void *opaque, hwaddr offset)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     uint32_t res;
     int irq;
     int i;
@@ -324,11 +325,12 @@ static uint32_t gic_dist_readb(void *opaque, target_phys_addr_t offset)
     }
     return res;
 bad_reg:
-    hw_error("gic_dist_readb: Bad offset %x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "gic_dist_readb: Bad offset %x\n", (int)offset);
     return 0;
 }
 
-static uint32_t gic_dist_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readw(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = gic_dist_readb(opaque, offset);
@@ -336,7 +338,7 @@ static uint32_t gic_dist_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint32_t gic_dist_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readl(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = gic_dist_readw(opaque, offset);
@@ -344,10 +346,10 @@ static uint32_t gic_dist_readl(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writeb(void *opaque, hwaddr offset,
                             uint32_t value)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int irq;
     int i;
     int cpu;
@@ -373,7 +375,8 @@ static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
           value = 0xff;
         for (i = 0; i < 8; i++) {
             if (value & (1 << i)) {
-                int mask = (irq < GIC_INTERNAL) ? (1 << cpu) : GIC_TARGET(irq);
+                int mask =
+                    (irq < GIC_INTERNAL) ? (1 << cpu) : GIC_TARGET(irq + i);
                 int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
 
                 if (!GIC_TEST_ENABLED(irq + i, cm)) {
@@ -416,7 +419,7 @@ static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
 
         for (i = 0; i < 8; i++) {
             if (value & (1 << i)) {
-                GIC_SET_PENDING(irq + i, GIC_TARGET(irq));
+                GIC_SET_PENDING(irq + i, GIC_TARGET(irq + i));
             }
         }
     } else if (offset < 0x300) {
@@ -487,20 +490,21 @@ static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
     gic_update(s);
     return;
 bad_reg:
-    hw_error("gic_dist_writeb: Bad offset %x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "gic_dist_writeb: Bad offset %x\n", (int)offset);
 }
 
-static void gic_dist_writew(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writew(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     gic_dist_writeb(opaque, offset, value & 0xff);
     gic_dist_writeb(opaque, offset + 1, value >> 8);
 }
 
-static void gic_dist_writel(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writel(void *opaque, hwaddr offset,
                             uint32_t value)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     if (offset == 0xf00) {
         int cpu;
         int irq;
@@ -539,7 +543,7 @@ static const MemoryRegionOps gic_dist_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint32_t gic_cpu_read(gic_state *s, int cpu, int offset)
+static uint32_t gic_cpu_read(GICState *s, int cpu, int offset)
 {
     switch (offset) {
     case 0x00: /* Control */
@@ -556,17 +560,18 @@ static uint32_t gic_cpu_read(gic_state *s, int cpu, int offset)
     case 0x18: /* Highest Pending Interrupt */
         return s->current_pending[cpu];
     default:
-        hw_error("gic_cpu_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_cpu_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void gic_cpu_write(gic_state *s, int cpu, int offset, uint32_t value)
+static void gic_cpu_write(GICState *s, int cpu, int offset, uint32_t value)
 {
     switch (offset) {
     case 0x00: /* Control */
         s->cpu_enabled[cpu] = (value & 1);
-        DPRINTF("CPU %d %sabled\n", cpu, s->cpu_enabled ? "En" : "Dis");
+        DPRINTF("CPU %d %sabled\n", cpu, s->cpu_enabled[cpu] ? "En" : "Dis");
         break;
     case 0x04: /* Priority mask */
         s->priority_mask[cpu] = (value & 0xff);
@@ -577,44 +582,45 @@ static void gic_cpu_write(gic_state *s, int cpu, int offset, uint32_t value)
     case 0x10: /* End Of Interrupt */
         return gic_complete_irq(s, cpu, value & 0x3ff);
     default:
-        hw_error("gic_cpu_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_cpu_write: Bad offset %x\n", (int)offset);
         return;
     }
     gic_update(s);
 }
 
 /* Wrappers to read/write the GIC CPU interface for the current CPU */
-static uint64_t gic_thiscpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t gic_thiscpu_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     return gic_cpu_read(s, gic_get_current_cpu(s), addr);
 }
 
-static void gic_thiscpu_write(void *opaque, target_phys_addr_t addr,
+static void gic_thiscpu_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     gic_cpu_write(s, gic_get_current_cpu(s), addr, value);
 }
 
 /* Wrappers to read/write the GIC CPU interface for a specific CPU.
- * These just decode the opaque pointer into gic_state* + cpu id.
+ * These just decode the opaque pointer into GICState* + cpu id.
  */
-static uint64_t gic_do_cpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t gic_do_cpu_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
-    gic_state **backref = (gic_state **)opaque;
-    gic_state *s = *backref;
+    GICState **backref = (GICState **)opaque;
+    GICState *s = *backref;
     int id = (backref - s->backref);
     return gic_cpu_read(s, id, addr);
 }
 
-static void gic_do_cpu_write(void *opaque, target_phys_addr_t addr,
+static void gic_do_cpu_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
-    gic_state **backref = (gic_state **)opaque;
-    gic_state *s = *backref;
+    GICState **backref = (GICState **)opaque;
+    GICState *s = *backref;
     int id = (backref - s->backref);
     gic_cpu_write(s, id, addr, value);
 }
@@ -631,7 +637,7 @@ static const MemoryRegionOps gic_cpu_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void gic_init_irqs_and_distributor(gic_state *s, int num_irq)
+void gic_init_irqs_and_distributor(GICState *s, int num_irq)
 {
     int i;
 
@@ -657,7 +663,7 @@ static int arm_gic_init(SysBusDevice *dev)
 {
     /* Device instance init function for the GIC sysbus device */
     int i;
-    gic_state *s = FROM_SYSBUS(gic_state, dev);
+    GICState *s = FROM_SYSBUS(GICState, dev);
     ARMGICClass *agc = ARM_GIC_GET_CLASS(s);
 
     agc->parent_init(dev);
@@ -698,10 +704,10 @@ static void arm_gic_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo arm_gic_info = {
+static const TypeInfo arm_gic_info = {
     .name = TYPE_ARM_GIC,
     .parent = TYPE_ARM_GIC_COMMON,
-    .instance_size = sizeof(gic_state),
+    .instance_size = sizeof(GICState),
     .class_init = arm_gic_class_init,
     .class_size = sizeof(ARMGICClass),
 };
diff --git a/hw/arm_gic_common.c b/hw/arm_gic_common.c
index 360e7823f76..40e8dd70459 100644
--- a/hw/arm_gic_common.c
+++ b/hw/arm_gic_common.c
@@ -22,7 +22,7 @@
 
 static void gic_save(QEMUFile *f, void *opaque)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int i;
     int j;
 
@@ -56,7 +56,7 @@ static void gic_save(QEMUFile *f, void *opaque)
 
 static int gic_load(QEMUFile *f, void *opaque, int version_id)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int i;
     int j;
 
@@ -96,7 +96,7 @@ static int gic_load(QEMUFile *f, void *opaque, int version_id)
 
 static int arm_gic_common_init(SysBusDevice *dev)
 {
-    gic_state *s = FROM_SYSBUS(gic_state, dev);
+    GICState *s = FROM_SYSBUS(GICState, dev);
     int num_irq = s->num_irq;
 
     if (s->num_cpu > NCPU) {
@@ -123,11 +123,15 @@ static int arm_gic_common_init(SysBusDevice *dev)
 
 static void arm_gic_common_reset(DeviceState *dev)
 {
-    gic_state *s = FROM_SYSBUS(gic_state, sysbus_from_qdev(dev));
+    GICState *s = FROM_SYSBUS(GICState, SYS_BUS_DEVICE(dev));
     int i;
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
-        s->priority_mask[i] = 0xf0;
+        if (s->revision == REV_11MPCORE) {
+            s->priority_mask[i] = 0xf0;
+        } else {
+            s->priority_mask[i] = 0;
+        }
         s->current_pending[i] = 1023;
         s->running_irq[i] = 1023;
         s->running_priority[i] = 0x100;
@@ -147,13 +151,13 @@ static void arm_gic_common_reset(DeviceState *dev)
 }
 
 static Property arm_gic_common_properties[] = {
-    DEFINE_PROP_UINT32("num-cpu", gic_state, num_cpu, 1),
-    DEFINE_PROP_UINT32("num-irq", gic_state, num_irq, 32),
+    DEFINE_PROP_UINT32("num-cpu", GICState, num_cpu, 1),
+    DEFINE_PROP_UINT32("num-irq", GICState, num_irq, 32),
     /* Revision can be 1 or 2 for GIC architecture specification
      * versions 1 or 2, or 0 to indicate the legacy 11MPCore GIC.
      * (Internally, 0xffffffff also indicates "not a GIC but an NVIC".)
      */
-    DEFINE_PROP_UINT32("revision", gic_state, revision, 1),
+    DEFINE_PROP_UINT32("revision", GICState, revision, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -167,10 +171,10 @@ static void arm_gic_common_class_init(ObjectClass *klass, void *data)
     sc->init = arm_gic_common_init;
 }
 
-static TypeInfo arm_gic_common_type = {
+static const TypeInfo arm_gic_common_type = {
     .name = TYPE_ARM_GIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(gic_state),
+    .instance_size = sizeof(GICState),
     .class_size = sizeof(ARMGICCommonClass),
     .class_init = arm_gic_common_class_init,
     .abstract = true,
diff --git a/hw/arm_gic_internal.h b/hw/arm_gic_internal.h
index db4fad564f9..699352ca8b0 100644
--- a/hw/arm_gic_internal.h
+++ b/hw/arm_gic_internal.h
@@ -69,7 +69,7 @@ typedef struct gic_irq_state {
     unsigned trigger:1; /* nonzero = edge triggered.  */
 } gic_irq_state;
 
-typedef struct gic_state {
+typedef struct GICState {
     SysBusDevice busdev;
     qemu_irq parent_irq[NCPU];
     int enabled;
@@ -92,25 +92,25 @@ typedef struct gic_state {
     /* This is just so we can have an opaque pointer which identifies
      * both this GIC and which CPU interface we should be accessing.
      */
-    struct gic_state *backref[NCPU];
+    struct GICState *backref[NCPU];
     MemoryRegion cpuiomem[NCPU+1]; /* CPU interfaces */
     uint32_t num_irq;
     uint32_t revision;
-} gic_state;
+} GICState;
 
 /* The special cases for the revision property: */
 #define REV_11MPCORE 0
 #define REV_NVIC 0xffffffff
 
-void gic_set_pending_private(gic_state *s, int cpu, int irq);
-uint32_t gic_acknowledge_irq(gic_state *s, int cpu);
-void gic_complete_irq(gic_state *s, int cpu, int irq);
-void gic_update(gic_state *s);
-void gic_init_irqs_and_distributor(gic_state *s, int num_irq);
+void gic_set_pending_private(GICState *s, int cpu, int irq);
+uint32_t gic_acknowledge_irq(GICState *s, int cpu);
+void gic_complete_irq(GICState *s, int cpu, int irq);
+void gic_update(GICState *s);
+void gic_init_irqs_and_distributor(GICState *s, int num_irq);
 
 #define TYPE_ARM_GIC_COMMON "arm_gic_common"
 #define ARM_GIC_COMMON(obj) \
-     OBJECT_CHECK(gic_state, (obj), TYPE_ARM_GIC_COMMON)
+     OBJECT_CHECK(GICState, (obj), TYPE_ARM_GIC_COMMON)
 #define ARM_GIC_COMMON_CLASS(klass) \
      OBJECT_CLASS_CHECK(ARMGICCommonClass, (klass), TYPE_ARM_GIC_COMMON)
 #define ARM_GIC_COMMON_GET_CLASS(obj) \
@@ -122,7 +122,7 @@ typedef struct ARMGICCommonClass {
 
 #define TYPE_ARM_GIC "arm_gic"
 #define ARM_GIC(obj) \
-     OBJECT_CHECK(gic_state, (obj), TYPE_ARM_GIC)
+     OBJECT_CHECK(GICState, (obj), TYPE_ARM_GIC)
 #define ARM_GIC_CLASS(klass) \
      OBJECT_CLASS_CHECK(ARMGICClass, (klass), TYPE_ARM_GIC)
 #define ARM_GIC_GET_CLASS(obj) \
diff --git a/hw/arm_l2x0.c b/hw/arm_l2x0.c
index de6a0863d83..ae1e51d0093 100644
--- a/hw/arm_l2x0.c
+++ b/hw/arm_l2x0.c
@@ -51,7 +51,7 @@ static const VMStateDescription vmstate_l2x0 = {
 };
 
 
-static uint64_t l2x0_priv_read(void *opaque, target_phys_addr_t offset,
+static uint64_t l2x0_priv_read(void *opaque, hwaddr offset,
                                unsigned size)
 {
     uint32_t cache_data;
@@ -87,13 +87,14 @@ static uint64_t l2x0_priv_read(void *opaque, target_phys_addr_t offset,
     case 0xF80:
         return 0;
     default:
-        fprintf(stderr, "l2x0_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "l2x0_priv_read: Bad offset %x\n", (int)offset);
         break;
     }
     return 0;
 }
 
-static void l2x0_priv_write(void *opaque, target_phys_addr_t offset,
+static void l2x0_priv_write(void *opaque, hwaddr offset,
                             uint64_t value, unsigned size)
 {
     l2x0_state *s = (l2x0_state *)opaque;
@@ -128,7 +129,8 @@ static void l2x0_priv_write(void *opaque, target_phys_addr_t offset,
     case 0xF80:
         return;
     default:
-        fprintf(stderr, "l2x0_priv_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "l2x0_priv_write: Bad offset %x\n", (int)offset);
         break;
     }
 }
@@ -177,7 +179,7 @@ static void l2x0_class_init(ObjectClass *klass, void *data)
     dc->reset = l2x0_priv_reset;
 }
 
-static TypeInfo l2x0_info = {
+static const TypeInfo l2x0_info = {
     .name = "l2x0",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(l2x0_state),
diff --git a/hw/arm_mptimer.c b/hw/arm_mptimer.c
index fe43cbb5f1e..32817d38149 100644
--- a/hw/arm_mptimer.c
+++ b/hw/arm_mptimer.c
@@ -20,7 +20,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* This device implements the per-cpu private timer and watchdog block
  * which is used in both the ARM11MPCore and Cortex-A9MP.
@@ -49,11 +49,13 @@ typedef struct {
 
 static inline int get_current_cpu(arm_mptimer_state *s)
 {
-    if (cpu_single_env->cpu_index >= s->num_cpu) {
+    CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+
+    if (cpu_single_cpu->cpu_index >= s->num_cpu) {
         hw_error("arm_mptimer: num-cpu %d but this cpu is %d!\n",
-                 s->num_cpu, cpu_single_env->cpu_index);
+                 s->num_cpu, cpu_single_cpu->cpu_index);
     }
-    return cpu_single_env->cpu_index;
+    return cpu_single_cpu->cpu_index;
 }
 
 static inline void timerblock_update_irq(timerblock *tb)
@@ -92,7 +94,7 @@ static void timerblock_tick(void *opaque)
     timerblock_update_irq(tb);
 }
 
-static uint64_t timerblock_read(void *opaque, target_phys_addr_t addr,
+static uint64_t timerblock_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     timerblock *tb = (timerblock *)opaque;
@@ -120,7 +122,7 @@ static uint64_t timerblock_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void timerblock_write(void *opaque, target_phys_addr_t addr,
+static void timerblock_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     timerblock *tb = (timerblock *)opaque;
@@ -159,7 +161,7 @@ static void timerblock_write(void *opaque, target_phys_addr_t addr,
 /* Wrapper functions to implement the "read timer/watchdog for
  * the current CPU" memory regions.
  */
-static uint64_t arm_thistimer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t arm_thistimer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -167,7 +169,7 @@ static uint64_t arm_thistimer_read(void *opaque, target_phys_addr_t addr,
     return timerblock_read(&s->timerblock[id * 2], addr, size);
 }
 
-static void arm_thistimer_write(void *opaque, target_phys_addr_t addr,
+static void arm_thistimer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -175,7 +177,7 @@ static void arm_thistimer_write(void *opaque, target_phys_addr_t addr,
     timerblock_write(&s->timerblock[id * 2], addr, value, size);
 }
 
-static uint64_t arm_thiswdog_read(void *opaque, target_phys_addr_t addr,
+static uint64_t arm_thiswdog_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -183,7 +185,7 @@ static uint64_t arm_thiswdog_read(void *opaque, target_phys_addr_t addr,
     return timerblock_read(&s->timerblock[id * 2 + 1], addr, size);
 }
 
-static void arm_thiswdog_write(void *opaque, target_phys_addr_t addr,
+static void arm_thiswdog_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -236,7 +238,7 @@ static void timerblock_reset(timerblock *tb)
 static void arm_mptimer_reset(DeviceState *dev)
 {
     arm_mptimer_state *s =
-        FROM_SYSBUS(arm_mptimer_state, sysbus_from_qdev(dev));
+        FROM_SYSBUS(arm_mptimer_state, SYS_BUS_DEVICE(dev));
     int i;
     /* We reset every timer in the array, not just the ones we're using,
      * because vmsave will look at every array element.
@@ -329,7 +331,7 @@ static void arm_mptimer_class_init(ObjectClass *klass, void *data)
     dc->props = arm_mptimer_properties;
 }
 
-static TypeInfo arm_mptimer_info = {
+static const TypeInfo arm_mptimer_info = {
     .name          = "arm_mptimer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(arm_mptimer_state),
diff --git a/hw/arm_sysctl.c b/hw/arm_sysctl.c
index 5f1237b8c25..da36f8a4357 100644
--- a/hw/arm_sysctl.c
+++ b/hw/arm_sysctl.c
@@ -8,10 +8,10 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
 #include "primecell.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define LOCK_VALUE 0xa05f
 
@@ -75,7 +75,7 @@ static int board_id(arm_sysctl_state *s)
 
 static void arm_sysctl_reset(DeviceState *d)
 {
-    arm_sysctl_state *s = FROM_SYSBUS(arm_sysctl_state, sysbus_from_qdev(d));
+    arm_sysctl_state *s = FROM_SYSBUS(arm_sysctl_state, SYS_BUS_DEVICE(d));
 
     s->leds = 0;
     s->lockval = 0;
@@ -92,7 +92,7 @@ static void arm_sysctl_reset(DeviceState *d)
     }
 }
 
-static uint64_t arm_sysctl_read(void *opaque, target_phys_addr_t offset,
+static uint64_t arm_sysctl_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     arm_sysctl_state *s = (arm_sysctl_state *)opaque;
@@ -184,12 +184,14 @@ static uint64_t arm_sysctl_read(void *opaque, target_phys_addr_t offset,
         return s->sys_cfgstat;
     default:
     bad_reg:
-        printf ("arm_sysctl_read: Bad register offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "arm_sysctl_read: Bad register offset 0x%x\n",
+                      (int)offset);
         return 0;
     }
 }
 
-static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
+static void arm_sysctl_write(void *opaque, hwaddr offset,
                              uint64_t val, unsigned size)
 {
     arm_sysctl_state *s = (arm_sysctl_state *)opaque;
@@ -197,6 +199,7 @@ static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
     switch (offset) {
     case 0x08: /* LED */
         s->leds = val;
+        break;
     case 0x0c: /* OSC0 */
     case 0x10: /* OSC1 */
     case 0x14: /* OSC2 */
@@ -293,6 +296,7 @@ static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
             /* On VExpress this register is unimplemented and will RAZ/WI */
             break;
         }
+        break;
     case 0x54: /* CLCDSER */
     case 0x64: /* DMAPSR0 */
     case 0x68: /* DMAPSR1 */
@@ -339,7 +343,9 @@ static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
         return;
     default:
     bad_reg:
-        printf ("arm_sysctl_write: Bad register offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "arm_sysctl_write: Bad register offset 0x%x\n",
+                      (int)offset);
         return;
     }
 }
@@ -406,7 +412,7 @@ static void arm_sysctl_class_init(ObjectClass *klass, void *data)
     dc->props = arm_sysctl_properties;
 }
 
-static TypeInfo arm_sysctl_info = {
+static const TypeInfo arm_sysctl_info = {
     .name          = "realview_sysctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(arm_sysctl_state),
diff --git a/hw/arm_timer.c b/hw/arm_timer.c
index e3ecce29f0f..c1e56be74e5 100644
--- a/hw/arm_timer.c
+++ b/hw/arm_timer.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "qdev.h"
 #include "ptimer.h"
@@ -45,7 +45,7 @@ static void arm_timer_update(arm_timer_state *s)
     }
 }
 
-static uint32_t arm_timer_read(void *opaque, target_phys_addr_t offset)
+static uint32_t arm_timer_read(void *opaque, hwaddr offset)
 {
     arm_timer_state *s = (arm_timer_state *)opaque;
 
@@ -64,7 +64,8 @@ static uint32_t arm_timer_read(void *opaque, target_phys_addr_t offset)
             return 0;
         return s->int_level;
     default:
-        hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset %x\n", __func__, (int)offset);
         return 0;
     }
 }
@@ -87,7 +88,7 @@ static void arm_timer_recalibrate(arm_timer_state *s, int reload)
     ptimer_set_limit(s->timer, limit, reload);
 }
 
-static void arm_timer_write(void *opaque, target_phys_addr_t offset,
+static void arm_timer_write(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     arm_timer_state *s = (arm_timer_state *)opaque;
@@ -131,7 +132,8 @@ static void arm_timer_write(void *opaque, target_phys_addr_t offset,
         arm_timer_recalibrate(s, 0);
         break;
     default:
-        hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset %x\n", __func__, (int)offset);
     }
     arm_timer_update(s);
 }
@@ -202,7 +204,7 @@ static void sp804_set_irq(void *opaque, int irq, int level)
     qemu_set_irq(s->irq, s->level[0] || s->level[1]);
 }
 
-static uint64_t sp804_read(void *opaque, target_phys_addr_t offset,
+static uint64_t sp804_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     sp804_state *s = (sp804_state *)opaque;
@@ -223,14 +225,18 @@ static uint64_t sp804_read(void *opaque, target_phys_addr_t offset,
     /* Integration Test control registers, which we won't support */
     case 0xf00: /* TimerITCR */
     case 0xf04: /* TimerITOP (strictly write only but..) */
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: integration test registers unimplemented\n",
+                      __func__);
         return 0;
     }
 
-    hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "%s: Bad offset %x\n", __func__, (int)offset);
     return 0;
 }
 
-static void sp804_write(void *opaque, target_phys_addr_t offset,
+static void sp804_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     sp804_state *s = (sp804_state *)opaque;
@@ -246,7 +252,8 @@ static void sp804_write(void *opaque, target_phys_addr_t offset,
     }
 
     /* Technically we could be writing to the Test Registers, but not likely */
-    hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %x\n",
+                  __func__, (int)offset);
 }
 
 static const MemoryRegionOps sp804_ops = {
@@ -291,7 +298,7 @@ typedef struct {
     arm_timer_state *timer[3];
 } icp_pit_state;
 
-static uint64_t icp_pit_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_pit_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     icp_pit_state *s = (icp_pit_state *)opaque;
@@ -300,13 +307,13 @@ static uint64_t icp_pit_read(void *opaque, target_phys_addr_t offset,
     /* ??? Don't know the PrimeCell ID for this device.  */
     n = offset >> 8;
     if (n > 2) {
-        hw_error("%s: Bad timer %d\n", __func__, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad timer %d\n", __func__, n);
     }
 
     return arm_timer_read(s->timer[n], offset & 0xff);
 }
 
-static void icp_pit_write(void *opaque, target_phys_addr_t offset,
+static void icp_pit_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     icp_pit_state *s = (icp_pit_state *)opaque;
@@ -314,7 +321,7 @@ static void icp_pit_write(void *opaque, target_phys_addr_t offset,
 
     n = offset >> 8;
     if (n > 2) {
-        hw_error("%s: Bad timer %d\n", __func__, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad timer %d\n", __func__, n);
     }
 
     arm_timer_write(s->timer[n], offset & 0xff, value);
@@ -354,7 +361,7 @@ static void icp_pit_class_init(ObjectClass *klass, void *data)
     sdc->init = icp_pit_init;
 }
 
-static TypeInfo icp_pit_info = {
+static const TypeInfo icp_pit_info = {
     .name          = "integrator_pit",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pit_state),
@@ -376,7 +383,7 @@ static void sp804_class_init(ObjectClass *klass, void *data)
     k->props = sp804_properties;
 }
 
-static TypeInfo sp804_info = {
+static const TypeInfo sp804_info = {
     .name          = "sp804",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(sp804_state),
diff --git a/hw/armv7m.c b/hw/armv7m.c
index 9f66667c6d4..904696ca7f2 100644
--- a/hw/armv7m.c
+++ b/hw/armv7m.c
@@ -25,14 +25,14 @@ static inline uint32_t bitband_addr(void * opaque, uint32_t addr)
 
 }
 
-static uint32_t bitband_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readb(void *opaque, hwaddr offset)
 {
     uint8_t v;
     cpu_physical_memory_read(bitband_addr(opaque, offset), &v, 1);
     return (v & (1 << ((offset >> 2) & 7))) != 0;
 }
 
-static void bitband_writeb(void *opaque, target_phys_addr_t offset,
+static void bitband_writeb(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
@@ -48,7 +48,7 @@ static void bitband_writeb(void *opaque, target_phys_addr_t offset,
     cpu_physical_memory_write(addr, &v, 1);
 }
 
-static uint32_t bitband_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readw(void *opaque, hwaddr offset)
 {
     uint32_t addr;
     uint16_t mask;
@@ -60,7 +60,7 @@ static uint32_t bitband_readw(void *opaque, target_phys_addr_t offset)
     return (v & mask) != 0;
 }
 
-static void bitband_writew(void *opaque, target_phys_addr_t offset,
+static void bitband_writew(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
@@ -77,7 +77,7 @@ static void bitband_writew(void *opaque, target_phys_addr_t offset,
     cpu_physical_memory_write(addr, (uint8_t *)&v, 2);
 }
 
-static uint32_t bitband_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readl(void *opaque, hwaddr offset)
 {
     uint32_t addr;
     uint32_t mask;
@@ -89,7 +89,7 @@ static uint32_t bitband_readl(void *opaque, target_phys_addr_t offset)
     return (v & mask) != 0;
 }
 
-static void bitband_writel(void *opaque, target_phys_addr_t offset,
+static void bitband_writel(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
@@ -137,12 +137,12 @@ static void armv7m_bitband_init(void)
     dev = qdev_create(NULL, "ARM,bitband-memory");
     qdev_prop_set_uint32(dev, "base", 0x20000000);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0x22000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0x22000000);
 
     dev = qdev_create(NULL, "ARM,bitband-memory");
     qdev_prop_set_uint32(dev, "base", 0x40000000);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0x42000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0x42000000);
 }
 
 /* Board init.  */
@@ -216,7 +216,7 @@ qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
     env->nvic = nvic;
     qdev_init_nofail(nvic);
     cpu_pic = arm_pic_init_cpu(cpu);
-    sysbus_connect_irq(sysbus_from_qdev(nvic), 0, cpu_pic[ARM_PIC_CPU_IRQ]);
+    sysbus_connect_irq(SYS_BUS_DEVICE(nvic), 0, cpu_pic[ARM_PIC_CPU_IRQ]);
     for (i = 0; i < 64; i++) {
         pic[i] = qdev_get_gpio_in(nvic, i);
     }
@@ -269,7 +269,7 @@ static void bitband_class_init(ObjectClass *klass, void *data)
     dc->props = bitband_properties;
 }
 
-static TypeInfo bitband_info = {
+static const TypeInfo bitband_info = {
     .name          = "ARM,bitband-memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(BitBandState),
diff --git a/hw/armv7m_nvic.c b/hw/armv7m_nvic.c
index 6a0832eb3fd..d5798d0309c 100644
--- a/hw/armv7m_nvic.c
+++ b/hw/armv7m_nvic.c
@@ -11,13 +11,13 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "arm-misc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "arm_gic_internal.h"
 
 typedef struct {
-    gic_state gic;
+    GICState gic;
     struct {
         uint32_t control;
         uint32_t reload;
@@ -138,9 +138,8 @@ void armv7m_nvic_complete_irq(void *opaque, int irq)
     gic_complete_irq(&s->gic, 0, irq);
 }
 
-static uint32_t nvic_readl(void *opaque, uint32_t offset)
+static uint32_t nvic_readl(nvic_state *s, uint32_t offset)
 {
-    nvic_state *s = (nvic_state *)opaque;
     uint32_t val;
     int irq;
 
@@ -216,14 +215,6 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
     case 0xd14: /* Configuration Control.  */
         /* TODO: Implement Configuration Control bits.  */
         return 0;
-    case 0xd18: case 0xd1c: case 0xd20: /* System Handler Priority.  */
-        irq = offset - 0xd14;
-        val = 0;
-        val |= s->gic.priority1[irq++][0];
-        val |= s->gic.priority1[irq++][0] << 8;
-        val |= s->gic.priority1[irq++][0] << 16;
-        val |= s->gic.priority1[irq][0] << 24;
-        return val;
     case 0xd24: /* System Handler Status.  */
         val = 0;
         if (s->gic.irq_state[ARMV7M_EXCP_MEM].active) val |= (1 << 0);
@@ -243,7 +234,7 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
         return val;
     case 0xd28: /* Configurable Fault Status.  */
         /* TODO: Implement Fault Status.  */
-        hw_error("Not implemented: Configurable Fault Status.");
+        qemu_log_mask(LOG_UNIMP, "Configurable Fault Status unimplemented\n");
         return 0;
     case 0xd2c: /* Hard Fault Status.  */
     case 0xd30: /* Debug Fault Status.  */
@@ -251,7 +242,8 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
     case 0xd38: /* Bus Fault Address.  */
     case 0xd3c: /* Aux Fault Status.  */
         /* TODO: Implement fault status registers.  */
-        goto bad_reg;
+        qemu_log_mask(LOG_UNIMP, "Fault status registers unimplemented\n");
+        return 0;
     case 0xd40: /* PFR0.  */
         return 0x00000030;
     case 0xd44: /* PRF1.  */
@@ -280,14 +272,13 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
         return 0x01310102;
     /* TODO: Implement debug registers.  */
     default:
-    bad_reg:
-        hw_error("NVIC: Bad read offset 0x%x\n", offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "NVIC: Bad read offset 0x%x\n", offset);
+        return 0;
     }
 }
 
-static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
+static void nvic_writel(nvic_state *s, uint32_t offset, uint32_t value)
 {
-    nvic_state *s = (nvic_state *)opaque;
     uint32_t oldval;
     switch (offset) {
     case 0x10: /* SysTick Control and Status.  */
@@ -345,27 +336,17 @@ static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         if ((value >> 16) == 0x05fa) {
             if (value & 2) {
-                hw_error("VECTCLRACTIVE not implemented");
+                qemu_log_mask(LOG_UNIMP, "VECTCLRACTIVE unimplemented\n");
             }
             if (value & 5) {
-                hw_error("System reset");
+                qemu_log_mask(LOG_UNIMP, "AIRCR system reset unimplemented\n");
             }
         }
         break;
     case 0xd10: /* System Control.  */
     case 0xd14: /* Configuration Control.  */
         /* TODO: Implement control registers.  */
-        goto bad_reg;
-    case 0xd18: case 0xd1c: case 0xd20: /* System Handler Priority.  */
-        {
-            int irq;
-            irq = offset - 0xd14;
-            s->gic.priority1[irq++][0] = value & 0xff;
-            s->gic.priority1[irq++][0] = (value >> 8) & 0xff;
-            s->gic.priority1[irq++][0] = (value >> 16) & 0xff;
-            s->gic.priority1[irq][0] = (value >> 24) & 0xff;
-            gic_update(&s->gic);
-        }
+        qemu_log_mask(LOG_UNIMP, "NVIC: SCR and CCR unimplemented\n");
         break;
     case 0xd24: /* System Handler Control.  */
         /* TODO: Real hardware allows you to set/clear the active bits
@@ -380,47 +361,71 @@ static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
     case 0xd34: /* Mem Manage Address.  */
     case 0xd38: /* Bus Fault Address.  */
     case 0xd3c: /* Aux Fault Status.  */
-        goto bad_reg;
+        qemu_log_mask(LOG_UNIMP,
+                      "NVIC: fault status registers unimplemented\n");
+        break;
     case 0xf00: /* Software Triggered Interrupt Register */
         if ((value & 0x1ff) < s->num_irq) {
             gic_set_pending_private(&s->gic, 0, value & 0x1ff);
         }
         break;
     default:
-    bad_reg:
-        hw_error("NVIC: Bad write offset 0x%x\n", offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "NVIC: Bad write offset 0x%x\n", offset);
     }
 }
 
-static uint64_t nvic_sysreg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t nvic_sysreg_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
-    /* At the moment we only support the ID registers for byte/word access.
-     * This is not strictly correct as a few of the other registers also
-     * allow byte access.
-     */
+    nvic_state *s = (nvic_state *)opaque;
     uint32_t offset = addr;
-    if (offset >= 0xfe0) {
+    int i;
+    uint32_t val;
+
+    switch (offset) {
+    case 0xd18 ... 0xd23: /* System Handler Priority.  */
+        val = 0;
+        for (i = 0; i < size; i++) {
+            val |= s->gic.priority1[(offset - 0xd14) + i][0] << (i * 8);
+        }
+        return val;
+    case 0xfe0 ... 0xfff: /* ID.  */
         if (offset & 3) {
             return 0;
         }
         return nvic_id[(offset - 0xfe0) >> 2];
     }
     if (size == 4) {
-        return nvic_readl(opaque, offset);
+        return nvic_readl(s, offset);
     }
-    hw_error("NVIC: Bad read of size %d at offset 0x%x\n", size, offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "NVIC: Bad read of size %d at offset 0x%x\n", size, offset);
+    return 0;
 }
 
-static void nvic_sysreg_write(void *opaque, target_phys_addr_t addr,
+static void nvic_sysreg_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
+    nvic_state *s = (nvic_state *)opaque;
     uint32_t offset = addr;
+    int i;
+
+    switch (offset) {
+    case 0xd18 ... 0xd23: /* System Handler Priority.  */
+        for (i = 0; i < size; i++) {
+            s->gic.priority1[(offset - 0xd14) + i][0] =
+                (value >> (i * 8)) & 0xff;
+        }
+        gic_update(&s->gic);
+        return;
+    }
     if (size == 4) {
-        nvic_writel(opaque, offset, value);
+        nvic_writel(s, offset, value);
         return;
     }
-    hw_error("NVIC: Bad write of size %d at offset 0x%x\n", size, offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "NVIC: Bad write of size %d at offset 0x%x\n", size, offset);
 }
 
 static const MemoryRegionOps nvic_sysreg_ops = {
@@ -450,9 +455,11 @@ static void armv7m_nvic_reset(DeviceState *dev)
     nc->parent_reset(dev);
     /* Common GIC reset resets to disabled; the NVIC doesn't have
      * per-CPU interfaces so mark our non-existent CPU interface
-     * as enabled by default.
+     * as enabled by default, and with a priority mask which allows
+     * all interrupts through.
      */
     s->gic.cpu_enabled[0] = 1;
+    s->gic.priority_mask[0] = 0x100;
     /* The NVIC as a whole is always enabled. */
     s->gic.enabled = 1;
     systick_reset(s);
@@ -489,7 +496,8 @@ static int armv7m_nvic_init(SysBusDevice *dev)
      */
     memory_region_init_alias(&s->gic_iomem_alias, "nvic-gic", &s->gic.iomem,
                              0x100, 0xc00);
-    memory_region_add_subregion_overlap(&s->container, 0x100, &s->gic.iomem, 1);
+    memory_region_add_subregion_overlap(&s->container, 0x100,
+                                        &s->gic_iomem_alias, 1);
     /* Map the whole thing into system memory at the location required
      * by the v7M architecture.
      */
@@ -504,9 +512,9 @@ static void armv7m_nvic_instance_init(Object *obj)
      * than our superclass. This function runs after qdev init
      * has set the defaults from the Property array and before
      * any user-specified property setting, so just modify the
-     * value in the gic_state struct.
+     * value in the GICState struct.
      */
-    gic_state *s = ARM_GIC_COMMON(obj);
+    GICState *s = ARM_GIC_COMMON(obj);
     /* The ARM v7m may have anything from 0 to 496 external interrupt
      * IRQ lines. We default to 64. Other boards may differ and should
      * set the num-irq property appropriately.
@@ -527,7 +535,7 @@ static void armv7m_nvic_class_init(ObjectClass *klass, void *data)
     dc->reset = armv7m_nvic_reset;
 }
 
-static TypeInfo armv7m_nvic_info = {
+static const TypeInfo armv7m_nvic_info = {
     .name          = TYPE_NVIC,
     .parent        = TYPE_ARM_GIC_COMMON,
     .instance_init = armv7m_nvic_instance_init,
diff --git a/hw/audiodev.h b/hw/audiodev.h
index ed2790f5f60..428274f9291 100644
--- a/hw/audiodev.h
+++ b/hw/audiodev.h
@@ -1,3 +1,6 @@
+#ifndef HW_AUDIODEV_H
+#define HW_AUDIODEV_H 1
+
 /* es1370.c */
 int es1370_init(PCIBus *bus);
 
@@ -18,3 +21,5 @@ int cs4231a_init(ISABus *bus);
 
 /* intel-hda.c + hda-audio.c */
 int intel_hda_and_codec_init(PCIBus *bus);
+
+#endif
diff --git a/hw/axis_dev88.c b/hw/axis_dev88.c
index eab6327bedb..dd37fa13e22 100644
--- a/hw/axis_dev88.c
+++ b/hw/axis_dev88.c
@@ -23,15 +23,15 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
 #include "boards.h"
 #include "etraxfs.h"
 #include "loader.h"
 #include "elf.h"
 #include "cris-boot.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define D(x)
 #define DNAND(x)
@@ -47,7 +47,7 @@ struct nand_state_t
 };
 
 static struct nand_state_t nand_state;
-static uint64_t nand_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t nand_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct nand_state_t *s = opaque;
     uint32_t r;
@@ -62,7 +62,7 @@ static uint64_t nand_read(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-nand_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+nand_write(void *opaque, hwaddr addr, uint64_t value,
            unsigned size)
 {
     struct nand_state_t *s = opaque;
@@ -166,7 +166,7 @@ static struct gpio_state_t
     uint32_t regs[0x5c / 4];
 } gpio_state;
 
-static uint64_t gpio_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t gpio_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct gpio_state_t *s = opaque;
     uint32_t r = 0;
@@ -195,7 +195,7 @@ static uint64_t gpio_read(void *opaque, target_phys_addr_t addr, unsigned size)
     D(printf("%s %x=%x\n", __func__, addr, r));
 }
 
-static void gpio_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void gpio_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     struct gpio_state_t *s = opaque;
@@ -242,11 +242,12 @@ static const MemoryRegionOps gpio_ops = {
 static struct cris_load_info li;
 
 static
-void axisdev88_init (ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+void axisdev88_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     CRISCPU *cpu;
     CPUCRISState *env;
     DeviceState *dev;
@@ -299,7 +300,7 @@ void axisdev88_init (ram_addr_t ram_size,
     /* FIXME: Is there a proper way to signal vectors to the CPU core?  */
     qdev_prop_set_ptr(dev, "interrupt_vector", &env->interrupt_vector);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, 0x3001c000);
     sysbus_connect_irq(s, 0, cpu_irq[0]);
     sysbus_connect_irq(s, 1, cpu_irq[1]);
@@ -354,6 +355,7 @@ static QEMUMachine axisdev88_machine = {
     .desc = "AXIS devboard 88",
     .init = axisdev88_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void axisdev88_machine_init(void)
diff --git a/hw/baum.c b/hw/baum.c
index 3e94f84e513..09dcb9cc741 100644
--- a/hw/baum.c
+++ b/hw/baum.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 #include "usb.h"
 #include "baum.h"
 #include <brlapi.h>
diff --git a/hw/baum.h b/hw/baum.h
index 8af710fa210..763588422ab 100644
--- a/hw/baum.h
+++ b/hw/baum.h
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_BAUM_H
+#define HW_BAUM_H 1
 
 /* char device */
 CharDriverState *chr_baum_init(QemuOpts *opts);
+
+#endif
diff --git a/hw/bitbang_i2c.c b/hw/bitbang_i2c.c
index 44ed7f4d618..114508fade9 100644
--- a/hw/bitbang_i2c.c
+++ b/hw/bitbang_i2c.c
@@ -230,7 +230,7 @@ static void gpio_i2c_class_init(ObjectClass *klass, void *data)
     dc->desc = "Virtual GPIO to I2C bridge";
 }
 
-static TypeInfo gpio_i2c_info = {
+static const TypeInfo gpio_i2c_info = {
     .name          = "gpio_i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GPIOI2CState),
diff --git a/hw/blizzard.c b/hw/blizzard.c
index 29074c4fa7e..24bde32e5af 100644
--- a/hw/blizzard.c
+++ b/hw/blizzard.c
@@ -19,10 +19,10 @@
  */
 
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 #include "devices.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
+#include "ui/pixel_ops.h"
 
 typedef void (*blizzard_fn_t)(uint8_t *, const uint8_t *, unsigned int);
 
@@ -878,8 +878,6 @@ void s1d13745_write_block(void *opaque, int dc,
         len -= 2;
         buf += 2;
     }
-
-    return;
 }
 
 static void blizzard_update_display(void *opaque)
@@ -923,8 +921,8 @@ static void blizzard_update_display(void *opaque)
     for (; y < s->my[1]; y ++, src += bypl, dst += bypl)
         memcpy(dst, src, bwidth);
 
-    dpy_update(s->state, s->mx[0], s->my[0],
-                    s->mx[1] - s->mx[0], y - s->my[0]);
+    dpy_gfx_update(s->state, s->mx[0], s->my[0],
+                   s->mx[1] - s->mx[0], y - s->my[0]);
 
     s->mx[0] = s->x;
     s->mx[1] = 0;
@@ -933,13 +931,13 @@ static void blizzard_update_display(void *opaque)
 }
 
 static void blizzard_screen_dump(void *opaque, const char *filename,
-                                 bool cswitch)
+                                 bool cswitch, Error **errp)
 {
     BlizzardState *s = (BlizzardState *) opaque;
 
     blizzard_update_display(opaque);
     if (s && ds_get_data(s->state))
-        ppm_save(filename, s->state->surface);
+        ppm_save(filename, s->state->surface, errp);
 }
 
 #define DEPTH 8
diff --git a/hw/block-common.c b/hw/block-common.c
index f0196d78dc4..0f1b64ec95a 100644
--- a/hw/block-common.c
+++ b/hw/block-common.c
@@ -7,9 +7,9 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 void blkconf_serial(BlockConf *conf, char **serial)
 {
diff --git a/hw/boards.h b/hw/boards.h
index a2e0a544973..3ff9665b1f0 100644
--- a/hw/boards.h
+++ b/hw/boards.h
@@ -3,14 +3,22 @@
 #ifndef HW_BOARDS_H
 #define HW_BOARDS_H
 
+#include "sysemu/blockdev.h"
 #include "qdev.h"
 
-typedef void QEMUMachineInitFunc(ram_addr_t ram_size,
-                                 const char *boot_device,
-                                 const char *kernel_filename,
-                                 const char *kernel_cmdline,
-                                 const char *initrd_filename,
-                                 const char *cpu_model);
+#define DEFAULT_MACHINE_OPTIONS \
+    .boot_order = "cad"
+
+typedef struct QEMUMachineInitArgs {
+    ram_addr_t ram_size;
+    const char *boot_device;
+    const char *kernel_filename;
+    const char *kernel_cmdline;
+    const char *initrd_filename;
+    const char *cpu_model;
+} QEMUMachineInitArgs;
+
+typedef void QEMUMachineInitFunc(QEMUMachineInitArgs *args);
 
 typedef void QEMUMachineResetFunc(void);
 
@@ -20,7 +28,7 @@ typedef struct QEMUMachine {
     const char *desc;
     QEMUMachineInitFunc *init;
     QEMUMachineResetFunc *reset;
-    int use_scsi;
+    BlockInterfaceType block_default_type;
     int max_cpus;
     unsigned int no_serial:1,
         no_parallel:1,
@@ -30,6 +38,7 @@ typedef struct QEMUMachine {
         no_sdcard:1;
     int is_default;
     const char *default_machine_opts;
+    const char *boot_order;
     GlobalProperty *compat_props;
     struct QEMUMachine *next;
     const char *hw_version;
diff --git a/hw/bonito.c b/hw/bonito.c
index 6084ac48fa9..0498c9be798 100644
--- a/hw/bonito.c
+++ b/hw/bonito.c
@@ -40,12 +40,12 @@
 #include <assert.h>
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "mips.h"
-#include "pci_host.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "pci/pci_host.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_BONITO
 
@@ -211,12 +211,12 @@ typedef struct PCIBonitoState
     MemoryRegion iomem_ldma;
     MemoryRegion iomem_cop;
 
-    target_phys_addr_t bonito_pciio_start;
-    target_phys_addr_t bonito_pciio_length;
+    hwaddr bonito_pciio_start;
+    hwaddr bonito_pciio_length;
     int bonito_pciio_handle;
 
-    target_phys_addr_t bonito_localio_start;
-    target_phys_addr_t bonito_localio_length;
+    hwaddr bonito_localio_start;
+    hwaddr bonito_localio_length;
     int bonito_localio_handle;
 
 } PCIBonitoState;
@@ -232,7 +232,7 @@ struct BonitoState {
     PCIBonitoState *pci_dev;
 };
 
-static void bonito_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_writel(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -295,7 +295,7 @@ static void bonito_writel(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t bonito_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_readl(void *opaque, hwaddr addr,
                              unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -322,7 +322,7 @@ static const MemoryRegionOps bonito_ops = {
     },
 };
 
-static void bonito_pciconf_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_pciconf_writel(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -332,7 +332,7 @@ static void bonito_pciconf_writel(void *opaque, target_phys_addr_t addr,
     d->config_write(d, addr, val, 4);
 }
 
-static uint64_t bonito_pciconf_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_pciconf_readl(void *opaque, hwaddr addr,
                                      unsigned size)
 {
 
@@ -355,7 +355,7 @@ static const MemoryRegionOps bonito_pciconf_ops = {
     },
 };
 
-static uint64_t bonito_ldma_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_ldma_readl(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     uint32_t val;
@@ -366,7 +366,7 @@ static uint64_t bonito_ldma_readl(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bonito_ldma_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_ldma_writel(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -384,7 +384,7 @@ static const MemoryRegionOps bonito_ldma_ops = {
     },
 };
 
-static uint64_t bonito_cop_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_cop_readl(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     uint32_t val;
@@ -395,7 +395,7 @@ static uint64_t bonito_cop_readl(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bonito_cop_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_cop_writel(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -413,7 +413,7 @@ static const MemoryRegionOps bonito_cop_ops = {
     },
 };
 
-static uint32_t bonito_sbridge_pciaddr(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_sbridge_pciaddr(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s->pcihost);
@@ -449,7 +449,7 @@ static uint32_t bonito_sbridge_pciaddr(void *opaque, target_phys_addr_t addr)
     return pciaddr;
 }
 
-static void bonito_spciconf_writeb(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writeb(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -475,7 +475,7 @@ static void bonito_spciconf_writeb(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static void bonito_spciconf_writew(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writew(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -503,7 +503,7 @@ static void bonito_spciconf_writew(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static void bonito_spciconf_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writel(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -531,7 +531,7 @@ static void bonito_spciconf_writel(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static uint32_t bonito_spciconf_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readb(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
@@ -557,7 +557,7 @@ static uint32_t bonito_spciconf_readb(void *opaque, target_phys_addr_t addr)
     return pci_data_read(phb->bus, phb->config_reg, 1);
 }
 
-static uint32_t bonito_spciconf_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readw(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
@@ -585,7 +585,7 @@ static uint32_t bonito_spciconf_readw(void *opaque, target_phys_addr_t addr)
     return pci_data_read(phb->bus, phb->config_reg, 2);
 }
 
-static uint32_t bonito_spciconf_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readl(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
diff --git a/hw/bt-hci-csr.c b/hw/bt-hci-csr.c
index 772b677ba17..2070bb940c7 100644
--- a/hw/bt-hci-csr.c
+++ b/hw/bt-hci-csr.c
@@ -19,10 +19,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 #include "irq.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 struct csrhci_s {
diff --git a/hw/bt-hci.c b/hw/bt-hci.c
index a3a7fb49e2e..69d2c73862c 100644
--- a/hw/bt-hci.c
+++ b/hw/bt-hci.c
@@ -19,9 +19,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "usb.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 struct bt_hci_s {
@@ -786,7 +786,6 @@ static void bt_hci_lmp_connection_request(struct bt_link_s *link)
     memcpy(&params.dev_class, &link->host->class, sizeof(params.dev_class));
     params.link_type	= ACL_LINK;
     bt_hci_event(hci, EVT_CONN_REQUEST, &params, EVT_CONN_REQUEST_SIZE);
-    return;
 }
 
 static void bt_hci_conn_accept_timeout(void *opaque)
@@ -943,7 +942,6 @@ static int bt_hci_name_req(struct bt_hci_s *hci, bdaddr_t *bdaddr)
 {
     struct bt_device_s *slave;
     evt_remote_name_req_complete params;
-    int len;
 
     for (slave = hci->device.net->slave; slave; slave = slave->next)
         if (slave->page_scan && !bacmp(&slave->bd_addr, bdaddr))
@@ -955,9 +953,7 @@ static int bt_hci_name_req(struct bt_hci_s *hci, bdaddr_t *bdaddr)
 
     params.status       = HCI_SUCCESS;
     bacpy(&params.bdaddr, &slave->bd_addr);
-    len = snprintf(params.name, sizeof(params.name),
-                    "%s", slave->lmp_name ?: "");
-    memset(params.name + len, 0, sizeof(params.name) - len);
+    pstrcpy(params.name, sizeof(params.name), slave->lmp_name ?: "");
     bt_hci_event(hci, EVT_REMOTE_NAME_REQ_COMPLETE,
                     &params, EVT_REMOTE_NAME_REQ_COMPLETE_SIZE);
 
@@ -1388,7 +1384,7 @@ static inline void bt_hci_event_complete_read_local_name(struct bt_hci_s *hci)
     params.status = HCI_SUCCESS;
     memset(params.name, 0, sizeof(params.name));
     if (hci->device.lmp_name)
-        strncpy(params.name, hci->device.lmp_name, sizeof(params.name));
+        pstrcpy(params.name, sizeof(params.name), hci->device.lmp_name);
 
     bt_hci_event_complete(hci, &params, READ_LOCAL_NAME_RP_SIZE);
 }
diff --git a/hw/bt-hid.c b/hw/bt-hid.c
index 8d7a3dae219..cfa7c145b88 100644
--- a/hw/bt-hid.c
+++ b/hw/bt-hid.c
@@ -19,8 +19,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "hid.h"
 #include "bt.h"
 
diff --git a/hw/bt-l2cap.c b/hw/bt-l2cap.c
index cb43ee77334..ba061c0da32 100644
--- a/hw/bt-l2cap.c
+++ b/hw/bt-l2cap.c
@@ -18,7 +18,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "bt.h"
 
 #define L2CAP_CID_MAX	0x100	/* Between 0x40 and 0x10000 */
diff --git a/hw/bt.c b/hw/bt.c
index dc99fc28fab..4f2372d7943 100644
--- a/hw/bt.c
+++ b/hw/bt.c
@@ -18,7 +18,7 @@
  */
 
 #include "qemu-common.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 /* Slave implementations can ignore this */
diff --git a/hw/bt.h b/hw/bt.h
index a48b8d4b133..830af947355 100644
--- a/hw/bt.h
+++ b/hw/bt.h
@@ -23,6 +23,11 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifndef HW_BT_H
+#define HW_BT_H 1
+
+#include "hw/irq.h"
+
 /* BD Address */
 typedef struct {
     uint8_t b[6];
@@ -2181,3 +2186,5 @@ enum bt_sdp_attribute_id {
     SDP_ATTR_NORMALLY_CONNECTABLE		= 0x020d,
     SDP_ATTR_BOOT_DEVICE			= 0x020e,
 };
+
+#endif
diff --git a/hw/cadence_gem.c b/hw/cadence_gem.c
index 967f62513e7..0d834422df1 100644
--- a/hw/cadence_gem.c
+++ b/hw/cadence_gem.c
@@ -25,7 +25,7 @@
 #include <zlib.h> /* For crc32 */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #ifdef CADENCE_GEM_ERR_DEBUG
@@ -605,7 +605,7 @@ static int gem_mac_address_filter(GemState *s, const uint8_t *packet)
 static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     unsigned    desc[2];
-    target_phys_addr_t packet_desc_addr, last_desc_addr;
+    hwaddr packet_desc_addr, last_desc_addr;
     GemState *s;
     unsigned   rxbufsize, bytes_to_copy;
     unsigned   rxbuf_offset;
@@ -824,7 +824,7 @@ static void gem_transmit_updatestats(GemState *s, const uint8_t *packet,
 static void gem_transmit(GemState *s)
 {
     unsigned    desc[2];
-    target_phys_addr_t packet_desc_addr;
+    hwaddr packet_desc_addr;
     uint8_t     tx_packet[2048];
     uint8_t     *p;
     unsigned    total_bytes;
@@ -959,7 +959,7 @@ static void gem_phy_reset(GemState *s)
 
 static void gem_reset(DeviceState *d)
 {
-    GemState *s = FROM_SYSBUS(GemState, sysbus_from_qdev(d));
+    GemState *s = FROM_SYSBUS(GemState, SYS_BUS_DEVICE(d));
 
     DB_PRINT("\n");
 
@@ -1021,7 +1021,7 @@ static void gem_phy_write(GemState *s, unsigned reg_num, uint16_t val)
  * gem_read32:
  * Read a GEM register.
  */
-static uint64_t gem_read(void *opaque, target_phys_addr_t offset, unsigned size)
+static uint64_t gem_read(void *opaque, hwaddr offset, unsigned size)
 {
     GemState *s;
     uint32_t retval;
@@ -1067,7 +1067,7 @@ static uint64_t gem_read(void *opaque, target_phys_addr_t offset, unsigned size)
  * gem_write32:
  * Write a GEM register.
  */
-static void gem_write(void *opaque, target_phys_addr_t offset, uint64_t val,
+static void gem_write(void *opaque, hwaddr offset, uint64_t val,
         unsigned size)
 {
     GemState *s = (GemState *)opaque;
@@ -1218,7 +1218,7 @@ static void gem_class_init(ObjectClass *klass, void *data)
     dc->reset = gem_reset;
 }
 
-static TypeInfo gem_info = {
+static const TypeInfo gem_info = {
     .class_init = gem_class_init,
     .name  = "cadence_gem",
     .parent = TYPE_SYS_BUS_DEVICE,
diff --git a/hw/cadence_ttc.c b/hw/cadence_ttc.c
index dd02f86eb98..2a8fadd8109 100644
--- a/hw/cadence_ttc.c
+++ b/hw/cadence_ttc.c
@@ -17,7 +17,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #ifdef CADENCE_TTC_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -76,7 +76,7 @@ static void cadence_timer_update(CadenceTimerState *s)
 }
 
 static CadenceTimerState *cadence_timer_from_addr(void *opaque,
-                                        target_phys_addr_t offset)
+                                        hwaddr offset)
 {
     unsigned int index;
     CadenceTTCState *s = (CadenceTTCState *)opaque;
@@ -224,7 +224,7 @@ static void cadence_timer_tick(void *opaque)
     cadence_timer_run(s);
 }
 
-static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
+static uint32_t cadence_ttc_read_imp(void *opaque, hwaddr offset)
 {
     CadenceTimerState *s = cadence_timer_from_addr(opaque, offset);
     uint32_t value;
@@ -274,6 +274,7 @@ static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
         /* cleared after read */
         value = s->reg_intr;
         s->reg_intr = 0;
+        cadence_timer_update(s);
         return value;
 
     case 0x60: /* interrupt enable */
@@ -296,7 +297,7 @@ static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
     }
 }
 
-static uint64_t cadence_ttc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t cadence_ttc_read(void *opaque, hwaddr offset,
     unsigned size)
 {
     uint32_t ret = cadence_ttc_read_imp(opaque, offset);
@@ -305,7 +306,7 @@ static uint64_t cadence_ttc_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void cadence_ttc_write(void *opaque, target_phys_addr_t offset,
+static void cadence_ttc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     CadenceTimerState *s = cadence_timer_from_addr(opaque, offset);
@@ -355,7 +356,6 @@ static void cadence_ttc_write(void *opaque, target_phys_addr_t offset,
     case 0x54: /* interrupt register */
     case 0x58:
     case 0x5c:
-        s->reg_intr &= (~value & 0xfff);
         break;
 
     case 0x60: /* interrupt enable */
@@ -474,7 +474,7 @@ static void cadence_ttc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_cadence_ttc;
 }
 
-static TypeInfo cadence_ttc_info = {
+static const TypeInfo cadence_ttc_info = {
     .name  = "cadence_ttc",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size  = sizeof(CadenceTTCState),
diff --git a/hw/cadence_uart.c b/hw/cadence_uart.c
index d98e5313725..cf2f53c81c9 100644
--- a/hw/cadence_uart.c
+++ b/hw/cadence_uart.c
@@ -17,8 +17,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 
 #ifdef CADENCE_UART_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -354,12 +354,12 @@ static void uart_read_rx_fifo(UartState *s, uint32_t *c)
     uart_update_status(s);
 }
 
-static void uart_write(void *opaque, target_phys_addr_t offset,
+static void uart_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     UartState *s = (UartState *)opaque;
 
-    DB_PRINT(" offset:%x data:%08x\n", offset, (unsigned)value);
+    DB_PRINT(" offset:%x data:%08x\n", (unsigned)offset, (unsigned)value);
     offset >>= 2;
     switch (offset) {
     case R_IER: /* ier (wts imr) */
@@ -397,20 +397,23 @@ static void uart_write(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t offset,
+static uint64_t uart_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     UartState *s = (UartState *)opaque;
     uint32_t c = 0;
 
     offset >>= 2;
-    if (offset > R_MAX) {
-        return 0;
+    if (offset >= R_MAX) {
+        c = 0;
     } else if (offset == R_TX_RX) {
         uart_read_rx_fifo(s, &c);
-        return c;
+    } else {
+       c = s->r[offset];
     }
-    return s->r[offset];
+
+    DB_PRINT(" offset:%x data:%08x\n", (unsigned)(offset << 2), (unsigned)c);
+    return c;
 }
 
 static const MemoryRegionOps uart_ops = {
@@ -498,7 +501,7 @@ static void cadence_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_cadence_uart;
 }
 
-static TypeInfo cadence_uart_info = {
+static const TypeInfo cadence_uart_info = {
     .name          = "cadence_uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(UartState),
diff --git a/hw/cbus.c b/hw/cbus.c
index 7216899a093..6fd39054486 100644
--- a/hw/cbus.c
+++ b/hw/cbus.c
@@ -23,7 +23,7 @@
 #include "qemu-common.h"
 #include "irq.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG
 
diff --git a/hw/ccid-card-emulated.c b/hw/ccid-card-emulated.c
index f4a6da4283a..c8f8ba3792b 100644
--- a/hw/ccid-card-emulated.c
+++ b/hw/ccid-card-emulated.c
@@ -31,9 +31,9 @@
 #include <vreader.h>
 #include <vcard_emul.h>
 
-#include "qemu-thread.h"
-#include "qemu-char.h"
-#include "monitor.h"
+#include "qemu/thread.h"
+#include "char/char.h"
+#include "monitor/monitor.h"
 #include "hw/ccid.h"
 
 #define DPRINTF(card, lvl, fmt, ...) \
@@ -587,7 +587,7 @@ static void emulated_class_initfn(ObjectClass *klass, void *data)
     dc->props = emulated_card_properties;
 }
 
-static TypeInfo emulated_card_info = {
+static const TypeInfo emulated_card_info = {
     .name          = EMULATED_DEV_NAME,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(EmulatedState),
diff --git a/hw/ccid-card-passthru.c b/hw/ccid-card-passthru.c
index bd6c77777da..984bd0bf4c7 100644
--- a/hw/ccid-card-passthru.c
+++ b/hw/ccid-card-passthru.c
@@ -8,9 +8,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu-char.h"
-#include "qemu_socket.h"
-#include "monitor.h"
+#include "char/char.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
 #include "hw/ccid.h"
 #include "libcacard/vscard_common.h"
 
@@ -336,7 +336,7 @@ static void passthru_class_initfn(ObjectClass *klass, void *data)
     dc->props = passthru_card_properties;
 }
 
-static TypeInfo passthru_card_info = {
+static const TypeInfo passthru_card_info = {
     .name          = PASSTHRU_DEV_NAME,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(PassthruState),
diff --git a/hw/cirrus_vga.c b/hw/cirrus_vga.c
index e8dcc6b883a..2a2c8dad628 100644
--- a/hw/cirrus_vga.c
+++ b/hw/cirrus_vga.c
@@ -27,9 +27,8 @@
  *   available at http://home.worldonline.dk/~finth/
  */
 #include "hw.h"
-#include "pci.h"
-#include "vga-pci.h"
-#include "console.h"
+#include "pci/pci.h"
+#include "ui/console.h"
 #include "vga_int.h"
 #include "loader.h"
 
@@ -43,8 +42,6 @@
 //#define DEBUG_CIRRUS
 //#define DEBUG_BITBLT
 
-#define VGA_RAM_SIZE (8192 * 1024)
-
 /***************************************
  *
  *  definitions
@@ -200,6 +197,7 @@ typedef void (*cirrus_fill_t)(struct CirrusVGAState *s,
 typedef struct CirrusVGAState {
     VGACommonState vga;
 
+    MemoryRegion cirrus_vga_io;
     MemoryRegion cirrus_linear_io;
     MemoryRegion cirrus_linear_bitblt_io;
     MemoryRegion cirrus_mmio_io;
@@ -1953,7 +1951,7 @@ static void cirrus_mem_writeb_mode4and5_16bpp(CirrusVGAState * s,
  ***************************************/
 
 static uint64_t cirrus_vga_mem_read(void *opaque,
-                                    target_phys_addr_t addr,
+                                    hwaddr addr,
                                     uint32_t size)
 {
     CirrusVGAState *s = opaque;
@@ -1997,7 +1995,7 @@ static uint64_t cirrus_vga_mem_read(void *opaque,
 }
 
 static void cirrus_vga_mem_write(void *opaque,
-                                 target_phys_addr_t addr,
+                                 hwaddr addr,
                                  uint64_t mem_value,
                                  uint32_t size)
 {
@@ -2256,7 +2254,7 @@ static void cirrus_cursor_draw_line(VGACommonState *s1, uint8_t *d1, int scr_y)
  *
  ***************************************/
 
-static uint64_t cirrus_linear_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cirrus_linear_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2285,7 +2283,7 @@ static uint64_t cirrus_linear_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void cirrus_linear_write(void *opaque, target_phys_addr_t addr,
+static void cirrus_linear_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2334,7 +2332,7 @@ static void cirrus_linear_write(void *opaque, target_phys_addr_t addr,
 
 
 static uint64_t cirrus_linear_bitblt_read(void *opaque,
-                                          target_phys_addr_t addr,
+                                          hwaddr addr,
                                           unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2347,7 +2345,7 @@ static uint64_t cirrus_linear_bitblt_read(void *opaque,
 }
 
 static void cirrus_linear_bitblt_write(void *opaque,
-                                       target_phys_addr_t addr,
+                                       hwaddr addr,
                                        uint64_t val,
                                        unsigned size)
 {
@@ -2435,12 +2433,16 @@ static void cirrus_update_memory_access(CirrusVGAState *s)
 
 /* I/O ports */
 
-static uint32_t cirrus_vga_ioport_read(void *opaque, uint32_t addr)
+static uint64_t cirrus_vga_ioport_read(void *opaque, hwaddr addr,
+                                       unsigned size)
 {
     CirrusVGAState *c = opaque;
     VGACommonState *s = &c->vga;
     int val, index;
 
+    qemu_flush_coalesced_mmio_buffer();
+    addr += 0x3b0;
+
     if (vga_ioport_invalid(s, addr)) {
 	val = 0xff;
     } else {
@@ -2528,12 +2530,16 @@ static uint32_t cirrus_vga_ioport_read(void *opaque, uint32_t addr)
     return val;
 }
 
-static void cirrus_vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void cirrus_vga_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                    unsigned size)
 {
     CirrusVGAState *c = opaque;
     VGACommonState *s = &c->vga;
     int index;
 
+    qemu_flush_coalesced_mmio_buffer();
+    addr += 0x3b0;
+
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
 	return;
@@ -2637,7 +2643,7 @@ static void cirrus_vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
  *
  ***************************************/
 
-static uint64_t cirrus_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cirrus_mmio_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2645,11 +2651,11 @@ static uint64_t cirrus_mmio_read(void *opaque, target_phys_addr_t addr,
     if (addr >= 0x100) {
         return cirrus_mmio_blt_read(s, addr - 0x100);
     } else {
-        return cirrus_vga_ioport_read(s, addr + 0x3c0);
+        return cirrus_vga_ioport_read(s, addr + 0x10, size);
     }
 }
 
-static void cirrus_mmio_write(void *opaque, target_phys_addr_t addr,
+static void cirrus_mmio_write(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2657,7 +2663,7 @@ static void cirrus_mmio_write(void *opaque, target_phys_addr_t addr,
     if (addr >= 0x100) {
 	cirrus_mmio_blt_write(s, addr - 0x100, val);
     } else {
-        cirrus_vga_ioport_write(s, addr + 0x3c0, val);
+        cirrus_vga_ioport_write(s, addr + 0x10, val, size);
     }
 }
 
@@ -2783,8 +2789,19 @@ static const MemoryRegionOps cirrus_linear_io_ops = {
     },
 };
 
+static const MemoryRegionOps cirrus_vga_io_ops = {
+    .read = cirrus_vga_ioport_read,
+    .write = cirrus_vga_ioport_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
-                               MemoryRegion *system_memory)
+                               MemoryRegion *system_memory,
+                               MemoryRegion *system_io)
 {
     int i;
     static int inited;
@@ -2816,19 +2833,10 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
             s->bustype = CIRRUS_BUSTYPE_ISA;
     }
 
-    register_ioport_write(0x3c0, 16, 1, cirrus_vga_ioport_write, s);
-
-    register_ioport_write(0x3b4, 2, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3d4, 2, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3ba, 1, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3da, 1, 1, cirrus_vga_ioport_write, s);
-
-    register_ioport_read(0x3c0, 16, 1, cirrus_vga_ioport_read, s);
-
-    register_ioport_read(0x3b4, 2, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3d4, 2, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3ba, 1, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3da, 1, 1, cirrus_vga_ioport_read, s);
+    /* Register ioport 0x3b0 - 0x3df */
+    memory_region_init_io(&s->cirrus_vga_io, &cirrus_vga_io_ops, s,
+                          "cirrus-io", 0x30);
+    memory_region_add_subregion(system_io, 0x3b0, &s->cirrus_vga_io);
 
     memory_region_init(&s->low_mem_container,
                        "cirrus-lowmem-container",
@@ -2853,7 +2861,9 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
 
     /* I/O handler for LFB */
     memory_region_init_io(&s->cirrus_linear_io, &cirrus_linear_io_ops, s,
-                          "cirrus-linear-io", VGA_RAM_SIZE);
+                          "cirrus-linear-io", s->vga.vram_size_mb
+                                              * 1024 * 1024);
+    memory_region_set_flush_coalesced(&s->cirrus_linear_io);
 
     /* I/O handler for LFB */
     memory_region_init_io(&s->cirrus_linear_bitblt_io,
@@ -2861,10 +2871,12 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
                           s,
                           "cirrus-bitblt-mmio",
                           0x400000);
+    memory_region_set_flush_coalesced(&s->cirrus_linear_bitblt_io);
 
     /* I/O handler for memory-mapped I/O */
     memory_region_init_io(&s->cirrus_mmio_io, &cirrus_mmio_io_ops, s,
                           "cirrus-mmio", CIRRUS_PNPMMIO_SIZE);
+    memory_region_set_flush_coalesced(&s->cirrus_mmio_io);
 
     s->real_vram_size =
         (s->device_id == CIRRUS_ID_CLGD5446) ? 4096 * 1024 : 2048 * 1024;
@@ -2893,10 +2905,9 @@ static int vga_initfn(ISADevice *dev)
     ISACirrusVGAState *d = DO_UPCAST(ISACirrusVGAState, dev, dev);
     VGACommonState *s = &d->cirrus_vga.vga;
 
-    s->vram_size_mb = VGA_RAM_SIZE >> 20;
     vga_common_init(s);
     cirrus_init_common(&d->cirrus_vga, CIRRUS_ID_CLGD5430, 0,
-                       isa_address_space(dev));
+                       isa_address_space(dev), isa_address_space_io(dev));
     s->ds = graphic_console_init(s->update, s->invalidate,
                                  s->screen_dump, s->text_update,
                                  s);
@@ -2906,6 +2917,12 @@ static int vga_initfn(ISADevice *dev)
     return 0;
 }
 
+static Property isa_vga_cirrus_properties[] = {
+    DEFINE_PROP_UINT32("vgamem_mb", struct ISACirrusVGAState,
+                       cirrus_vga.vga.vram_size_mb, 8),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void isa_cirrus_vga_class_init(ObjectClass *klass, void *data)
 {
     ISADeviceClass *k = ISA_DEVICE_CLASS(klass);
@@ -2913,9 +2930,10 @@ static void isa_cirrus_vga_class_init(ObjectClass *klass, void *data)
 
     dc->vmsd  = &vmstate_cirrus_vga;
     k->init   = vga_initfn;
+    dc->props = isa_vga_cirrus_properties;
 }
 
-static TypeInfo isa_cirrus_vga_info = {
+static const TypeInfo isa_cirrus_vga_info = {
     .name          = "isa-cirrus-vga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISACirrusVGAState),
@@ -2936,9 +2954,9 @@ static int pci_cirrus_vga_initfn(PCIDevice *dev)
      int16_t device_id = pc->device_id;
 
      /* setup VGA */
-     s->vga.vram_size_mb = VGA_RAM_SIZE >> 20;
      vga_common_init(&s->vga);
-     cirrus_init_common(s, device_id, 1, pci_address_space(dev));
+     cirrus_init_common(s, device_id, 1, pci_address_space(dev),
+                        pci_address_space_io(dev));
      s->vga.ds = graphic_console_init(s->vga.update, s->vga.invalidate,
                                       s->vga.screen_dump, s->vga.text_update,
                                       &s->vga);
@@ -2963,10 +2981,11 @@ static int pci_cirrus_vga_initfn(PCIDevice *dev)
      return 0;
 }
 
-DeviceState *pci_cirrus_vga_init(PCIBus *bus)
-{
-    return &pci_create_simple(bus, -1, "cirrus-vga")->qdev;
-}
+static Property pci_vga_cirrus_properties[] = {
+    DEFINE_PROP_UINT32("vgamem_mb", struct PCICirrusVGAState,
+                       cirrus_vga.vga.vram_size_mb, 8),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
 static void cirrus_vga_class_init(ObjectClass *klass, void *data)
 {
@@ -2981,9 +3000,10 @@ static void cirrus_vga_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_DISPLAY_VGA;
     dc->desc = "Cirrus CLGD 54xx VGA";
     dc->vmsd = &vmstate_pci_cirrus_vga;
+    dc->props = pci_vga_cirrus_properties;
 }
 
-static TypeInfo cirrus_vga_info = {
+static const TypeInfo cirrus_vga_info = {
     .name          = "cirrus-vga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCICirrusVGAState),
diff --git a/hw/collie.c b/hw/collie.c
index 56f89a9f2e8..d19db590feb 100644
--- a/hw/collie.c
+++ b/hw/collie.c
@@ -15,19 +15,20 @@
 #include "strongarm.h"
 #include "arm-misc.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 static struct arm_boot_info collie_binfo = {
     .loader_start = SA_SDCS0,
     .ram_size = 0x20000000,
 };
 
-static void collie_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void collie_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     StrongARMState *s;
     DriveInfo *dinfo;
     MemoryRegion *sysmem = get_system_memory();
@@ -61,6 +62,7 @@ static QEMUMachine collie_machine = {
     .name = "collie",
     .desc = "Collie PDA (SA-1110)",
     .init = collie_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void collie_machine_init(void)
diff --git a/hw/cris-boot.h b/hw/cris-boot.h
index 0a2c2424115..c4d3fa6f6f6 100644
--- a/hw/cris-boot.h
+++ b/hw/cris-boot.h
@@ -1,3 +1,5 @@
+#ifndef _CRIS_BOOT_H
+#define HW_CRIS_BOOT_H 1
 
 struct cris_load_info
 {
@@ -5,7 +7,9 @@ struct cris_load_info
     const char *cmdline;
     int image_size;
 
-    target_phys_addr_t entry;
+    hwaddr entry;
 };
 
 void cris_load_image(CRISCPU *cpu, struct cris_load_info *li);
+
+#endif
diff --git a/hw/cs4231.c b/hw/cs4231.c
index cfec1d9cd15..ae384b90fd2 100644
--- a/hw/cs4231.c
+++ b/hw/cs4231.c
@@ -55,7 +55,7 @@ static void cs_reset(DeviceState *d)
     s->dregs[25] = CS_VER;
 }
 
-static uint64_t cs_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cs_mem_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     CSState *s = opaque;
@@ -82,7 +82,7 @@ static uint64_t cs_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void cs_mem_write(void *opaque, target_phys_addr_t addr,
+static void cs_mem_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     CSState *s = opaque;
@@ -166,7 +166,7 @@ static void cs4231_class_init(ObjectClass *klass, void *data)
     dc->props = cs4231_properties;
 }
 
-static TypeInfo cs4231_info = {
+static const TypeInfo cs4231_info = {
     .name          = "SUNW,CS4231",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(CSState),
diff --git a/hw/cs4231a.c b/hw/cs4231a.c
index e07b9d62371..73f08594bf4 100644
--- a/hw/cs4231a.c
+++ b/hw/cs4231a.c
@@ -26,7 +26,7 @@
 #include "audio/audio.h"
 #include "isa.h"
 #include "qdev.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /*
   Missing features:
@@ -346,7 +346,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
     }
 }
 
-static uint64_t cs_read (void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t cs_read (void *opaque, hwaddr addr, unsigned size)
 {
     CSState *s = opaque;
     uint32_t saddr, iaddr, ret;
@@ -383,7 +383,7 @@ static uint64_t cs_read (void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static void cs_write (void *opaque, target_phys_addr_t addr,
+static void cs_write (void *opaque, hwaddr addr,
                       uint64_t val64, unsigned size)
 {
     CSState *s = opaque;
@@ -682,7 +682,7 @@ static void cs4231a_class_initfn (ObjectClass *klass, void *data)
     dc->props = cs4231a_properties;
 }
 
-static TypeInfo cs4231a_info = {
+static const TypeInfo cs4231a_info = {
     .name          = "cs4231a",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (CSState),
diff --git a/hw/cuda.c b/hw/cuda.c
index 233ab666dab..b36c53527ab 100644
--- a/hw/cuda.c
+++ b/hw/cuda.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
+#include "ppc/mac.h"
 #include "adb.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 /* XXX: implement all timer modes */
 
@@ -108,50 +108,6 @@
 /* CUDA returns time_t's offset from Jan 1, 1904, not 1970 */
 #define RTC_OFFSET                      2082844800
 
-typedef struct CUDATimer {
-    int index;
-    uint16_t latch;
-    uint16_t counter_value; /* counter value at load time */
-    int64_t load_time;
-    int64_t next_irq_time;
-    QEMUTimer *timer;
-} CUDATimer;
-
-typedef struct CUDAState {
-    MemoryRegion mem;
-    /* cuda registers */
-    uint8_t b;      /* B-side data */
-    uint8_t a;      /* A-side data */
-    uint8_t dirb;   /* B-side direction (1=output) */
-    uint8_t dira;   /* A-side direction (1=output) */
-    uint8_t sr;     /* Shift register */
-    uint8_t acr;    /* Auxiliary control register */
-    uint8_t pcr;    /* Peripheral control register */
-    uint8_t ifr;    /* Interrupt flag register */
-    uint8_t ier;    /* Interrupt enable register */
-    uint8_t anh;    /* A-side data, no handshake */
-
-    CUDATimer timers[2];
-
-    uint32_t tick_offset;
-
-    uint8_t last_b; /* last value of B register */
-    uint8_t last_acr; /* last value of B register */
-
-    int data_in_size;
-    int data_in_index;
-    int data_out_index;
-
-    qemu_irq irq;
-    uint8_t autopoll;
-    uint8_t data_in[128];
-    uint8_t data_out[16];
-    QEMUTimer *adb_poll_timer;
-} CUDAState;
-
-static CUDAState cuda_state;
-ADBBusState adb_bus;
-
 static void cuda_update(CUDAState *s);
 static void cuda_receive_packet_from_host(CUDAState *s,
                                           const uint8_t *data, int len);
@@ -252,7 +208,7 @@ static void cuda_timer1(void *opaque)
     cuda_update_irq(s);
 }
 
-static uint32_t cuda_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readb(void *opaque, hwaddr addr)
 {
     CUDAState *s = opaque;
     uint32_t val;
@@ -325,7 +281,7 @@ static uint32_t cuda_readb(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void cuda_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     CUDAState *s = opaque;
 
@@ -501,7 +457,7 @@ static void cuda_adb_poll(void *opaque)
     uint8_t obuf[ADB_MAX_OUT_LEN + 2];
     int olen;
 
-    olen = adb_poll(&adb_bus, obuf + 2);
+    olen = adb_poll(&s->adb_bus, obuf + 2);
     if (olen > 0) {
         obuf[0] = ADB_PACKET;
         obuf[1] = 0x40; /* polled data */
@@ -597,7 +553,7 @@ static void cuda_receive_packet_from_host(CUDAState *s,
         {
             uint8_t obuf[ADB_MAX_OUT_LEN + 2];
             int olen;
-            olen = adb_request(&adb_bus, obuf + 2, data + 1, len - 1);
+            olen = adb_request(&s->adb_bus, obuf + 2, data + 1, len - 1);
             if (olen > 0) {
                 obuf[0] = ADB_PACKET;
                 obuf[1] = 0x00;
@@ -616,20 +572,20 @@ static void cuda_receive_packet_from_host(CUDAState *s,
     }
 }
 
-static void cuda_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void cuda_writew (void *opaque, hwaddr addr, uint32_t value)
 {
 }
 
-static void cuda_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void cuda_writel (void *opaque, hwaddr addr, uint32_t value)
 {
 }
 
-static uint32_t cuda_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readw (void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static uint32_t cuda_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readl (void *opaque, hwaddr addr)
 {
     return 0;
 }
@@ -701,9 +657,9 @@ static const VMStateDescription vmstate_cuda = {
     }
 };
 
-static void cuda_reset(void *opaque)
+static void cuda_reset(DeviceState *dev)
 {
-    CUDAState *s = opaque;
+    CUDAState *s = CUDA(dev);
 
     s->b = 0;
     s->a = 0;
@@ -728,25 +684,57 @@ static void cuda_reset(void *opaque)
     set_counter(s, &s->timers[1], 0xffff);
 }
 
-void cuda_init (MemoryRegion **cuda_mem, qemu_irq irq)
+static void cuda_realizefn(DeviceState *dev, Error **errp)
 {
+    CUDAState *s = CUDA(dev);
     struct tm tm;
-    CUDAState *s = &cuda_state;
-
-    s->irq = irq;
 
-    s->timers[0].index = 0;
     s->timers[0].timer = qemu_new_timer_ns(vm_clock, cuda_timer1, s);
 
-    s->timers[1].index = 1;
-
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
 
     s->adb_poll_timer = qemu_new_timer_ns(vm_clock, cuda_adb_poll, s);
+}
+
+static void cuda_initfn(Object *obj)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(obj);
+    CUDAState *s = CUDA(obj);
+    int i;
+
     memory_region_init_io(&s->mem, &cuda_ops, s, "cuda", 0x2000);
+    sysbus_init_mmio(d, &s->mem);
+    sysbus_init_irq(d, &s->irq);
+
+    for (i = 0; i < ARRAY_SIZE(s->timers); i++) {
+        s->timers[i].index = i;
+    }
 
-    *cuda_mem = &s->mem;
-    vmstate_register(NULL, -1, &vmstate_cuda, s);
-    qemu_register_reset(cuda_reset, s);
+    qbus_create_inplace((BusState *)&s->adb_bus, TYPE_ADB_BUS, DEVICE(obj),
+                        "adb.0");
 }
+
+static void cuda_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = cuda_realizefn;
+    dc->reset = cuda_reset;
+    dc->vmsd = &vmstate_cuda;
+}
+
+static const TypeInfo cuda_type_info = {
+    .name = TYPE_CUDA,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(CUDAState),
+    .instance_init = cuda_initfn,
+    .class_init = cuda_class_init,
+};
+
+static void cuda_register_types(void)
+{
+    type_register_static(&cuda_type_info);
+}
+
+type_init(cuda_register_types)
diff --git a/hw/dataplane/Makefile.objs b/hw/dataplane/Makefile.objs
new file mode 100644
index 00000000000..3e47d0537e8
--- /dev/null
+++ b/hw/dataplane/Makefile.objs
@@ -0,0 +1 @@
+obj-$(CONFIG_VIRTIO_BLK_DATA_PLANE) += hostmem.o vring.o event-poll.o ioq.o virtio-blk.o
diff --git a/hw/dataplane/event-poll.c b/hw/dataplane/event-poll.c
new file mode 100644
index 00000000000..2b55c6e255e
--- /dev/null
+++ b/hw/dataplane/event-poll.c
@@ -0,0 +1,100 @@
+/*
+ * Event loop with file descriptor polling
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <sys/epoll.h>
+#include "hw/dataplane/event-poll.h"
+
+/* Add an event notifier and its callback for polling */
+void event_poll_add(EventPoll *poll, EventHandler *handler,
+                    EventNotifier *notifier, EventCallback *callback)
+{
+    struct epoll_event event = {
+        .events = EPOLLIN,
+        .data.ptr = handler,
+    };
+    handler->notifier = notifier;
+    handler->callback = callback;
+    if (epoll_ctl(poll->epoll_fd, EPOLL_CTL_ADD,
+                  event_notifier_get_fd(notifier), &event) != 0) {
+        fprintf(stderr, "failed to add event handler to epoll: %m\n");
+        exit(1);
+    }
+}
+
+/* Event callback for stopping event_poll() */
+static void handle_stop(EventHandler *handler)
+{
+    /* Do nothing */
+}
+
+void event_poll_init(EventPoll *poll)
+{
+    /* Create epoll file descriptor */
+    poll->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+    if (poll->epoll_fd < 0) {
+        fprintf(stderr, "epoll_create1 failed: %m\n");
+        exit(1);
+    }
+
+    /* Set up stop notifier */
+    if (event_notifier_init(&poll->stop_notifier, 0) < 0) {
+        fprintf(stderr, "failed to init stop notifier\n");
+        exit(1);
+    }
+    event_poll_add(poll, &poll->stop_handler,
+                   &poll->stop_notifier, handle_stop);
+}
+
+void event_poll_cleanup(EventPoll *poll)
+{
+    event_notifier_cleanup(&poll->stop_notifier);
+    close(poll->epoll_fd);
+    poll->epoll_fd = -1;
+}
+
+/* Block until the next event and invoke its callback */
+void event_poll(EventPoll *poll)
+{
+    EventHandler *handler;
+    struct epoll_event event;
+    int nevents;
+
+    /* Wait for the next event.  Only do one event per call to keep the
+     * function simple, this could be changed later. */
+    do {
+        nevents = epoll_wait(poll->epoll_fd, &event, 1, -1);
+    } while (nevents < 0 && errno == EINTR);
+    if (unlikely(nevents != 1)) {
+        fprintf(stderr, "epoll_wait failed: %m\n");
+        exit(1); /* should never happen */
+    }
+
+    /* Find out which event handler has become active */
+    handler = event.data.ptr;
+
+    /* Clear the eventfd */
+    event_notifier_test_and_clear(handler->notifier);
+
+    /* Handle the event */
+    handler->callback(handler);
+}
+
+/* Stop event_poll()
+ *
+ * This function can be used from another thread.
+ */
+void event_poll_notify(EventPoll *poll)
+{
+    event_notifier_set(&poll->stop_notifier);
+}
diff --git a/hw/dataplane/event-poll.h b/hw/dataplane/event-poll.h
new file mode 100644
index 00000000000..3e8d3ec7d59
--- /dev/null
+++ b/hw/dataplane/event-poll.h
@@ -0,0 +1,40 @@
+/*
+ * Event loop with file descriptor polling
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef EVENT_POLL_H
+#define EVENT_POLL_H
+
+#include "qemu/event_notifier.h"
+
+typedef struct EventHandler EventHandler;
+typedef void EventCallback(EventHandler *handler);
+struct EventHandler {
+    EventNotifier *notifier;        /* eventfd */
+    EventCallback *callback;        /* callback function */
+};
+
+typedef struct {
+    int epoll_fd;                   /* epoll(2) file descriptor */
+    EventNotifier stop_notifier;    /* stop poll notifier */
+    EventHandler stop_handler;      /* stop poll handler */
+} EventPoll;
+
+void event_poll_add(EventPoll *poll, EventHandler *handler,
+                    EventNotifier *notifier, EventCallback *callback);
+void event_poll_init(EventPoll *poll);
+void event_poll_cleanup(EventPoll *poll);
+void event_poll(EventPoll *poll);
+void event_poll_notify(EventPoll *poll);
+
+#endif /* EVENT_POLL_H */
diff --git a/hw/dataplane/hostmem.c b/hw/dataplane/hostmem.c
new file mode 100644
index 00000000000..380537e06d8
--- /dev/null
+++ b/hw/dataplane/hostmem.c
@@ -0,0 +1,176 @@
+/*
+ * Thread-safe guest to host memory mapping
+ *
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "exec/address-spaces.h"
+#include "hostmem.h"
+
+static int hostmem_lookup_cmp(const void *phys_, const void *region_)
+{
+    hwaddr phys = *(const hwaddr *)phys_;
+    const HostMemRegion *region = region_;
+
+    if (phys < region->guest_addr) {
+        return -1;
+    } else if (phys >= region->guest_addr + region->size) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/**
+ * Map guest physical address to host pointer
+ */
+void *hostmem_lookup(HostMem *hostmem, hwaddr phys, hwaddr len, bool is_write)
+{
+    HostMemRegion *region;
+    void *host_addr = NULL;
+    hwaddr offset_within_region;
+
+    qemu_mutex_lock(&hostmem->current_regions_lock);
+    region = bsearch(&phys, hostmem->current_regions,
+                     hostmem->num_current_regions,
+                     sizeof(hostmem->current_regions[0]),
+                     hostmem_lookup_cmp);
+    if (!region) {
+        goto out;
+    }
+    if (is_write && region->readonly) {
+        goto out;
+    }
+    offset_within_region = phys - region->guest_addr;
+    if (len <= region->size - offset_within_region) {
+        host_addr = region->host_addr + offset_within_region;
+    }
+out:
+    qemu_mutex_unlock(&hostmem->current_regions_lock);
+
+    return host_addr;
+}
+
+/**
+ * Install new regions list
+ */
+static void hostmem_listener_commit(MemoryListener *listener)
+{
+    HostMem *hostmem = container_of(listener, HostMem, listener);
+
+    qemu_mutex_lock(&hostmem->current_regions_lock);
+    g_free(hostmem->current_regions);
+    hostmem->current_regions = hostmem->new_regions;
+    hostmem->num_current_regions = hostmem->num_new_regions;
+    qemu_mutex_unlock(&hostmem->current_regions_lock);
+
+    /* Reset new regions list */
+    hostmem->new_regions = NULL;
+    hostmem->num_new_regions = 0;
+}
+
+/**
+ * Add a MemoryRegionSection to the new regions list
+ */
+static void hostmem_append_new_region(HostMem *hostmem,
+                                      MemoryRegionSection *section)
+{
+    void *ram_ptr = memory_region_get_ram_ptr(section->mr);
+    size_t num = hostmem->num_new_regions;
+    size_t new_size = (num + 1) * sizeof(hostmem->new_regions[0]);
+
+    hostmem->new_regions = g_realloc(hostmem->new_regions, new_size);
+    hostmem->new_regions[num] = (HostMemRegion){
+        .host_addr = ram_ptr + section->offset_within_region,
+        .guest_addr = section->offset_within_address_space,
+        .size = section->size,
+        .readonly = section->readonly,
+    };
+    hostmem->num_new_regions++;
+}
+
+static void hostmem_listener_append_region(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    HostMem *hostmem = container_of(listener, HostMem, listener);
+
+    /* Ignore non-RAM regions, we may not be able to map them */
+    if (!memory_region_is_ram(section->mr)) {
+        return;
+    }
+
+    /* Ignore regions with dirty logging, we cannot mark them dirty */
+    if (memory_region_is_logging(section->mr)) {
+        return;
+    }
+
+    hostmem_append_new_region(hostmem, section);
+}
+
+/* We don't implement most MemoryListener callbacks, use these nop stubs */
+static void hostmem_listener_dummy(MemoryListener *listener)
+{
+}
+
+static void hostmem_listener_section_dummy(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+}
+
+static void hostmem_listener_eventfd_dummy(MemoryListener *listener,
+                                           MemoryRegionSection *section,
+                                           bool match_data, uint64_t data,
+                                           EventNotifier *e)
+{
+}
+
+static void hostmem_listener_coalesced_mmio_dummy(MemoryListener *listener,
+                                                  MemoryRegionSection *section,
+                                                  hwaddr addr, hwaddr len)
+{
+}
+
+void hostmem_init(HostMem *hostmem)
+{
+    memset(hostmem, 0, sizeof(*hostmem));
+
+    qemu_mutex_init(&hostmem->current_regions_lock);
+
+    hostmem->listener = (MemoryListener){
+        .begin = hostmem_listener_dummy,
+        .commit = hostmem_listener_commit,
+        .region_add = hostmem_listener_append_region,
+        .region_del = hostmem_listener_section_dummy,
+        .region_nop = hostmem_listener_append_region,
+        .log_start = hostmem_listener_section_dummy,
+        .log_stop = hostmem_listener_section_dummy,
+        .log_sync = hostmem_listener_section_dummy,
+        .log_global_start = hostmem_listener_dummy,
+        .log_global_stop = hostmem_listener_dummy,
+        .eventfd_add = hostmem_listener_eventfd_dummy,
+        .eventfd_del = hostmem_listener_eventfd_dummy,
+        .coalesced_mmio_add = hostmem_listener_coalesced_mmio_dummy,
+        .coalesced_mmio_del = hostmem_listener_coalesced_mmio_dummy,
+        .priority = 10,
+    };
+
+    memory_listener_register(&hostmem->listener, &address_space_memory);
+    if (hostmem->num_new_regions > 0) {
+        hostmem_listener_commit(&hostmem->listener);
+    }
+}
+
+void hostmem_finalize(HostMem *hostmem)
+{
+    memory_listener_unregister(&hostmem->listener);
+    g_free(hostmem->new_regions);
+    g_free(hostmem->current_regions);
+    qemu_mutex_destroy(&hostmem->current_regions_lock);
+}
diff --git a/hw/dataplane/hostmem.h b/hw/dataplane/hostmem.h
new file mode 100644
index 00000000000..b2cf09333fb
--- /dev/null
+++ b/hw/dataplane/hostmem.h
@@ -0,0 +1,57 @@
+/*
+ * Thread-safe guest to host memory mapping
+ *
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HOSTMEM_H
+#define HOSTMEM_H
+
+#include "exec/memory.h"
+#include "qemu/thread.h"
+
+typedef struct {
+    void *host_addr;
+    hwaddr guest_addr;
+    uint64_t size;
+    bool readonly;
+} HostMemRegion;
+
+typedef struct {
+    /* The listener is invoked when regions change and a new list of regions is
+     * built up completely before they are installed.
+     */
+    MemoryListener listener;
+    HostMemRegion *new_regions;
+    size_t num_new_regions;
+
+    /* Current regions are accessed from multiple threads either to lookup
+     * addresses or to install a new list of regions.  The lock protects the
+     * pointer and the regions.
+     */
+    QemuMutex current_regions_lock;
+    HostMemRegion *current_regions;
+    size_t num_current_regions;
+} HostMem;
+
+void hostmem_init(HostMem *hostmem);
+void hostmem_finalize(HostMem *hostmem);
+
+/**
+ * Map a guest physical address to a pointer
+ *
+ * Note that there is map/unmap mechanism here.  The caller must ensure that
+ * mapped memory is no longer used across events like hot memory unplug.  This
+ * can be done with other mechanisms like bdrv_drain_all() that quiesce
+ * in-flight I/O.
+ */
+void *hostmem_lookup(HostMem *hostmem, hwaddr phys, hwaddr len, bool is_write);
+
+#endif /* HOSTMEM_H */
diff --git a/hw/dataplane/ioq.c b/hw/dataplane/ioq.c
new file mode 100644
index 00000000000..0c9f5c4d608
--- /dev/null
+++ b/hw/dataplane/ioq.c
@@ -0,0 +1,117 @@
+/*
+ * Linux AIO request queue
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "hw/dataplane/ioq.h"
+
+void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs)
+{
+    int rc;
+
+    ioq->fd = fd;
+    ioq->max_reqs = max_reqs;
+
+    memset(&ioq->io_ctx, 0, sizeof ioq->io_ctx);
+    rc = io_setup(max_reqs, &ioq->io_ctx);
+    if (rc != 0) {
+        fprintf(stderr, "ioq io_setup failed %d\n", rc);
+        exit(1);
+    }
+
+    rc = event_notifier_init(&ioq->io_notifier, 0);
+    if (rc != 0) {
+        fprintf(stderr, "ioq io event notifier creation failed %d\n", rc);
+        exit(1);
+    }
+
+    ioq->freelist = g_malloc0(sizeof ioq->freelist[0] * max_reqs);
+    ioq->freelist_idx = 0;
+
+    ioq->queue = g_malloc0(sizeof ioq->queue[0] * max_reqs);
+    ioq->queue_idx = 0;
+}
+
+void ioq_cleanup(IOQueue *ioq)
+{
+    g_free(ioq->freelist);
+    g_free(ioq->queue);
+
+    event_notifier_cleanup(&ioq->io_notifier);
+    io_destroy(ioq->io_ctx);
+}
+
+EventNotifier *ioq_get_notifier(IOQueue *ioq)
+{
+    return &ioq->io_notifier;
+}
+
+struct iocb *ioq_get_iocb(IOQueue *ioq)
+{
+    /* Underflow cannot happen since ioq is sized for max_reqs */
+    assert(ioq->freelist_idx != 0);
+
+    struct iocb *iocb = ioq->freelist[--ioq->freelist_idx];
+    ioq->queue[ioq->queue_idx++] = iocb;
+    return iocb;
+}
+
+void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb)
+{
+    /* Overflow cannot happen since ioq is sized for max_reqs */
+    assert(ioq->freelist_idx != ioq->max_reqs);
+
+    ioq->freelist[ioq->freelist_idx++] = iocb;
+}
+
+struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
+                      unsigned int count, long long offset)
+{
+    struct iocb *iocb = ioq_get_iocb(ioq);
+
+    if (read) {
+        io_prep_preadv(iocb, ioq->fd, iov, count, offset);
+    } else {
+        io_prep_pwritev(iocb, ioq->fd, iov, count, offset);
+    }
+    io_set_eventfd(iocb, event_notifier_get_fd(&ioq->io_notifier));
+    return iocb;
+}
+
+int ioq_submit(IOQueue *ioq)
+{
+    int rc = io_submit(ioq->io_ctx, ioq->queue_idx, ioq->queue);
+    ioq->queue_idx = 0; /* reset */
+    return rc;
+}
+
+int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
+                       void *opaque)
+{
+    struct io_event events[ioq->max_reqs];
+    int nevents, i;
+
+    do {
+        nevents = io_getevents(ioq->io_ctx, 0, ioq->max_reqs, events, NULL);
+    } while (nevents < 0 && errno == EINTR);
+    if (nevents < 0) {
+        return nevents;
+    }
+
+    for (i = 0; i < nevents; i++) {
+        ssize_t ret = ((uint64_t)events[i].res2 << 32) | events[i].res;
+
+        completion(events[i].obj, ret, opaque);
+        ioq_put_iocb(ioq, events[i].obj);
+    }
+    return nevents;
+}
diff --git a/hw/dataplane/ioq.h b/hw/dataplane/ioq.h
new file mode 100644
index 00000000000..b49b5de7f49
--- /dev/null
+++ b/hw/dataplane/ioq.h
@@ -0,0 +1,57 @@
+/*
+ * Linux AIO request queue
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef IOQ_H
+#define IOQ_H
+
+#include <libaio.h>
+#include "qemu/event_notifier.h"
+
+typedef struct {
+    int fd;                         /* file descriptor */
+    unsigned int max_reqs;          /* max length of freelist and queue */
+
+    io_context_t io_ctx;            /* Linux AIO context */
+    EventNotifier io_notifier;      /* Linux AIO eventfd */
+
+    /* Requests can complete in any order so a free list is necessary to manage
+     * available iocbs.
+     */
+    struct iocb **freelist;         /* free iocbs */
+    unsigned int freelist_idx;
+
+    /* Multiple requests are queued up before submitting them all in one go */
+    struct iocb **queue;            /* queued iocbs */
+    unsigned int queue_idx;
+} IOQueue;
+
+void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs);
+void ioq_cleanup(IOQueue *ioq);
+EventNotifier *ioq_get_notifier(IOQueue *ioq);
+struct iocb *ioq_get_iocb(IOQueue *ioq);
+void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb);
+struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
+                      unsigned int count, long long offset);
+int ioq_submit(IOQueue *ioq);
+
+static inline unsigned int ioq_num_queued(IOQueue *ioq)
+{
+    return ioq->queue_idx;
+}
+
+typedef void IOQueueCompletion(struct iocb *iocb, ssize_t ret, void *opaque);
+int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
+                       void *opaque);
+
+#endif /* IOQ_H */
diff --git a/hw/dataplane/virtio-blk.c b/hw/dataplane/virtio-blk.c
new file mode 100644
index 00000000000..3f2da226696
--- /dev/null
+++ b/hw/dataplane/virtio-blk.c
@@ -0,0 +1,517 @@
+/*
+ * Dedicated thread for virtio-blk I/O processing
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "qemu/iov.h"
+#include "event-poll.h"
+#include "qemu/thread.h"
+#include "vring.h"
+#include "ioq.h"
+#include "migration/migration.h"
+#include "hw/virtio-blk.h"
+#include "hw/dataplane/virtio-blk.h"
+
+enum {
+    SEG_MAX = 126,                  /* maximum number of I/O segments */
+    VRING_MAX = SEG_MAX + 2,        /* maximum number of vring descriptors */
+    REQ_MAX = VRING_MAX,            /* maximum number of requests in the vring,
+                                     * is VRING_MAX / 2 with traditional and
+                                     * VRING_MAX with indirect descriptors */
+};
+
+typedef struct {
+    struct iocb iocb;               /* Linux AIO control block */
+    QEMUIOVector *inhdr;            /* iovecs for virtio_blk_inhdr */
+    unsigned int head;              /* vring descriptor index */
+    struct iovec *bounce_iov;       /* used if guest buffers are unaligned */
+    QEMUIOVector *read_qiov;        /* for read completion /w bounce buffer */
+} VirtIOBlockRequest;
+
+struct VirtIOBlockDataPlane {
+    bool started;
+    bool stopping;
+    QEMUBH *start_bh;
+    QemuThread thread;
+
+    VirtIOBlkConf *blk;
+    int fd;                         /* image file descriptor */
+
+    VirtIODevice *vdev;
+    Vring vring;                    /* virtqueue vring */
+    EventNotifier *guest_notifier;  /* irq */
+
+    EventPoll event_poll;           /* event poller */
+    EventHandler io_handler;        /* Linux AIO completion handler */
+    EventHandler notify_handler;    /* virtqueue notify handler */
+
+    IOQueue ioqueue;                /* Linux AIO queue (should really be per
+                                       dataplane thread) */
+    VirtIOBlockRequest requests[REQ_MAX]; /* pool of requests, managed by the
+                                             queue */
+
+    unsigned int num_reqs;
+
+    Error *migration_blocker;
+};
+
+/* Raise an interrupt to signal guest, if necessary */
+static void notify_guest(VirtIOBlockDataPlane *s)
+{
+    if (!vring_should_notify(s->vdev, &s->vring)) {
+        return;
+    }
+
+    event_notifier_set(s->guest_notifier);
+}
+
+static void complete_request(struct iocb *iocb, ssize_t ret, void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+    VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+    struct virtio_blk_inhdr hdr;
+    int len;
+
+    if (likely(ret >= 0)) {
+        hdr.status = VIRTIO_BLK_S_OK;
+        len = ret;
+    } else {
+        hdr.status = VIRTIO_BLK_S_IOERR;
+        len = 0;
+    }
+
+    trace_virtio_blk_data_plane_complete_request(s, req->head, ret);
+
+    if (req->read_qiov) {
+        assert(req->bounce_iov);
+        qemu_iovec_from_buf(req->read_qiov, 0, req->bounce_iov->iov_base, len);
+        qemu_iovec_destroy(req->read_qiov);
+        g_slice_free(QEMUIOVector, req->read_qiov);
+    }
+
+    if (req->bounce_iov) {
+        qemu_vfree(req->bounce_iov->iov_base);
+        g_slice_free(struct iovec, req->bounce_iov);
+    }
+
+    qemu_iovec_from_buf(req->inhdr, 0, &hdr, sizeof(hdr));
+    qemu_iovec_destroy(req->inhdr);
+    g_slice_free(QEMUIOVector, req->inhdr);
+
+    /* According to the virtio specification len should be the number of bytes
+     * written to, but for virtio-blk it seems to be the number of bytes
+     * transferred plus the status bytes.
+     */
+    vring_push(&s->vring, req->head, len + sizeof(hdr));
+
+    s->num_reqs--;
+}
+
+static void complete_request_early(VirtIOBlockDataPlane *s, unsigned int head,
+                                   QEMUIOVector *inhdr, unsigned char status)
+{
+    struct virtio_blk_inhdr hdr = {
+        .status = status,
+    };
+
+    qemu_iovec_from_buf(inhdr, 0, &hdr, sizeof(hdr));
+    qemu_iovec_destroy(inhdr);
+    g_slice_free(QEMUIOVector, inhdr);
+
+    vring_push(&s->vring, head, sizeof(hdr));
+    notify_guest(s);
+}
+
+/* Get disk serial number */
+static void do_get_id_cmd(VirtIOBlockDataPlane *s,
+                          struct iovec *iov, unsigned int iov_cnt,
+                          unsigned int head, QEMUIOVector *inhdr)
+{
+    char id[VIRTIO_BLK_ID_BYTES];
+
+    /* Serial number not NUL-terminated when shorter than buffer */
+    strncpy(id, s->blk->serial ? s->blk->serial : "", sizeof(id));
+    iov_from_buf(iov, iov_cnt, 0, id, sizeof(id));
+    complete_request_early(s, head, inhdr, VIRTIO_BLK_S_OK);
+}
+
+static int do_rdwr_cmd(VirtIOBlockDataPlane *s, bool read,
+                       struct iovec *iov, unsigned int iov_cnt,
+                       long long offset, unsigned int head,
+                       QEMUIOVector *inhdr)
+{
+    struct iocb *iocb;
+    QEMUIOVector qiov;
+    struct iovec *bounce_iov = NULL;
+    QEMUIOVector *read_qiov = NULL;
+
+    qemu_iovec_init_external(&qiov, iov, iov_cnt);
+    if (!bdrv_qiov_is_aligned(s->blk->conf.bs, &qiov)) {
+        void *bounce_buffer = qemu_blockalign(s->blk->conf.bs, qiov.size);
+
+        if (read) {
+            /* Need to copy back from bounce buffer on completion */
+            read_qiov = g_slice_new(QEMUIOVector);
+            qemu_iovec_init(read_qiov, iov_cnt);
+            qemu_iovec_concat_iov(read_qiov, iov, iov_cnt, 0, qiov.size);
+        } else {
+            qemu_iovec_to_buf(&qiov, 0, bounce_buffer, qiov.size);
+        }
+
+        /* Redirect I/O to aligned bounce buffer */
+        bounce_iov = g_slice_new(struct iovec);
+        bounce_iov->iov_base = bounce_buffer;
+        bounce_iov->iov_len = qiov.size;
+        iov = bounce_iov;
+        iov_cnt = 1;
+    }
+
+    iocb = ioq_rdwr(&s->ioqueue, read, iov, iov_cnt, offset);
+
+    /* Fill in virtio block metadata needed for completion */
+    VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+    req->head = head;
+    req->inhdr = inhdr;
+    req->bounce_iov = bounce_iov;
+    req->read_qiov = read_qiov;
+    return 0;
+}
+
+static int process_request(IOQueue *ioq, struct iovec iov[],
+                           unsigned int out_num, unsigned int in_num,
+                           unsigned int head)
+{
+    VirtIOBlockDataPlane *s = container_of(ioq, VirtIOBlockDataPlane, ioqueue);
+    struct iovec *in_iov = &iov[out_num];
+    struct virtio_blk_outhdr outhdr;
+    QEMUIOVector *inhdr;
+    size_t in_size;
+
+    /* Copy in outhdr */
+    if (unlikely(iov_to_buf(iov, out_num, 0, &outhdr,
+                            sizeof(outhdr)) != sizeof(outhdr))) {
+        error_report("virtio-blk request outhdr too short");
+        return -EFAULT;
+    }
+    iov_discard_front(&iov, &out_num, sizeof(outhdr));
+
+    /* Grab inhdr for later */
+    in_size = iov_size(in_iov, in_num);
+    if (in_size < sizeof(struct virtio_blk_inhdr)) {
+        error_report("virtio_blk request inhdr too short");
+        return -EFAULT;
+    }
+    inhdr = g_slice_new(QEMUIOVector);
+    qemu_iovec_init(inhdr, 1);
+    qemu_iovec_concat_iov(inhdr, in_iov, in_num,
+            in_size - sizeof(struct virtio_blk_inhdr),
+            sizeof(struct virtio_blk_inhdr));
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+    /* TODO Linux sets the barrier bit even when not advertised! */
+    outhdr.type &= ~VIRTIO_BLK_T_BARRIER;
+
+    switch (outhdr.type) {
+    case VIRTIO_BLK_T_IN:
+        do_rdwr_cmd(s, true, in_iov, in_num, outhdr.sector * 512, head, inhdr);
+        return 0;
+
+    case VIRTIO_BLK_T_OUT:
+        do_rdwr_cmd(s, false, iov, out_num, outhdr.sector * 512, head, inhdr);
+        return 0;
+
+    case VIRTIO_BLK_T_SCSI_CMD:
+        /* TODO support SCSI commands */
+        complete_request_early(s, head, inhdr, VIRTIO_BLK_S_UNSUPP);
+        return 0;
+
+    case VIRTIO_BLK_T_FLUSH:
+        /* TODO fdsync not supported by Linux AIO, do it synchronously here! */
+        if (qemu_fdatasync(s->fd) < 0) {
+            complete_request_early(s, head, inhdr, VIRTIO_BLK_S_IOERR);
+        } else {
+            complete_request_early(s, head, inhdr, VIRTIO_BLK_S_OK);
+        }
+        return 0;
+
+    case VIRTIO_BLK_T_GET_ID:
+        do_get_id_cmd(s, in_iov, in_num, head, inhdr);
+        return 0;
+
+    default:
+        error_report("virtio-blk unsupported request type %#x", outhdr.type);
+        qemu_iovec_destroy(inhdr);
+        g_slice_free(QEMUIOVector, inhdr);
+        return -EFAULT;
+    }
+}
+
+static void handle_notify(EventHandler *handler)
+{
+    VirtIOBlockDataPlane *s = container_of(handler, VirtIOBlockDataPlane,
+                                           notify_handler);
+
+    /* There is one array of iovecs into which all new requests are extracted
+     * from the vring.  Requests are read from the vring and the translated
+     * descriptors are written to the iovecs array.  The iovecs do not have to
+     * persist across handle_notify() calls because the kernel copies the
+     * iovecs on io_submit().
+     *
+     * Handling io_submit() EAGAIN may require storing the requests across
+     * handle_notify() calls until the kernel has sufficient resources to
+     * accept more I/O.  This is not implemented yet.
+     */
+    struct iovec iovec[VRING_MAX];
+    struct iovec *end = &iovec[VRING_MAX];
+    struct iovec *iov = iovec;
+
+    /* When a request is read from the vring, the index of the first descriptor
+     * (aka head) is returned so that the completed request can be pushed onto
+     * the vring later.
+     *
+     * The number of hypervisor read-only iovecs is out_num.  The number of
+     * hypervisor write-only iovecs is in_num.
+     */
+    int head;
+    unsigned int out_num = 0, in_num = 0;
+    unsigned int num_queued;
+
+    for (;;) {
+        /* Disable guest->host notifies to avoid unnecessary vmexits */
+        vring_disable_notification(s->vdev, &s->vring);
+
+        for (;;) {
+            head = vring_pop(s->vdev, &s->vring, iov, end, &out_num, &in_num);
+            if (head < 0) {
+                break; /* no more requests */
+            }
+
+            trace_virtio_blk_data_plane_process_request(s, out_num, in_num,
+                                                        head);
+
+            if (process_request(&s->ioqueue, iov, out_num, in_num, head) < 0) {
+                vring_set_broken(&s->vring);
+                break;
+            }
+            iov += out_num + in_num;
+        }
+
+        if (likely(head == -EAGAIN)) { /* vring emptied */
+            /* Re-enable guest->host notifies and stop processing the vring.
+             * But if the guest has snuck in more descriptors, keep processing.
+             */
+            if (vring_enable_notification(s->vdev, &s->vring)) {
+                break;
+            }
+        } else { /* head == -ENOBUFS or fatal error, iovecs[] is depleted */
+            /* Since there are no iovecs[] left, stop processing for now.  Do
+             * not re-enable guest->host notifies since the I/O completion
+             * handler knows to check for more vring descriptors anyway.
+             */
+            break;
+        }
+    }
+
+    num_queued = ioq_num_queued(&s->ioqueue);
+    if (num_queued > 0) {
+        s->num_reqs += num_queued;
+
+        int rc = ioq_submit(&s->ioqueue);
+        if (unlikely(rc < 0)) {
+            fprintf(stderr, "ioq_submit failed %d\n", rc);
+            exit(1);
+        }
+    }
+}
+
+static void handle_io(EventHandler *handler)
+{
+    VirtIOBlockDataPlane *s = container_of(handler, VirtIOBlockDataPlane,
+                                           io_handler);
+
+    if (ioq_run_completion(&s->ioqueue, complete_request, s) > 0) {
+        notify_guest(s);
+    }
+
+    /* If there were more requests than iovecs, the vring will not be empty yet
+     * so check again.  There should now be enough resources to process more
+     * requests.
+     */
+    if (unlikely(vring_more_avail(&s->vring))) {
+        handle_notify(&s->notify_handler);
+    }
+}
+
+static void *data_plane_thread(void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+
+    do {
+        event_poll(&s->event_poll);
+    } while (!s->stopping || s->num_reqs > 0);
+    return NULL;
+}
+
+static void start_data_plane_bh(void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+
+    qemu_bh_delete(s->start_bh);
+    s->start_bh = NULL;
+    qemu_thread_create(&s->thread, data_plane_thread,
+                       s, QEMU_THREAD_JOINABLE);
+}
+
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+                                  VirtIOBlockDataPlane **dataplane)
+{
+    VirtIOBlockDataPlane *s;
+    int fd;
+
+    *dataplane = NULL;
+
+    if (!blk->data_plane) {
+        return true;
+    }
+
+    if (blk->scsi) {
+        error_report("device is incompatible with x-data-plane, use scsi=off");
+        return false;
+    }
+
+    if (blk->config_wce) {
+        error_report("device is incompatible with x-data-plane, "
+                     "use config-wce=off");
+        return false;
+    }
+
+    fd = raw_get_aio_fd(blk->conf.bs);
+    if (fd < 0) {
+        error_report("drive is incompatible with x-data-plane, "
+                     "use format=raw,cache=none,aio=native");
+        return false;
+    }
+
+    s = g_new0(VirtIOBlockDataPlane, 1);
+    s->vdev = vdev;
+    s->fd = fd;
+    s->blk = blk;
+
+    /* Prevent block operations that conflict with data plane thread */
+    bdrv_set_in_use(blk->conf.bs, 1);
+
+    error_setg(&s->migration_blocker,
+            "x-data-plane does not support migration");
+    migrate_add_blocker(s->migration_blocker);
+
+    *dataplane = s;
+    return true;
+}
+
+void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
+{
+    if (!s) {
+        return;
+    }
+
+    virtio_blk_data_plane_stop(s);
+    migrate_del_blocker(s->migration_blocker);
+    error_free(s->migration_blocker);
+    bdrv_set_in_use(s->blk->conf.bs, 0);
+    g_free(s);
+}
+
+void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
+{
+    VirtQueue *vq;
+    int i;
+
+    if (s->started) {
+        return;
+    }
+
+    vq = virtio_get_queue(s->vdev, 0);
+    if (!vring_setup(&s->vring, s->vdev, 0)) {
+        return;
+    }
+
+    event_poll_init(&s->event_poll);
+
+    /* Set up guest notifier (irq) */
+    if (s->vdev->binding->set_guest_notifiers(s->vdev->binding_opaque, 1,
+                                              true) != 0) {
+        fprintf(stderr, "virtio-blk failed to set guest notifier, "
+                "ensure -enable-kvm is set\n");
+        exit(1);
+    }
+    s->guest_notifier = virtio_queue_get_guest_notifier(vq);
+
+    /* Set up virtqueue notify */
+    if (s->vdev->binding->set_host_notifier(s->vdev->binding_opaque,
+                                            0, true) != 0) {
+        fprintf(stderr, "virtio-blk failed to set host notifier\n");
+        exit(1);
+    }
+    event_poll_add(&s->event_poll, &s->notify_handler,
+                   virtio_queue_get_host_notifier(vq),
+                   handle_notify);
+
+    /* Set up ioqueue */
+    ioq_init(&s->ioqueue, s->fd, REQ_MAX);
+    for (i = 0; i < ARRAY_SIZE(s->requests); i++) {
+        ioq_put_iocb(&s->ioqueue, &s->requests[i].iocb);
+    }
+    event_poll_add(&s->event_poll, &s->io_handler,
+                   ioq_get_notifier(&s->ioqueue), handle_io);
+
+    s->started = true;
+    trace_virtio_blk_data_plane_start(s);
+
+    /* Kick right away to begin processing requests already in vring */
+    event_notifier_set(virtio_queue_get_host_notifier(vq));
+
+    /* Spawn thread in BH so it inherits iothread cpusets */
+    s->start_bh = qemu_bh_new(start_data_plane_bh, s);
+    qemu_bh_schedule(s->start_bh);
+}
+
+void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
+{
+    if (!s->started || s->stopping) {
+        return;
+    }
+    s->stopping = true;
+    trace_virtio_blk_data_plane_stop(s);
+
+    /* Stop thread or cancel pending thread creation BH */
+    if (s->start_bh) {
+        qemu_bh_delete(s->start_bh);
+        s->start_bh = NULL;
+    } else {
+        event_poll_notify(&s->event_poll);
+        qemu_thread_join(&s->thread);
+    }
+
+    ioq_cleanup(&s->ioqueue);
+
+    s->vdev->binding->set_host_notifier(s->vdev->binding_opaque, 0, false);
+
+    event_poll_cleanup(&s->event_poll);
+
+    /* Clean up guest notifier (irq) */
+    s->vdev->binding->set_guest_notifiers(s->vdev->binding_opaque, 1, false);
+
+    vring_teardown(&s->vring);
+    s->started = false;
+    s->stopping = false;
+}
diff --git a/hw/dataplane/virtio-blk.h b/hw/dataplane/virtio-blk.h
new file mode 100644
index 00000000000..1e8fdfe4183
--- /dev/null
+++ b/hw/dataplane/virtio-blk.h
@@ -0,0 +1,29 @@
+/*
+ * Dedicated thread for virtio-blk I/O processing
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_DATAPLANE_VIRTIO_BLK_H
+#define HW_DATAPLANE_VIRTIO_BLK_H
+
+#include "hw/virtio.h"
+
+typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane;
+
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+                                  VirtIOBlockDataPlane **dataplane);
+void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_drain(VirtIOBlockDataPlane *s);
+
+#endif /* HW_DATAPLANE_VIRTIO_BLK_H */
diff --git a/hw/dataplane/vring.c b/hw/dataplane/vring.c
new file mode 100644
index 00000000000..d5d4ef45d17
--- /dev/null
+++ b/hw/dataplane/vring.c
@@ -0,0 +1,362 @@
+/* Copyright 2012 Red Hat, Inc.
+ * Copyright IBM, Corp. 2012
+ *
+ * Based on Linux 2.6.39 vhost code:
+ * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2006 Rusty Russell IBM Corporation
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *         Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * Inspiration, some code, and most witty comments come from
+ * Documentation/virtual/lguest/lguest.c, by Rusty Russell
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#include "trace.h"
+#include "hw/dataplane/vring.h"
+
+/* Map the guest's vring to host memory */
+bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
+{
+    hwaddr vring_addr = virtio_queue_get_ring_addr(vdev, n);
+    hwaddr vring_size = virtio_queue_get_ring_size(vdev, n);
+    void *vring_ptr;
+
+    vring->broken = false;
+
+    hostmem_init(&vring->hostmem);
+    vring_ptr = hostmem_lookup(&vring->hostmem, vring_addr, vring_size, true);
+    if (!vring_ptr) {
+        error_report("Failed to map vring "
+                     "addr %#" HWADDR_PRIx " size %" HWADDR_PRIu,
+                     vring_addr, vring_size);
+        vring->broken = true;
+        return false;
+    }
+
+    vring_init(&vring->vr, virtio_queue_get_num(vdev, n), vring_ptr, 4096);
+
+    vring->last_avail_idx = 0;
+    vring->last_used_idx = 0;
+    vring->signalled_used = 0;
+    vring->signalled_used_valid = false;
+
+    trace_vring_setup(virtio_queue_get_ring_addr(vdev, n),
+                      vring->vr.desc, vring->vr.avail, vring->vr.used);
+    return true;
+}
+
+void vring_teardown(Vring *vring)
+{
+    hostmem_finalize(&vring->hostmem);
+}
+
+/* Disable guest->host notifies */
+void vring_disable_notification(VirtIODevice *vdev, Vring *vring)
+{
+    if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
+        vring->vr.used->flags |= VRING_USED_F_NO_NOTIFY;
+    }
+}
+
+/* Enable guest->host notifies
+ *
+ * Return true if the vring is empty, false if there are more requests.
+ */
+bool vring_enable_notification(VirtIODevice *vdev, Vring *vring)
+{
+    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
+        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
+    } else {
+        vring->vr.used->flags &= ~VRING_USED_F_NO_NOTIFY;
+    }
+    smp_mb(); /* ensure update is seen before reading avail_idx */
+    return !vring_more_avail(vring);
+}
+
+/* This is stolen from linux/drivers/vhost/vhost.c:vhost_notify() */
+bool vring_should_notify(VirtIODevice *vdev, Vring *vring)
+{
+    uint16_t old, new;
+    bool v;
+    /* Flush out used index updates. This is paired
+     * with the barrier that the Guest executes when enabling
+     * interrupts. */
+    smp_mb();
+
+    if ((vdev->guest_features & VIRTIO_F_NOTIFY_ON_EMPTY) &&
+        unlikely(vring->vr.avail->idx == vring->last_avail_idx)) {
+        return true;
+    }
+
+    if (!(vdev->guest_features & VIRTIO_RING_F_EVENT_IDX)) {
+        return !(vring->vr.avail->flags & VRING_AVAIL_F_NO_INTERRUPT);
+    }
+    old = vring->signalled_used;
+    v = vring->signalled_used_valid;
+    new = vring->signalled_used = vring->last_used_idx;
+    vring->signalled_used_valid = true;
+
+    if (unlikely(!v)) {
+        return true;
+    }
+
+    return vring_need_event(vring_used_event(&vring->vr), new, old);
+}
+
+/* This is stolen from linux/drivers/vhost/vhost.c. */
+static int get_indirect(Vring *vring,
+                        struct iovec iov[], struct iovec *iov_end,
+                        unsigned int *out_num, unsigned int *in_num,
+                        struct vring_desc *indirect)
+{
+    struct vring_desc desc;
+    unsigned int i = 0, count, found = 0;
+
+    /* Sanity check */
+    if (unlikely(indirect->len % sizeof(desc))) {
+        error_report("Invalid length in indirect descriptor: "
+                     "len %#x not multiple of %#zx",
+                     indirect->len, sizeof(desc));
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    count = indirect->len / sizeof(desc);
+    /* Buffers are chained via a 16 bit next field, so
+     * we can have at most 2^16 of these. */
+    if (unlikely(count > USHRT_MAX + 1)) {
+        error_report("Indirect buffer length too big: %d", indirect->len);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    do {
+        struct vring_desc *desc_ptr;
+
+        /* Translate indirect descriptor */
+        desc_ptr = hostmem_lookup(&vring->hostmem,
+                                  indirect->addr + found * sizeof(desc),
+                                  sizeof(desc), false);
+        if (!desc_ptr) {
+            error_report("Failed to map indirect descriptor "
+                         "addr %#" PRIx64 " len %zu",
+                         (uint64_t)indirect->addr + found * sizeof(desc),
+                         sizeof(desc));
+            vring->broken = true;
+            return -EFAULT;
+        }
+        desc = *desc_ptr;
+
+        /* Ensure descriptor has been loaded before accessing fields */
+        barrier(); /* read_barrier_depends(); */
+
+        if (unlikely(++found > count)) {
+            error_report("Loop detected: last one at %u "
+                         "indirect size %u", i, count);
+            vring->broken = true;
+            return -EFAULT;
+        }
+
+        if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
+            error_report("Nested indirect descriptor");
+            vring->broken = true;
+            return -EFAULT;
+        }
+
+        /* Stop for now if there are not enough iovecs available. */
+        if (iov >= iov_end) {
+            return -ENOBUFS;
+        }
+
+        iov->iov_base = hostmem_lookup(&vring->hostmem, desc.addr, desc.len,
+                                       desc.flags & VRING_DESC_F_WRITE);
+        if (!iov->iov_base) {
+            error_report("Failed to map indirect descriptor"
+                         "addr %#" PRIx64 " len %u",
+                         (uint64_t)desc.addr, desc.len);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        iov->iov_len = desc.len;
+        iov++;
+
+        /* If this is an input descriptor, increment that count. */
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            *in_num += 1;
+        } else {
+            /* If it's an output descriptor, they're all supposed
+             * to come before any input descriptors. */
+            if (unlikely(*in_num)) {
+                error_report("Indirect descriptor "
+                             "has out after in: idx %u", i);
+                vring->broken = true;
+                return -EFAULT;
+            }
+            *out_num += 1;
+        }
+        i = desc.next;
+    } while (desc.flags & VRING_DESC_F_NEXT);
+    return 0;
+}
+
+/* This looks in the virtqueue and for the first available buffer, and converts
+ * it to an iovec for convenient access.  Since descriptors consist of some
+ * number of output then some number of input descriptors, it's actually two
+ * iovecs, but we pack them into one and note how many of each there were.
+ *
+ * This function returns the descriptor number found, or vq->num (which is
+ * never a valid descriptor number) if none was found.  A negative code is
+ * returned on error.
+ *
+ * Stolen from linux/drivers/vhost/vhost.c.
+ */
+int vring_pop(VirtIODevice *vdev, Vring *vring,
+              struct iovec iov[], struct iovec *iov_end,
+              unsigned int *out_num, unsigned int *in_num)
+{
+    struct vring_desc desc;
+    unsigned int i, head, found = 0, num = vring->vr.num;
+    uint16_t avail_idx, last_avail_idx;
+
+    /* If there was a fatal error then refuse operation */
+    if (vring->broken) {
+        return -EFAULT;
+    }
+
+    /* Check it isn't doing very strange things with descriptor numbers. */
+    last_avail_idx = vring->last_avail_idx;
+    avail_idx = vring->vr.avail->idx;
+    barrier(); /* load indices now and not again later */
+
+    if (unlikely((uint16_t)(avail_idx - last_avail_idx) > num)) {
+        error_report("Guest moved used index from %u to %u",
+                     last_avail_idx, avail_idx);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    /* If there's nothing new since last we looked. */
+    if (avail_idx == last_avail_idx) {
+        return -EAGAIN;
+    }
+
+    /* Only get avail ring entries after they have been exposed by guest. */
+    smp_rmb();
+
+    /* Grab the next descriptor number they're advertising, and increment
+     * the index we've seen. */
+    head = vring->vr.avail->ring[last_avail_idx % num];
+
+    /* If their number is silly, that's an error. */
+    if (unlikely(head >= num)) {
+        error_report("Guest says index %u > %u is available", head, num);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
+        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
+    }
+
+    /* When we start there are none of either input nor output. */
+    *out_num = *in_num = 0;
+
+    i = head;
+    do {
+        if (unlikely(i >= num)) {
+            error_report("Desc index is %u > %u, head = %u", i, num, head);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        if (unlikely(++found > num)) {
+            error_report("Loop detected: last one at %u vq size %u head %u",
+                         i, num, head);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        desc = vring->vr.desc[i];
+
+        /* Ensure descriptor is loaded before accessing fields */
+        barrier();
+
+        if (desc.flags & VRING_DESC_F_INDIRECT) {
+            int ret = get_indirect(vring, iov, iov_end, out_num, in_num, &desc);
+            if (ret < 0) {
+                return ret;
+            }
+            continue;
+        }
+
+        /* If there are not enough iovecs left, stop for now.  The caller
+         * should check if there are more descs available once they have dealt
+         * with the current set.
+         */
+        if (iov >= iov_end) {
+            return -ENOBUFS;
+        }
+
+        /* TODO handle non-contiguous memory across region boundaries */
+        iov->iov_base = hostmem_lookup(&vring->hostmem, desc.addr, desc.len,
+                                       desc.flags & VRING_DESC_F_WRITE);
+        if (!iov->iov_base) {
+            error_report("Failed to map vring desc addr %#" PRIx64 " len %u",
+                         (uint64_t)desc.addr, desc.len);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        iov->iov_len  = desc.len;
+        iov++;
+
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            /* If this is an input descriptor,
+             * increment that count. */
+            *in_num += 1;
+        } else {
+            /* If it's an output descriptor, they're all supposed
+             * to come before any input descriptors. */
+            if (unlikely(*in_num)) {
+                error_report("Descriptor has out after in: idx %d", i);
+                vring->broken = true;
+                return -EFAULT;
+            }
+            *out_num += 1;
+        }
+        i = desc.next;
+    } while (desc.flags & VRING_DESC_F_NEXT);
+
+    /* On success, increment avail index. */
+    vring->last_avail_idx++;
+    return head;
+}
+
+/* After we've used one of their buffers, we tell them about it.
+ *
+ * Stolen from linux/drivers/vhost/vhost.c.
+ */
+void vring_push(Vring *vring, unsigned int head, int len)
+{
+    struct vring_used_elem *used;
+    uint16_t new;
+
+    /* Don't touch vring if a fatal error occurred */
+    if (vring->broken) {
+        return;
+    }
+
+    /* The virtqueue contains a ring of used buffers.  Get a pointer to the
+     * next entry in that used ring. */
+    used = &vring->vr.used->ring[vring->last_used_idx % vring->vr.num];
+    used->id = head;
+    used->len = len;
+
+    /* Make sure buffer is written before we update index. */
+    smp_wmb();
+
+    new = vring->vr.used->idx = ++vring->last_used_idx;
+    if (unlikely((int16_t)(new - vring->signalled_used) < (uint16_t)1)) {
+        vring->signalled_used_valid = false;
+    }
+}
diff --git a/hw/dataplane/vring.h b/hw/dataplane/vring.h
new file mode 100644
index 00000000000..3274f623f52
--- /dev/null
+++ b/hw/dataplane/vring.h
@@ -0,0 +1,62 @@
+/* Copyright 2012 Red Hat, Inc. and/or its affiliates
+ * Copyright IBM, Corp. 2012
+ *
+ * Based on Linux 2.6.39 vhost code:
+ * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2006 Rusty Russell IBM Corporation
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *         Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * Inspiration, some code, and most witty comments come from
+ * Documentation/virtual/lguest/lguest.c, by Rusty Russell
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#ifndef VRING_H
+#define VRING_H
+
+#include <linux/virtio_ring.h>
+#include "qemu-common.h"
+#include "hw/dataplane/hostmem.h"
+#include "hw/virtio.h"
+
+typedef struct {
+    HostMem hostmem;                /* guest memory mapper */
+    struct vring vr;                /* virtqueue vring mapped to host memory */
+    uint16_t last_avail_idx;        /* last processed avail ring index */
+    uint16_t last_used_idx;         /* last processed used ring index */
+    uint16_t signalled_used;        /* EVENT_IDX state */
+    bool signalled_used_valid;
+    bool broken;                    /* was there a fatal error? */
+} Vring;
+
+static inline unsigned int vring_get_num(Vring *vring)
+{
+    return vring->vr.num;
+}
+
+/* Are there more descriptors available? */
+static inline bool vring_more_avail(Vring *vring)
+{
+    return vring->vr.avail->idx != vring->last_avail_idx;
+}
+
+/* Fail future vring_pop() and vring_push() calls until reset */
+static inline void vring_set_broken(Vring *vring)
+{
+    vring->broken = true;
+}
+
+bool vring_setup(Vring *vring, VirtIODevice *vdev, int n);
+void vring_teardown(Vring *vring);
+void vring_disable_notification(VirtIODevice *vdev, Vring *vring);
+bool vring_enable_notification(VirtIODevice *vdev, Vring *vring);
+bool vring_should_notify(VirtIODevice *vdev, Vring *vring);
+int vring_pop(VirtIODevice *vdev, Vring *vring,
+              struct iovec iov[], struct iovec *iov_end,
+              unsigned int *out_num, unsigned int *in_num);
+void vring_push(Vring *vring, unsigned int head, int len);
+
+#endif /* VRING_H */
diff --git a/hw/debugcon.c b/hw/debugcon.c
index 14ab326be36..81b2bb00fd0 100644
--- a/hw/debugcon.c
+++ b/hw/debugcon.c
@@ -25,24 +25,31 @@
  */
 
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "isa.h"
 #include "pc.h"
 
+#define TYPE_ISA_DEBUGCON_DEVICE "isa-debugcon"
+#define ISA_DEBUGCON_DEVICE(obj) \
+     OBJECT_CHECK(ISADebugconState, (obj), TYPE_ISA_DEBUGCON_DEVICE)
+
 //#define DEBUG_DEBUGCON
 
 typedef struct DebugconState {
+    MemoryRegion io;
     CharDriverState *chr;
     uint32_t readback;
 } DebugconState;
 
 typedef struct ISADebugconState {
-    ISADevice dev;
+    ISADevice parent_obj;
+
     uint32_t iobase;
     DebugconState state;
 } ISADebugconState;
 
-static void debugcon_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void debugcon_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned width)
 {
     DebugconState *s = opaque;
     unsigned char ch = val;
@@ -55,7 +62,7 @@ static void debugcon_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 }
 
 
-static uint32_t debugcon_ioport_read(void *opaque, uint32_t addr)
+static uint64_t debugcon_ioport_read(void *opaque, hwaddr addr, unsigned width)
 {
     DebugconState *s = opaque;
 
@@ -66,6 +73,14 @@ static uint32_t debugcon_ioport_read(void *opaque, uint32_t addr)
     return s->readback;
 }
 
+static const MemoryRegionOps debugcon_ops = {
+    .read = debugcon_ioport_read,
+    .write = debugcon_ioport_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 static void debugcon_init_core(DebugconState *s)
 {
     if (!s->chr) {
@@ -78,12 +93,14 @@ static void debugcon_init_core(DebugconState *s)
 
 static int debugcon_isa_initfn(ISADevice *dev)
 {
-    ISADebugconState *isa = DO_UPCAST(ISADebugconState, dev, dev);
+    ISADebugconState *isa = ISA_DEBUGCON_DEVICE(dev);
     DebugconState *s = &isa->state;
 
     debugcon_init_core(s);
-    register_ioport_write(isa->iobase, 1, 1, debugcon_ioport_write, s);
-    register_ioport_read(isa->iobase, 1, 1, debugcon_ioport_read, s);
+    memory_region_init_io(&s->io, &debugcon_ops, s,
+                          TYPE_ISA_DEBUGCON_DEVICE, 1);
+    memory_region_add_subregion(isa_address_space_io(dev),
+                                isa->iobase, &s->io);
     return 0;
 }
 
@@ -102,8 +119,8 @@ static void debugcon_isa_class_initfn(ObjectClass *klass, void *data)
     dc->props = debugcon_isa_properties;
 }
 
-static TypeInfo debugcon_isa_info = {
-    .name          = "isa-debugcon",
+static const TypeInfo debugcon_isa_info = {
+    .name          = TYPE_ISA_DEBUGCON_DEVICE,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISADebugconState),
     .class_init    = debugcon_isa_class_initfn,
diff --git a/hw/debugexit.c b/hw/debugexit.c
new file mode 100644
index 00000000000..c1b489ddcb8
--- /dev/null
+++ b/hw/debugexit.c
@@ -0,0 +1,75 @@
+/*
+ * debug exit port emulation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) any later version.
+ */
+
+#include "hw.h"
+#include "isa.h"
+
+#define TYPE_ISA_DEBUG_EXIT_DEVICE "isa-debug-exit"
+#define ISA_DEBUG_EXIT_DEVICE(obj) \
+     OBJECT_CHECK(ISADebugExitState, (obj), TYPE_ISA_DEBUG_EXIT_DEVICE)
+
+typedef struct ISADebugExitState {
+    ISADevice parent_obj;
+
+    uint32_t iobase;
+    uint32_t iosize;
+    MemoryRegion io;
+} ISADebugExitState;
+
+static void debug_exit_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned width)
+{
+    exit((val << 1) | 1);
+}
+
+static const MemoryRegionOps debug_exit_ops = {
+    .write = debug_exit_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int debug_exit_initfn(ISADevice *dev)
+{
+    ISADebugExitState *isa = ISA_DEBUG_EXIT_DEVICE(dev);
+
+    memory_region_init_io(&isa->io, &debug_exit_ops, isa,
+                          TYPE_ISA_DEBUG_EXIT_DEVICE, isa->iosize);
+    memory_region_add_subregion(isa_address_space_io(dev),
+                                isa->iobase, &isa->io);
+    return 0;
+}
+
+static Property debug_exit_properties[] = {
+    DEFINE_PROP_HEX32("iobase", ISADebugExitState, iobase, 0x501),
+    DEFINE_PROP_HEX32("iosize", ISADebugExitState, iosize, 0x02),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void debug_exit_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+    ic->init = debug_exit_initfn;
+    dc->props = debug_exit_properties;
+}
+
+static const TypeInfo debug_exit_info = {
+    .name          = TYPE_ISA_DEBUG_EXIT_DEVICE,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISADebugExitState),
+    .class_init    = debug_exit_class_initfn,
+};
+
+static void debug_exit_register_types(void)
+{
+    type_register_static(&debug_exit_info);
+}
+
+type_init(debug_exit_register_types)
diff --git a/hw/dec_pci.c b/hw/dec_pci.c
index c30ade38bde..ee3f4ca8344 100644
--- a/hw/dec_pci.c
+++ b/hw/dec_pci.c
@@ -25,10 +25,10 @@
 
 #include "dec_pci.h"
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "pci/pci_bridge.h"
+#include "pci/pci_bus.h"
 
 /* debug DEC */
 //#define DEBUG_DEC
diff --git a/hw/device-hotplug.c b/hw/device-hotplug.c
index 2bdc615b495..88da145a89e 100644
--- a/hw/device-hotplug.c
+++ b/hw/device-hotplug.c
@@ -24,11 +24,10 @@
 
 #include "hw.h"
 #include "boards.h"
-#include "net.h"
-#include "blockdev.h"
-#include "qemu-config.h"
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/blockdev.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 
 DriveInfo *add_init_drive(const char *optstr)
 {
@@ -39,7 +38,7 @@ DriveInfo *add_init_drive(const char *optstr)
     if (!opts)
         return NULL;
 
-    dinfo = drive_init(opts, current_machine->use_scsi);
+    dinfo = drive_init(opts, current_machine->block_default_type);
     if (!dinfo) {
         qemu_opts_del(opts);
         return NULL;
@@ -49,18 +48,16 @@ DriveInfo *add_init_drive(const char *optstr)
 }
 
 #if !defined(TARGET_I386)
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type)
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
 {
     /* On non-x86 we don't do PCI hotplug */
-    monitor_printf(mon, "Can't hot-add drive to type %d\n", type);
+    monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
     return -1;
 }
 #endif
 
 void drive_hot_add(Monitor *mon, const QDict *qdict)
 {
-    int type;
     DriveInfo *dinfo = NULL;
     const char *opts = qdict_get_str(qdict, "opts");
 
@@ -72,14 +69,13 @@ void drive_hot_add(Monitor *mon, const QDict *qdict)
         monitor_printf(mon, "Parameter addr not supported\n");
         goto err;
     }
-    type = dinfo->type;
 
-    switch (type) {
+    switch (dinfo->type) {
     case IF_NONE:
         monitor_printf(mon, "OK\n");
         break;
     default:
-        if (pci_drive_hot_add(mon, qdict, dinfo, type)) {
+        if (pci_drive_hot_add(mon, qdict, dinfo)) {
             goto err;
         }
     }
@@ -89,5 +85,4 @@ err:
     if (dinfo) {
         drive_put_ref(dinfo);
     }
-    return;
 }
diff --git a/hw/devices.h b/hw/devices.h
index 1a55c1e9053..c60bcabae33 100644
--- a/hw/devices.h
+++ b/hw/devices.h
@@ -1,6 +1,8 @@
 #ifndef QEMU_DEVICES_H
 #define QEMU_DEVICES_H
 
+#include "hw/irq.h"
+
 /* ??? Not all users of this file can include cpu-common.h.  */
 struct MemoryRegion;
 
diff --git a/hw/dma.c b/hw/dma.c
index 0a9322daa3d..5bdf4358e3d 100644
--- a/hw/dma.c
+++ b/hw/dma.c
@@ -23,6 +23,7 @@
  */
 #include "hw.h"
 #include "isa.h"
+#include "qemu/main-loop.h"
 
 /* #define DEBUG_DMA */
 
@@ -58,6 +59,8 @@ static struct dma_cont {
     int dshift;
     struct dma_regs regs[4];
     qemu_irq *cpu_request_exit;
+    MemoryRegion channel_io;
+    MemoryRegion cont_io;
 } dma_controllers[2];
 
 enum {
@@ -149,7 +152,7 @@ static inline int getff (struct dma_cont *d)
     return ff;
 }
 
-static uint32_t read_chan (void *opaque, uint32_t nport)
+static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
 {
     struct dma_cont *d = opaque;
     int ichan, nreg, iport, ff, val, dir;
@@ -171,7 +174,8 @@ static uint32_t read_chan (void *opaque, uint32_t nport)
     return (val >> (d->dshift + (ff << 3))) & 0xff;
 }
 
-static void write_chan (void *opaque, uint32_t nport, uint32_t data)
+static void write_chan(void *opaque, hwaddr nport, uint64_t data,
+                       unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, ichan, nreg;
@@ -189,22 +193,23 @@ static void write_chan (void *opaque, uint32_t nport, uint32_t data)
     }
 }
 
-static void write_cont (void *opaque, uint32_t nport, uint32_t data)
+static void write_cont(void *opaque, hwaddr nport, uint64_t data,
+                       unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, ichan = 0;
 
     iport = (nport >> d->dshift) & 0x0f;
     switch (iport) {
-    case 0x08:                  /* command */
+    case 0x00:                  /* command */
         if ((data != 0) && (data & CMD_NOT_SUPPORTED)) {
-            dolog ("command %#x not supported\n", data);
+            dolog("command %"PRIx64" not supported\n", data);
             return;
         }
         d->command = data;
         break;
 
-    case 0x09:
+    case 0x01:
         ichan = data & 3;
         if (data & 4) {
             d->status |= 1 << (ichan + 4);
@@ -216,7 +221,7 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
         DMA_run();
         break;
 
-    case 0x0a:                  /* single mask */
+    case 0x02:                  /* single mask */
         if (data & 4)
             d->mask |= 1 << (data & 3);
         else
@@ -224,7 +229,7 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
         DMA_run();
         break;
 
-    case 0x0b:                  /* mode */
+    case 0x03:                  /* mode */
         {
             ichan = data & 3;
 #ifdef DEBUG_DMA
@@ -243,23 +248,23 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
             break;
         }
 
-    case 0x0c:                  /* clear flip flop */
+    case 0x04:                  /* clear flip flop */
         d->flip_flop = 0;
         break;
 
-    case 0x0d:                  /* reset */
+    case 0x05:                  /* reset */
         d->flip_flop = 0;
         d->mask = ~0;
         d->status = 0;
         d->command = 0;
         break;
 
-    case 0x0e:                  /* clear mask for all channels */
+    case 0x06:                  /* clear mask for all channels */
         d->mask = 0;
         DMA_run();
         break;
 
-    case 0x0f:                  /* write mask for all channels */
+    case 0x07:                  /* write mask for all channels */
         d->mask = data;
         DMA_run();
         break;
@@ -277,18 +282,18 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
 #endif
 }
 
-static uint32_t read_cont (void *opaque, uint32_t nport)
+static uint64_t read_cont(void *opaque, hwaddr nport, unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, val;
 
     iport = (nport >> d->dshift) & 0x0f;
     switch (iport) {
-    case 0x08:                  /* status */
+    case 0x00:                  /* status */
         val = d->status;
         d->status &= 0xf0;
         break;
-    case 0x0f:                  /* mask */
+    case 0x01:                  /* mask */
         val = d->mask;
         break;
     default:
@@ -411,7 +416,7 @@ void DMA_register_channel (int nchan,
 int DMA_read_memory (int nchan, void *buf, int pos, int len)
 {
     struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
-    target_phys_addr_t addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
+    hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
         int i;
@@ -433,7 +438,7 @@ int DMA_read_memory (int nchan, void *buf, int pos, int len)
 int DMA_write_memory (int nchan, void *buf, int pos, int len)
 {
     struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
-    target_phys_addr_t addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
+    hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
         int i;
@@ -463,7 +468,7 @@ void DMA_schedule(int nchan)
 static void dma_reset(void *opaque)
 {
     struct dma_cont *d = opaque;
-    write_cont (d, (0x0d << d->dshift), 0);
+    write_cont(d, (0x05 << d->dshift), 0, 1);
 }
 
 static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
@@ -473,38 +478,68 @@ static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
     return dma_pos;
 }
 
+
+static const MemoryRegionOps channel_io_ops = {
+    .read = read_chan,
+    .write = write_chan,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+/* IOport from page_base */
+static const MemoryRegionPortio page_portio_list[] = {
+    { 0x01, 3, 1, .write = write_page, .read = read_page, },
+    { 0x07, 1, 1, .write = write_page, .read = read_page, },
+    PORTIO_END_OF_LIST(),
+};
+
+/* IOport from pageh_base */
+static const MemoryRegionPortio pageh_portio_list[] = {
+    { 0x01, 3, 1, .write = write_pageh, .read = read_pageh, },
+    { 0x07, 3, 1, .write = write_pageh, .read = read_pageh, },
+    PORTIO_END_OF_LIST(),
+};
+
+static const MemoryRegionOps cont_io_ops = {
+    .read = read_cont,
+    .write = write_cont,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 /* dshift = 0: 8 bit DMA, 1 = 16 bit DMA */
 static void dma_init2(struct dma_cont *d, int base, int dshift,
                       int page_base, int pageh_base,
                       qemu_irq *cpu_request_exit)
 {
-    static const int page_port_list[] = { 0x1, 0x2, 0x3, 0x7 };
     int i;
 
     d->dshift = dshift;
     d->cpu_request_exit = cpu_request_exit;
-    for (i = 0; i < 8; i++) {
-        register_ioport_write (base + (i << dshift), 1, 1, write_chan, d);
-        register_ioport_read (base + (i << dshift), 1, 1, read_chan, d);
-    }
-    for (i = 0; i < ARRAY_SIZE (page_port_list); i++) {
-        register_ioport_write (page_base + page_port_list[i], 1, 1,
-                               write_page, d);
-        register_ioport_read (page_base + page_port_list[i], 1, 1,
-                              read_page, d);
-        if (pageh_base >= 0) {
-            register_ioport_write (pageh_base + page_port_list[i], 1, 1,
-                                   write_pageh, d);
-            register_ioport_read (pageh_base + page_port_list[i], 1, 1,
-                                  read_pageh, d);
-        }
-    }
-    for (i = 0; i < 8; i++) {
-        register_ioport_write (base + ((i + 8) << dshift), 1, 1,
-                               write_cont, d);
-        register_ioport_read (base + ((i + 8) << dshift), 1, 1,
-                              read_cont, d);
+
+    memory_region_init_io(&d->channel_io, &channel_io_ops, d,
+                          "dma-chan", 8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(NULL),
+                                base, &d->channel_io);
+
+    isa_register_portio_list(NULL, page_base, page_portio_list, d,
+                             "dma-page");
+    if (pageh_base >= 0) {
+        isa_register_portio_list(NULL, pageh_base, pageh_portio_list, d,
+                                 "dma-pageh");
     }
+
+    memory_region_init_io(&d->cont_io, &cont_io_ops, d, "dma-cont",
+                          8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(NULL),
+                                base + (8 << d->dshift), &d->cont_io);
+
     qemu_register_reset(dma_reset, d);
     dma_reset(d);
     for (i = 0; i < ARRAY_SIZE (d->regs); ++i) {
diff --git a/hw/dp8393x.c b/hw/dp8393x.c
index 4fa6eccba49..b5014501df3 100644
--- a/hw/dp8393x.c
+++ b/hw/dp8393x.c
@@ -18,8 +18,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "net.h"
+#include "qemu/timer.h"
+#include "net/net.h"
 #include "mips.h"
 
 //#define DEBUG_SONIC
@@ -168,7 +168,7 @@ typedef struct dp8393xState {
     int loopback_packet;
 
     /* Memory access */
-    void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write);
+    void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
     void* mem_opaque;
 } dp8393xState;
 
@@ -603,7 +603,7 @@ static void dp8393x_watchdog(void *opaque)
     dp8393x_update_irq(s);
 }
 
-static uint32_t dp8393x_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readw(void *opaque, hwaddr addr)
 {
     dp8393xState *s = opaque;
     int reg;
@@ -616,13 +616,13 @@ static uint32_t dp8393x_readw(void *opaque, target_phys_addr_t addr)
     return read_register(s, reg);
 }
 
-static uint32_t dp8393x_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readb(void *opaque, hwaddr addr)
 {
     uint16_t v = dp8393x_readw(opaque, addr & ~0x1);
     return (v >> (8 * (addr & 0x1))) & 0xff;
 }
 
-static uint32_t dp8393x_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readl(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = dp8393x_readw(opaque, addr);
@@ -630,7 +630,7 @@ static uint32_t dp8393x_readl(void *opaque, target_phys_addr_t addr)
     return v;
 }
 
-static void dp8393x_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     dp8393xState *s = opaque;
     int reg;
@@ -644,7 +644,7 @@ static void dp8393x_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     write_register(s, reg, (uint16_t)val);
 }
 
-static void dp8393x_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint16_t old_val = dp8393x_readw(opaque, addr & ~0x1);
 
@@ -659,7 +659,7 @@ static void dp8393x_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
     dp8393x_writew(opaque, addr & ~0x1, val);
 }
 
-static void dp8393x_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     dp8393x_writew(opaque, addr, val & 0xffff);
     dp8393x_writew(opaque, addr + 2, (val >> 16) & 0xffff);
@@ -879,10 +879,10 @@ static NetClientInfo net_dp83932_info = {
     .cleanup = nic_cleanup,
 };
 
-void dp83932_init(NICInfo *nd, target_phys_addr_t base, int it_shift,
+void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
                   MemoryRegion *address_space,
                   qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write))
+                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write))
 {
     dp8393xState *s;
 
diff --git a/hw/ds1225y.c b/hw/ds1225y.c
index 2cd355bd0a2..a6219a7908c 100644
--- a/hw/ds1225y.c
+++ b/hw/ds1225y.c
@@ -34,7 +34,7 @@ typedef struct {
     uint8_t *contents;
 } NvRamState;
 
-static uint64_t nvram_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t nvram_read(void *opaque, hwaddr addr, unsigned size)
 {
     NvRamState *s = opaque;
     uint32_t val;
@@ -44,7 +44,7 @@ static uint64_t nvram_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return val;
 }
 
-static void nvram_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void nvram_write(void *opaque, hwaddr addr, uint64_t val,
                         unsigned size)
 {
     NvRamState *s = opaque;
@@ -150,7 +150,7 @@ static void nvram_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = nvram_sysbus_properties;
 }
 
-static TypeInfo nvram_sysbus_info = {
+static const TypeInfo nvram_sysbus_info = {
     .name          = "ds1225y",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusNvRamState),
diff --git a/hw/ds1338.c b/hw/ds1338.c
index d590d9c0078..379220638e3 100644
--- a/hw/ds1338.c
+++ b/hw/ds1338.c
@@ -12,39 +12,97 @@
 
 #include "i2c.h"
 
+/* Size of NVRAM including both the user-accessible area and the
+ * secondary register area.
+ */
+#define NVRAM_SIZE 64
+
+/* Flags definitions */
+#define SECONDS_CH 0x80
+#define HOURS_12   0x40
+#define HOURS_PM   0x20
+#define CTRL_OSF   0x20
+
 typedef struct {
     I2CSlave i2c;
-    time_t offset;
-    struct tm now;
-    uint8_t nvram[56];
-    int ptr;
-    int addr_byte;
+    int64_t offset;
+    uint8_t wday_offset;
+    uint8_t nvram[NVRAM_SIZE];
+    int32_t ptr;
+    bool addr_byte;
 } DS1338State;
 
+static const VMStateDescription vmstate_ds1338 = {
+    .name = "ds1338",
+    .version_id = 2,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_I2C_SLAVE(i2c, DS1338State),
+        VMSTATE_INT64(offset, DS1338State),
+        VMSTATE_UINT8_V(wday_offset, DS1338State, 2),
+        VMSTATE_UINT8_ARRAY(nvram, DS1338State, NVRAM_SIZE),
+        VMSTATE_INT32(ptr, DS1338State),
+        VMSTATE_BOOL(addr_byte, DS1338State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void capture_current_time(DS1338State *s)
+{
+    /* Capture the current time into the secondary registers
+     * which will be actually read by the data transfer operation.
+     */
+    struct tm now;
+    qemu_get_timedate(&now, s->offset);
+    s->nvram[0] = to_bcd(now.tm_sec);
+    s->nvram[1] = to_bcd(now.tm_min);
+    if (s->nvram[2] & HOURS_12) {
+        int tmp = now.tm_hour;
+        if (tmp == 0) {
+            tmp = 24;
+        }
+        if (tmp <= 12) {
+            s->nvram[2] = HOURS_12 | to_bcd(tmp);
+        } else {
+            s->nvram[2] = HOURS_12 | HOURS_PM | to_bcd(tmp - 12);
+        }
+    } else {
+        s->nvram[2] = to_bcd(now.tm_hour);
+    }
+    s->nvram[3] = (now.tm_wday + s->wday_offset) % 7 + 1;
+    s->nvram[4] = to_bcd(now.tm_mday);
+    s->nvram[5] = to_bcd(now.tm_mon + 1);
+    s->nvram[6] = to_bcd(now.tm_year - 100);
+}
+
+static void inc_regptr(DS1338State *s)
+{
+    /* The register pointer wraps around after 0x3F; wraparound
+     * causes the current time/date to be retransferred into
+     * the secondary registers.
+     */
+    s->ptr = (s->ptr + 1) & (NVRAM_SIZE - 1);
+    if (!s->ptr) {
+        capture_current_time(s);
+    }
+}
+
 static void ds1338_event(I2CSlave *i2c, enum i2c_event event)
 {
     DS1338State *s = FROM_I2C_SLAVE(DS1338State, i2c);
 
     switch (event) {
     case I2C_START_RECV:
-        qemu_get_timedate(&s->now, s->offset);
-        s->nvram[0] = to_bcd(s->now.tm_sec);
-        s->nvram[1] = to_bcd(s->now.tm_min);
-        if (s->nvram[2] & 0x40) {
-            s->nvram[2] = (to_bcd((s->now.tm_hour % 12)) + 1) | 0x40;
-            if (s->now.tm_hour >= 12) {
-                s->nvram[2] |= 0x20;
-            }
-        } else {
-            s->nvram[2] = to_bcd(s->now.tm_hour);
-        }
-        s->nvram[3] = to_bcd(s->now.tm_wday) + 1;
-        s->nvram[4] = to_bcd(s->now.tm_mday);
-        s->nvram[5] = to_bcd(s->now.tm_mon) + 1;
-        s->nvram[6] = to_bcd(s->now.tm_year - 100);
+        /* In h/w, capture happens on any START condition, not just a
+         * START_RECV, but there is no need to actually capture on
+         * START_SEND, because the guest can't get at that data
+         * without going through a START_RECV which would overwrite it.
+         */
+        capture_current_time(s);
         break;
     case I2C_START_SEND:
-        s->addr_byte = 1;
+        s->addr_byte = true;
         break;
     default:
         break;
@@ -57,7 +115,7 @@ static int ds1338_recv(I2CSlave *i2c)
     uint8_t res;
 
     res  = s->nvram[s->ptr];
-    s->ptr = (s->ptr + 1) & 0xff;
+    inc_regptr(s);
     return res;
 }
 
@@ -65,52 +123,71 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
 {
     DS1338State *s = FROM_I2C_SLAVE(DS1338State, i2c);
     if (s->addr_byte) {
-        s->ptr = data;
-        s->addr_byte = 0;
+        s->ptr = data & (NVRAM_SIZE - 1);
+        s->addr_byte = false;
         return 0;
     }
-    s->nvram[s->ptr - 8] = data;
-    if (data < 8) {
-        qemu_get_timedate(&s->now, s->offset);
-        switch(data) {
+    if (s->ptr < 7) {
+        /* Time register. */
+        struct tm now;
+        qemu_get_timedate(&now, s->offset);
+        switch(s->ptr) {
         case 0:
             /* TODO: Implement CH (stop) bit.  */
-            s->now.tm_sec = from_bcd(data & 0x7f);
+            now.tm_sec = from_bcd(data & 0x7f);
             break;
         case 1:
-            s->now.tm_min = from_bcd(data & 0x7f);
+            now.tm_min = from_bcd(data & 0x7f);
             break;
         case 2:
-            if (data & 0x40) {
-                if (data & 0x20) {
-                    data = from_bcd(data & 0x4f) + 11;
-                } else {
-                    data = from_bcd(data & 0x1f) - 1;
+            if (data & HOURS_12) {
+                int tmp = from_bcd(data & (HOURS_PM - 1));
+                if (data & HOURS_PM) {
+                    tmp += 12;
                 }
+                if (tmp == 24) {
+                    tmp = 0;
+                }
+                now.tm_hour = tmp;
             } else {
-                data = from_bcd(data);
+                now.tm_hour = from_bcd(data & (HOURS_12 - 1));
             }
-            s->now.tm_hour = data;
             break;
         case 3:
-            s->now.tm_wday = from_bcd(data & 7) - 1;
+            {
+                /* The day field is supposed to contain a value in
+                   the range 1-7. Otherwise behavior is undefined.
+                 */
+                int user_wday = (data & 7) - 1;
+                s->wday_offset = (user_wday - now.tm_wday + 7) % 7;
+            }
             break;
         case 4:
-            s->now.tm_mday = from_bcd(data & 0x3f);
+            now.tm_mday = from_bcd(data & 0x3f);
             break;
         case 5:
-            s->now.tm_mon = from_bcd(data & 0x1f) - 1;
+            now.tm_mon = from_bcd(data & 0x1f) - 1;
             break;
         case 6:
-            s->now.tm_year = from_bcd(data) + 100;
-            break;
-        case 7:
-            /* Control register. Currently ignored.  */
+            now.tm_year = from_bcd(data) + 100;
             break;
         }
-        s->offset = qemu_timedate_diff(&s->now);
+        s->offset = qemu_timedate_diff(&now);
+    } else if (s->ptr == 7) {
+        /* Control register. */
+
+        /* Ensure bits 2, 3 and 6 will read back as zero. */
+        data &= 0xB3;
+
+        /* Attempting to write the OSF flag to logic 1 leaves the
+           value unchanged. */
+        data = (data & ~CTRL_OSF) | (data & s->nvram[s->ptr] & CTRL_OSF);
+
+        s->nvram[s->ptr] = data;
+    } else {
+        s->nvram[s->ptr] = data;
     }
-    s->ptr = (s->ptr + 1) & 0xff;
+    inc_regptr(s);
     return 0;
 }
 
@@ -119,17 +196,32 @@ static int ds1338_init(I2CSlave *i2c)
     return 0;
 }
 
+static void ds1338_reset(DeviceState *dev)
+{
+    DS1338State *s = FROM_I2C_SLAVE(DS1338State, I2C_SLAVE_FROM_QDEV(dev));
+
+    /* The clock is running and synchronized with the host */
+    s->offset = 0;
+    s->wday_offset = 0;
+    memset(s->nvram, 0, NVRAM_SIZE);
+    s->ptr = 0;
+    s->addr_byte = false;
+}
+
 static void ds1338_class_init(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     I2CSlaveClass *k = I2C_SLAVE_CLASS(klass);
 
     k->init = ds1338_init;
     k->event = ds1338_event;
     k->recv = ds1338_recv;
     k->send = ds1338_send;
+    dc->reset = ds1338_reset;
+    dc->vmsd = &vmstate_ds1338;
 }
 
-static TypeInfo ds1338_info = {
+static const TypeInfo ds1338_info = {
     .name          = "ds1338",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(DS1338State),
diff --git a/hw/dummy_m68k.c b/hw/dummy_m68k.c
index 7cc7a99bfb0..3a88805d0f0 100644
--- a/hw/dummy_m68k.c
+++ b/hw/dummy_m68k.c
@@ -10,23 +10,23 @@
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
 
 /* Board init.  */
 
-static void dummy_m68k_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void dummy_m68k_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
 
     if (!cpu_model)
         cpu_model = "cfv4e";
@@ -73,6 +73,7 @@ static QEMUMachine dummy_m68k_machine = {
     .name = "dummy",
     .desc = "Dummy board",
     .init = dummy_m68k_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void dummy_m68k_machine_init(void)
diff --git a/hw/e1000.c b/hw/e1000.c
index ae8a6c5523e..ef06ca18941 100644
--- a/hw/e1000.c
+++ b/hw/e1000.c
@@ -26,12 +26,12 @@
 
 
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "loader.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include "e1000_hw.h"
 
@@ -59,6 +59,11 @@ static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 #define PNPMMIO_SIZE      0x20000
 #define MIN_BUF_SIZE      60 /* Min. octets in an ethernet frame sans FCS */
 
+/* this is the size past which hardware will drop packets when setting LPE=0 */
+#define MAXIMUM_ETHERNET_VLAN_SIZE 1522
+/* this is the size past which hardware will drop packets when setting LPE=1 */
+#define MAXIMUM_ETHERNET_LPE_SIZE 16384
+
 /*
  * HW models:
  *  E1000_DEV_ID_82540EM works with Windows and Linux
@@ -92,7 +97,6 @@ typedef struct E1000State_st {
 
     uint32_t rxbuf_size;
     uint32_t rxbuf_min_shift;
-    int check_rxov;
     struct e1000_tx {
         unsigned char header[256];
         unsigned char vlan_header[4];
@@ -162,6 +166,11 @@ static void
 set_phy_ctrl(E1000State *s, int index, uint16_t val)
 {
     if ((val & MII_CR_AUTO_NEG_EN) && (val & MII_CR_RESTART_AUTO_NEG)) {
+        /* no need auto-negotiation if link was down */
+        if (s->nic->nc.link_down) {
+            s->phy_reg[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
+            return;
+        }
         s->nic->nc.link_down = true;
         e1000_link_down(s);
         s->phy_reg[PHY_STATUS] &= ~MII_SR_AUTONEG_COMPLETE;
@@ -266,6 +275,8 @@ rxbufsize(uint32_t v)
 static void e1000_reset(void *opaque)
 {
     E1000State *d = opaque;
+    uint8_t *macaddr = d->conf.macaddr.a;
+    int i;
 
     qemu_del_timer(d->autoneg_timer);
     memset(d->phy_reg, 0, sizeof d->phy_reg);
@@ -278,6 +289,14 @@ static void e1000_reset(void *opaque)
     if (d->nic->nc.link_down) {
         e1000_link_down(d);
     }
+
+    /* Some guests expect pre-initialized RAH/RAL (AddrValid flag + MACaddr) */
+    d->mac_reg[RA] = 0;
+    d->mac_reg[RA + 1] = E1000_RAH_AV;
+    for (i = 0; i < 4; i++) {
+        d->mac_reg[RA] |= macaddr[i] << (8 * i);
+        d->mac_reg[RA + 1] |= (i < 2) ? macaddr[i + 4] << (8 * i) : 0;
+    }
 }
 
 static void
@@ -295,6 +314,7 @@ set_rx_control(E1000State *s, int index, uint32_t val)
     s->rxbuf_min_shift = ((val / E1000_RCTL_RDMTS_QUAT) & 3) + 1;
     DBGOUT(RX, "RCTL: %d, mac_reg[RCTL] = 0x%x\n", s->mac_reg[RDT],
            s->mac_reg[RCTL]);
+    qemu_flush_queued_packets(&s->nic->nc);
 }
 
 static void
@@ -740,11 +760,11 @@ static bool e1000_has_rxbufs(E1000State *s, size_t total_size)
     int bufs;
     /* Fast-path short packets */
     if (total_size <= s->rxbuf_size) {
-        return s->mac_reg[RDH] != s->mac_reg[RDT] || !s->check_rxov;
+        return s->mac_reg[RDH] != s->mac_reg[RDT];
     }
     if (s->mac_reg[RDH] < s->mac_reg[RDT]) {
         bufs = s->mac_reg[RDT] - s->mac_reg[RDH];
-    } else if (s->mac_reg[RDH] > s->mac_reg[RDT] || !s->check_rxov) {
+    } else if (s->mac_reg[RDH] > s->mac_reg[RDT]) {
         bufs = s->mac_reg[RDLEN] /  sizeof(struct e1000_rx_desc) +
             s->mac_reg[RDT] - s->mac_reg[RDH];
     } else {
@@ -795,6 +815,14 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         size = sizeof(min_buf);
     }
 
+    /* Discard oversized packets if !LPE and !SBP. */
+    if ((size > MAXIMUM_ETHERNET_LPE_SIZE ||
+        (size > MAXIMUM_ETHERNET_VLAN_SIZE
+        && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)))
+        && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
+        return size;
+    }
+
     if (!receive_filter(s, buf, size))
         return size;
 
@@ -847,7 +875,6 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 
         if (++s->mac_reg[RDH] * sizeof(desc) >= s->mac_reg[RDLEN])
             s->mac_reg[RDH] = 0;
-        s->check_rxov = 1;
         /* see comment in start_xmit; same here */
         if (s->mac_reg[RDH] == rdh_start) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
@@ -924,8 +951,10 @@ mac_writereg(E1000State *s, int index, uint32_t val)
 static void
 set_rdt(E1000State *s, int index, uint32_t val)
 {
-    s->check_rxov = 0;
     s->mac_reg[index] = val & 0xffff;
+    if (e1000_has_rxbufs(s, 1)) {
+        qemu_flush_queued_packets(&s->nic->nc);
+    }
 }
 
 static void
@@ -1007,7 +1036,7 @@ static void (*macreg_writeops[])(E1000State *, int, uint32_t) = {
 enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) };
 
 static void
-e1000_mmio_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned size)
 {
     E1000State *s = opaque;
@@ -1024,7 +1053,7 @@ e1000_mmio_write(void *opaque, target_phys_addr_t addr, uint64_t val,
 }
 
 static uint64_t
-e1000_mmio_read(void *opaque, target_phys_addr_t addr, unsigned size)
+e1000_mmio_read(void *opaque, hwaddr addr, unsigned size)
 {
     E1000State *s = opaque;
     unsigned int index = (addr & 0x1ffff) >> 2;
@@ -1047,7 +1076,7 @@ static const MemoryRegionOps e1000_mmio_ops = {
     },
 };
 
-static uint64_t e1000_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t e1000_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     E1000State *s = opaque;
@@ -1056,7 +1085,7 @@ static uint64_t e1000_io_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void e1000_io_write(void *opaque, target_phys_addr_t addr,
+static void e1000_io_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     E1000State *s = opaque;
@@ -1075,11 +1104,23 @@ static bool is_version_1(void *opaque, int version_id)
     return version_id == 1;
 }
 
+static int e1000_post_load(void *opaque, int version_id)
+{
+    E1000State *s = opaque;
+
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in mac_reg[STATUS] */
+    s->nic->nc.link_down = (s->mac_reg[STATUS] & E1000_STATUS_LU) == 0;
+
+    return 0;
+}
+
 static const VMStateDescription vmstate_e1000 = {
     .name = "e1000",
     .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .post_load = e1000_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_PCI_DEVICE(dev, E1000State),
         VMSTATE_UNUSED_TEST(is_version_1, 4), /* was instance id */
@@ -1286,7 +1327,7 @@ static void e1000_class_init(ObjectClass *klass, void *data)
     dc->props = e1000_properties;
 }
 
-static TypeInfo e1000_info = {
+static const TypeInfo e1000_info = {
     .name          = "e1000",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(E1000State),
diff --git a/hw/eccmemctl.c b/hw/eccmemctl.c
index fe1cd900075..dbac2c2bbcc 100644
--- a/hw/eccmemctl.c
+++ b/hw/eccmemctl.c
@@ -129,7 +129,7 @@ typedef struct ECCState {
     uint32_t version;
 } ECCState;
 
-static void ecc_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void ecc_mem_write(void *opaque, hwaddr addr, uint64_t val,
                           unsigned size)
 {
     ECCState *s = opaque;
@@ -172,7 +172,7 @@ static void ecc_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
     }
 }
 
-static uint64_t ecc_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ecc_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     ECCState *s = opaque;
@@ -229,7 +229,7 @@ static const MemoryRegionOps ecc_mem_ops = {
     },
 };
 
-static void ecc_diag_mem_write(void *opaque, target_phys_addr_t addr,
+static void ecc_diag_mem_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     ECCState *s = opaque;
@@ -238,7 +238,7 @@ static void ecc_diag_mem_write(void *opaque, target_phys_addr_t addr,
     s->diag[addr & ECC_DIAG_MASK] = val;
 }
 
-static uint64_t ecc_diag_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ecc_diag_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     ECCState *s = opaque;
@@ -324,7 +324,7 @@ static void ecc_class_init(ObjectClass *klass, void *data)
     dc->props = ecc_properties;
 }
 
-static TypeInfo ecc_info = {
+static const TypeInfo ecc_info = {
     .name          = "eccmemctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ECCState),
diff --git a/hw/eepro100.c b/hw/eepro100.c
index 50d117e35e8..6bbefb505fd 100644
--- a/hw/eepro100.c
+++ b/hw/eepro100.c
@@ -42,11 +42,11 @@
 
 #include <stddef.h>             /* offsetof */
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "eeprom93xx.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 /* QEMU sends frames smaller than 60 bytes to ethernet nics.
  * Such frames are rejected by real nics and their emulations.
@@ -1036,6 +1036,7 @@ static void eepro100_ru_command(EEPRO100State * s, uint8_t val)
         }
         set_ru_state(s, ru_ready);
         s->ru_offset = e100_read_reg4(s, SCBPointer);
+        qemu_flush_queued_packets(&s->nic->nc);
         TRACE(OTHER, logout("val=0x%02x (rx start)\n", val));
         break;
     case RX_RESUME:
@@ -1577,7 +1578,7 @@ static void eepro100_write4(EEPRO100State * s, uint32_t addr, uint32_t val)
     }
 }
 
-static uint64_t eepro100_read(void *opaque, target_phys_addr_t addr,
+static uint64_t eepro100_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     EEPRO100State *s = opaque;
@@ -1590,7 +1591,7 @@ static uint64_t eepro100_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void eepro100_write(void *opaque, target_phys_addr_t addr,
+static void eepro100_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     EEPRO100State *s = opaque;
@@ -1770,7 +1771,8 @@ static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
     if (rfd_command & COMMAND_EL) {
         /* EL bit is set, so this was the last frame. */
         logout("receive: Running out of frames\n");
-        set_ru_state(s, ru_suspended);
+        set_ru_state(s, ru_no_resources);
+        eepro100_rnr_interrupt(s);
     }
     if (rfd_command & COMMAND_S) {
         /* S bit is set. */
diff --git a/hw/elf_ops.h b/hw/elf_ops.h
index fa65ce2f930..531a42553bf 100644
--- a/hw/elf_ops.h
+++ b/hw/elf_ops.h
@@ -62,7 +62,7 @@ static struct elf_shdr *glue(find_section, SZ)(struct elf_shdr *shdr_table,
 
 static int glue(symfind, SZ)(const void *s0, const void *s1)
 {
-    target_phys_addr_t addr = *(target_phys_addr_t *)s0;
+    hwaddr addr = *(hwaddr *)s0;
     struct elf_sym *sym = (struct elf_sym *)s1;
     int result = 0;
     if (addr < sym->st_value) {
@@ -74,7 +74,7 @@ static int glue(symfind, SZ)(const void *s0, const void *s1)
 }
 
 static const char *glue(lookup_symbol, SZ)(struct syminfo *s,
-                                           target_phys_addr_t orig_addr)
+                                           hwaddr orig_addr)
 {
     struct elf_sym *syms = glue(s->disas_symtab.elf, SZ);
     struct elf_sym *sym;
@@ -269,6 +269,17 @@ static int glue(load_elf, SZ)(const char *name, int fd,
                 addr = ph->p_paddr;
             }
 
+            /* the entry pointer in the ELF header is a virtual
+             * address, if the text segments paddr and vaddr differ
+             * we need to adjust the entry */
+            if (pentry && !translate_fn &&
+                    ph->p_vaddr != ph->p_paddr &&
+                    ehdr.e_entry >= ph->p_vaddr &&
+                    ehdr.e_entry < ph->p_vaddr + ph->p_filesz &&
+                    ph->p_flags & PF_X) {
+                *pentry = ehdr.e_entry - ph->p_vaddr + ph->p_paddr;
+            }
+
             snprintf(label, sizeof(label), "phdr #%d: %s", i, name);
             rom_add_blob_fixed(label, data, mem_size, addr);
 
diff --git a/hw/empty_slot.c b/hw/empty_slot.c
index 099c85e5833..d7b54973a42 100644
--- a/hw/empty_slot.c
+++ b/hw/empty_slot.c
@@ -28,14 +28,14 @@ typedef struct EmptySlot {
     uint64_t size;
 } EmptySlot;
 
-static uint64_t empty_slot_read(void *opaque, target_phys_addr_t addr,
+static uint64_t empty_slot_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     DPRINTF("read from " TARGET_FMT_plx "\n", addr);
     return 0;
 }
 
-static void empty_slot_write(void *opaque, target_phys_addr_t addr,
+static void empty_slot_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     DPRINTF("write 0x%x to " TARGET_FMT_plx "\n", (unsigned)val, addr);
@@ -47,7 +47,7 @@ static const MemoryRegionOps empty_slot_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void empty_slot_init(target_phys_addr_t addr, uint64_t slot_size)
+void empty_slot_init(hwaddr addr, uint64_t slot_size)
 {
     if (slot_size > 0) {
         /* Only empty slots larger than 0 byte need handling. */
@@ -56,7 +56,7 @@ void empty_slot_init(target_phys_addr_t addr, uint64_t slot_size)
         EmptySlot *e;
 
         dev = qdev_create(NULL, "empty_slot");
-        s = sysbus_from_qdev(dev);
+        s = SYS_BUS_DEVICE(dev);
         e = FROM_SYSBUS(EmptySlot, s);
         e->size = slot_size;
 
@@ -83,7 +83,7 @@ static void empty_slot_class_init(ObjectClass *klass, void *data)
     k->init = empty_slot_init1;
 }
 
-static TypeInfo empty_slot_info = {
+static const TypeInfo empty_slot_info = {
     .name          = "empty_slot",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(EmptySlot),
diff --git a/hw/empty_slot.h b/hw/empty_slot.h
index 78dc91de102..6079602cdf5 100644
--- a/hw/empty_slot.h
+++ b/hw/empty_slot.h
@@ -1,2 +1,7 @@
+#ifndef HW_EMPTY_SLOT_H
+#define HW_EMPTY_SLOT_H 1
+
 /* empty_slot.c */
-void empty_slot_init(target_phys_addr_t addr, uint64_t slot_size);
+void empty_slot_init(hwaddr addr, uint64_t slot_size);
+
+#endif
diff --git a/hw/es1370.c b/hw/es1370.c
index e34234c3502..977d2e37676 100644
--- a/hw/es1370.c
+++ b/hw/es1370.c
@@ -29,8 +29,8 @@
 #include "hw.h"
 #include "audiodev.h"
 #include "audio/audio.h"
-#include "pci.h"
-#include "dma.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
 
 /* Missing stuff:
    SCTRL_P[12](END|ST)INC
@@ -908,18 +908,44 @@ static void es1370_adc_callback (void *opaque, int avail)
     es1370_run_channel (s, ADC_CHANNEL, avail);
 }
 
-static const MemoryRegionPortio es1370_portio[] = {
-    { 0, 0x40 * 4, 1, .write = es1370_writeb, },
-    { 0, 0x40 * 2, 2, .write = es1370_writew, },
-    { 0, 0x40, 4, .write = es1370_writel, },
-    { 0, 0x40 * 4, 1, .read = es1370_readb, },
-    { 0, 0x40 * 2, 2, .read = es1370_readw, },
-    { 0, 0x40, 4, .read = es1370_readl, },
-    PORTIO_END_OF_LIST ()
-};
+static uint64_t es1370_read(void *opaque, hwaddr addr,
+                            unsigned size)
+{
+    switch (size) {
+    case 1:
+        return es1370_readb(opaque, addr);
+    case 2:
+        return es1370_readw(opaque, addr);
+    case 4:
+        return es1370_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void es1370_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    switch (size) {
+    case 1:
+        es1370_writeb(opaque, addr, val);
+        break;
+    case 2:
+        es1370_writew(opaque, addr, val);
+        break;
+    case 4:
+        es1370_writel(opaque, addr, val);
+        break;
+    }
+}
 
 static const MemoryRegionOps es1370_io_ops = {
-    .old_portio = es1370_portio,
+    .read = es1370_read,
+    .write = es1370_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
@@ -1047,7 +1073,7 @@ static void es1370_class_init (ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_es1370;
 }
 
-static TypeInfo es1370_info = {
+static const TypeInfo es1370_info = {
     .name          = "ES1370",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (ES1370State),
diff --git a/hw/escc.c b/hw/escc.c
index e1f5e73ba2c..18c02921e30 100644
--- a/hw/escc.c
+++ b/hw/escc.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "escc.h"
-#include "qemu-char.h"
-#include "console.h"
+#include "char/char.h"
+#include "ui/console.h"
 #include "trace.h"
 
 /*
@@ -463,7 +463,7 @@ static void escc_update_parameters(ChannelState *s)
     qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
 }
 
-static void escc_mem_write(void *opaque, target_phys_addr_t addr,
+static void escc_mem_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     SerialState *serial = opaque;
@@ -565,7 +565,7 @@ static void escc_mem_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t escc_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t escc_mem_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     SerialState *serial = opaque;
@@ -683,7 +683,7 @@ static const VMStateDescription vmstate_escc = {
     }
 };
 
-MemoryRegion *escc_init(target_phys_addr_t base, qemu_irq irqA, qemu_irq irqB,
+MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
               CharDriverState *chrA, CharDriverState *chrB,
               int clock, int it_shift)
 {
@@ -700,7 +700,7 @@ MemoryRegion *escc_init(target_phys_addr_t base, qemu_irq irqA, qemu_irq irqB,
     qdev_prop_set_uint32(dev, "chnBtype", ser);
     qdev_prop_set_uint32(dev, "chnAtype", ser);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irqB);
     sysbus_connect_irq(s, 1, irqA);
     if (base) {
@@ -846,7 +846,7 @@ static void sunmouse_event(void *opaque,
     put_queue(s, 0);
 }
 
-void slavio_serial_ms_kbd_init(target_phys_addr_t base, qemu_irq irq,
+void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
                                int disabled, int clock, int it_shift)
 {
     DeviceState *dev;
@@ -861,7 +861,7 @@ void slavio_serial_ms_kbd_init(target_phys_addr_t base, qemu_irq irq,
     qdev_prop_set_uint32(dev, "chnBtype", mouse);
     qdev_prop_set_uint32(dev, "chnAtype", kbd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_connect_irq(s, 1, irq);
     sysbus_mmio_map(s, 0, base);
@@ -923,7 +923,7 @@ static void escc_class_init(ObjectClass *klass, void *data)
     dc->props = escc_properties;
 }
 
-static TypeInfo escc_info = {
+static const TypeInfo escc_info = {
     .name          = "escc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SerialState),
diff --git a/hw/escc.h b/hw/escc.h
index d1da46fd06c..bda32133179 100644
--- a/hw/escc.h
+++ b/hw/escc.h
@@ -1,8 +1,13 @@
+#ifndef HW_ESCC_H
+#define HW_ESCC_H 1
+
 /* escc.c */
 #define ESCC_SIZE 4
-MemoryRegion *escc_init(target_phys_addr_t base, qemu_irq irqA, qemu_irq irqB,
+MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
               CharDriverState *chrA, CharDriverState *chrB,
               int clock, int it_shift);
 
-void slavio_serial_ms_kbd_init(target_phys_addr_t base, qemu_irq irq,
+void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
                                int disabled, int clock, int it_shift);
+
+#endif
diff --git a/hw/esp-pci.c b/hw/esp-pci.c
index 170e007be9f..c949e6e0d9e 100644
--- a/hw/esp-pci.c
+++ b/hw/esp-pci.c
@@ -23,11 +23,11 @@
  * THE SOFTWARE.
  */
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "eeprom93xx.h"
 #include "esp.h"
 #include "trace.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #define TYPE_AM53C974_DEVICE "am53c974"
 
@@ -159,7 +159,7 @@ static uint32_t esp_pci_dma_read(PCIESPState *pci, uint32_t saddr)
     return val;
 }
 
-static void esp_pci_io_write(void *opaque, target_phys_addr_t addr,
+static void esp_pci_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned int size)
 {
     PCIESPState *pci = opaque;
@@ -202,7 +202,7 @@ static void esp_pci_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t esp_pci_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t esp_pci_io_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     PCIESPState *pci = opaque;
diff --git a/hw/esp.c b/hw/esp.c
index 84a4e74e253..2af48aac4bb 100644
--- a/hw/esp.c
+++ b/hw/esp.c
@@ -26,7 +26,7 @@
 #include "sysbus.h"
 #include "esp.h"
 #include "trace.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 /*
  * On Sparc32, this is the ESP (NCR53C90) part of chip STP2000 (Master I/O),
@@ -550,7 +550,7 @@ void esp_reg_write(ESPState *s, uint32_t saddr, uint64_t val)
     s->wregs[saddr] = val;
 }
 
-static bool esp_mem_accepts(void *opaque, target_phys_addr_t addr,
+static bool esp_mem_accepts(void *opaque, hwaddr addr,
                             unsigned size, bool is_write)
 {
     return (size == 1) || (is_write && size == 4);
@@ -585,7 +585,7 @@ typedef struct {
     ESPState esp;
 } SysBusESPState;
 
-static void sysbus_esp_mem_write(void *opaque, target_phys_addr_t addr,
+static void sysbus_esp_mem_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned int size)
 {
     SysBusESPState *sysbus = opaque;
@@ -595,7 +595,7 @@ static void sysbus_esp_mem_write(void *opaque, target_phys_addr_t addr,
     esp_reg_write(&sysbus->esp, saddr, val);
 }
 
-static uint64_t sysbus_esp_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sysbus_esp_mem_read(void *opaque, hwaddr addr,
                                     unsigned int size)
 {
     SysBusESPState *sysbus = opaque;
@@ -612,7 +612,7 @@ static const MemoryRegionOps sysbus_esp_mem_ops = {
     .valid.accepts = esp_mem_accepts,
 };
 
-void esp_init(target_phys_addr_t espaddr, int it_shift,
+void esp_init(hwaddr espaddr, int it_shift,
               ESPDMAMemoryReadWriteFunc dma_memory_read,
               ESPDMAMemoryReadWriteFunc dma_memory_write,
               void *dma_opaque, qemu_irq irq, qemu_irq *reset,
@@ -633,7 +633,7 @@ void esp_init(target_phys_addr_t espaddr, int it_shift,
     /* XXX for now until rc4030 has been changed to use DMA enable signal */
     esp->dma_enabled = 1;
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, espaddr);
     *reset = qdev_get_gpio_in(dev, 0);
diff --git a/hw/esp.h b/hw/esp.h
index fa855e2fdf3..f15cc7b5bdb 100644
--- a/hw/esp.h
+++ b/hw/esp.h
@@ -6,7 +6,7 @@
 /* esp.c */
 #define ESP_MAX_DEVS 7
 typedef void (*ESPDMAMemoryReadWriteFunc)(void *opaque, uint8_t *buf, int len);
-void esp_init(target_phys_addr_t espaddr, int it_shift,
+void esp_init(hwaddr espaddr, int it_shift,
               ESPDMAMemoryReadWriteFunc dma_memory_read,
               ESPDMAMemoryReadWriteFunc dma_memory_write,
               void *dma_opaque, qemu_irq irq, qemu_irq *reset,
diff --git a/hw/etraxfs.h b/hw/etraxfs.h
index c62f94b7b79..180de5a0887 100644
--- a/hw/etraxfs.h
+++ b/hw/etraxfs.h
@@ -22,14 +22,17 @@
  * THE SOFTWARE.
  */
 
-#include "net.h"
+#ifndef HW_EXTRAXFS_H
+#define HW_EXTRAXFS_H 1
+
+#include "net/net.h"
 #include "etraxfs_dma.h"
 
 qemu_irq *cris_pic_init_cpu(CPUCRISState *env);
 
 /* Instantiate an ETRAXFS Ethernet MAC.  */
 static inline DeviceState *
-etraxfs_eth_init(NICInfo *nd, target_phys_addr_t base, int phyaddr,
+etraxfs_eth_init(NICInfo *nd, hwaddr base, int phyaddr,
                  void *dma_out, void *dma_in)
 {
     DeviceState *dev;
@@ -41,6 +44,8 @@ etraxfs_eth_init(NICInfo *nd, target_phys_addr_t base, int phyaddr,
     qdev_prop_set_ptr(dev, "dma_out", dma_out);
     qdev_prop_set_ptr(dev, "dma_in", dma_in);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     return dev;
 }
+
+#endif
diff --git a/hw/etraxfs_dma.c b/hw/etraxfs_dma.c
index 332525cab18..d41500316fb 100644
--- a/hw/etraxfs_dma.c
+++ b/hw/etraxfs_dma.c
@@ -24,9 +24,9 @@
 #include <stdio.h>
 #include <sys/time.h>
 #include "hw.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #include "etraxfs_dma.h"
 
@@ -212,7 +212,7 @@ static inline int channel_en(struct fs_dma_ctrl *ctrl, int c)
 		&& ctrl->channels[c].client;
 }
 
-static inline int fs_channel(target_phys_addr_t addr)
+static inline int fs_channel(hwaddr addr)
 {
 	/* Every channel has a 0x2000 ctrl register map.  */
 	return addr >> 13;
@@ -221,7 +221,7 @@ static inline int fs_channel(target_phys_addr_t addr)
 #ifdef USE_THIS_DEAD_CODE
 static void channel_load_g(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	cpu_physical_memory_read (addr, 
@@ -253,7 +253,7 @@ static void dump_d(int ch, struct dma_descr_data *d)
 
 static void channel_load_c(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	cpu_physical_memory_read (addr, 
@@ -270,7 +270,7 @@ static void channel_load_c(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_load_d(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_SAVED_DATA);
+	hwaddr addr = channel_reg(ctrl, c, RW_SAVED_DATA);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -284,7 +284,7 @@ static void channel_load_d(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_store_c(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
 
 	/* Encode and store. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -296,7 +296,7 @@ static void channel_store_c(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_store_d(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_SAVED_DATA);
+	hwaddr addr = channel_reg(ctrl, c, RW_SAVED_DATA);
 
 	/* Encode and store. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -573,14 +573,14 @@ static inline int channel_in_run(struct fs_dma_ctrl *ctrl, int c)
 		return 0;
 }
 
-static uint32_t dma_rinvalid (void *opaque, target_phys_addr_t addr)
+static uint32_t dma_rinvalid (void *opaque, hwaddr addr)
 {
         hw_error("Unsupported short raccess. reg=" TARGET_FMT_plx "\n", addr);
         return 0;
 }
 
 static uint64_t
-dma_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+dma_read(void *opaque, hwaddr addr, unsigned int size)
 {
         struct fs_dma_ctrl *ctrl = opaque;
 	int c;
@@ -612,7 +612,7 @@ dma_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-dma_winvalid (void *opaque, target_phys_addr_t addr, uint32_t value)
+dma_winvalid (void *opaque, hwaddr addr, uint32_t value)
 {
         hw_error("Unsupported short waccess. reg=" TARGET_FMT_plx "\n", addr);
 }
@@ -627,7 +627,7 @@ dma_update_state(struct fs_dma_ctrl *ctrl, int c)
 }
 
 static void
-dma_write(void *opaque, target_phys_addr_t addr,
+dma_write(void *opaque, hwaddr addr,
 	  uint64_t val64, unsigned int size)
 {
         struct fs_dma_ctrl *ctrl = opaque;
@@ -762,7 +762,7 @@ static void DMA_run(void *opaque)
         qemu_bh_schedule_idle(etraxfs_dmac->bh);
 }
 
-void *etraxfs_dmac_init(target_phys_addr_t base, int nr_channels)
+void *etraxfs_dmac_init(hwaddr base, int nr_channels)
 {
 	struct fs_dma_ctrl *ctrl = NULL;
 
diff --git a/hw/etraxfs_dma.h b/hw/etraxfs_dma.h
index 021c52ae7d7..38104a67b53 100644
--- a/hw/etraxfs_dma.h
+++ b/hw/etraxfs_dma.h
@@ -1,3 +1,6 @@
+#ifndef HW_ETRAXFS_DMA_H
+#define HW_ETRAXFS_DMA_H 1
+
 struct dma_context_metadata {
 	/* data descriptor md */
 	uint16_t metadata;
@@ -20,10 +23,12 @@ struct etraxfs_dma_client
 	} client;
 };
 
-void *etraxfs_dmac_init(target_phys_addr_t base, int nr_channels);
+void *etraxfs_dmac_init(hwaddr base, int nr_channels);
 void etraxfs_dmac_connect(void *opaque, int channel, qemu_irq *line,
 			  int input);
 void etraxfs_dmac_connect_client(void *opaque, int c, 
 				 struct etraxfs_dma_client *cl);
 int etraxfs_dmac_input(struct etraxfs_dma_client *client, 
 		       void *buf, int len, int eop);
+
+#endif
diff --git a/hw/etraxfs_eth.c b/hw/etraxfs_eth.c
index b124f5bb3aa..0b474c08439 100644
--- a/hw/etraxfs_eth.c
+++ b/hw/etraxfs_eth.c
@@ -24,7 +24,7 @@
 
 #include <stdio.h>
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "etraxfs.h"
 
 #define D(x)
@@ -35,582 +35,592 @@
 #define ADVERTISE_100HALF       0x0080  /* Try for 100mbps half-duplex */
 #define ADVERTISE_100FULL       0x0100  /* Try for 100mbps full-duplex */
 
-/* 
- * The MDIO extensions in the TDK PHY model were reversed engineered from the 
+/*
+ * The MDIO extensions in the TDK PHY model were reversed engineered from the
  * linux driver (PHYID and Diagnostics reg).
  * TODO: Add friendly names for the register nums.
  */
 struct qemu_phy
 {
-	uint32_t regs[32];
+    uint32_t regs[32];
 
-	int link;
+    int link;
 
-	unsigned int (*read)(struct qemu_phy *phy, unsigned int req);
-	void (*write)(struct qemu_phy *phy, unsigned int req, 
-		      unsigned int data);
+    unsigned int (*read)(struct qemu_phy *phy, unsigned int req);
+    void (*write)(struct qemu_phy *phy, unsigned int req, unsigned int data);
 };
 
 static unsigned int tdk_read(struct qemu_phy *phy, unsigned int req)
 {
-	int regnum;
-	unsigned r = 0;
-
-	regnum = req & 0x1f;
-
-	switch (regnum) {
-		case 1:
-			if (!phy->link)
-				break;
-			/* MR1.	 */
-			/* Speeds and modes.  */
-			r |= (1 << 13) | (1 << 14);
-			r |= (1 << 11) | (1 << 12);
-			r |= (1 << 5); /* Autoneg complete.  */
-			r |= (1 << 3); /* Autoneg able.	 */
-			r |= (1 << 2); /* link.	 */
-			break;
-		case 5:
-			/* Link partner ability.
-			   We are kind; always agree with whatever best mode
-			   the guest advertises.  */
-			r = 1 << 14; /* Success.  */
-			/* Copy advertised modes.  */
-			r |= phy->regs[4] & (15 << 5);
-			/* Autoneg support.  */
-			r |= 1;
-			break;
-		case 18:
-		{
-			/* Diagnostics reg.  */
-			int duplex = 0;
-			int speed_100 = 0;
-
-			if (!phy->link)
-				break;
-
-			/* Are we advertising 100 half or 100 duplex ? */
-			speed_100 = !!(phy->regs[4] & ADVERTISE_100HALF);
-			speed_100 |= !!(phy->regs[4] & ADVERTISE_100FULL);
-
-			/* Are we advertising 10 duplex or 100 duplex ? */
-			duplex = !!(phy->regs[4] & ADVERTISE_100FULL);
-			duplex |= !!(phy->regs[4] & ADVERTISE_10FULL);
-			r = (speed_100 << 10) | (duplex << 11);
-		}
-		break;
-
-		default:
-			r = phy->regs[regnum];
-			break;
-	}
-	D(printf("\n%s %x = reg[%d]\n", __func__, r, regnum));
-	return r;
+    int regnum;
+    unsigned r = 0;
+
+    regnum = req & 0x1f;
+
+    switch (regnum) {
+    case 1:
+        if (!phy->link) {
+            break;
+        }
+        /* MR1.     */
+        /* Speeds and modes.  */
+        r |= (1 << 13) | (1 << 14);
+        r |= (1 << 11) | (1 << 12);
+        r |= (1 << 5); /* Autoneg complete.  */
+        r |= (1 << 3); /* Autoneg able.     */
+        r |= (1 << 2); /* link.     */
+        break;
+    case 5:
+        /* Link partner ability.
+           We are kind; always agree with whatever best mode
+           the guest advertises.  */
+        r = 1 << 14; /* Success.  */
+        /* Copy advertised modes.  */
+        r |= phy->regs[4] & (15 << 5);
+        /* Autoneg support.  */
+        r |= 1;
+        break;
+    case 18:
+    {
+        /* Diagnostics reg.  */
+        int duplex = 0;
+        int speed_100 = 0;
+
+        if (!phy->link) {
+            break;
+        }
+
+        /* Are we advertising 100 half or 100 duplex ? */
+        speed_100 = !!(phy->regs[4] & ADVERTISE_100HALF);
+        speed_100 |= !!(phy->regs[4] & ADVERTISE_100FULL);
+
+        /* Are we advertising 10 duplex or 100 duplex ? */
+        duplex = !!(phy->regs[4] & ADVERTISE_100FULL);
+        duplex |= !!(phy->regs[4] & ADVERTISE_10FULL);
+        r = (speed_100 << 10) | (duplex << 11);
+    }
+    break;
+
+    default:
+        r = phy->regs[regnum];
+        break;
+    }
+    D(printf("\n%s %x = reg[%d]\n", __func__, r, regnum));
+    return r;
 }
 
-static void 
+static void
 tdk_write(struct qemu_phy *phy, unsigned int req, unsigned int data)
 {
-	int regnum;
-
-	regnum = req & 0x1f;
-	D(printf("%s reg[%d] = %x\n", __func__, regnum, data));
-	switch (regnum) {
-		default:
-			phy->regs[regnum] = data;
-			break;
-	}
+    int regnum;
+
+    regnum = req & 0x1f;
+    D(printf("%s reg[%d] = %x\n", __func__, regnum, data));
+    switch (regnum) {
+    default:
+        phy->regs[regnum] = data;
+        break;
+    }
 }
 
-static void 
+static void
 tdk_init(struct qemu_phy *phy)
 {
-	phy->regs[0] = 0x3100;
-	/* PHY Id.  */
-	phy->regs[2] = 0x0300;
-	phy->regs[3] = 0xe400;
-	/* Autonegotiation advertisement reg.  */
-	phy->regs[4] = 0x01E1;
-	phy->link = 1;
-
-	phy->read = tdk_read;
-	phy->write = tdk_write;
+    phy->regs[0] = 0x3100;
+    /* PHY Id.  */
+    phy->regs[2] = 0x0300;
+    phy->regs[3] = 0xe400;
+    /* Autonegotiation advertisement reg.  */
+    phy->regs[4] = 0x01E1;
+    phy->link = 1;
+
+    phy->read = tdk_read;
+    phy->write = tdk_write;
 }
 
 struct qemu_mdio
 {
-	/* bus.	 */
-	int mdc;
-	int mdio;
-
-	/* decoder.  */
-	enum {
-		PREAMBLE,
-		SOF,
-		OPC,
-		ADDR,
-		REQ,
-		TURNAROUND,
-		DATA
-	} state;
-	unsigned int drive;
-
-	unsigned int cnt;
-	unsigned int addr;
-	unsigned int opc;
-	unsigned int req;
-	unsigned int data;
-
-	struct qemu_phy *devs[32];
+    /* bus.     */
+    int mdc;
+    int mdio;
+
+    /* decoder.  */
+    enum {
+        PREAMBLE,
+        SOF,
+        OPC,
+        ADDR,
+        REQ,
+        TURNAROUND,
+        DATA
+    } state;
+    unsigned int drive;
+
+    unsigned int cnt;
+    unsigned int addr;
+    unsigned int opc;
+    unsigned int req;
+    unsigned int data;
+
+    struct qemu_phy *devs[32];
 };
 
-static void 
+static void
 mdio_attach(struct qemu_mdio *bus, struct qemu_phy *phy, unsigned int addr)
 {
-	bus->devs[addr & 0x1f] = phy;
+    bus->devs[addr & 0x1f] = phy;
 }
 
 #ifdef USE_THIS_DEAD_CODE
-static void 
+static void
 mdio_detach(struct qemu_mdio *bus, struct qemu_phy *phy, unsigned int addr)
 {
-	bus->devs[addr & 0x1f] = NULL;	
+    bus->devs[addr & 0x1f] = NULL;
 }
 #endif
 
 static void mdio_read_req(struct qemu_mdio *bus)
 {
-	struct qemu_phy *phy;
-
-	phy = bus->devs[bus->addr];
-	if (phy && phy->read)
-		bus->data = phy->read(phy, bus->req);
-	else 
-		bus->data = 0xffff;
+    struct qemu_phy *phy;
+
+    phy = bus->devs[bus->addr];
+    if (phy && phy->read) {
+        bus->data = phy->read(phy, bus->req);
+    } else {
+        bus->data = 0xffff;
+    }
 }
 
 static void mdio_write_req(struct qemu_mdio *bus)
 {
-	struct qemu_phy *phy;
+    struct qemu_phy *phy;
 
-	phy = bus->devs[bus->addr];
-	if (phy && phy->write)
-		phy->write(phy, bus->req, bus->data);
+    phy = bus->devs[bus->addr];
+    if (phy && phy->write) {
+        phy->write(phy, bus->req, bus->data);
+    }
 }
 
 static void mdio_cycle(struct qemu_mdio *bus)
 {
-	bus->cnt++;
+    bus->cnt++;
 
-	D(printf("mdc=%d mdio=%d state=%d cnt=%d drv=%d\n",
-		bus->mdc, bus->mdio, bus->state, bus->cnt, bus->drive));
+    D(printf("mdc=%d mdio=%d state=%d cnt=%d drv=%d\n",
+        bus->mdc, bus->mdio, bus->state, bus->cnt, bus->drive));
 #if 0
-	if (bus->mdc)
-		printf("%d", bus->mdio);
+    if (bus->mdc) {
+        printf("%d", bus->mdio);
+    }
 #endif
-	switch (bus->state)
-	{
-		case PREAMBLE:
-			if (bus->mdc) {
-				if (bus->cnt >= (32 * 2) && !bus->mdio) {
-					bus->cnt = 0;
-					bus->state = SOF;
-					bus->data = 0;
-				}
-			}
-			break;
-		case SOF:
-			if (bus->mdc) {
-				if (bus->mdio != 1)
-					printf("WARNING: no SOF\n");
-				if (bus->cnt == 1*2) {
-					bus->cnt = 0;
-					bus->opc = 0;
-					bus->state = OPC;
-				}
-			}
-			break;
-		case OPC:
-			if (bus->mdc) {
-				bus->opc <<= 1;
-				bus->opc |= bus->mdio & 1;
-				if (bus->cnt == 2*2) {
-					bus->cnt = 0;
-					bus->addr = 0;
-					bus->state = ADDR;
-				}
-			}
-			break;
-		case ADDR:
-			if (bus->mdc) {
-				bus->addr <<= 1;
-				bus->addr |= bus->mdio & 1;
-
-				if (bus->cnt == 5*2) {
-					bus->cnt = 0;
-					bus->req = 0;
-					bus->state = REQ;
-				}
-			}
-			break;
-		case REQ:
-			if (bus->mdc) {
-				bus->req <<= 1;
-				bus->req |= bus->mdio & 1;
-				if (bus->cnt == 5*2) {
-					bus->cnt = 0;
-					bus->state = TURNAROUND;
-				}
-			}
-			break;
-		case TURNAROUND:
-			if (bus->mdc && bus->cnt == 2*2) {
-				bus->mdio = 0;
-				bus->cnt = 0;
-
-				if (bus->opc == 2) {
-					bus->drive = 1;
-					mdio_read_req(bus);
-					bus->mdio = bus->data & 1;
-				}
-				bus->state = DATA;
-			}
-			break;
-		case DATA:			
-			if (!bus->mdc) {
-				if (bus->drive) {
-					bus->mdio = !!(bus->data & (1 << 15));
-					bus->data <<= 1;
-				}
-			} else {
-				if (!bus->drive) {
-					bus->data <<= 1;
-					bus->data |= bus->mdio;
-				}
-				if (bus->cnt == 16 * 2) {
-					bus->cnt = 0;
-					bus->state = PREAMBLE;
-					if (!bus->drive)
-						mdio_write_req(bus);
-					bus->drive = 0;
-				}
-			}
-			break;
-		default:
-			break;
-	}
+    switch (bus->state) {
+    case PREAMBLE:
+        if (bus->mdc) {
+            if (bus->cnt >= (32 * 2) && !bus->mdio) {
+                bus->cnt = 0;
+                bus->state = SOF;
+                bus->data = 0;
+            }
+        }
+        break;
+    case SOF:
+        if (bus->mdc) {
+            if (bus->mdio != 1) {
+                printf("WARNING: no SOF\n");
+            }
+            if (bus->cnt == 1*2) {
+                bus->cnt = 0;
+                bus->opc = 0;
+                bus->state = OPC;
+            }
+        }
+        break;
+    case OPC:
+        if (bus->mdc) {
+            bus->opc <<= 1;
+            bus->opc |= bus->mdio & 1;
+            if (bus->cnt == 2*2) {
+                bus->cnt = 0;
+                bus->addr = 0;
+                bus->state = ADDR;
+            }
+        }
+        break;
+    case ADDR:
+        if (bus->mdc) {
+            bus->addr <<= 1;
+            bus->addr |= bus->mdio & 1;
+
+            if (bus->cnt == 5*2) {
+                bus->cnt = 0;
+                bus->req = 0;
+                bus->state = REQ;
+            }
+        }
+        break;
+    case REQ:
+        if (bus->mdc) {
+            bus->req <<= 1;
+            bus->req |= bus->mdio & 1;
+            if (bus->cnt == 5*2) {
+                bus->cnt = 0;
+                bus->state = TURNAROUND;
+            }
+        }
+        break;
+    case TURNAROUND:
+        if (bus->mdc && bus->cnt == 2*2) {
+            bus->mdio = 0;
+            bus->cnt = 0;
+
+            if (bus->opc == 2) {
+                bus->drive = 1;
+                mdio_read_req(bus);
+                bus->mdio = bus->data & 1;
+            }
+            bus->state = DATA;
+        }
+        break;
+    case DATA:
+        if (!bus->mdc) {
+            if (bus->drive) {
+                bus->mdio = !!(bus->data & (1 << 15));
+                bus->data <<= 1;
+            }
+        } else {
+            if (!bus->drive) {
+                bus->data <<= 1;
+                bus->data |= bus->mdio;
+            }
+            if (bus->cnt == 16 * 2) {
+                bus->cnt = 0;
+                bus->state = PREAMBLE;
+                if (!bus->drive) {
+                    mdio_write_req(bus);
+                }
+                bus->drive = 0;
+            }
+        }
+        break;
+    default:
+        break;
+    }
 }
 
 /* ETRAX-FS Ethernet MAC block starts here.  */
 
-#define RW_MA0_LO	  0x00
-#define RW_MA0_HI	  0x01
-#define RW_MA1_LO	  0x02
-#define RW_MA1_HI	  0x03
-#define RW_GA_LO	  0x04
-#define RW_GA_HI	  0x05
-#define RW_GEN_CTRL	  0x06
-#define RW_REC_CTRL	  0x07
-#define RW_TR_CTRL	  0x08
-#define RW_CLR_ERR	  0x09
-#define RW_MGM_CTRL	  0x0a
-#define R_STAT		  0x0b
-#define FS_ETH_MAX_REGS	  0x17
+#define RW_MA0_LO      0x00
+#define RW_MA0_HI      0x01
+#define RW_MA1_LO      0x02
+#define RW_MA1_HI      0x03
+#define RW_GA_LO      0x04
+#define RW_GA_HI      0x05
+#define RW_GEN_CTRL      0x06
+#define RW_REC_CTRL      0x07
+#define RW_TR_CTRL      0x08
+#define RW_CLR_ERR      0x09
+#define RW_MGM_CTRL      0x0a
+#define R_STAT          0x0b
+#define FS_ETH_MAX_REGS      0x17
 
 struct fs_eth
 {
-	SysBusDevice busdev;
-	MemoryRegion mmio;
-	NICState *nic;
-	NICConf conf;
-
-	/* Two addrs in the filter.  */
-	uint8_t macaddr[2][6];
-	uint32_t regs[FS_ETH_MAX_REGS];
-
-	union {
-		void *vdma_out;
-		struct etraxfs_dma_client *dma_out;
-	};
-	union {
-		void *vdma_in;
-		struct etraxfs_dma_client *dma_in;
-	};
-
-	/* MDIO bus.  */
-	struct qemu_mdio mdio_bus;
-	unsigned int phyaddr;
-	int duplex_mismatch;
-
-	/* PHY.	 */
-	struct qemu_phy phy;
+    SysBusDevice busdev;
+    MemoryRegion mmio;
+    NICState *nic;
+    NICConf conf;
+
+    /* Two addrs in the filter.  */
+    uint8_t macaddr[2][6];
+    uint32_t regs[FS_ETH_MAX_REGS];
+
+    union {
+        void *vdma_out;
+        struct etraxfs_dma_client *dma_out;
+    };
+    union {
+        void *vdma_in;
+        struct etraxfs_dma_client *dma_in;
+    };
+
+    /* MDIO bus.  */
+    struct qemu_mdio mdio_bus;
+    unsigned int phyaddr;
+    int duplex_mismatch;
+
+    /* PHY.     */
+    struct qemu_phy phy;
 };
 
 static void eth_validate_duplex(struct fs_eth *eth)
 {
-	struct qemu_phy *phy;
-	unsigned int phy_duplex;
-	unsigned int mac_duplex;
-	int new_mm = 0;
-
-	phy = eth->mdio_bus.devs[eth->phyaddr];
-	phy_duplex = !!(phy->read(phy, 18) & (1 << 11));
-	mac_duplex = !!(eth->regs[RW_REC_CTRL] & 128);
-
-	if (mac_duplex != phy_duplex)
-		new_mm = 1;
-
-	if (eth->regs[RW_GEN_CTRL] & 1) {
-		if (new_mm != eth->duplex_mismatch) {
-			if (new_mm)
-				printf("HW: WARNING "
-				       "ETH duplex mismatch MAC=%d PHY=%d\n",
-				       mac_duplex, phy_duplex);
-			else
-				printf("HW: ETH duplex ok.\n");
-		}
-		eth->duplex_mismatch = new_mm;
-	}
+    struct qemu_phy *phy;
+    unsigned int phy_duplex;
+    unsigned int mac_duplex;
+    int new_mm = 0;
+
+    phy = eth->mdio_bus.devs[eth->phyaddr];
+    phy_duplex = !!(phy->read(phy, 18) & (1 << 11));
+    mac_duplex = !!(eth->regs[RW_REC_CTRL] & 128);
+
+    if (mac_duplex != phy_duplex) {
+        new_mm = 1;
+    }
+
+    if (eth->regs[RW_GEN_CTRL] & 1) {
+        if (new_mm != eth->duplex_mismatch) {
+            if (new_mm) {
+                printf("HW: WARNING ETH duplex mismatch MAC=%d PHY=%d\n",
+                       mac_duplex, phy_duplex);
+            } else {
+                printf("HW: ETH duplex ok.\n");
+            }
+        }
+        eth->duplex_mismatch = new_mm;
+    }
 }
 
 static uint64_t
-eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+eth_read(void *opaque, hwaddr addr, unsigned int size)
 {
-	struct fs_eth *eth = opaque;
-	uint32_t r = 0;
-
-	addr >>= 2;
-
-	switch (addr) {
-		case R_STAT:
-			r = eth->mdio_bus.mdio & 1;
-			break;
-	default:
-		r = eth->regs[addr];
-		D(printf ("%s %x\n", __func__, addr * 4));
-		break;
-	}
-	return r;
+    struct fs_eth *eth = opaque;
+    uint32_t r = 0;
+
+    addr >>= 2;
+
+    switch (addr) {
+    case R_STAT:
+        r = eth->mdio_bus.mdio & 1;
+        break;
+    default:
+        r = eth->regs[addr];
+        D(printf("%s %x\n", __func__, addr * 4));
+        break;
+    }
+    return r;
 }
 
 static void eth_update_ma(struct fs_eth *eth, int ma)
 {
-	int reg;
-	int i = 0;
-
-	ma &= 1;
-
-	reg = RW_MA0_LO;
-	if (ma)
-		reg = RW_MA1_LO;
-
-	eth->macaddr[ma][i++] = eth->regs[reg];
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 8;
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 16;
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 24;
-	eth->macaddr[ma][i++] = eth->regs[reg + 1];
-	eth->macaddr[ma][i] = eth->regs[reg + 1] >> 8;
-
-	D(printf("set mac%d=%x.%x.%x.%x.%x.%x\n", ma,
-		 eth->macaddr[ma][0], eth->macaddr[ma][1],
-		 eth->macaddr[ma][2], eth->macaddr[ma][3],
-		 eth->macaddr[ma][4], eth->macaddr[ma][5]));
+    int reg;
+    int i = 0;
+
+    ma &= 1;
+
+    reg = RW_MA0_LO;
+    if (ma) {
+        reg = RW_MA1_LO;
+    }
+
+    eth->macaddr[ma][i++] = eth->regs[reg];
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 8;
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 16;
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 24;
+    eth->macaddr[ma][i++] = eth->regs[reg + 1];
+    eth->macaddr[ma][i] = eth->regs[reg + 1] >> 8;
+
+    D(printf("set mac%d=%x.%x.%x.%x.%x.%x\n", ma,
+             eth->macaddr[ma][0], eth->macaddr[ma][1],
+             eth->macaddr[ma][2], eth->macaddr[ma][3],
+             eth->macaddr[ma][4], eth->macaddr[ma][5]));
 }
 
 static void
-eth_write(void *opaque, target_phys_addr_t addr,
+eth_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
-	struct fs_eth *eth = opaque;
-	uint32_t value = val64;
-
-	addr >>= 2;
-	switch (addr)
-	{
-		case RW_MA0_LO:
-		case RW_MA0_HI:
-			eth->regs[addr] = value;
-			eth_update_ma(eth, 0);
-			break;
-		case RW_MA1_LO:
-		case RW_MA1_HI:
-			eth->regs[addr] = value;
-			eth_update_ma(eth, 1);
-			break;
-
-		case RW_MGM_CTRL:
-			/* Attach an MDIO/PHY abstraction.  */
-			if (value & 2)
-				eth->mdio_bus.mdio = value & 1;
-			if (eth->mdio_bus.mdc != (value & 4)) {
-				mdio_cycle(&eth->mdio_bus);
-				eth_validate_duplex(eth);
-			}
-			eth->mdio_bus.mdc = !!(value & 4);
-			eth->regs[addr] = value;
-			break;
-
-		case RW_REC_CTRL:
-			eth->regs[addr] = value;
-			eth_validate_duplex(eth);
-			break;
-
-		default:
-			eth->regs[addr] = value;
-			D(printf ("%s %x %x\n",
-				  __func__, addr, value));
-			break;
-	}
+    struct fs_eth *eth = opaque;
+    uint32_t value = val64;
+
+    addr >>= 2;
+    switch (addr) {
+    case RW_MA0_LO:
+    case RW_MA0_HI:
+        eth->regs[addr] = value;
+        eth_update_ma(eth, 0);
+        break;
+    case RW_MA1_LO:
+    case RW_MA1_HI:
+        eth->regs[addr] = value;
+        eth_update_ma(eth, 1);
+        break;
+
+    case RW_MGM_CTRL:
+        /* Attach an MDIO/PHY abstraction.  */
+        if (value & 2) {
+            eth->mdio_bus.mdio = value & 1;
+        }
+        if (eth->mdio_bus.mdc != (value & 4)) {
+            mdio_cycle(&eth->mdio_bus);
+            eth_validate_duplex(eth);
+        }
+        eth->mdio_bus.mdc = !!(value & 4);
+        eth->regs[addr] = value;
+        break;
+
+    case RW_REC_CTRL:
+        eth->regs[addr] = value;
+        eth_validate_duplex(eth);
+        break;
+
+    default:
+        eth->regs[addr] = value;
+        D(printf("%s %x %x\n", __func__, addr, value));
+        break;
+    }
 }
 
 /* The ETRAX FS has a groupt address table (GAT) which works like a k=1 bloom
-   filter dropping group addresses we have not joined.	The filter has 64
-   bits (m). The has function is a simple nible xor of the group addr.	*/
+   filter dropping group addresses we have not joined.    The filter has 64
+   bits (m). The has function is a simple nible xor of the group addr.    */
 static int eth_match_groupaddr(struct fs_eth *eth, const unsigned char *sa)
 {
-	unsigned int hsh;
-	int m_individual = eth->regs[RW_REC_CTRL] & 4;
-	int match;
-
-	/* First bit on the wire of a MAC address signals multicast or
-	   physical address.  */
-	if (!m_individual && !(sa[0] & 1))
-		return 0;
-
-	/* Calculate the hash index for the GA registers. */
-	hsh = 0;
-	hsh ^= (*sa) & 0x3f;
-	hsh ^= ((*sa) >> 6) & 0x03;
-	++sa;
-	hsh ^= ((*sa) << 2) & 0x03c;
-	hsh ^= ((*sa) >> 4) & 0xf;
-	++sa;
-	hsh ^= ((*sa) << 4) & 0x30;
-	hsh ^= ((*sa) >> 2) & 0x3f;
-	++sa;
-	hsh ^= (*sa) & 0x3f;
-	hsh ^= ((*sa) >> 6) & 0x03;
-	++sa;
-	hsh ^= ((*sa) << 2) & 0x03c;
-	hsh ^= ((*sa) >> 4) & 0xf;
-	++sa;
-	hsh ^= ((*sa) << 4) & 0x30;
-	hsh ^= ((*sa) >> 2) & 0x3f;
-
-	hsh &= 63;
-	if (hsh > 31)
-		match = eth->regs[RW_GA_HI] & (1 << (hsh - 32));
-	else
-		match = eth->regs[RW_GA_LO] & (1 << hsh);
-	D(printf("hsh=%x ga=%x.%x mtch=%d\n", hsh,
-		 eth->regs[RW_GA_HI], eth->regs[RW_GA_LO], match));
-	return match;
+    unsigned int hsh;
+    int m_individual = eth->regs[RW_REC_CTRL] & 4;
+    int match;
+
+    /* First bit on the wire of a MAC address signals multicast or
+       physical address.  */
+    if (!m_individual && !(sa[0] & 1)) {
+        return 0;
+    }
+
+    /* Calculate the hash index for the GA registers. */
+    hsh = 0;
+    hsh ^= (*sa) & 0x3f;
+    hsh ^= ((*sa) >> 6) & 0x03;
+    ++sa;
+    hsh ^= ((*sa) << 2) & 0x03c;
+    hsh ^= ((*sa) >> 4) & 0xf;
+    ++sa;
+    hsh ^= ((*sa) << 4) & 0x30;
+    hsh ^= ((*sa) >> 2) & 0x3f;
+    ++sa;
+    hsh ^= (*sa) & 0x3f;
+    hsh ^= ((*sa) >> 6) & 0x03;
+    ++sa;
+    hsh ^= ((*sa) << 2) & 0x03c;
+    hsh ^= ((*sa) >> 4) & 0xf;
+    ++sa;
+    hsh ^= ((*sa) << 4) & 0x30;
+    hsh ^= ((*sa) >> 2) & 0x3f;
+
+    hsh &= 63;
+    if (hsh > 31) {
+        match = eth->regs[RW_GA_HI] & (1 << (hsh - 32));
+    } else {
+        match = eth->regs[RW_GA_LO] & (1 << hsh);
+    }
+    D(printf("hsh=%x ga=%x.%x mtch=%d\n", hsh,
+             eth->regs[RW_GA_HI], eth->regs[RW_GA_LO], match));
+    return match;
 }
 
 static int eth_can_receive(NetClientState *nc)
 {
-	return 1;
+    return 1;
 }
 
 static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-	unsigned char sa_bcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
-	int use_ma0 = eth->regs[RW_REC_CTRL] & 1;
-	int use_ma1 = eth->regs[RW_REC_CTRL] & 2;
-	int r_bcast = eth->regs[RW_REC_CTRL] & 8;
-
-	if (size < 12)
-		return -1;
-
-	D(printf("%x.%x.%x.%x.%x.%x ma=%d %d bc=%d\n",
-		 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
-		 use_ma0, use_ma1, r_bcast));
-	       
-	/* Does the frame get through the address filters?  */
-	if ((!use_ma0 || memcmp(buf, eth->macaddr[0], 6))
-	    && (!use_ma1 || memcmp(buf, eth->macaddr[1], 6))
-	    && (!r_bcast || memcmp(buf, sa_bcast, 6))
-	    && !eth_match_groupaddr(eth, buf))
-		return size;
-
-	/* FIXME: Find another way to pass on the fake csum.  */
-	etraxfs_dmac_input(eth->dma_in, (void *)buf, size + 4, 1);
+    unsigned char sa_bcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+    struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
+    int use_ma0 = eth->regs[RW_REC_CTRL] & 1;
+    int use_ma1 = eth->regs[RW_REC_CTRL] & 2;
+    int r_bcast = eth->regs[RW_REC_CTRL] & 8;
+
+    if (size < 12) {
+        return -1;
+    }
+
+    D(printf("%x.%x.%x.%x.%x.%x ma=%d %d bc=%d\n",
+         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
+         use_ma0, use_ma1, r_bcast));
+
+    /* Does the frame get through the address filters?  */
+    if ((!use_ma0 || memcmp(buf, eth->macaddr[0], 6))
+        && (!use_ma1 || memcmp(buf, eth->macaddr[1], 6))
+        && (!r_bcast || memcmp(buf, sa_bcast, 6))
+        && !eth_match_groupaddr(eth, buf)) {
+        return size;
+    }
+
+    /* FIXME: Find another way to pass on the fake csum.  */
+    etraxfs_dmac_input(eth->dma_in, (void *)buf, size + 4, 1);
 
         return size;
 }
 
 static int eth_tx_push(void *opaque, unsigned char *buf, int len, bool eop)
 {
-	struct fs_eth *eth = opaque;
+    struct fs_eth *eth = opaque;
 
-	D(printf("%s buf=%p len=%d\n", __func__, buf, len));
-	qemu_send_packet(&eth->nic->nc, buf, len);
-	return len;
+    D(printf("%s buf=%p len=%d\n", __func__, buf, len));
+    qemu_send_packet(&eth->nic->nc, buf, len);
+    return len;
 }
 
 static void eth_set_link(NetClientState *nc)
 {
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
-	D(printf("%s %d\n", __func__, nc->link_down));
-	eth->phy.link = !nc->link_down;
+    struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
+    D(printf("%s %d\n", __func__, nc->link_down));
+    eth->phy.link = !nc->link_down;
 }
 
 static const MemoryRegionOps eth_ops = {
-	.read = eth_read,
-	.write = eth_write,
-	.endianness = DEVICE_LITTLE_ENDIAN,
-	.valid = {
-		.min_access_size = 4,
-		.max_access_size = 4
-	}
+    .read = eth_read,
+    .write = eth_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
 };
 
 static void eth_cleanup(NetClientState *nc)
 {
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
 
-	/* Disconnect the client.  */
-	eth->dma_out->client.push = NULL;
-	eth->dma_out->client.opaque = NULL;
-	eth->dma_in->client.opaque = NULL;
-	eth->dma_in->client.pull = NULL;
+    /* Disconnect the client.  */
+    eth->dma_out->client.push = NULL;
+    eth->dma_out->client.opaque = NULL;
+    eth->dma_in->client.opaque = NULL;
+    eth->dma_in->client.pull = NULL;
         g_free(eth);
 }
 
 static NetClientInfo net_etraxfs_info = {
-	.type = NET_CLIENT_OPTIONS_KIND_NIC,
-	.size = sizeof(NICState),
-	.can_receive = eth_can_receive,
-	.receive = eth_receive,
-	.cleanup = eth_cleanup,
-	.link_status_changed = eth_set_link,
+    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .size = sizeof(NICState),
+    .can_receive = eth_can_receive,
+    .receive = eth_receive,
+    .cleanup = eth_cleanup,
+    .link_status_changed = eth_set_link,
 };
 
 static int fs_eth_init(SysBusDevice *dev)
 {
-	struct fs_eth *s = FROM_SYSBUS(typeof(*s), dev);
+    struct fs_eth *s = FROM_SYSBUS(typeof(*s), dev);
 
-	if (!s->dma_out || !s->dma_in) {
-		hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
-	}
+    if (!s->dma_out || !s->dma_in) {
+        hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
+    }
 
-	s->dma_out->client.push = eth_tx_push;
-	s->dma_out->client.opaque = s;
-	s->dma_in->client.opaque = s;
-	s->dma_in->client.pull = NULL;
+    s->dma_out->client.push = eth_tx_push;
+    s->dma_out->client.opaque = s;
+    s->dma_in->client.opaque = s;
+    s->dma_in->client.pull = NULL;
 
-	memory_region_init_io(&s->mmio, &eth_ops, s, "etraxfs-eth", 0x5c);
-	sysbus_init_mmio(dev, &s->mmio);
+    memory_region_init_io(&s->mmio, &eth_ops, s, "etraxfs-eth", 0x5c);
+    sysbus_init_mmio(dev, &s->mmio);
 
-	qemu_macaddr_default_if_unset(&s->conf.macaddr);
-	s->nic = qemu_new_nic(&net_etraxfs_info, &s->conf,
-			      object_get_typename(OBJECT(s)), dev->qdev.id, s);
-	qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+    s->nic = qemu_new_nic(&net_etraxfs_info, &s->conf,
+                          object_get_typename(OBJECT(s)), dev->qdev.id, s);
+    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
 
-	tdk_init(&s->phy);
-	mdio_attach(&s->mdio_bus, &s->phy, s->phyaddr);
-	return 0;
+    tdk_init(&s->phy);
+    mdio_attach(&s->mdio_bus, &s->phy, s->phyaddr);
+    return 0;
 }
 
 static Property etraxfs_eth_properties[] = {
@@ -630,7 +640,7 @@ static void etraxfs_eth_class_init(ObjectClass *klass, void *data)
     dc->props = etraxfs_eth_properties;
 }
 
-static TypeInfo etraxfs_eth_info = {
+static const TypeInfo etraxfs_eth_info = {
     .name          = "etraxfs-eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct fs_eth),
diff --git a/hw/etraxfs_pic.c b/hw/etraxfs_pic.c
index dc27f88ac95..64af31c46e9 100644
--- a/hw/etraxfs_pic.c
+++ b/hw/etraxfs_pic.c
@@ -79,7 +79,7 @@ static void pic_update(struct etrax_pic *fs)
 }
 
 static uint64_t
-pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+pic_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_pic *fs = opaque;
     uint32_t rval;
@@ -89,7 +89,7 @@ pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
     return rval;
 }
 
-static void pic_write(void *opaque, target_phys_addr_t addr,
+static void pic_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned int size)
 {
     struct etrax_pic *fs = opaque;
@@ -165,7 +165,7 @@ static void etraxfs_pic_class_init(ObjectClass *klass, void *data)
     dc->props = etraxfs_pic_properties;
 }
 
-static TypeInfo etraxfs_pic_info = {
+static const TypeInfo etraxfs_pic_info = {
     .name          = "etraxfs,pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct etrax_pic),
diff --git a/hw/etraxfs_ser.c b/hw/etraxfs_ser.c
index 5f16b178355..72c8868639b 100644
--- a/hw/etraxfs_ser.c
+++ b/hw/etraxfs_ser.c
@@ -23,8 +23,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
+#include "char/char.h"
+#include "qemu/log.h"
 
 #define D(x)
 
@@ -75,7 +75,7 @@ static void ser_update_irq(struct etrax_serial *s)
 }
 
 static uint64_t
-ser_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+ser_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_serial *s = opaque;
     D(CPUCRISState *env = s->env);
@@ -110,7 +110,7 @@ ser_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-ser_write(void *opaque, target_phys_addr_t addr,
+ser_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct etrax_serial *s = opaque;
@@ -233,7 +233,7 @@ static void etraxfs_ser_class_init(ObjectClass *klass, void *data)
     dc->reset = etraxfs_ser_reset;
 }
 
-static TypeInfo etraxfs_ser_info = {
+static const TypeInfo etraxfs_ser_info = {
     .name          = "etraxfs,serial",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct etrax_serial),
diff --git a/hw/etraxfs_timer.c b/hw/etraxfs_timer.c
index 9076a49884e..d3dac52315c 100644
--- a/hw/etraxfs_timer.c
+++ b/hw/etraxfs_timer.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 
 #define D(x)
@@ -75,7 +75,7 @@ struct etrax_timer {
 };
 
 static uint64_t
-timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+timer_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_timer *t = opaque;
     uint32_t r = 0;
@@ -242,7 +242,7 @@ static inline void timer_watchdog_update(struct etrax_timer *t, uint32_t value)
 }
 
 static void
-timer_write(void *opaque, target_phys_addr_t addr,
+timer_write(void *opaque, hwaddr addr,
             uint64_t val64, unsigned int size)
 {
     struct etrax_timer *t = opaque;
@@ -336,7 +336,7 @@ static void etraxfs_timer_class_init(ObjectClass *klass, void *data)
     sdc->init = etraxfs_timer_init;
 }
 
-static TypeInfo etraxfs_timer_info = {
+static const TypeInfo etraxfs_timer_info = {
     .name          = "etraxfs,timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (struct etrax_timer),
diff --git a/hw/exynos4210.c b/hw/exynos4210.c
index 00d4db8871e..fa54e42a478 100644
--- a/hw/exynos4210.c
+++ b/hw/exynos4210.c
@@ -22,11 +22,12 @@
  */
 
 #include "boards.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "loader.h"
 #include "exynos4210.h"
+#include "usb/hcd-ehci.h"
 
 #define EXYNOS4210_CHIPID_ADDR         0x10000000
 
@@ -72,6 +73,9 @@
 /* Display controllers (FIMD) */
 #define EXYNOS4210_FIMD0_BASE_ADDR          0x11C00000
 
+/* EHCI */
+#define EXYNOS4210_EHCI_BASE_ADDR           0x12580000
+
 static uint8_t chipid_and_omr[] = { 0x11, 0x02, 0x21, 0x43,
                                     0x09, 0x00, 0x00, 0x00 };
 
@@ -80,12 +84,16 @@ void exynos4210_write_secondary(ARMCPU *cpu,
 {
     int n;
     uint32_t smpboot[] = {
-        0xe59f3024, /* ldr r3, External gic_cpu_if */
-        0xe59f2024, /* ldr r2, Internal gic_cpu_if */
-        0xe59f0024, /* ldr r0, startaddr */
+        0xe59f3034, /* ldr r3, External gic_cpu_if */
+        0xe59f2034, /* ldr r2, Internal gic_cpu_if */
+        0xe59f0034, /* ldr r0, startaddr */
         0xe3a01001, /* mov r1, #1 */
         0xe5821000, /* str r1, [r2] */
         0xe5831000, /* str r1, [r3] */
+        0xe3a010ff, /* mov r1, #0xff */
+        0xe5821004, /* str r1, [r2, #4] */
+        0xe5831004, /* str r1, [r3, #4] */
+        0xf57ff04f, /* dsb */
         0xe320f003, /* wfi */
         0xe5901000, /* ldr     r1, [r0] */
         0xe1110001, /* tst     r1, r1 */
@@ -146,7 +154,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
         for (n = 0; n < EXYNOS4210_IRQ_GATE_NINPUTS; n++) {
             gate_irq[i][n] = qdev_get_gpio_in(dev, n);
         }
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
 
         /* Connect IRQ Gate output to cpu_irq */
         sysbus_connect_irq(busdev, 0, cpu_irq[i]);
@@ -156,7 +164,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", EXYNOS4210_NCPUS);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, EXYNOS4210_SMP_PRIVATE_BASE_ADDR);
     for (n = 0; n < EXYNOS4210_NCPUS; n++) {
         sysbus_connect_irq(busdev, n, gate_irq[n][0]);
@@ -172,7 +180,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "exynos4210.gic");
     qdev_prop_set_uint32(dev, "num-cpu", EXYNOS4210_NCPUS);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     /* Map CPU interface */
     sysbus_mmio_map(busdev, 0, EXYNOS4210_EXT_GIC_CPU_BASE_ADDR);
     /* Map Distributer interface */
@@ -187,7 +195,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     /* Internal Interrupt Combiner */
     dev = qdev_create(NULL, "exynos4210.combiner");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < EXYNOS4210_MAX_INT_COMBINER_OUT_IRQ; n++) {
         sysbus_connect_irq(busdev, n, s->irqs.int_gic_irq[n]);
     }
@@ -198,7 +206,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "exynos4210.combiner");
     qdev_prop_set_uint32(dev, "external", 1);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < EXYNOS4210_MAX_INT_COMBINER_OUT_IRQ; n++) {
         sysbus_connect_irq(busdev, n, s->irqs.ext_gic_irq[n]);
     }
@@ -277,7 +285,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     /* Multi Core Timer */
     dev = qdev_create(NULL, "exynos4210.mct");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < 4; n++) {
         /* Connect global timer interrupts to Combiner gpio_in */
         sysbus_connect_irq(busdev, n,
@@ -303,7 +311,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
         dev = qdev_create(NULL, "exynos4210.i2c");
         qdev_init_nofail(dev);
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         sysbus_connect_irq(busdev, 0, i2c_irq);
         sysbus_mmio_map(busdev, 0, addr);
         s->i2c_if[n] = (i2c_bus *)qdev_get_child_bus(dev, "i2c");
@@ -334,5 +342,8 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
             s->irq_table[exynos4210_get_irq(11, 2)],
             NULL);
 
+    sysbus_create_simple(TYPE_EXYNOS4210_EHCI, EXYNOS4210_EHCI_BASE_ADDR,
+            s->irq_table[exynos4210_get_irq(28, 3)]);
+
     return s;
 }
diff --git a/hw/exynos4210.h b/hw/exynos4210.h
index a43ba3aedc9..bb9a1dddc8d 100644
--- a/hw/exynos4210.h
+++ b/hw/exynos4210.h
@@ -27,7 +27,7 @@
 #define EXYNOS4210_H_
 
 #include "qemu-common.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 #define EXYNOS4210_NCPUS                    2
 
@@ -128,7 +128,7 @@ void exynos4210_combiner_get_gpioin(Exynos4210Irq *irqs, DeviceState *dev,
 /*
  * exynos4210 UART
  */
-DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
+DeviceState *exynos4210_uart_create(hwaddr addr,
                                     int fifo_size,
                                     int channel,
                                     CharDriverState *chr,
diff --git a/hw/exynos4210_combiner.c b/hw/exynos4210_combiner.c
index 80af22cc33f..ba644b43c22 100644
--- a/hw/exynos4210_combiner.c
+++ b/hw/exynos4210_combiner.c
@@ -174,7 +174,7 @@ void exynos4210_combiner_get_gpioin(Exynos4210Irq *irqs, DeviceState *dev,
 }
 
 static uint64_t
-exynos4210_combiner_read(void *opaque, target_phys_addr_t offset, unsigned size)
+exynos4210_combiner_read(void *opaque, hwaddr offset, unsigned size)
 {
     struct Exynos4210CombinerState *s =
             (struct Exynos4210CombinerState *)opaque;
@@ -266,7 +266,7 @@ static void exynos4210_combiner_update(void *opaque, uint8_t group_n)
     }
 }
 
-static void exynos4210_combiner_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_combiner_write(void *opaque, hwaddr offset,
         uint64_t val, unsigned size)
 {
     struct Exynos4210CombinerState *s =
@@ -347,8 +347,6 @@ static void exynos4210_combiner_write(void *opaque, target_phys_addr_t offset,
                 TARGET_FMT_plx "\n", offset);
         break;
     }
-
-    return;
 }
 
 /* Get combiner group and bit from irq number */
@@ -380,8 +378,6 @@ static void exynos4210_combiner_handler(void *opaque, int irq, int level)
     }
 
     exynos4210_combiner_update(s, group_n);
-
-    return;
 }
 
 static void exynos4210_combiner_reset(DeviceState *d)
@@ -444,7 +440,7 @@ static void exynos4210_combiner_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_combiner;
 }
 
-static TypeInfo exynos4210_combiner_info = {
+static const TypeInfo exynos4210_combiner_info = {
     .name          = "exynos4210.combiner",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210CombinerState),
diff --git a/hw/exynos4210_fimd.c b/hw/exynos4210_fimd.c
index 3313f00a71c..3d498b77f84 100644
--- a/hw/exynos4210_fimd.c
+++ b/hw/exynos4210_fimd.c
@@ -23,11 +23,11 @@
  */
 
 #include "qemu-common.h"
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "sysbus.h"
-#include "console.h"
-#include "pixel_ops.h"
-#include "bswap.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
+#include "qemu/bswap.h"
 
 /* Debug messages configuration */
 #define EXYNOS4210_FIMD_DEBUG              0
@@ -290,7 +290,7 @@ struct Exynos4210fimdWindow {
     uint16_t virtpage_offsize;       /* VIDWADD2 register */
     MemoryRegionSection mem_section; /* RAM fragment containing framebuffer */
     uint8_t *host_fb_addr;           /* Host pointer to window's framebuffer */
-    target_phys_addr_t fb_len;       /* Framebuffer length */
+    hwaddr fb_len;       /* Framebuffer length */
 };
 
 typedef struct {
@@ -1110,7 +1110,7 @@ static inline int fimd_get_buffer_id(Exynos4210fimdWindow *w)
 static void fimd_update_memory_section(Exynos4210fimdState *s, unsigned win)
 {
     Exynos4210fimdWindow *w = &s->window[win];
-    target_phys_addr_t fb_start_addr, fb_mapped_len;
+    hwaddr fb_start_addr, fb_mapped_len;
 
     if (!s->enabled || !(w->wincon & FIMD_WINCON_ENWIN) ||
             FIMD_WINDOW_PROTECTED(s->shadowcon, win)) {
@@ -1243,7 +1243,7 @@ static void exynos4210_fimd_update(void *opaque)
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
     Exynos4210fimdWindow *w;
     int i, line;
-    target_phys_addr_t fb_line_addr, inc_size;
+    hwaddr fb_line_addr, inc_size;
     int scrn_height;
     int first_line = -1, last_line = -1, scrn_width;
     bool blend = false;
@@ -1307,7 +1307,7 @@ static void exynos4210_fimd_update(void *opaque)
             fimd_copy_line_toqemu(global_width, s->ifb + global_width * line *
                     RGBA_SIZE, d + global_width * line * bpp);
         }
-        dpy_update(s->console, 0, 0, global_width, global_height);
+        dpy_gfx_update(s->console, 0, 0, global_width, global_height);
     }
     s->invalidate = false;
     s->vidintcon[1] |= FIMD_VIDINT_INTFRMPEND;
@@ -1348,7 +1348,7 @@ static void exynos4210_fimd_reset(DeviceState *d)
     s->hueoffset = 0x01800080;
 }
 
-static void exynos4210_fimd_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_fimd_write(void *opaque, hwaddr offset,
                               uint64_t val, unsigned size)
 {
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
@@ -1649,7 +1649,7 @@ static void exynos4210_fimd_write(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t exynos4210_fimd_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_fimd_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
@@ -1913,7 +1913,7 @@ static void exynos4210_fimd_class_init(ObjectClass *klass, void *data)
     k->init = exynos4210_fimd_init;
 }
 
-static TypeInfo exynos4210_fimd_info = {
+static const TypeInfo exynos4210_fimd_info = {
     .name = "exynos4210.fimd",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210fimdState),
diff --git a/hw/exynos4210_gic.c b/hw/exynos4210_gic.c
index 7d03dd9ae3f..94b138fa467 100644
--- a/hw/exynos4210_gic.c
+++ b/hw/exynos4210_gic.c
@@ -140,7 +140,7 @@ combiner_grp_to_gic_id[64-EXYNOS4210_MAX_EXT_COMBINER_OUT_IRQ][8] = {
             EXT_GIC_ID_I2C4, EXT_GIC_ID_I2C5, EXT_GIC_ID_I2C6,
             EXT_GIC_ID_I2C7 },
     /* int combiner group 28 */
-    { EXT_GIC_ID_SPI0, EXT_GIC_ID_SPI1, EXT_GIC_ID_SPI2 },
+    { EXT_GIC_ID_SPI0, EXT_GIC_ID_SPI1, EXT_GIC_ID_SPI2 , EXT_GIC_ID_USB_HOST},
     /* int combiner group 29 */
     { EXT_GIC_ID_HSMMC0, EXT_GIC_ID_HSMMC1, EXT_GIC_ID_HSMMC2,
      EXT_GIC_ID_HSMMC3, EXT_GIC_ID_SDMMC },
@@ -193,8 +193,6 @@ static void exynos4210_irq_handler(void *opaque, int irq, int level)
 
     /* Bypass */
     qemu_set_irq(s->board_irqs[irq], level);
-
-    return;
 }
 
 /*
@@ -292,7 +290,7 @@ static int exynos4210_gic_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(s->gic, "num-irq", EXYNOS4210_GIC_NIRQ);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -348,7 +346,7 @@ static void exynos4210_gic_class_init(ObjectClass *klass, void *data)
     dc->props = exynos4210_gic_properties;
 }
 
-static TypeInfo exynos4210_gic_info = {
+static const TypeInfo exynos4210_gic_info = {
     .name          = "exynos4210.gic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210GicState),
@@ -410,8 +408,6 @@ static void exynos4210_irq_gate_handler(void *opaque, int irq, int level)
     }
 
     qemu_irq_lower(s->out);
-
-    return;
 }
 
 static void exynos4210_irq_gate_reset(DeviceState *d)
@@ -451,7 +447,7 @@ static void exynos4210_irq_gate_class_init(ObjectClass *klass, void *data)
     dc->props = exynos4210_irq_gate_properties;
 }
 
-static TypeInfo exynos4210_irq_gate_info = {
+static const TypeInfo exynos4210_irq_gate_info = {
     .name          = "exynos4210.irq_gate",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210IRQGateState),
diff --git a/hw/exynos4210_i2c.c b/hw/exynos4210_i2c.c
index 3f72a5c464d..cefd736092d 100644
--- a/hw/exynos4210_i2c.c
+++ b/hw/exynos4210_i2c.c
@@ -20,7 +20,7 @@
  *
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
 #include "i2c.h"
 
@@ -129,7 +129,7 @@ static void exynos4210_i2c_data_send(void *opaque)
     exynos4210_i2c_raise_interrupt(s);
 }
 
-static uint64_t exynos4210_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_i2c_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     Exynos4210I2CState *s = (Exynos4210I2CState *)opaque;
@@ -168,7 +168,7 @@ static uint64_t exynos4210_i2c_read(void *opaque, target_phys_addr_t offset,
     return value;
 }
 
-static void exynos4210_i2c_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_i2c_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     Exynos4210I2CState *s = (Exynos4210I2CState *)opaque;
diff --git a/hw/exynos4210_mct.c b/hw/exynos4210_mct.c
index 7a22b1f900a..d7d5904cc00 100644
--- a/hw/exynos4210_mct.c
+++ b/hw/exynos4210_mct.c
@@ -53,7 +53,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
@@ -568,14 +568,12 @@ static void exynos4210_gfrc_event(void *opaque)
     /* Reload FRC to reach nearest comparator */
     s->g_timer.curr_comp = exynos4210_gcomp_find(s);
     distance = exynos4210_gcomp_get_distance(s, s->g_timer.curr_comp);
-    if (distance > MCT_GT_COUNTER_STEP) {
+    if (distance > MCT_GT_COUNTER_STEP || !distance) {
         distance = MCT_GT_COUNTER_STEP;
     }
     exynos4210_gfrc_set_count(&s->g_timer, distance);
 
     exynos4210_gfrc_start(&s->g_timer);
-
-    return;
 }
 
 /*
@@ -987,7 +985,7 @@ static void exynos4210_mct_reset(DeviceState *d)
 }
 
 /* Multi Core Timer read */
-static uint64_t exynos4210_mct_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_mct_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     Exynos4210MCTState *s = (Exynos4210MCTState *)opaque;
@@ -1100,7 +1098,7 @@ static uint64_t exynos4210_mct_read(void *opaque, target_phys_addr_t offset,
 }
 
 /* MCT write */
-static void exynos4210_mct_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_mct_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210MCTState *s = (Exynos4210MCTState *)opaque;
@@ -1469,7 +1467,7 @@ static void exynos4210_mct_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_mct_state;
 }
 
-static TypeInfo exynos4210_mct_info = {
+static const TypeInfo exynos4210_mct_info = {
     .name          = "exynos4210.mct",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210MCTState),
diff --git a/hw/exynos4210_pmu.c b/hw/exynos4210_pmu.c
index c12d7501cc5..7c81a1b6283 100644
--- a/hw/exynos4210_pmu.c
+++ b/hw/exynos4210_pmu.c
@@ -392,7 +392,7 @@ typedef struct Exynos4210PmuState {
     uint32_t reg[PMU_NUM_OF_REGISTERS];
 } Exynos4210PmuState;
 
-static uint64_t exynos4210_pmu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_pmu_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     Exynos4210PmuState *s = (Exynos4210PmuState *)opaque;
@@ -411,7 +411,7 @@ static uint64_t exynos4210_pmu_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void exynos4210_pmu_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_pmu_write(void *opaque, hwaddr offset,
                                  uint64_t val, unsigned size)
 {
     Exynos4210PmuState *s = (Exynos4210PmuState *)opaque;
@@ -484,7 +484,7 @@ static void exynos4210_pmu_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &exynos4210_pmu_vmstate;
 }
 
-static TypeInfo exynos4210_pmu_info = {
+static const TypeInfo exynos4210_pmu_info = {
     .name          = "exynos4210.pmu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PmuState),
diff --git a/hw/exynos4210_pwm.c b/hw/exynos4210_pwm.c
index 0c228280a91..c8656248a81 100644
--- a/hw/exynos4210_pwm.c
+++ b/hw/exynos4210_pwm.c
@@ -21,7 +21,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
@@ -208,7 +208,7 @@ static void exynos4210_pwm_tick(void *opaque)
 /*
  * PWM Read
  */
-static uint64_t exynos4210_pwm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_pwm_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     Exynos4210PWMState *s = (Exynos4210PWMState *)opaque;
@@ -259,7 +259,7 @@ static uint64_t exynos4210_pwm_read(void *opaque, target_phys_addr_t offset,
 /*
  * PWM Write
  */
-static void exynos4210_pwm_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_pwm_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210PWMState *s = (Exynos4210PWMState *)opaque;
@@ -407,7 +407,7 @@ static void exynos4210_pwm_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_pwm_state;
 }
 
-static TypeInfo exynos4210_pwm_info = {
+static const TypeInfo exynos4210_pwm_info = {
     .name          = "exynos4210.pwm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PWMState),
diff --git a/hw/exynos4210_rtc.c b/hw/exynos4210_rtc.c
index 42a4ddc3271..5694a6207b2 100644
--- a/hw/exynos4210_rtc.c
+++ b/hw/exynos4210_rtc.c
@@ -26,13 +26,13 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 #include "exynos4210.h"
 
@@ -299,7 +299,7 @@ static void exynos4210_rtc_1Hz_tick(void *opaque)
 /*
  * RTC Read
  */
-static uint64_t exynos4210_rtc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_rtc_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     uint32_t value = 0;
@@ -376,7 +376,7 @@ static uint64_t exynos4210_rtc_read(void *opaque, target_phys_addr_t offset,
 /*
  * RTC Write
  */
-static void exynos4210_rtc_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_rtc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210RTCState *s = (Exynos4210RTCState *)opaque;
diff --git a/hw/exynos4210_uart.c b/hw/exynos4210_uart.c
index ccc47804f97..bdf797a0291 100644
--- a/hw/exynos4210_uart.c
+++ b/hw/exynos4210_uart.c
@@ -20,8 +20,8 @@
  */
 
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 
 #include "exynos4210.h"
 
@@ -96,7 +96,7 @@
 
 typedef struct Exynos4210UartReg {
     const char         *name; /* the only reason is the debug output */
-    target_phys_addr_t  offset;
+    hwaddr  offset;
     uint32_t            reset_value;
 } Exynos4210UartReg;
 
@@ -184,7 +184,7 @@ typedef struct {
 
 #if DEBUG_UART
 /* Used only for debugging inside PRINT_DEBUG_... macros */
-static const char *exynos4210_uart_regname(target_phys_addr_t  offset)
+static const char *exynos4210_uart_regname(hwaddr  offset)
 {
 
     int regs_number = sizeof(exynos4210_uart_regs) / sizeof(Exynos4210UartReg);
@@ -348,7 +348,7 @@ static void exynos4210_uart_update_parameters(Exynos4210UartState *s)
                 s->channel, speed, parity, data_bits, stop_bits);
 }
 
-static void exynos4210_uart_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_uart_write(void *opaque, hwaddr offset,
                                uint64_t val, unsigned size)
 {
     Exynos4210UartState *s = (Exynos4210UartState *)opaque;
@@ -423,7 +423,7 @@ static void exynos4210_uart_write(void *opaque, target_phys_addr_t offset,
         break;
     }
 }
-static uint64_t exynos4210_uart_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_uart_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     Exynos4210UartState *s = (Exynos4210UartState *)opaque;
@@ -581,7 +581,7 @@ static const VMStateDescription vmstate_exynos4210_uart = {
     }
 };
 
-DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
+DeviceState *exynos4210_uart_create(hwaddr addr,
                                  int fifo_size,
                                  int channel,
                                  CharDriverState *chr,
@@ -615,9 +615,9 @@ DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
     qdev_prop_set_uint32(dev, "rx-size", fifo_size);
     qdev_prop_set_uint32(dev, "tx-size", fifo_size);
 
-    bus = sysbus_from_qdev(dev);
+    bus = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
     }
     sysbus_connect_irq(bus, 0, irq);
@@ -661,7 +661,7 @@ static void exynos4210_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_uart;
 }
 
-static TypeInfo exynos4210_uart_info = {
+static const TypeInfo exynos4210_uart_info = {
     .name          = "exynos4210.uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210UartState),
diff --git a/hw/exynos4_boards.c b/hw/exynos4_boards.c
index 4bb0a60cb12..b59e6aabf36 100644
--- a/hw/exynos4_boards.c
+++ b/hw/exynos4_boards.c
@@ -21,11 +21,11 @@
  *
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "arm-misc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "exynos4210.h"
 #include "boards.h"
 
@@ -87,17 +87,14 @@ static void lan9215_init(uint32_t base, qemu_irq irq)
         qdev_set_nic_properties(dev, &nd_table[0]);
         qdev_prop_set_uint32(dev, "mode_16bit", 1);
         qdev_init_nofail(dev);
-        s = sysbus_from_qdev(dev);
+        s = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(s, 0, base);
         sysbus_connect_irq(s, 0, irq);
     }
 }
 
-static Exynos4210State *exynos4_boards_init_common(
-        const char *kernel_filename,
-        const char *kernel_cmdline,
-        const char *initrd_filename,
-        Exynos4BoardType board_type)
+static Exynos4210State *exynos4_boards_init_common(QEMUMachineInitArgs *args,
+                                                   Exynos4BoardType board_type)
 {
     if (smp_cpus != EXYNOS4210_NCPUS) {
         fprintf(stderr, "%s board supports only %d CPU cores. Ignoring smp_cpus"
@@ -110,9 +107,9 @@ static Exynos4210State *exynos4_boards_init_common(
     exynos4_board_binfo.board_id = exynos4_board_id[board_type];
     exynos4_board_binfo.smp_bootreg_addr =
             exynos4_board_smp_bootreg_addr[board_type];
-    exynos4_board_binfo.kernel_filename = kernel_filename;
-    exynos4_board_binfo.initrd_filename = initrd_filename;
-    exynos4_board_binfo.kernel_cmdline = kernel_cmdline;
+    exynos4_board_binfo.kernel_filename = args->kernel_filename;
+    exynos4_board_binfo.initrd_filename = args->initrd_filename;
+    exynos4_board_binfo.kernel_cmdline = args->kernel_cmdline;
     exynos4_board_binfo.gic_cpu_if_addr =
             EXYNOS4210_SMP_PRIVATE_BASE_ADDR + 0x100;
 
@@ -122,32 +119,25 @@ static Exynos4210State *exynos4_boards_init_common(
             " initrd_filename: %s\n",
             exynos4_board_ram_size[board_type] / 1048576,
             exynos4_board_ram_size[board_type],
-            kernel_filename,
-            kernel_cmdline,
-            initrd_filename);
+            args->kernel_filename,
+            args->kernel_cmdline,
+            args->initrd_filename);
 
     return exynos4210_init(get_system_memory(),
             exynos4_board_ram_size[board_type]);
 }
 
-static void nuri_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void nuri_init(QEMUMachineInitArgs *args)
 {
-    exynos4_boards_init_common(kernel_filename, kernel_cmdline,
-                initrd_filename, EXYNOS4_BOARD_NURI);
+    exynos4_boards_init_common(args, EXYNOS4_BOARD_NURI);
 
     arm_load_kernel(arm_env_get_cpu(first_cpu), &exynos4_board_binfo);
 }
 
-static void smdkc210_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void smdkc210_init(QEMUMachineInitArgs *args)
 {
-    Exynos4210State *s = exynos4_boards_init_common(kernel_filename,
-            kernel_cmdline, initrd_filename, EXYNOS4_BOARD_SMDKC210);
+    Exynos4210State *s = exynos4_boards_init_common(args,
+                                                    EXYNOS4_BOARD_SMDKC210);
 
     lan9215_init(SMDK_LAN9118_BASE_ADDR,
             qemu_irq_invert(s->irq_table[exynos4210_get_irq(37, 1)]));
@@ -160,12 +150,14 @@ static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS] = {
         .desc = "Samsung NURI board (Exynos4210)",
         .init = nuri_init,
         .max_cpus = EXYNOS4210_NCPUS,
+        DEFAULT_MACHINE_OPTIONS,
     },
     [EXYNOS4_BOARD_SMDKC210] = {
         .name = "smdkc210",
         .desc = "Samsung SMDKC210 board (Exynos4210)",
         .init = smdkc210_init,
         .max_cpus = EXYNOS4210_NCPUS,
+        DEFAULT_MACHINE_OPTIONS,
     },
 };
 
diff --git a/hw/fdc.c b/hw/fdc.c
index 08830c1ba2c..976a587c42f 100644
--- a/hw/fdc.c
+++ b/hw/fdc.c
@@ -29,14 +29,14 @@
 
 #include "hw.h"
 #include "fdc.h"
-#include "qemu-error.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
-#include "blockdev.h"
-#include "sysemu.h"
-#include "qemu-log.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 /********************************************************/
 /* debug Floppy devices */
@@ -327,7 +327,7 @@ static void fdctrl_reset(FDCtrl *fdctrl, int do_irq);
 static void fdctrl_reset_fifo(FDCtrl *fdctrl);
 static int fdctrl_transfer_handler (void *opaque, int nchan,
                                     int dma_pos, int dma_len);
-static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0);
+static void fdctrl_raise_irq(FDCtrl *fdctrl);
 static FDrive *get_cur_drv(FDCtrl *fdctrl);
 
 static uint32_t fdctrl_read_statusA(FDCtrl *fdctrl);
@@ -349,12 +349,12 @@ enum {
     FD_DIR_SCANE   = 2,
     FD_DIR_SCANL   = 3,
     FD_DIR_SCANH   = 4,
+    FD_DIR_VERIFY  = 5,
 };
 
 enum {
     FD_STATE_MULTI  = 0x01,	/* multi track flag */
     FD_STATE_FORMAT = 0x02,	/* format flag */
-    FD_STATE_SEEK   = 0x04,	/* seek flag */
 };
 
 enum {
@@ -496,7 +496,6 @@ enum {
 };
 
 #define FD_MULTI_TRACK(state) ((state) & FD_STATE_MULTI)
-#define FD_DID_SEEK(state) ((state) & FD_STATE_SEEK)
 #define FD_FORMAT_CMD(state) ((state) & FD_STATE_FORMAT)
 
 struct FDCtrl {
@@ -626,13 +625,13 @@ static void fdctrl_write (void *opaque, uint32_t reg, uint32_t value)
     }
 }
 
-static uint64_t fdctrl_read_mem (void *opaque, target_phys_addr_t reg,
+static uint64_t fdctrl_read_mem (void *opaque, hwaddr reg,
                                  unsigned ize)
 {
     return fdctrl_read(opaque, (uint32_t)reg);
 }
 
-static void fdctrl_write_mem (void *opaque, target_phys_addr_t reg,
+static void fdctrl_write_mem (void *opaque, hwaddr reg,
                               uint64_t value, unsigned size)
 {
     fdctrl_write(opaque, (uint32_t)reg, value);
@@ -799,6 +798,7 @@ static void fdctrl_handle_tc(void *opaque, int irq, int level)
 /* Change IRQ state */
 static void fdctrl_reset_irq(FDCtrl *fdctrl)
 {
+    fdctrl->status0 = 0;
     if (!(fdctrl->sra & FD_SRA_INTPEND))
         return;
     FLOPPY_DPRINTF("Reset interrupt\n");
@@ -806,14 +806,13 @@ static void fdctrl_reset_irq(FDCtrl *fdctrl)
     fdctrl->sra &= ~FD_SRA_INTPEND;
 }
 
-static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0)
+static void fdctrl_raise_irq(FDCtrl *fdctrl)
 {
     /* Sparc mutation */
     if (fdctrl->sun4m && (fdctrl->msr & FD_MSR_CMDBUSY)) {
         /* XXX: not sure */
         fdctrl->msr &= ~FD_MSR_CMDBUSY;
         fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
-        fdctrl->status0 = status0;
         return;
     }
     if (!(fdctrl->sra & FD_SRA_INTPEND)) {
@@ -822,7 +821,6 @@ static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0)
     }
 
     fdctrl->reset_sensei = 0;
-    fdctrl->status0 = status0;
     FLOPPY_DPRINTF("Set interrupt status to 0x%02x\n", fdctrl->status0);
 }
 
@@ -851,7 +849,8 @@ static void fdctrl_reset(FDCtrl *fdctrl, int do_irq)
         fd_recalibrate(&fdctrl->drives[i]);
     fdctrl_reset_fifo(fdctrl);
     if (do_irq) {
-        fdctrl_raise_irq(fdctrl, FD_SR0_RDYCHG);
+        fdctrl->status0 |= FD_SR0_RDYCHG;
+        fdctrl_raise_irq(fdctrl);
         fdctrl->reset_sensei = FD_RESET_SENSEI_COUNT;
     }
 }
@@ -1079,15 +1078,12 @@ static void fdctrl_reset_fifo(FDCtrl *fdctrl)
 }
 
 /* Set FIFO status for the host to read */
-static void fdctrl_set_fifo(FDCtrl *fdctrl, int fifo_len, uint8_t status0)
+static void fdctrl_set_fifo(FDCtrl *fdctrl, int fifo_len)
 {
     fdctrl->data_dir = FD_DIR_READ;
     fdctrl->data_len = fifo_len;
     fdctrl->data_pos = 0;
     fdctrl->msr |= FD_MSR_CMDBUSY | FD_MSR_RQM | FD_MSR_DIO;
-    if (status0) {
-        fdctrl_raise_irq(fdctrl, status0);
-    }
 }
 
 /* Set an error: unimplemented/unknown command */
@@ -1096,7 +1092,7 @@ static void fdctrl_unimplemented(FDCtrl *fdctrl, int direction)
     qemu_log_mask(LOG_UNIMP, "fdc: unimplemented command 0x%02x\n",
                   fdctrl->fifo[0]);
     fdctrl->fifo[0] = FD_SR0_INVCMD;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 /* Seek to next sector
@@ -1126,11 +1122,13 @@ static int fdctrl_seek_to_next_sect(FDCtrl *fdctrl, FDrive *cur_drv)
             } else {
                 new_head = 0;
                 new_track++;
+                fdctrl->status0 |= FD_SR0_SEEK;
                 if ((cur_drv->flags & FDISK_DBL_SIDES) == 0) {
                     ret = 0;
                 }
             }
         } else {
+            fdctrl->status0 |= FD_SR0_SEEK;
             new_track++;
             ret = 0;
         }
@@ -1150,10 +1148,14 @@ static void fdctrl_stop_transfer(FDCtrl *fdctrl, uint8_t status0,
                                  uint8_t status1, uint8_t status2)
 {
     FDrive *cur_drv;
-
     cur_drv = get_cur_drv(fdctrl);
-    fdctrl->status0 = status0 | FD_SR0_SEEK | (cur_drv->head << 2) |
-                      GET_CUR_DRV(fdctrl);
+
+    fdctrl->status0 &= ~(FD_SR0_DS0 | FD_SR0_DS1 | FD_SR0_HEAD);
+    fdctrl->status0 |= GET_CUR_DRV(fdctrl);
+    if (cur_drv->head) {
+        fdctrl->status0 |= FD_SR0_HEAD;
+    }
+    fdctrl->status0 |= status0;
 
     FLOPPY_DPRINTF("transfer status: %02x %02x %02x (%02x)\n",
                    status0, status1, status2, fdctrl->status0);
@@ -1170,7 +1172,9 @@ static void fdctrl_stop_transfer(FDCtrl *fdctrl, uint8_t status0,
     }
     fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
     fdctrl->msr &= ~FD_MSR_NONDMA;
-    fdctrl_set_fifo(fdctrl, 7, fdctrl->status0);
+
+    fdctrl_set_fifo(fdctrl, 7);
+    fdctrl_raise_irq(fdctrl);
 }
 
 /* Prepare a data transfer (either DMA or FIFO) */
@@ -1178,7 +1182,6 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
 {
     FDrive *cur_drv;
     uint8_t kh, kt, ks;
-    int did_seek = 0;
 
     SET_CUR_DRV(fdctrl, fdctrl->fifo[1] & FD_DOR_SELMASK);
     cur_drv = get_cur_drv(fdctrl);
@@ -1212,7 +1215,7 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
         fdctrl->fifo[5] = ks;
         return;
     case 1:
-        did_seek = 1;
+        fdctrl->status0 |= FD_SR0_SEEK;
         break;
     default:
         break;
@@ -1234,16 +1237,12 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
     /* Set the FIFO state */
     fdctrl->data_dir = direction;
     fdctrl->data_pos = 0;
-    fdctrl->msr |= FD_MSR_CMDBUSY;
+    assert(fdctrl->msr & FD_MSR_CMDBUSY);
     if (fdctrl->fifo[0] & 0x80)
         fdctrl->data_state |= FD_STATE_MULTI;
     else
         fdctrl->data_state &= ~FD_STATE_MULTI;
-    if (did_seek)
-        fdctrl->data_state |= FD_STATE_SEEK;
-    else
-        fdctrl->data_state &= ~FD_STATE_SEEK;
-    if (fdctrl->fifo[5] == 00) {
+    if (fdctrl->fifo[5] == 0) {
         fdctrl->data_len = fdctrl->fifo[8];
     } else {
         int tmp;
@@ -1266,14 +1265,21 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
         if (((direction == FD_DIR_SCANE || direction == FD_DIR_SCANL ||
               direction == FD_DIR_SCANH) && dma_mode == 0) ||
             (direction == FD_DIR_WRITE && dma_mode == 2) ||
-            (direction == FD_DIR_READ && dma_mode == 1)) {
+            (direction == FD_DIR_READ && dma_mode == 1) ||
+            (direction == FD_DIR_VERIFY)) {
             /* No access is allowed until DMA transfer has completed */
             fdctrl->msr &= ~FD_MSR_RQM;
-            /* Now, we just have to wait for the DMA controller to
-             * recall us...
-             */
-            DMA_hold_DREQ(fdctrl->dma_chann);
-            DMA_schedule(fdctrl->dma_chann);
+            if (direction != FD_DIR_VERIFY) {
+                /* Now, we just have to wait for the DMA controller to
+                 * recall us...
+                 */
+                DMA_hold_DREQ(fdctrl->dma_chann);
+                DMA_schedule(fdctrl->dma_chann);
+            } else {
+                /* Start transfer */
+                fdctrl_transfer_handler(fdctrl, fdctrl->dma_chann, 0,
+                                        fdctrl->data_len);
+            }
             return;
         } else {
             FLOPPY_DPRINTF("bad dma_mode=%d direction=%d\n", dma_mode,
@@ -1285,9 +1291,7 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
     if (direction != FD_DIR_WRITE)
         fdctrl->msr |= FD_MSR_DIO;
     /* IO based transfer: calculate len */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
-
-    return;
+    fdctrl_raise_irq(fdctrl);
 }
 
 /* Prepare a transfer of deleted data */
@@ -1378,6 +1382,9 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
                 goto transfer_error;
             }
             break;
+        case FD_DIR_VERIFY:
+            /* VERIFY commands */
+            break;
         default:
             /* SCAN commands */
             {
@@ -1413,8 +1420,6 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
         fdctrl->data_dir == FD_DIR_SCANL ||
         fdctrl->data_dir == FD_DIR_SCANH)
         status2 = FD_SR2_SEH;
-    if (FD_DID_SEEK(fdctrl->data_state))
-        status0 |= FD_SR0_SEEK;
     fdctrl->data_len -= len;
     fdctrl_stop_transfer(fdctrl, status0, status1, status2);
  transfer_error:
@@ -1460,7 +1465,7 @@ static uint32_t fdctrl_read_data(FDCtrl *fdctrl)
          * then from status mode to command mode
          */
         if (fdctrl->msr & FD_MSR_NONDMA) {
-            fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         } else {
             fdctrl_reset_fifo(fdctrl);
             fdctrl_reset_irq(fdctrl);
@@ -1508,7 +1513,7 @@ static void fdctrl_format_sector(FDCtrl *fdctrl)
         fdctrl->fifo[5] = ks;
         return;
     case 1:
-        fdctrl->data_state |= FD_STATE_SEEK;
+        fdctrl->status0 |= FD_SR0_SEEK;
         break;
     default:
         break;
@@ -1522,10 +1527,7 @@ static void fdctrl_format_sector(FDCtrl *fdctrl)
         if (cur_drv->sect == cur_drv->last_sect) {
             fdctrl->data_state &= ~FD_STATE_FORMAT;
             /* Last sector done */
-            if (FD_DID_SEEK(fdctrl->data_state))
-                fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
-            else
-                fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         } else {
             /* More to do */
             fdctrl->data_pos = 0;
@@ -1538,7 +1540,7 @@ static void fdctrl_handle_lock(FDCtrl *fdctrl, int direction)
 {
     fdctrl->lock = (fdctrl->fifo[0] & 0x80) ? 1 : 0;
     fdctrl->fifo[0] = fdctrl->lock << 4;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_dumpreg(FDCtrl *fdctrl, int direction)
@@ -1563,20 +1565,20 @@ static void fdctrl_handle_dumpreg(FDCtrl *fdctrl, int direction)
         (cur_drv->perpendicular << 2);
     fdctrl->fifo[8] = fdctrl->config;
     fdctrl->fifo[9] = fdctrl->precomp_trk;
-    fdctrl_set_fifo(fdctrl, 10, 0);
+    fdctrl_set_fifo(fdctrl, 10);
 }
 
 static void fdctrl_handle_version(FDCtrl *fdctrl, int direction)
 {
     /* Controller's version */
     fdctrl->fifo[0] = fdctrl->version;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_partid(FDCtrl *fdctrl, int direction)
 {
     fdctrl->fifo[0] = 0x41; /* Stepping 1 */
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_restore(FDCtrl *fdctrl, int direction)
@@ -1629,7 +1631,7 @@ static void fdctrl_handle_save(FDCtrl *fdctrl, int direction)
     fdctrl->fifo[12] = fdctrl->pwrd;
     fdctrl->fifo[13] = 0;
     fdctrl->fifo[14] = 0;
-    fdctrl_set_fifo(fdctrl, 15, 0);
+    fdctrl_set_fifo(fdctrl, 15);
 }
 
 static void fdctrl_handle_readid(FDCtrl *fdctrl, int direction)
@@ -1652,7 +1654,6 @@ static void fdctrl_handle_format_track(FDCtrl *fdctrl, int direction)
         fdctrl->data_state |= FD_STATE_MULTI;
     else
         fdctrl->data_state &= ~FD_STATE_MULTI;
-    fdctrl->data_state &= ~FD_STATE_SEEK;
     cur_drv->bps =
         fdctrl->fifo[2] > 7 ? 16384 : 128 << fdctrl->fifo[2];
 #if 0
@@ -1695,7 +1696,7 @@ static void fdctrl_handle_sense_drive_status(FDCtrl *fdctrl, int direction)
         (cur_drv->head << 2) |
         GET_CUR_DRV(fdctrl) |
         0x28;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_recalibrate(FDCtrl *fdctrl, int direction)
@@ -1707,7 +1708,8 @@ static void fdctrl_handle_recalibrate(FDCtrl *fdctrl, int direction)
     fd_recalibrate(cur_drv);
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
@@ -1720,7 +1722,7 @@ static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
         fdctrl->reset_sensei--;
     } else if (!(fdctrl->sra & FD_SRA_INTPEND)) {
         fdctrl->fifo[0] = FD_SR0_INVCMD;
-        fdctrl_set_fifo(fdctrl, 1, 0);
+        fdctrl_set_fifo(fdctrl, 1);
         return;
     } else {
         fdctrl->fifo[0] =
@@ -1729,7 +1731,7 @@ static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
     }
 
     fdctrl->fifo[1] = cur_drv->track;
-    fdctrl_set_fifo(fdctrl, 2, 0);
+    fdctrl_set_fifo(fdctrl, 2);
     fdctrl_reset_irq(fdctrl);
     fdctrl->status0 = FD_SR0_RDYCHG;
 }
@@ -1746,7 +1748,8 @@ static void fdctrl_handle_seek(FDCtrl *fdctrl, int direction)
      */
     fd_seek(cur_drv, cur_drv->head, fdctrl->fifo[2], cur_drv->sect, 1);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_perpendicular_mode(FDCtrl *fdctrl, int direction)
@@ -1771,7 +1774,7 @@ static void fdctrl_handle_powerdown_mode(FDCtrl *fdctrl, int direction)
 {
     fdctrl->pwrd = fdctrl->fifo[1];
     fdctrl->fifo[0] = fdctrl->fifo[1];
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_option(FDCtrl *fdctrl, int direction)
@@ -1790,7 +1793,7 @@ static void fdctrl_handle_drive_specification_command(FDCtrl *fdctrl, int direct
             fdctrl->fifo[0] = fdctrl->fifo[1];
             fdctrl->fifo[2] = 0;
             fdctrl->fifo[3] = 0;
-            fdctrl_set_fifo(fdctrl, 4, 0);
+            fdctrl_set_fifo(fdctrl, 4);
         } else {
             fdctrl_reset_fifo(fdctrl);
         }
@@ -1798,7 +1801,7 @@ static void fdctrl_handle_drive_specification_command(FDCtrl *fdctrl, int direct
         /* ERROR */
         fdctrl->fifo[0] = 0x80 |
             (cur_drv->head << 2) | GET_CUR_DRV(fdctrl);
-        fdctrl_set_fifo(fdctrl, 1, 0);
+        fdctrl_set_fifo(fdctrl, 1);
     }
 }
 
@@ -1817,7 +1820,8 @@ static void fdctrl_handle_relative_seek_in(FDCtrl *fdctrl, int direction)
     }
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_relative_seek_out(FDCtrl *fdctrl, int direction)
@@ -1834,7 +1838,8 @@ static void fdctrl_handle_relative_seek_out(FDCtrl *fdctrl, int direction)
     }
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static const struct {
@@ -1856,7 +1861,7 @@ static const struct {
     { FD_CMD_SAVE, 0xff, "SAVE", 0, fdctrl_handle_save }, /* part of READ DELETED DATA */
     { FD_CMD_READ_DELETED, 0x1f, "READ DELETED DATA", 8, fdctrl_start_transfer_del, FD_DIR_READ },
     { FD_CMD_SCAN_EQUAL, 0x1f, "SCAN EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANE },
-    { FD_CMD_VERIFY, 0x1f, "VERIFY", 8, fdctrl_unimplemented },
+    { FD_CMD_VERIFY, 0x1f, "VERIFY", 8, fdctrl_start_transfer, FD_DIR_VERIFY },
     { FD_CMD_SCAN_LOW_OR_EQUAL, 0x1f, "SCAN LOW OR EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANL },
     { FD_CMD_SCAN_HIGH_OR_EQUAL, 0x1f, "SCAN HIGH OR EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANH },
     { FD_CMD_WRITE_DELETED, 0x3f, "WRITE DELETED DATA", 8, fdctrl_start_transfer_del, FD_DIR_WRITE },
@@ -1920,7 +1925,7 @@ static void fdctrl_write_data(FDCtrl *fdctrl, uint32_t value)
          * then from status mode to command mode
          */
         if (fdctrl->data_pos == fdctrl->data_len)
-            fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         return;
     }
     if (fdctrl->data_pos == 0) {
@@ -1994,11 +1999,11 @@ static int fdctrl_connect_drives(FDCtrl *fdctrl)
         drive->fdctrl = fdctrl;
 
         if (drive->bs) {
-            if (bdrv_get_on_error(drive->bs, 0) != BLOCK_ERR_STOP_ENOSPC) {
+            if (bdrv_get_on_error(drive->bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
                 error_report("fdc doesn't support drive option werror");
                 return -1;
             }
-            if (bdrv_get_on_error(drive->bs, 1) != BLOCK_ERR_REPORT) {
+            if (bdrv_get_on_error(drive->bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
                 error_report("fdc doesn't support drive option rerror");
                 return -1;
             }
@@ -2034,7 +2039,7 @@ ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds)
 }
 
 void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
-                        target_phys_addr_t mmio_base, DriveInfo **fds)
+                        hwaddr mmio_base, DriveInfo **fds)
 {
     FDCtrl *fdctrl;
     DeviceState *dev;
@@ -2055,7 +2060,7 @@ void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
     sysbus_mmio_map(&sys->busdev, 0, mmio_base);
 }
 
-void sun4m_fdctrl_init(qemu_irq irq, target_phys_addr_t io_base,
+void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc)
 {
     DeviceState *dev;
@@ -2205,7 +2210,7 @@ static void isabus_fdc_class_init1(ObjectClass *klass, void *data)
     dc->props = isa_fdc_properties;
 }
 
-static TypeInfo isa_fdc_info = {
+static const TypeInfo isa_fdc_info = {
     .name          = "isa-fdc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(FDCtrlISABus),
@@ -2239,7 +2244,7 @@ static void sysbus_fdc_class_init(ObjectClass *klass, void *data)
     dc->props = sysbus_fdc_properties;
 }
 
-static TypeInfo sysbus_fdc_info = {
+static const TypeInfo sysbus_fdc_info = {
     .name          = "sysbus-fdc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FDCtrlSysBus),
@@ -2262,7 +2267,7 @@ static void sun4m_fdc_class_init(ObjectClass *klass, void *data)
     dc->props = sun4m_fdc_properties;
 }
 
-static TypeInfo sun4m_fdc_info = {
+static const TypeInfo sun4m_fdc_info = {
     .name          = "SUNW,fdtwo",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FDCtrlSysBus),
diff --git a/hw/fdc.h b/hw/fdc.h
index b5c9f31074d..a8f6f7c8502 100644
--- a/hw/fdc.h
+++ b/hw/fdc.h
@@ -15,8 +15,8 @@ typedef enum FDriveType {
 
 ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds);
 void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
-                        target_phys_addr_t mmio_base, DriveInfo **fds);
-void sun4m_fdctrl_init(qemu_irq irq, target_phys_addr_t io_base,
+                        hwaddr mmio_base, DriveInfo **fds);
+void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc);
 
 FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
diff --git a/hw/fifo.c b/hw/fifo.c
new file mode 100644
index 00000000000..68a955a77b3
--- /dev/null
+++ b/hw/fifo.c
@@ -0,0 +1,78 @@
+/*
+ * Generic FIFO component, implemented as a circular buffer.
+ *
+ * Copyright (c) 2012 Peter A. G. Crosthwaite
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "fifo.h"
+
+void fifo8_create(Fifo8 *fifo, uint32_t capacity)
+{
+    fifo->data = g_new(uint8_t, capacity);
+    fifo->capacity = capacity;
+    fifo->head = 0;
+    fifo->num = 0;
+}
+
+void fifo8_destroy(Fifo8 *fifo)
+{
+    g_free(fifo->data);
+}
+
+void fifo8_push(Fifo8 *fifo, uint8_t data)
+{
+    if (fifo->num == fifo->capacity) {
+        abort();
+    }
+    fifo->data[(fifo->head + fifo->num) % fifo->capacity] = data;
+    fifo->num++;
+}
+
+uint8_t fifo8_pop(Fifo8 *fifo)
+{
+    uint8_t ret;
+
+    if (fifo->num == 0) {
+        abort();
+    }
+    ret = fifo->data[fifo->head++];
+    fifo->head %= fifo->capacity;
+    fifo->num--;
+    return ret;
+}
+
+void fifo8_reset(Fifo8 *fifo)
+{
+    fifo->num = 0;
+}
+
+bool fifo8_is_empty(Fifo8 *fifo)
+{
+    return (fifo->num == 0);
+}
+
+bool fifo8_is_full(Fifo8 *fifo)
+{
+    return (fifo->num == fifo->capacity);
+}
+
+const VMStateDescription vmstate_fifo8 = {
+    .name = "Fifo8",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_VBUFFER_UINT32(data, Fifo8, 1, NULL, 0, capacity),
+        VMSTATE_UINT32(head, Fifo8),
+        VMSTATE_UINT32(num, Fifo8),
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/hw/fifo.h b/hw/fifo.h
new file mode 100644
index 00000000000..f23890abf45
--- /dev/null
+++ b/hw/fifo.h
@@ -0,0 +1,99 @@
+#ifndef FIFO_H
+#define FIFO_H
+
+#include "hw.h"
+
+typedef struct {
+    /* All fields are private */
+    uint8_t *data;
+    uint32_t capacity;
+    uint32_t head;
+    uint32_t num;
+} Fifo8;
+
+/**
+ * fifo8_create:
+ * @fifo: struct Fifo8 to initialise with new FIFO
+ * @capacity: capacity of the newly created FIFO
+ *
+ * Create a FIFO of the specified size. Clients should call fifo8_destroy()
+ * when finished using the fifo. The FIFO is initially empty.
+ */
+
+void fifo8_create(Fifo8 *fifo, uint32_t capacity);
+
+/**
+ * fifo8_destroy:
+ * @fifo: FIFO to cleanup
+ *
+ * Cleanup a FIFO created with fifo8_create(). Frees memory created for FIFO
+  *storage. The FIFO is no longer usable after this has been called.
+ */
+
+void fifo8_destroy(Fifo8 *fifo);
+
+/**
+ * fifo8_push:
+ * @fifo: FIFO to push to
+ * @data: data byte to push
+ *
+ * Push a data byte to the FIFO. Behaviour is undefined if the FIFO is full.
+ * Clients are responsible for checking for fullness using fifo8_is_full().
+ */
+
+void fifo8_push(Fifo8 *fifo, uint8_t data);
+
+/**
+ * fifo8_pop:
+ * @fifo: fifo to pop from
+ *
+ * Pop a data byte from the FIFO. Behaviour is undefined if the FIFO is empty.
+ * Clients are responsible for checking for emptyness using fifo8_is_empty().
+ *
+ * Returns: The popped data byte.
+ */
+
+uint8_t fifo8_pop(Fifo8 *fifo);
+
+/**
+ * fifo8_reset:
+ * @fifo: FIFO to reset
+ *
+ * Reset a FIFO. All data is discarded and the FIFO is emptied.
+ */
+
+void fifo8_reset(Fifo8 *fifo);
+
+/**
+ * fifo8_is_empty:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is empty.
+ *
+ * Returns: True if the fifo is empty, false otherwise.
+ */
+
+bool fifo8_is_empty(Fifo8 *fifo);
+
+/**
+ * fifo8_is_full:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is full.
+ *
+ * Returns: True if the fifo is full, false otherwise.
+ */
+
+bool fifo8_is_full(Fifo8 *fifo);
+
+extern const VMStateDescription vmstate_fifo8;
+
+#define VMSTATE_FIFO8(_field, _state) {                              \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(Fifo8),                                     \
+    .vmsd       = &vmstate_fifo8,                                    \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, Fifo8),       \
+}
+
+#endif /* FIFO_H */
diff --git a/hw/flash.h b/hw/flash.h
index 9c9e5265b70..920d7596e36 100644
--- a/hw/flash.h
+++ b/hw/flash.h
@@ -1,22 +1,25 @@
+#ifndef HW_FLASH_H
+#define HW_FLASH_H 1
+
 /* NOR flash devices */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 typedef struct pflash_t pflash_t;
 
 /* pflash_cfi01.c */
-pflash_t *pflash_cfi01_register(target_phys_addr_t base,
+pflash_t *pflash_cfi01_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
+                                hwaddr size,
                                 BlockDriverState *bs,
                                 uint32_t sector_len, int nb_blocs, int width,
                                 uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3, int be);
 
 /* pflash_cfi02.c */
-pflash_t *pflash_cfi02_register(target_phys_addr_t base,
+pflash_t *pflash_cfi02_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
+                                hwaddr size,
                                 BlockDriverState *bs, uint32_t sector_len,
                                 int nb_blocs, int nb_mappings, int width,
                                 uint16_t id0, uint16_t id1,
@@ -57,3 +60,5 @@ typedef struct {
 uint8_t ecc_digest(ECCState *s, uint8_t sample);
 void ecc_reset(ECCState *s);
 extern VMStateDescription vmstate_ecc_state;
+
+#endif
diff --git a/hw/framebuffer.c b/hw/framebuffer.c
index 85a00a57986..2a870961bc9 100644
--- a/hw/framebuffer.c
+++ b/hw/framebuffer.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
 
 /* Render an image from a shared memory framebuffer.  */
@@ -26,7 +26,7 @@
 void framebuffer_update_display(
     DisplayState *ds,
     MemoryRegion *address_space,
-    target_phys_addr_t base,
+    hwaddr base,
     int cols, /* Width in pixels.  */
     int rows, /* Height in pixels.  */
     int src_width, /* Length of source line, in bytes.  */
@@ -38,7 +38,7 @@ void framebuffer_update_display(
     int *first_row, /* Input and output.  */
     int *last_row /* Output only */)
 {
-    target_phys_addr_t src_len;
+    hwaddr src_len;
     uint8_t *dest;
     uint8_t *src;
     uint8_t *src_base;
@@ -107,5 +107,4 @@ void framebuffer_update_display(
                               DIRTY_MEMORY_VGA);
     *first_row = first;
     *last_row = last;
-    return;
 }
diff --git a/hw/framebuffer.h b/hw/framebuffer.h
index 527a6b85f81..11f53edec05 100644
--- a/hw/framebuffer.h
+++ b/hw/framebuffer.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_FRAMEBUFFER_H
 #define QEMU_FRAMEBUFFER_H
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Framebuffer device helper routines.  */
 
@@ -10,7 +10,7 @@ typedef void (*drawfn)(void *, uint8_t *, const uint8_t *, int, int);
 void framebuffer_update_display(
     DisplayState *ds,
     MemoryRegion *address_space,
-    target_phys_addr_t base,
+    hwaddr base,
     int cols,
     int rows,
     int src_width,
diff --git a/hw/fw_cfg.c b/hw/fw_cfg.c
index 7b3b5769a26..02618f24800 100644
--- a/hw/fw_cfg.c
+++ b/hw/fw_cfg.c
@@ -22,21 +22,13 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "isa.h"
 #include "fw_cfg.h"
 #include "sysbus.h"
-#include "qemu-error.h"
-
-/* debug firmware config */
-//#define DEBUG_FW_CFG
-
-#ifdef DEBUG_FW_CFG
-#define FW_CFG_DPRINTF(fmt, ...)                        \
-    do { printf("FW_CFG: " fmt , ## __VA_ARGS__); } while (0)
-#else
-#define FW_CFG_DPRINTF(fmt, ...)
-#endif
+#include "trace.h"
+#include "qemu/error-report.h"
+#include "qemu/config-file.h"
 
 #define FW_CFG_SIZE 2
 #define FW_CFG_DATA_SIZE 1
@@ -62,16 +54,17 @@ struct FWCfgState {
 #define JPG_FILE 0
 #define BMP_FILE 1
 
-static char *read_splashfile(char *filename, int *file_sizep, int *file_typep)
+static char *read_splashfile(char *filename, size_t *file_sizep,
+                             int *file_typep)
 {
     GError *err = NULL;
     gboolean res;
     gchar *content;
-    int file_type = -1;
-    unsigned int filehead = 0;
+    int file_type;
+    unsigned int filehead;
     int bmp_bpp;
 
-    res = g_file_get_contents(filename, &content, (gsize *)file_sizep, &err);
+    res = g_file_get_contents(filename, &content, file_sizep, &err);
     if (res == FALSE) {
         error_report("failed to read splash file '%s'", filename);
         g_error_free(err);
@@ -119,8 +112,8 @@ static void fw_cfg_bootsplash(FWCfgState *s)
     const char *boot_splash_filename = NULL;
     char *p;
     char *filename, *file_data;
-    int file_size;
-    int file_type = -1;
+    size_t file_size;
+    int file_type;
     const char *temp;
 
     /* get user configuration */
@@ -183,12 +176,36 @@ static void fw_cfg_bootsplash(FWCfgState *s)
     }
 }
 
+static void fw_cfg_reboot(FWCfgState *s)
+{
+    int reboot_timeout = -1;
+    char *p;
+    const char *temp;
+
+    /* get user configuration */
+    QemuOptsList *plist = qemu_find_opts("boot-opts");
+    QemuOpts *opts = QTAILQ_FIRST(&plist->head);
+    if (opts != NULL) {
+        temp = qemu_opt_get(opts, "reboot-timeout");
+        if (temp != NULL) {
+            p = (char *)temp;
+            reboot_timeout = strtol(p, (char **)&p, 10);
+        }
+    }
+    /* validate the input */
+    if (reboot_timeout > 0xffff) {
+        error_report("reboot timeout is larger than 65535, force it to 65535.");
+        reboot_timeout = 0xffff;
+    }
+    fw_cfg_add_file(s, "etc/boot-fail-wait", g_memdup(&reboot_timeout, 4), 4);
+}
+
 static void fw_cfg_write(FWCfgState *s, uint8_t value)
 {
     int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
     FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
 
-    FW_CFG_DPRINTF("write %d\n", value);
+    trace_fw_cfg_write(s, value);
 
     if (s->cur_entry & FW_CFG_WRITE_CHANNEL && e->callback &&
         s->cur_offset < e->len) {
@@ -213,8 +230,7 @@ static int fw_cfg_select(FWCfgState *s, uint16_t key)
         ret = 1;
     }
 
-    FW_CFG_DPRINTF("select key %d (%sfound)\n", key, ret ? "" : "not ");
-
+    trace_fw_cfg_select(s, key, ret);
     return ret;
 }
 
@@ -229,42 +245,41 @@ static uint8_t fw_cfg_read(FWCfgState *s)
     else
         ret = e->data[s->cur_offset++];
 
-    FW_CFG_DPRINTF("read %d\n", ret);
-
+    trace_fw_cfg_read(s, ret);
     return ret;
 }
 
-static uint64_t fw_cfg_data_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t fw_cfg_data_mem_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     return fw_cfg_read(opaque);
 }
 
-static void fw_cfg_data_mem_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_data_mem_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     fw_cfg_write(opaque, (uint8_t)value);
 }
 
-static void fw_cfg_ctl_mem_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_ctl_mem_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     fw_cfg_select(opaque, (uint16_t)value);
 }
 
-static bool fw_cfg_ctl_mem_valid(void *opaque, target_phys_addr_t addr,
+static bool fw_cfg_ctl_mem_valid(void *opaque, hwaddr addr,
                                  unsigned size, bool is_write)
 {
     return is_write && size == 2;
 }
 
-static uint64_t fw_cfg_comb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t fw_cfg_comb_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     return fw_cfg_read(opaque);
 }
 
-static void fw_cfg_comb_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_comb_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     switch (size) {
@@ -277,7 +292,7 @@ static void fw_cfg_comb_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static bool fw_cfg_comb_valid(void *opaque, target_phys_addr_t addr,
+static bool fw_cfg_comb_valid(void *opaque, hwaddr addr,
                                   unsigned size, bool is_write)
 {
     return (size == 1) || (is_write && size == 2);
@@ -359,85 +374,83 @@ static const VMStateDescription vmstate_fw_cfg = {
     }
 };
 
-int fw_cfg_add_bytes(FWCfgState *s, uint16_t key, uint8_t *data, uint32_t len)
+void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len)
 {
     int arch = !!(key & FW_CFG_ARCH_LOCAL);
 
     key &= FW_CFG_ENTRY_MASK;
 
-    if (key >= FW_CFG_MAX_ENTRY)
-        return 0;
+    assert(key < FW_CFG_MAX_ENTRY && len < UINT32_MAX);
 
     s->entries[arch][key].data = data;
-    s->entries[arch][key].len = len;
+    s->entries[arch][key].len = (uint32_t)len;
+}
 
-    return 1;
+void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value)
+{
+    size_t sz = strlen(value) + 1;
+
+    return fw_cfg_add_bytes(s, key, g_memdup(value, sz), sz);
 }
 
-int fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value)
+void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value)
 {
     uint16_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le16(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value)
+void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value)
 {
     uint32_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le32(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value)
+void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value)
 {
     uint64_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le64(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                        void *callback_opaque, uint8_t *data, size_t len)
+void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
+                         void *callback_opaque, void *data, size_t len)
 {
     int arch = !!(key & FW_CFG_ARCH_LOCAL);
 
-    if (!(key & FW_CFG_WRITE_CHANNEL))
-        return 0;
+    assert(key & FW_CFG_WRITE_CHANNEL);
 
     key &= FW_CFG_ENTRY_MASK;
 
-    if (key >= FW_CFG_MAX_ENTRY || len > 65535)
-        return 0;
+    assert(key < FW_CFG_MAX_ENTRY && len <= UINT32_MAX);
 
     s->entries[arch][key].data = data;
-    s->entries[arch][key].len = len;
+    s->entries[arch][key].len = (uint32_t)len;
     s->entries[arch][key].callback_opaque = callback_opaque;
     s->entries[arch][key].callback = callback;
-
-    return 1;
 }
 
-int fw_cfg_add_file(FWCfgState *s,  const char *filename, uint8_t *data,
-                    uint32_t len)
+void fw_cfg_add_file(FWCfgState *s,  const char *filename,
+                     void *data, size_t len)
 {
     int i, index;
+    size_t dsize;
 
     if (!s->files) {
-        int dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
+        dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
         s->files = g_malloc0(dsize);
-        fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, (uint8_t*)s->files, dsize);
+        fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, s->files, dsize);
     }
 
     index = be32_to_cpu(s->files->count);
-    if (index == FW_CFG_FILE_SLOTS) {
-        fprintf(stderr, "fw_cfg: out of file slots\n");
-        return 0;
-    }
+    assert(index < FW_CFG_FILE_SLOTS);
 
     fw_cfg_add_bytes(s, FW_CFG_FILE_FIRST + index, data, len);
 
@@ -445,24 +458,21 @@ int fw_cfg_add_file(FWCfgState *s,  const char *filename, uint8_t *data,
             filename);
     for (i = 0; i < index; i++) {
         if (strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
-            FW_CFG_DPRINTF("%s: skip duplicate: %s\n", __FUNCTION__,
-                           s->files->f[index].name);
-            return 1;
+            trace_fw_cfg_add_file_dupe(s, s->files->f[index].name);
+            return;
         }
     }
 
     s->files->f[index].size   = cpu_to_be32(len);
     s->files->f[index].select = cpu_to_be16(FW_CFG_FILE_FIRST + index);
-    FW_CFG_DPRINTF("%s: #%d: %s (%d bytes)\n", __FUNCTION__,
-                   index, s->files->f[index].name, len);
+    trace_fw_cfg_add_file(s, index, s->files->f[index].name, len);
 
     s->files->count = cpu_to_be32(index+1);
-    return 1;
 }
 
 static void fw_cfg_machine_ready(struct Notifier *n, void *data)
 {
-    uint32_t len;
+    size_t len;
     FWCfgState *s = container_of(n, FWCfgState, machine_ready);
     char *bootindex = get_boot_devices_list(&len);
 
@@ -470,7 +480,7 @@ static void fw_cfg_machine_ready(struct Notifier *n, void *data)
 }
 
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        target_phys_addr_t ctl_addr, target_phys_addr_t data_addr)
+                        hwaddr ctl_addr, hwaddr data_addr)
 {
     DeviceState *dev;
     SysBusDevice *d;
@@ -480,7 +490,7 @@ FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
     qdev_prop_set_uint32(dev, "ctl_iobase", ctl_port);
     qdev_prop_set_uint32(dev, "data_iobase", data_port);
     qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
+    d = SYS_BUS_DEVICE(dev);
 
     s = DO_UPCAST(FWCfgState, busdev.qdev, dev);
 
@@ -490,13 +500,13 @@ FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
     if (data_addr) {
         sysbus_mmio_map(d, 1, data_addr);
     }
-    fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (uint8_t *)"QEMU", 4);
+    fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (char *)"QEMU", 4);
     fw_cfg_add_bytes(s, FW_CFG_UUID, qemu_uuid, 16);
     fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)(display_type == DT_NOGRAPHIC));
     fw_cfg_add_i16(s, FW_CFG_NB_CPUS, (uint16_t)smp_cpus);
-    fw_cfg_add_i16(s, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i16(s, FW_CFG_BOOT_MENU, (uint16_t)boot_menu);
     fw_cfg_bootsplash(s);
+    fw_cfg_reboot(s);
 
     s->machine_ready.notify = fw_cfg_machine_ready;
     qemu_add_machine_init_done_notifier(&s->machine_ready);
@@ -549,7 +559,7 @@ static void fw_cfg_class_init(ObjectClass *klass, void *data)
     dc->props = fw_cfg_properties;
 }
 
-static TypeInfo fw_cfg_info = {
+static const TypeInfo fw_cfg_info = {
     .name          = "fw_cfg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FWCfgState),
diff --git a/hw/fw_cfg.h b/hw/fw_cfg.h
index 856bf9199dc..05c8df186fc 100644
--- a/hw/fw_cfg.h
+++ b/hw/fw_cfg.h
@@ -54,16 +54,17 @@ typedef struct FWCfgFiles {
 typedef void (*FWCfgCallback)(void *opaque, uint8_t *data);
 
 typedef struct FWCfgState FWCfgState;
-int fw_cfg_add_bytes(FWCfgState *s, uint16_t key, uint8_t *data, uint32_t len);
-int fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
-int fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
-int fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
-int fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                        void *callback_opaque, uint8_t *data, size_t len);
-int fw_cfg_add_file(FWCfgState *s, const char *filename, uint8_t *data,
-                    uint32_t len);
+void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len);
+void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value);
+void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
+void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
+void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
+void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
+                         void *callback_opaque, void *data, size_t len);
+void fw_cfg_add_file(FWCfgState *s, const char *filename, void *data,
+                     size_t len);
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        target_phys_addr_t crl_addr, target_phys_addr_t data_addr);
+                        hwaddr crl_addr, hwaddr data_addr);
 
 #endif /* NO_QEMU_PROTOS */
 
diff --git a/hw/g364fb.c b/hw/g364fb.c
index 3a0b68fbae1..0c0c8ba3028 100644
--- a/hw/g364fb.c
+++ b/hw/g364fb.c
@@ -18,8 +18,8 @@
  */
 
 #include "hw.h"
-#include "console.h"
-#include "pixel_ops.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -197,7 +197,8 @@ static void g364fb_draw_graphic8(G364State *s)
                 reset_dirty(s, page_min, page_max);
                 page_min = (ram_addr_t)-1;
                 page_max = 0;
-                dpy_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
+                dpy_gfx_update(s->ds, xmin, ymin,
+                               xmax - xmin + 1, ymax - ymin + 1);
                 xmin = s->width;
                 xmax = 0;
                 ymin = s->height;
@@ -216,7 +217,7 @@ static void g364fb_draw_graphic8(G364State *s)
 
 done:
     if (page_min != (ram_addr_t)-1) {
-        dpy_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
+        dpy_gfx_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
         reset_dirty(s, page_min, page_max);
     }
 }
@@ -238,7 +239,7 @@ static void g364fb_draw_blank(G364State *s)
         d += ds_get_linesize(s->ds);
     }
 
-    dpy_update(s->ds, 0, 0, s->width, s->height);
+    dpy_gfx_update(s->ds, 0, 0, s->width, s->height);
     s->blanked = 1;
 }
 
@@ -289,10 +290,11 @@ static void g364fb_reset(G364State *s)
     g364fb_invalidate_display(s);
 }
 
-static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     G364State *s = opaque;
-    int y, x;
+    int ret, y, x;
     uint8_t index;
     uint8_t *data_buffer;
     FILE *f;
@@ -300,40 +302,68 @@ static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch)
     qemu_flush_coalesced_mmio_buffer();
 
     if (s->depth != 8) {
-        error_report("g364: unknown guest depth %d", s->depth);
+        error_setg(errp, "g364: unknown guest depth %d", s->depth);
         return;
     }
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
+    }
 
     if (s->ctla & CTLA_FORCE_BLANK) {
         /* blank screen */
-        fprintf(f, "P4\n%d %d\n",
-            s->width, s->height);
+        ret = fprintf(f, "P4\n%d %d\n", s->width, s->height);
+        if (ret < 0) {
+            goto write_err;
+        }
         for (y = 0; y < s->height; y++)
-            for (x = 0; x < s->width; x++)
-                fputc(0, f);
+            for (x = 0; x < s->width; x++) {
+                ret = fputc(0, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+            }
     } else {
         data_buffer = s->vram + s->top_of_screen;
-        fprintf(f, "P6\n%d %d\n%d\n",
-            s->width, s->height, 255);
+        ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+        if (ret < 0) {
+            goto write_err;
+        }
         for (y = 0; y < s->height; y++)
             for (x = 0; x < s->width; x++, data_buffer++) {
                 index = *data_buffer;
-                fputc(s->color_palette[index][0], f);
-                fputc(s->color_palette[index][1], f);
-                fputc(s->color_palette[index][2], f);
+                ret = fputc(s->color_palette[index][0], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->color_palette[index][1], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->color_palette[index][2], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
         }
     }
 
+out:
     fclose(f);
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 /* called for accesses to io ports */
 static uint64_t g364fb_ctrl_read(void *opaque,
-                                 target_phys_addr_t addr,
+                                 hwaddr addr,
                                  unsigned int size)
 {
     G364State *s = opaque;
@@ -395,7 +425,7 @@ static void g364_invalidate_cursor_position(G364State *s)
 }
 
 static void g364fb_ctrl_write(void *opaque,
-                              target_phys_addr_t addr,
+                              hwaddr addr,
                               uint64_t val,
                               unsigned int size)
 {
@@ -567,7 +597,7 @@ static void g364fb_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = g364fb_sysbus_properties;
 }
 
-static TypeInfo g364fb_sysbus_info = {
+static const TypeInfo g364fb_sysbus_info = {
     .name          = "sysbus-g364",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(G364SysBusState),
diff --git a/hw/grackle_pci.c b/hw/grackle_pci.c
index 67da3072848..95639d5735d 100644
--- a/hw/grackle_pci.c
+++ b/hw/grackle_pci.c
@@ -23,9 +23,9 @@
  * THE SOFTWARE.
  */
 
-#include "pci_host.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "pci/pci_host.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
 
 /* debug Grackle */
 //#define DEBUG_GRACKLE
diff --git a/hw/grlib.h b/hw/grlib.h
index e1c41378d46..afd53892b05 100644
--- a/hw/grlib.h
+++ b/hw/grlib.h
@@ -41,7 +41,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level);
 void grlib_irqmp_ack(DeviceState *dev, int intno);
 
 static inline
-DeviceState *grlib_irqmp_create(target_phys_addr_t   base,
+DeviceState *grlib_irqmp_create(hwaddr   base,
                                 CPUSPARCState            *env,
                                 qemu_irq           **cpu_irqs,
                                 uint32_t             nr_irqs,
@@ -61,7 +61,7 @@ DeviceState *grlib_irqmp_create(target_phys_addr_t   base,
 
     env->irq_manager = dev;
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     *cpu_irqs = qemu_allocate_irqs(grlib_irqmp_set_irq,
                                    dev,
@@ -73,7 +73,7 @@ DeviceState *grlib_irqmp_create(target_phys_addr_t   base,
 /* GPTimer */
 
 static inline
-DeviceState *grlib_gptimer_create(target_phys_addr_t  base,
+DeviceState *grlib_gptimer_create(hwaddr  base,
                                   uint32_t            nr_timers,
                                   uint32_t            freq,
                                   qemu_irq           *cpu_irqs,
@@ -91,10 +91,10 @@ DeviceState *grlib_gptimer_create(target_phys_addr_t  base,
         return NULL;
     }
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     for (i = 0; i < nr_timers; i++) {
-        sysbus_connect_irq(sysbus_from_qdev(dev), i, cpu_irqs[base_irq + i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, cpu_irqs[base_irq + i]);
     }
 
     return dev;
@@ -103,7 +103,7 @@ DeviceState *grlib_gptimer_create(target_phys_addr_t  base,
 /* APB UART */
 
 static inline
-DeviceState *grlib_apbuart_create(target_phys_addr_t  base,
+DeviceState *grlib_apbuart_create(hwaddr  base,
                                   CharDriverState    *serial,
                                   qemu_irq            irq)
 {
@@ -116,9 +116,9 @@ DeviceState *grlib_apbuart_create(target_phys_addr_t  base,
         return NULL;
     }
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
diff --git a/hw/grlib_apbuart.c b/hw/grlib_apbuart.c
index 73fc9894db3..760bed0b72a 100644
--- a/hw/grlib_apbuart.c
+++ b/hw/grlib_apbuart.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #include "trace.h"
 
@@ -151,7 +151,7 @@ static void grlib_apbuart_event(void *opaque, int event)
 }
 
 
-static uint64_t grlib_apbuart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_apbuart_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     UART     *uart = opaque;
@@ -181,7 +181,7 @@ static uint64_t grlib_apbuart_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void grlib_apbuart_write(void *opaque, target_phys_addr_t addr,
+static void grlib_apbuart_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     UART          *uart = opaque;
@@ -256,7 +256,7 @@ static void grlib_gptimer_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_gptimer_properties;
 }
 
-static TypeInfo grlib_gptimer_info = {
+static const TypeInfo grlib_gptimer_info = {
     .name          = "grlib,apbuart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(UART),
diff --git a/hw/grlib_gptimer.c b/hw/grlib_gptimer.c
index 41770a9e6c7..7962b74f2c4 100644
--- a/hw/grlib_gptimer.c
+++ b/hw/grlib_gptimer.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 
 #include "trace.h"
@@ -155,11 +155,11 @@ static void grlib_gptimer_hit(void *opaque)
     }
 }
 
-static uint64_t grlib_gptimer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_gptimer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     GPTimerUnit        *unit  = opaque;
-    target_phys_addr_t  timer_addr;
+    hwaddr  timer_addr;
     int                 id;
     uint32_t            value = 0;
 
@@ -214,11 +214,11 @@ static uint64_t grlib_gptimer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void grlib_gptimer_write(void *opaque, target_phys_addr_t addr,
+static void grlib_gptimer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     GPTimerUnit        *unit = opaque;
-    target_phys_addr_t  timer_addr;
+    hwaddr  timer_addr;
     int                 id;
 
     addr &= 0xff;
@@ -389,7 +389,7 @@ static void grlib_gptimer_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_gptimer_properties;
 }
 
-static TypeInfo grlib_gptimer_info = {
+static const TypeInfo grlib_gptimer_info = {
     .name          = "grlib,gptimer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GPTimerUnit),
diff --git a/hw/grlib_irqmp.c b/hw/grlib_irqmp.c
index 0f6e65cf204..ef8dd95ac9b 100644
--- a/hw/grlib_irqmp.c
+++ b/hw/grlib_irqmp.c
@@ -109,7 +109,7 @@ void grlib_irqmp_ack(DeviceState *dev, int intno)
 
     assert(dev != NULL);
 
-    sdev = sysbus_from_qdev(dev);
+    sdev = SYS_BUS_DEVICE(dev);
     assert(sdev != NULL);
 
     irqmp = FROM_SYSBUS(typeof(*irqmp), sdev);
@@ -138,7 +138,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level)
 
     assert(opaque != NULL);
 
-    irqmp = FROM_SYSBUS(typeof(*irqmp), sysbus_from_qdev(opaque));
+    irqmp = FROM_SYSBUS(typeof(*irqmp), SYS_BUS_DEVICE(opaque));
     assert(irqmp != NULL);
 
     s = irqmp->state;
@@ -162,7 +162,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level)
     }
 }
 
-static uint64_t grlib_irqmp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_irqmp_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     IRQMP      *irqmp = opaque;
@@ -226,7 +226,7 @@ static uint64_t grlib_irqmp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void grlib_irqmp_write(void *opaque, target_phys_addr_t addr,
+static void grlib_irqmp_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     IRQMP      *irqmp = opaque;
@@ -370,7 +370,7 @@ static void grlib_irqmp_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_irqmp_properties;
 }
 
-static TypeInfo grlib_irqmp_info = {
+static const TypeInfo grlib_irqmp_info = {
     .name          = "grlib,irqmp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IRQMP),
diff --git a/hw/gt64xxx.c b/hw/gt64xxx.c
index e95e6648338..977a2c5e692 100644
--- a/hw/gt64xxx.c
+++ b/hw/gt64xxx.c
@@ -24,10 +24,10 @@
 
 #include "hw.h"
 #include "mips.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "pc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG
 
@@ -225,8 +225,8 @@
 #define GT_PCI1_SERR1MASK    	(0xca8 >> 2)
 
 #define PCI_MAPPING_ENTRY(regname)            \
-    target_phys_addr_t regname ##_start;      \
-    target_phys_addr_t regname ##_length;     \
+    hwaddr regname ##_start;      \
+    hwaddr regname ##_length;     \
     MemoryRegion regname ##_mem
 
 #define TYPE_GT64120_PCI_HOST_BRIDGE "gt64120"
@@ -245,11 +245,11 @@ typedef struct GT64120State {
 /* Adjust range to avoid touching space which isn't mappable via PCI */
 /* XXX: Hardcoded values for Malta: 0x1e000000 - 0x1f100000
                                     0x1fc00000 - 0x1fd00000  */
-static void check_reserved_space (target_phys_addr_t *start,
-                                  target_phys_addr_t *length)
+static void check_reserved_space (hwaddr *start,
+                                  hwaddr *length)
 {
-    target_phys_addr_t begin = *start;
-    target_phys_addr_t end = *start + *length;
+    hwaddr begin = *start;
+    hwaddr end = *start + *length;
 
     if (end >= 0x1e000000LL && end < 0x1f100000LL)
         end = 0x1e000000LL;
@@ -271,8 +271,8 @@ static void check_reserved_space (target_phys_addr_t *start,
 
 static void gt64120_isd_mapping(GT64120State *s)
 {
-    target_phys_addr_t start = s->regs[GT_ISD] << 21;
-    target_phys_addr_t length = 0x1000;
+    hwaddr start = s->regs[GT_ISD] << 21;
+    hwaddr length = 0x1000;
 
     if (s->ISD_length) {
         memory_region_del_subregion(get_system_memory(), &s->ISD_mem);
@@ -311,7 +311,7 @@ static void gt64120_pci_mapping(GT64120State *s)
     }
 }
 
-static void gt64120_writel (void *opaque, target_phys_addr_t addr,
+static void gt64120_writel (void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     GT64120State *s = opaque;
@@ -594,7 +594,7 @@ static void gt64120_writel (void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t gt64120_readl (void *opaque,
-                               target_phys_addr_t addr, unsigned size)
+                               hwaddr addr, unsigned size)
 {
     GT64120State *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
diff --git a/hw/gumstix.c b/hw/gumstix.c
index 13a36ea5c52..bea16058f71 100644
--- a/hw/gumstix.c
+++ b/hw/gumstix.c
@@ -36,19 +36,16 @@
 
 #include "hw.h"
 #include "pxa.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
 #include "devices.h"
 #include "boards.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 static const int sector_len = 128 * 1024;
 
-static void connex_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void connex_init(QEMUMachineInitArgs *args)
 {
     PXA2xxState *cpu;
     DriveInfo *dinfo;
@@ -84,11 +81,9 @@ static void connex_init(ram_addr_t ram_size,
                     qdev_get_gpio_in(cpu->gpio, 36));
 }
 
-static void verdex_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void verdex_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
     PXA2xxState *cpu;
     DriveInfo *dinfo;
     int be;
@@ -127,12 +122,14 @@ static QEMUMachine connex_machine = {
     .name = "connex",
     .desc = "Gumstix Connex (PXA255)",
     .init = connex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine verdex_machine = {
     .name = "verdex",
     .desc = "Gumstix Verdex (PXA270)",
     .init = verdex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void gumstix_machine_init(void)
diff --git a/hw/gus.c b/hw/gus.c
index 840d098d6ae..aa13fccf0d5 100644
--- a/hw/gus.c
+++ b/hw/gus.c
@@ -317,7 +317,7 @@ static void gus_class_initfn (ObjectClass *klass, void *data)
     dc->props = gus_properties;
 }
 
-static TypeInfo gus_info = {
+static const TypeInfo gus_info = {
     .name          = "gus",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (GUSState),
diff --git a/hw/hd-geometry.c b/hw/hd-geometry.c
index 1cdb9fb7537..c30514364f1 100644
--- a/hw/hd-geometry.c
+++ b/hw/hd-geometry.c
@@ -30,7 +30,7 @@
  * THE SOFTWARE.
  */
 
-#include "block.h"
+#include "block/block.h"
 #include "hw/block-common.h"
 #include "trace.h"
 
diff --git a/hw/hda-audio.c b/hw/hda-audio.c
index 36761dd2dee..3190bd1cf84 100644
--- a/hw/hda-audio.c
+++ b/hw/hda-audio.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "intel-hda.h"
 #include "intel-hda-defs.h"
 #include "audio/audio.h"
@@ -1039,7 +1039,7 @@ static void hda_audio_output_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_output_info = {
+static const TypeInfo hda_audio_output_info = {
     .name          = "hda-output",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
@@ -1060,7 +1060,7 @@ static void hda_audio_duplex_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_duplex_info = {
+static const TypeInfo hda_audio_duplex_info = {
     .name          = "hda-duplex",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
@@ -1081,7 +1081,7 @@ static void hda_audio_micro_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_micro_info = {
+static const TypeInfo hda_audio_micro_info = {
     .name          = "hda-micro",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
diff --git a/hw/heathrow_pic.c b/hw/heathrow_pic.c
index 16f48d12e10..c0a71c3d5f2 100644
--- a/hw/heathrow_pic.c
+++ b/hw/heathrow_pic.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
+#include "ppc/mac.h"
 
 /* debug PIC */
 //#define DEBUG_PIC
@@ -63,7 +63,7 @@ static void heathrow_pic_update(HeathrowPICS *s)
     }
 }
 
-static void pic_write(void *opaque, target_phys_addr_t addr,
+static void pic_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned size)
 {
     HeathrowPICS *s = opaque;
@@ -91,7 +91,7 @@ static void pic_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t pic_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pic_read(void *opaque, hwaddr addr,
                          unsigned size)
 {
     HeathrowPICS *s = opaque;
diff --git a/hw/hid.c b/hw/hid.c
index 03761ab8b8e..89b5415c0fe 100644
--- a/hw/hid.c
+++ b/hw/hid.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 #include "hid.h"
 
 #define HID_USAGE_ERROR_ROLLOVER        0x01
@@ -71,12 +71,38 @@ static const uint8_t hid_usage_keys[0x100] = {
 
 bool hid_has_events(HIDState *hs)
 {
-    return hs->n > 0;
+    return hs->n > 0 || hs->idle_pending;
 }
 
-void hid_set_next_idle(HIDState *hs, int64_t curtime)
+static void hid_idle_timer(void *opaque)
 {
-    hs->next_idle_clock = curtime + (get_ticks_per_sec() * hs->idle * 4) / 1000;
+    HIDState *hs = opaque;
+
+    hs->idle_pending = true;
+    hs->event(hs);
+}
+
+static void hid_del_idle_timer(HIDState *hs)
+{
+    if (hs->idle_timer) {
+        qemu_del_timer(hs->idle_timer);
+        qemu_free_timer(hs->idle_timer);
+        hs->idle_timer = NULL;
+    }
+}
+
+void hid_set_next_idle(HIDState *hs)
+{
+    if (hs->idle) {
+        uint64_t expire_time = qemu_get_clock_ns(vm_clock) +
+                               get_ticks_per_sec() * hs->idle * 4 / 1000;
+        if (!hs->idle_timer) {
+            hs->idle_timer = qemu_new_timer_ns(vm_clock, hid_idle_timer, hs);
+        }
+        qemu_mod_timer_ns(hs->idle_timer, expire_time);
+    } else {
+        hid_del_idle_timer(hs);
+    }
 }
 
 static void hid_pointer_event_clear(HIDPointerEvent *e, int buttons)
@@ -232,6 +258,8 @@ int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len)
     int index;
     HIDPointerEvent *e;
 
+    hs->idle_pending = false;
+
     hid_pointer_activate(hs);
 
     /* When the buffer is empty, return the last event.  Relative
@@ -319,6 +347,8 @@ int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len)
 
 int hid_keyboard_poll(HIDState *hs, uint8_t *buf, int len)
 {
+    hs->idle_pending = false;
+
     if (len < 2) {
         return 0;
     }
@@ -377,6 +407,8 @@ void hid_reset(HIDState *hs)
     hs->n = 0;
     hs->protocol = 1;
     hs->idle = 0;
+    hs->idle_pending = false;
+    hid_del_idle_timer(hs);
 }
 
 void hid_free(HIDState *hs)
@@ -390,6 +422,7 @@ void hid_free(HIDState *hs)
         qemu_remove_mouse_event_handler(hs->ptr.eh_entry);
         break;
     }
+    hid_del_idle_timer(hs);
 }
 
 void hid_init(HIDState *hs, int kind, HIDEventFunc event)
@@ -412,9 +445,7 @@ static int hid_post_load(void *opaque, int version_id)
 {
     HIDState *s = opaque;
 
-    if (s->idle) {
-        hid_set_next_idle(s, qemu_get_clock_ns(vm_clock));
-    }
+    hid_set_next_idle(s);
     return 0;
 }
 
diff --git a/hw/hid.h b/hw/hid.h
index 5315cf7a31f..56c71ed5ae0 100644
--- a/hw/hid.h
+++ b/hw/hid.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_HID_H
 #define QEMU_HID_H
 
-#include "vmstate.h"
+#include "migration/vmstate.h"
 
 #define HID_MOUSE     1
 #define HID_TABLET    2
@@ -43,7 +43,8 @@ struct HIDState {
     int kind;
     int32_t protocol;
     uint8_t idle;
-    int64_t next_idle_clock;
+    bool idle_pending;
+    QEMUTimer *idle_timer;
     HIDEventFunc event;
 };
 
@@ -52,7 +53,7 @@ void hid_reset(HIDState *hs);
 void hid_free(HIDState *hs);
 
 bool hid_has_events(HIDState *hs);
-void hid_set_next_idle(HIDState *hs, int64_t curtime);
+void hid_set_next_idle(HIDState *hs);
 void hid_pointer_activate(HIDState *hs);
 int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len);
 int hid_keyboard_poll(HIDState *hs, uint8_t *buf, int len);
diff --git a/hw/highbank.c b/hw/highbank.c
index 11aa1312c0a..defcc092b4c 100644
--- a/hw/highbank.c
+++ b/hw/highbank.c
@@ -21,12 +21,12 @@
 #include "arm-misc.h"
 #include "devices.h"
 #include "loader.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "sysbus.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define SMP_BOOT_ADDR 0x100
 #define SMP_BOOT_REG  0x40
@@ -44,9 +44,12 @@ static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
         0xe210000f, /* ands r0, r0, #0x0f */
         0xe3a03040, /* mov r3, #0x40 - jump address is 0x40 + 0x10 * core id */
         0xe0830200, /* add r0, r3, r0, lsl #4 */
-        0xe59f2018, /* ldr r2, privbase */
+        0xe59f2024, /* ldr r2, privbase */
         0xe3a01001, /* mov r1, #1 */
-        0xe5821100, /* str r1, [r2, #256] */
+        0xe5821100, /* str r1, [r2, #256] - set GICC_CTLR.Enable */
+        0xe3a010ff, /* mov r1, #0xff */
+        0xe5821104, /* str r1, [r2, #260] - set GICC_PMR.Priority to 0xff */
+        0xf57ff04f, /* dsb */
         0xe320f003, /* wfi */
         0xe5901000, /* ldr     r1, [r0] */
         0xe1110001, /* tst     r1, r1 */
@@ -79,7 +82,7 @@ static void hb_reset_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
 }
 
 #define NUM_REGS      0x200
-static void hb_regs_write(void *opaque, target_phys_addr_t offset,
+static void hb_regs_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     uint32_t *regs = opaque;
@@ -95,7 +98,7 @@ static void hb_regs_write(void *opaque, target_phys_addr_t offset,
     regs[offset/4] = value;
 }
 
-static uint64_t hb_regs_read(void *opaque, target_phys_addr_t offset,
+static uint64_t hb_regs_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     uint32_t *regs = opaque;
@@ -133,7 +136,7 @@ static VMStateDescription vmstate_highbank_regs = {
 
 static void highbank_regs_reset(DeviceState *dev)
 {
-    SysBusDevice *sys_dev = sysbus_from_qdev(dev);
+    SysBusDevice *sys_dev = SYS_BUS_DEVICE(dev);
     HighbankRegsState *s = FROM_SYSBUS(HighbankRegsState, sys_dev);
 
     s->regs[0x40] = 0x05F20121;
@@ -165,7 +168,7 @@ static void highbank_regs_class_init(ObjectClass *klass, void *data)
     dc->reset = highbank_regs_reset;
 }
 
-static TypeInfo highbank_regs_info = {
+static const TypeInfo highbank_regs_info = {
     .name          = "highbank-regs",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(HighbankRegsState),
@@ -187,11 +190,13 @@ static struct arm_boot_info highbank_binfo;
  * 32-bit host, set the reg value of memory to 0xf7ff00000 in the
  * device tree and pass -m 2047 to QEMU.
  */
-static void highbank_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void highbank_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     DeviceState *dev;
     SysBusDevice *busdev;
     qemu_irq *irqp;
@@ -246,7 +251,7 @@ static void highbank_init(ram_addr_t ram_size,
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_prop_set_uint32(dev, "num-irq", NIRQ_GIC);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, GIC_BASE_ADDR);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -258,21 +263,21 @@ static void highbank_init(ram_addr_t ram_size,
 
     dev = qdev_create(NULL, "l2x0");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff12000);
 
     dev = qdev_create(NULL, "sp804");
     qdev_prop_set_uint32(dev, "freq0", 150000000);
     qdev_prop_set_uint32(dev, "freq1", 150000000);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff34000);
     sysbus_connect_irq(busdev, 0, pic[18]);
     sysbus_create_simple("pl011", 0xfff36000, pic[20]);
 
     dev = qdev_create(NULL, "highbank-regs");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff3c000);
 
     sysbus_create_simple("pl061", 0xfff30000, pic[14]);
@@ -289,19 +294,19 @@ static void highbank_init(ram_addr_t ram_size,
         dev = qdev_create(NULL, "xgmac");
         qdev_set_nic_properties(dev, &nd_table[0]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xfff50000);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[77]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 1, pic[78]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 2, pic[79]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xfff50000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[77]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, pic[78]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, pic[79]);
 
         qemu_check_nic_model(&nd_table[1], "xgmac");
         dev = qdev_create(NULL, "xgmac");
         qdev_set_nic_properties(dev, &nd_table[1]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xfff51000);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[80]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 1, pic[81]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 2, pic[82]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xfff51000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[80]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, pic[81]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, pic[82]);
     }
 
     highbank_binfo.ram_size = ram_size;
@@ -324,8 +329,9 @@ static QEMUMachine highbank_machine = {
     .name = "highbank",
     .desc = "Calxeda Highbank (ECX-1000)",
     .init = highbank_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void highbank_machine_init(void)
diff --git a/hw/hpet.c b/hw/hpet.c
index fd3ddca7f76..97eaa2f700a 100644
--- a/hw/hpet.c
+++ b/hw/hpet.c
@@ -26,8 +26,8 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 #include "hpet_emul.h"
 #include "sysbus.h"
 #include "mc146818rtc.h"
@@ -370,20 +370,20 @@ static void hpet_del_timer(HPETTimer *t)
 }
 
 #ifdef HPET_DEBUG
-static uint32_t hpet_ram_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t hpet_ram_readb(void *opaque, hwaddr addr)
 {
     printf("qemu: hpet_read b at %" PRIx64 "\n", addr);
     return 0;
 }
 
-static uint32_t hpet_ram_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t hpet_ram_readw(void *opaque, hwaddr addr)
 {
     printf("qemu: hpet_read w at %" PRIx64 "\n", addr);
     return 0;
 }
 #endif
 
-static uint64_t hpet_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t hpet_ram_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     HPETState *s = opaque;
@@ -455,7 +455,7 @@ static uint64_t hpet_ram_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void hpet_ram_write(void *opaque, target_phys_addr_t addr,
+static void hpet_ram_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     int i;
@@ -634,7 +634,7 @@ static const MemoryRegionOps hpet_ram_ops = {
 
 static void hpet_reset(DeviceState *d)
 {
-    HPETState *s = FROM_SYSBUS(HPETState, sysbus_from_qdev(d));
+    HPETState *s = FROM_SYSBUS(HPETState, SYS_BUS_DEVICE(d));
     int i;
 
     for (i = 0; i < s->num_timers; i++) {
@@ -657,7 +657,7 @@ static void hpet_reset(DeviceState *d)
     s->hpet_offset = 0ULL;
     s->config = 0ULL;
     hpet_cfg.hpet[s->hpet_id].event_timer_block_id = (uint32_t)s->capability;
-    hpet_cfg.hpet[s->hpet_id].address = sysbus_from_qdev(d)->mmio[0].addr;
+    hpet_cfg.hpet[s->hpet_id].address = SYS_BUS_DEVICE(d)->mmio[0].addr;
 
     /* to document that the RTC lowers its output on reset as well */
     s->rtc_irq_level = 0;
@@ -745,7 +745,7 @@ static void hpet_device_class_init(ObjectClass *klass, void *data)
     dc->props = hpet_device_properties;
 }
 
-static TypeInfo hpet_device_info = {
+static const TypeInfo hpet_device_info = {
     .name          = "hpet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(HPETState),
diff --git a/hw/hw.h b/hw/hw.h
index e5cb9bf94d4..dfced97bbc1 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -4,14 +4,16 @@
 
 #include "qemu-common.h"
 
-#if defined(TARGET_PHYS_ADDR_BITS) && !defined(NEED_CPU_H)
-#include "cpu-common.h"
+#if !defined(CONFIG_USER_ONLY) && !defined(NEED_CPU_H)
+#include "exec/cpu-common.h"
 #endif
 
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "irq.h"
-#include "qemu-file.h"
-#include "vmstate.h"
+#include "block/aio.h"
+#include "migration/qemu-file.h"
+#include "migration/vmstate.h"
+#include "qemu/log.h"
 
 #ifdef NEED_CPU_H
 #if TARGET_LONG_BITS == 64
diff --git a/hw/i2c.c b/hw/i2c.c
index 296bece119b..119e96bc0e4 100644
--- a/hw/i2c.c
+++ b/hw/i2c.c
@@ -228,7 +228,7 @@ static void i2c_slave_class_init(ObjectClass *klass, void *data)
     k->props = i2c_props;
 }
 
-static TypeInfo i2c_slave_type_info = {
+static const TypeInfo i2c_slave_type_info = {
     .name = TYPE_I2C_SLAVE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(I2CSlave),
diff --git a/hw/i2c.h b/hw/i2c.h
index 0f5682b4b0d..883b5c588d2 100644
--- a/hw/i2c.h
+++ b/hw/i2c.h
@@ -73,9 +73,6 @@ void *wm8750_dac_buffer(void *opaque, int samples);
 void wm8750_dac_commit(void *opaque);
 void wm8750_set_bclk_in(void *opaque, int new_hz);
 
-/* tmp105.c */
-void tmp105_set(I2CSlave *i2c, int temp);
-
 /* lm832x.c */
 void lm832x_key_event(DeviceState *dev, int key, int state);
 
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index 8c764bbfef5..025803aa666 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -2,14 +2,16 @@ obj-y += mc146818rtc.o pc.o
 obj-y += apic_common.o apic.o kvmvapic.o
 obj-y += sga.o ioapic_common.o ioapic.o piix_pci.o
 obj-y += vmport.o
-obj-y += pci-hotplug.o smbios.o wdt_ib700.o
-obj-y += debugcon.o multiboot.o
+obj-y += pci/pci-hotplug.o smbios.o wdt_ib700.o
+obj-y += debugcon.o debugexit.o multiboot.o
 obj-y += pc_piix.o
 obj-y += pc_sysfw.o
+obj-y += lpc_ich9.o q35.o pc_q35.o
 obj-$(CONFIG_XEN) += xen_platform.o xen_apic.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
 obj-y += kvm/
 obj-$(CONFIG_SPICE) += qxl.o qxl-logger.o qxl-render.o
+obj-y += pc-testdev.o
 
 obj-y := $(addprefix ../,$(obj-y))
diff --git a/hw/i82374.c b/hw/i82374.c
index 4a922c3f4e9..6a62ba2ab88 100644
--- a/hw/i82374.c
+++ b/hw/i82374.c
@@ -153,7 +153,7 @@ static void i82374_class_init(ObjectClass *klass, void *data)
     dc->props = i82374_properties;
 }
 
-static TypeInfo i82374_isa_info = {
+static const TypeInfo i82374_isa_info = {
     .name  = "i82374",
     .parent = TYPE_ISA_DEVICE,
     .instance_size  = sizeof(ISAi82374State),
diff --git a/hw/i82378.c b/hw/i82378.c
index 2123c142aae..0914d7bbfb6 100644
--- a/hw/i82378.c
+++ b/hw/i82378.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "i8254.h"
 #include "pcspk.h"
@@ -59,7 +59,7 @@ static const VMStateDescription vmstate_pci_i82378 = {
     },
 };
 
-static void i82378_io_write(void *opaque, target_phys_addr_t addr,
+static void i82378_io_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned int size)
 {
     switch (size) {
@@ -83,7 +83,7 @@ static void i82378_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t i82378_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t i82378_io_read(void *opaque, hwaddr addr,
                                unsigned int size)
 {
     DPRINTF("%s: " TARGET_FMT_plx "\n", __func__, addr);
@@ -105,7 +105,7 @@ static const MemoryRegionOps i82378_io_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void i82378_mem_write(void *opaque, target_phys_addr_t addr,
+static void i82378_mem_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned int size)
 {
     switch (size) {
@@ -129,7 +129,7 @@ static void i82378_mem_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t i82378_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t i82378_mem_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     DPRINTF("%s: " TARGET_FMT_plx "\n", __func__, addr);
@@ -262,7 +262,7 @@ static void pci_i82378_class_init(ObjectClass *klass, void *data)
     dc->props = i82378_properties;
 }
 
-static TypeInfo pci_i82378_info = {
+static const TypeInfo pci_i82378_info = {
     .name = "i82378",
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIi82378State),
diff --git a/hw/i8254.c b/hw/i8254.c
index 77bd5e82220..394b2e81d76 100644
--- a/hw/i8254.c
+++ b/hw/i8254.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "i8254_internal.h"
 
@@ -111,7 +111,8 @@ static void pit_latch_count(PITChannelState *s)
     }
 }
 
-static void pit_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void pit_ioport_write(void *opaque, hwaddr addr,
+                             uint64_t val, unsigned size)
 {
     PITCommonState *pit = opaque;
     int channel, access;
@@ -178,7 +179,8 @@ static void pit_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t pit_ioport_read(void *opaque, uint32_t addr)
+static uint64_t pit_ioport_read(void *opaque, hwaddr addr,
+                                unsigned size)
 {
     PITCommonState *pit = opaque;
     int ret, count;
@@ -290,14 +292,14 @@ static void pit_irq_control(void *opaque, int n, int enable)
     }
 }
 
-static const MemoryRegionPortio pit_portio[] = {
-    { 0, 4, 1, .write = pit_ioport_write },
-    { 0, 3, 1, .read = pit_ioport_read },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps pit_ioport_ops = {
-    .old_portio = pit_portio
+    .read = pit_ioport_read,
+    .write = pit_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void pit_post_load(PITCommonState *s)
@@ -345,7 +347,7 @@ static void pit_class_initfn(ObjectClass *klass, void *data)
     dc->props = pit_properties;
 }
 
-static TypeInfo pit_info = {
+static const TypeInfo pit_info = {
     .name          = "isa-pit",
     .parent        = TYPE_PIT_COMMON,
     .instance_size = sizeof(PITCommonState),
diff --git a/hw/i8254_common.c b/hw/i8254_common.c
index a03d7cd4580..8c2e45a92e7 100644
--- a/hw/i8254_common.c
+++ b/hw/i8254_common.c
@@ -25,7 +25,7 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "i8254_internal.h"
 
@@ -294,7 +294,7 @@ static void pit_common_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo pit_common_type = {
+static const TypeInfo pit_common_type = {
     .name          = TYPE_PIT_COMMON,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(PITCommonState),
diff --git a/hw/i8259.c b/hw/i8259.c
index 53daf78652e..54fe14447b7 100644
--- a/hw/i8259.c
+++ b/hw/i8259.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "monitor.h"
-#include "qemu-timer.h"
+#include "monitor/monitor.h"
+#include "qemu/timer.h"
 #include "i8259_internal.h"
 
 /* debug PIC */
@@ -235,7 +235,7 @@ static void pic_reset(DeviceState *dev)
     pic_init_reset(s);
 }
 
-static void pic_ioport_write(void *opaque, target_phys_addr_t addr64,
+static void pic_ioport_write(void *opaque, hwaddr addr64,
                              uint64_t val64, unsigned size)
 {
     PICCommonState *s = opaque;
@@ -329,7 +329,7 @@ static void pic_ioport_write(void *opaque, target_phys_addr_t addr64,
     }
 }
 
-static uint64_t pic_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pic_ioport_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PICCommonState *s = opaque;
@@ -366,14 +366,14 @@ int pic_get_output(DeviceState *d)
     return (pic_get_irq(s) >= 0);
 }
 
-static void elcr_ioport_write(void *opaque, target_phys_addr_t addr,
+static void elcr_ioport_write(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     PICCommonState *s = opaque;
     s->elcr = val & s->elcr_mask;
 }
 
-static uint64_t elcr_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t elcr_ioport_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     PICCommonState *s = opaque;
@@ -407,7 +407,7 @@ static void pic_init(PICCommonState *s)
     qdev_init_gpio_in(&s->dev.qdev, pic_set_irq, 8);
 }
 
-void pic_info(Monitor *mon)
+void pic_info(Monitor *mon, const QDict *qdict)
 {
     int i;
     PICCommonState *s;
@@ -425,7 +425,7 @@ void pic_info(Monitor *mon)
     }
 }
 
-void irq_info(Monitor *mon)
+void irq_info(Monitor *mon, const QDict *qdict)
 {
 #ifndef DEBUG_IRQ_COUNT
     monitor_printf(mon, "irq statistic code not compiled.\n");
@@ -481,7 +481,7 @@ static void i8259_class_init(ObjectClass *klass, void *data)
     dc->reset = pic_reset;
 }
 
-static TypeInfo i8259_info = {
+static const TypeInfo i8259_info = {
     .name       = "isa-i8259",
     .instance_size = sizeof(PICCommonState),
     .parent     = TYPE_PIC_COMMON,
diff --git a/hw/i8259_common.c b/hw/i8259_common.c
index ab3d98b2a17..fc91056afbf 100644
--- a/hw/i8259_common.c
+++ b/hw/i8259_common.c
@@ -144,7 +144,7 @@ static void pic_common_class_init(ObjectClass *klass, void *data)
     ic->init = pic_init_common;
 }
 
-static TypeInfo pic_common_type = {
+static const TypeInfo pic_common_type = {
     .name = TYPE_PIC_COMMON,
     .parent = TYPE_ISA_DEVICE,
     .instance_size = sizeof(PICCommonState),
diff --git a/hw/i8259_internal.h b/hw/i8259_internal.h
index 4137b617034..8785b1da3fd 100644
--- a/hw/i8259_internal.h
+++ b/hw/i8259_internal.h
@@ -33,7 +33,7 @@ typedef struct PICCommonState PICCommonState;
 
 #define TYPE_PIC_COMMON "pic-common"
 #define PIC_COMMON(obj) \
-     OBJECT_CHECK(PICCommon, (obj), TYPE_PIC_COMMON)
+     OBJECT_CHECK(PICCommonState, (obj), TYPE_PIC_COMMON)
 #define PIC_COMMON_CLASS(klass) \
      OBJECT_CLASS_CHECK(PICCommonClass, (klass), TYPE_PIC_COMMON)
 #define PIC_COMMON_GET_CLASS(obj) \
diff --git a/hw/i82801b11.c b/hw/i82801b11.c
new file mode 100644
index 00000000000..3dc10000a44
--- /dev/null
+++ b/hw/i82801b11.c
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+/*
+ * QEMU i82801b11 dmi-to-pci Bridge Emulation
+ *
+ *  Copyright (c) 2009, 2010, 2011
+ *                Isaku Yamahata <yamahata at valinux co jp>
+ *                VA Linux Systems Japan K.K.
+ *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "pci/pci.h"
+#include "ich9.h"
+
+
+/*****************************************************************************/
+/* ICH9 DMI-to-PCI bridge */
+#define I82801ba_SSVID_OFFSET   0x50
+#define I82801ba_SSVID_SVID     0
+#define I82801ba_SSVID_SSID     0
+
+typedef struct I82801b11Bridge {
+    PCIBridge br;
+} I82801b11Bridge;
+
+static int i82801b11_bridge_initfn(PCIDevice *d)
+{
+    int rc;
+
+    rc = pci_bridge_initfn(d);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = pci_bridge_ssvid_init(d, I82801ba_SSVID_OFFSET,
+                               I82801ba_SSVID_SVID, I82801ba_SSVID_SSID);
+    if (rc < 0) {
+        goto err_bridge;
+    }
+    pci_config_set_prog_interface(d->config, PCI_CLASS_BRDIGE_PCI_INF_SUB);
+    return 0;
+
+err_bridge:
+    pci_bridge_exitfn(d);
+
+    return rc;
+}
+
+static void i82801b11_bridge_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->is_bridge = 1;
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_82801BA_11;
+    k->revision = ICH9_D2P_A2_REVISION;
+    k->init = i82801b11_bridge_initfn;
+}
+
+static const TypeInfo i82801b11_bridge_info = {
+    .name          = "i82801b11-bridge",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(I82801b11Bridge),
+    .class_init    = i82801b11_bridge_class_init,
+};
+
+PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus)
+{
+    PCIDevice *d;
+    PCIBridge *br;
+    char buf[16];
+    DeviceState *qdev;
+
+    d = pci_create_multifunction(bus, devfn, true, "i82801b11-bridge");
+    if (!d) {
+        return NULL;
+    }
+    br = DO_UPCAST(PCIBridge, dev, d);
+    qdev = &br->dev.qdev;
+
+    snprintf(buf, sizeof(buf), "pci.%d", sec_bus);
+    pci_bridge_map_irq(br, buf, pci_swizzle_map_irq_fn);
+    qdev_init_nofail(qdev);
+
+    return pci_bridge_get_sec_bus(br);
+}
+
+static void d2pbr_register(void)
+{
+    type_register_static(&i82801b11_bridge_info);
+}
+
+type_init(d2pbr_register);
diff --git a/hw/ich9.h b/hw/ich9.h
new file mode 100644
index 00000000000..b8d8e6d3dfe
--- /dev/null
+++ b/hw/ich9.h
@@ -0,0 +1,208 @@
+#ifndef HW_ICH9_H
+#define HW_ICH9_H
+
+#include "hw.h"
+#include "qemu/range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "ioapic.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
+#include "pci/pci_bridge.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+#include "pci/pci_bus.h"
+
+void ich9_lpc_set_irq(void *opaque, int irq_num, int level);
+int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx);
+void ich9_lpc_pm_init(PCIDevice *pci_lpc, qemu_irq cmos_s3);
+PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus);
+i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base);
+
+#define ICH9_CC_SIZE                            (16 * 1024)     /* 16KB */
+
+#define TYPE_ICH9_LPC_DEVICE "ICH9 LPC"
+#define ICH9_LPC_DEVICE(obj) \
+     OBJECT_CHECK(ICH9LPCState, (obj), TYPE_ICH9_LPC_DEVICE)
+
+typedef struct ICH9LPCState {
+    /* ICH9 LPC PCI to ISA bridge */
+    PCIDevice d;
+
+    /* (pci device, intx) -> pirq
+     * In real chipset case, the unused slots are never used
+     * as ICH9 supports only D25-D32 irq routing.
+     * On the other hand in qemu case, any slot/function can be populated
+     * via command line option.
+     * So fallback interrupt routing for any devices in any slots is necessary.
+    */
+    uint8_t irr[PCI_SLOT_MAX][PCI_NUM_PINS];
+
+    APMState apm;
+    ICH9LPCPMRegs pm;
+    uint32_t sci_level; /* track sci level */
+
+    /* 10.1 Chipset Configuration registers(Memory Space)
+     which is pointed by RCBA */
+    uint8_t chip_config[ICH9_CC_SIZE];
+    /* isa bus */
+    ISABus *isa_bus;
+    MemoryRegion rbca_mem;
+    Notifier machine_ready;
+
+    qemu_irq *pic;
+    qemu_irq *ioapic;
+} ICH9LPCState;
+
+#define Q35_MASK(bit, ms_bit, ls_bit) \
+((uint##bit##_t)(((1ULL << ((ms_bit) + 1)) - 1) & ~((1ULL << ls_bit) - 1)))
+
+/* ICH9: Chipset Configuration Registers */
+#define ICH9_CC_ADDR_MASK                       (ICH9_CC_SIZE - 1)
+
+#define ICH9_CC
+#define ICH9_CC_D28IP                           0x310C
+#define ICH9_CC_D28IP_SHIFT                     4
+#define ICH9_CC_D28IP_MASK                      0xf
+#define ICH9_CC_D28IP_DEFAULT                   0x00214321
+#define ICH9_CC_D31IR                           0x3140
+#define ICH9_CC_D30IR                           0x3142
+#define ICH9_CC_D29IR                           0x3144
+#define ICH9_CC_D28IR                           0x3146
+#define ICH9_CC_D27IR                           0x3148
+#define ICH9_CC_D26IR                           0x314C
+#define ICH9_CC_D25IR                           0x3150
+#define ICH9_CC_DIR_DEFAULT                     0x3210
+#define ICH9_CC_D30IR_DEFAULT                   0x0
+#define ICH9_CC_DIR_SHIFT                       4
+#define ICH9_CC_DIR_MASK                        0x7
+#define ICH9_CC_OIC                             0x31FF
+#define ICH9_CC_OIC_AEN                         0x1
+
+/* D28:F[0-5] */
+#define ICH9_PCIE_DEV                           28
+#define ICH9_PCIE_FUNC_MAX                      6
+
+
+/* D29:F0 USB UHCI Controller #1 */
+#define ICH9_USB_UHCI1_DEV                      29
+#define ICH9_USB_UHCI1_FUNC                     0
+
+/* D30:F0 DMI-to-PCI brdige */
+#define ICH9_D2P_BRIDGE                         "ICH9 D2P BRIDGE"
+#define ICH9_D2P_BRIDGE_SAVEVM_VERSION          0
+
+#define ICH9_D2P_BRIDGE_DEV                     30
+#define ICH9_D2P_BRIDGE_FUNC                    0
+
+#define ICH9_D2P_SECONDARY_DEFAULT              (256 - 8)
+
+#define ICH9_D2P_A2_REVISION                    0x92
+
+
+/* D31:F1 LPC controller */
+#define ICH9_A2_LPC                             "ICH9 A2 LPC"
+#define ICH9_A2_LPC_SAVEVM_VERSION              0
+
+#define ICH9_LPC_DEV                            31
+#define ICH9_LPC_FUNC                           0
+
+#define ICH9_A2_LPC_REVISION                    0x2
+#define ICH9_LPC_NB_PIRQS                       8       /* PCI A-H */
+
+#define ICH9_LPC_PMBASE                         0x40
+#define ICH9_LPC_PMBASE_BASE_ADDRESS_MASK       Q35_MASK(32, 15, 7)
+#define ICH9_LPC_PMBASE_RTE                     0x1
+#define ICH9_LPC_PMBASE_DEFAULT                 0x1
+#define ICH9_LPC_ACPI_CTRL                      0x44
+#define ICH9_LPC_ACPI_CTRL_ACPI_EN              0x80
+#define ICH9_LPC_ACPI_CTRL_SCI_IRQ_SEL_MASK     Q35_MASK(8, 2, 0)
+#define ICH9_LPC_ACPI_CTRL_9                    0x0
+#define ICH9_LPC_ACPI_CTRL_10                   0x1
+#define ICH9_LPC_ACPI_CTRL_11                   0x2
+#define ICH9_LPC_ACPI_CTRL_20                   0x4
+#define ICH9_LPC_ACPI_CTRL_21                   0x5
+#define ICH9_LPC_ACPI_CTRL_DEFAULT              0x0
+
+#define ICH9_LPC_PIRQA_ROUT                     0x60
+#define ICH9_LPC_PIRQB_ROUT                     0x61
+#define ICH9_LPC_PIRQC_ROUT                     0x62
+#define ICH9_LPC_PIRQD_ROUT                     0x63
+
+#define ICH9_LPC_PIRQE_ROUT                     0x68
+#define ICH9_LPC_PIRQF_ROUT                     0x69
+#define ICH9_LPC_PIRQG_ROUT                     0x6a
+#define ICH9_LPC_PIRQH_ROUT                     0x6b
+
+#define ICH9_LPC_PIRQ_ROUT_IRQEN                0x80
+#define ICH9_LPC_PIRQ_ROUT_MASK                 Q35_MASK(8, 3, 0)
+#define ICH9_LPC_PIRQ_ROUT_DEFAULT              0x80
+
+#define ICH9_LPC_RCBA                           0xf0
+#define ICH9_LPC_RCBA_BA_MASK                   Q35_MASK(32, 31, 14)
+#define ICH9_LPC_RCBA_EN                        0x1
+#define ICH9_LPC_RCBA_DEFAULT                   0x0
+
+#define ICH9_LPC_PIC_NUM_PINS                   16
+#define ICH9_LPC_IOAPIC_NUM_PINS                24
+
+/* D31:F2 SATA Controller #1 */
+#define ICH9_SATA1_DEV                          31
+#define ICH9_SATA1_FUNC                         2
+
+/* D30:F1 power management I/O registers
+   offset from the address ICH9_LPC_PMBASE */
+
+/* ICH9 LPC PM I/O registers are 128 ports and 128-aligned */
+#define ICH9_PMIO_SIZE                          128
+#define ICH9_PMIO_MASK                          (ICH9_PMIO_SIZE - 1)
+
+#define ICH9_PMIO_PM1_STS                       0x00
+#define ICH9_PMIO_PM1_EN                        0x02
+#define ICH9_PMIO_PM1_CNT                       0x04
+#define ICH9_PMIO_PM1_TMR                       0x08
+#define ICH9_PMIO_GPE0_STS                      0x20
+#define ICH9_PMIO_GPE0_EN                       0x28
+#define ICH9_PMIO_GPE0_LEN                      16
+#define ICH9_PMIO_SMI_EN                        0x30
+#define ICH9_PMIO_SMI_EN_APMC_EN                (1 << 5)
+#define ICH9_PMIO_SMI_STS                       0x34
+
+/* FADT ACPI_ENABLE/ACPI_DISABLE */
+#define ICH9_APM_ACPI_ENABLE                    0x2
+#define ICH9_APM_ACPI_DISABLE                   0x3
+
+
+/* D31:F3 SMBus controller */
+#define ICH9_A2_SMB_REVISION                    0x02
+#define ICH9_SMB_PI                             0x00
+
+#define ICH9_SMB_SMBMBAR0                       0x10
+#define ICH9_SMB_SMBMBAR1                       0x14
+#define ICH9_SMB_SMBM_BAR                       0
+#define ICH9_SMB_SMBM_SIZE                      (1 << 8)
+#define ICH9_SMB_SMB_BASE                       0x20
+#define ICH9_SMB_SMB_BASE_BAR                   4
+#define ICH9_SMB_SMB_BASE_SIZE                  (1 << 5)
+#define ICH9_SMB_HOSTC                          0x40
+#define ICH9_SMB_HOSTC_SSRESET                  ((uint8_t)(1 << 3))
+#define ICH9_SMB_HOSTC_I2C_EN                   ((uint8_t)(1 << 2))
+#define ICH9_SMB_HOSTC_SMB_SMI_EN               ((uint8_t)(1 << 1))
+#define ICH9_SMB_HOSTC_HST_EN                   ((uint8_t)(1 << 0))
+
+/* D31:F3 SMBus I/O and memory mapped I/O registers */
+#define ICH9_SMB_DEV                            31
+#define ICH9_SMB_FUNC                           3
+
+#define ICH9_SMB_HST_STS                        0x00
+#define ICH9_SMB_HST_CNT                        0x02
+#define ICH9_SMB_HST_CMD                        0x03
+#define ICH9_SMB_XMIT_SLVA                      0x04
+#define ICH9_SMB_HST_D0                         0x05
+#define ICH9_SMB_HST_D1                         0x06
+#define ICH9_SMB_HOST_BLOCK_DB                  0x07
+
+#endif /* HW_ICH9_H */
diff --git a/hw/ide.h b/hw/ide.h
index 2db4079f68e..9b357c05a5d 100644
--- a/hw/ide.h
+++ b/hw/ide.h
@@ -2,8 +2,8 @@
 #define HW_IDE_H
 
 #include "isa.h"
-#include "pci.h"
-#include "memory.h"
+#include "pci/pci.h"
+#include "exec/memory.h"
 
 #define MAX_IDE_DEVS	2
 
@@ -19,12 +19,8 @@ PCIDevice *pci_piix3_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 PCIDevice *pci_piix4_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 void vt82c686b_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 
-/* ide-macio.c */
-MemoryRegion *pmac_ide_init (DriveInfo **hd_table, qemu_irq irq,
-		   void *dbdma, int channel, qemu_irq dma_irq);
-
 /* ide-mmio.c */
-void mmio_ide_init (target_phys_addr_t membase, target_phys_addr_t membase2,
+void mmio_ide_init (hwaddr membase, hwaddr membase2,
                     MemoryRegion *address_space,
                     qemu_irq irq, int shift,
                     DriveInfo *hd0, DriveInfo *hd1);
diff --git a/hw/ide/Makefile.objs b/hw/ide/Makefile.objs
index cf718dd0163..5c8c22aad7d 100644
--- a/hw/ide/Makefile.objs
+++ b/hw/ide/Makefile.objs
@@ -1,10 +1,10 @@
-hw-obj-$(CONFIG_IDE_CORE) += core.o atapi.o
-hw-obj-$(CONFIG_IDE_QDEV) += qdev.o
-hw-obj-$(CONFIG_IDE_PCI) += pci.o
-hw-obj-$(CONFIG_IDE_ISA) += isa.o
-hw-obj-$(CONFIG_IDE_PIIX) += piix.o
-hw-obj-$(CONFIG_IDE_CMD646) += cmd646.o
-hw-obj-$(CONFIG_IDE_MACIO) += macio.o
-hw-obj-$(CONFIG_IDE_VIA) += via.o
-hw-obj-$(CONFIG_AHCI) += ahci.o
-hw-obj-$(CONFIG_AHCI) += ich.o
+common-obj-$(CONFIG_IDE_CORE) += core.o atapi.o
+common-obj-$(CONFIG_IDE_QDEV) += qdev.o
+common-obj-$(CONFIG_IDE_PCI) += pci.o
+common-obj-$(CONFIG_IDE_ISA) += isa.o
+common-obj-$(CONFIG_IDE_PIIX) += piix.o
+common-obj-$(CONFIG_IDE_CMD646) += cmd646.o
+common-obj-$(CONFIG_IDE_MACIO) += macio.o
+common-obj-$(CONFIG_IDE_VIA) += via.o
+common-obj-$(CONFIG_AHCI) += ahci.o
+common-obj-$(CONFIG_AHCI) += ich.o
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 5ea3cadb014..ad0094f532e 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -22,14 +22,14 @@
  */
 
 #include <hw/hw.h>
-#include <hw/msi.h>
+#include <hw/pci/msi.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/sysbus.h>
 
-#include "monitor.h"
-#include "dma.h"
-#include "cpu-common.h"
+#include "monitor/monitor.h"
+#include "sysemu/dma.h"
+#include "exec/cpu-common.h"
 #include "internal.h"
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
@@ -174,7 +174,7 @@ static void ahci_trigger_irq(AHCIState *s, AHCIDevice *d,
 
 static void map_page(uint8_t **ptr, uint64_t addr, uint32_t wanted)
 {
-    target_phys_addr_t len = wanted;
+    hwaddr len = wanted;
 
     if (*ptr) {
         cpu_physical_memory_unmap(*ptr, len, 1, len);
@@ -241,7 +241,7 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             if ((pr->cmd & PORT_CMD_FIS_ON) &&
                 !s->dev[port].init_d2h_sent) {
                 ahci_init_d2h(&s->dev[port]);
-                s->dev[port].init_d2h_sent = 1;
+                s->dev[port].init_d2h_sent = true;
             }
 
             check_cmd(s, port);
@@ -279,7 +279,7 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
     }
 }
 
-static uint64_t ahci_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ahci_mem_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     AHCIState *s = opaque;
@@ -317,7 +317,7 @@ static uint64_t ahci_mem_read(void *opaque, target_phys_addr_t addr,
 
 
 
-static void ahci_mem_write(void *opaque, target_phys_addr_t addr,
+static void ahci_mem_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     AHCIState *s = opaque;
@@ -373,7 +373,7 @@ static const MemoryRegionOps ahci_mem_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t ahci_idp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ahci_idp_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     AHCIState *s = opaque;
@@ -389,7 +389,7 @@ static uint64_t ahci_idp_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void ahci_idp_write(void *opaque, target_phys_addr_t addr,
+static void ahci_idp_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     AHCIState *s = opaque;
@@ -494,7 +494,7 @@ static void ahci_reset_port(AHCIState *s, int port)
     pr->scr_err = 0;
     pr->scr_act = 0;
     d->busy_slot = -1;
-    d->init_d2h_sent = 0;
+    d->init_d2h_sent = false;
 
     ide_state = &s->dev[port].port.ifs[0];
     if (!ide_state->bs) {
@@ -946,7 +946,7 @@ static int handle_cmd(AHCIState *s, int port, int slot)
             ide_state->hcyl = 0xeb;
             debug_print_fis(ide_state->io_buffer, 0x10);
             ide_state->feature = IDE_FEATURE_DMA;
-            s->dev[port].done_atapi_packet = 0;
+            s->dev[port].done_atapi_packet = false;
             /* XXX send PIO setup FIS */
         }
 
@@ -991,7 +991,7 @@ static int ahci_start_transfer(IDEDMA *dma)
 
     if (is_atapi && !ad->done_atapi_packet) {
         /* already prepopulated iobuffer */
-        ad->done_atapi_packet = 1;
+        ad->done_atapi_packet = true;
         goto out;
     }
 
@@ -1035,11 +1035,10 @@ out:
 static void ahci_start_dma(IDEDMA *dma, IDEState *s,
                            BlockDriverCompletionFunc *dma_cb)
 {
+#ifdef DEBUG_AHCI
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-
+#endif
     DPRINTF(ad->port_no, "\n");
-    ad->dma_cb = dma_cb;
-    ad->dma_status |= BM_STATUS_DMAING;
     s->io_buffer_offset = 0;
     dma_cb(s, 0);
 }
@@ -1095,7 +1094,6 @@ static int ahci_dma_set_unit(IDEDMA *dma, int unit)
 static int ahci_dma_add_status(IDEDMA *dma, int status)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-    ad->dma_status |= status;
     DPRINTF(ad->port_no, "set status: %x\n", status);
 
     if (status & BM_STATUS_INT) {
@@ -1114,8 +1112,6 @@ static int ahci_dma_set_inactive(IDEDMA *dma)
     /* update d2h status */
     ahci_write_fis_d2h(ad, NULL);
 
-    ad->dma_cb = NULL;
-
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
         ad->check_bh = qemu_bh_new(ahci_check_cmd_bh, ad);
@@ -1175,7 +1171,6 @@ void ahci_init(AHCIState *s, DeviceState *qdev, DMAContext *dma, int ports)
         ad->port_no = i;
         ad->port.dma = &ad->dma;
         ad->port.dma->ops = &ahci_dma_ops;
-        ad->port_regs.cmd = PORT_CMD_SPIN_UP | PORT_CMD_POWER_ON;
     }
 }
 
@@ -1199,10 +1194,87 @@ void ahci_reset(AHCIState *s)
         pr->irq_stat = 0;
         pr->irq_mask = 0;
         pr->scr_ctl = 0;
+        pr->cmd = PORT_CMD_SPIN_UP | PORT_CMD_POWER_ON;
         ahci_reset_port(s, i);
     }
 }
 
+static const VMStateDescription vmstate_ahci_device = {
+    .name = "ahci port",
+    .version_id = 1,
+    .fields = (VMStateField []) {
+        VMSTATE_IDE_BUS(port, AHCIDevice),
+        VMSTATE_UINT32(port_state, AHCIDevice),
+        VMSTATE_UINT32(finished, AHCIDevice),
+        VMSTATE_UINT32(port_regs.lst_addr, AHCIDevice),
+        VMSTATE_UINT32(port_regs.lst_addr_hi, AHCIDevice),
+        VMSTATE_UINT32(port_regs.fis_addr, AHCIDevice),
+        VMSTATE_UINT32(port_regs.fis_addr_hi, AHCIDevice),
+        VMSTATE_UINT32(port_regs.irq_stat, AHCIDevice),
+        VMSTATE_UINT32(port_regs.irq_mask, AHCIDevice),
+        VMSTATE_UINT32(port_regs.cmd, AHCIDevice),
+        VMSTATE_UINT32(port_regs.tfdata, AHCIDevice),
+        VMSTATE_UINT32(port_regs.sig, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_stat, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_ctl, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_err, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_act, AHCIDevice),
+        VMSTATE_UINT32(port_regs.cmd_issue, AHCIDevice),
+        VMSTATE_BOOL(done_atapi_packet, AHCIDevice),
+        VMSTATE_INT32(busy_slot, AHCIDevice),
+        VMSTATE_BOOL(init_d2h_sent, AHCIDevice),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static int ahci_state_post_load(void *opaque, int version_id)
+{
+    int i;
+    struct AHCIDevice *ad;
+    AHCIState *s = opaque;
+
+    for (i = 0; i < s->ports; i++) {
+        ad = &s->dev[i];
+        AHCIPortRegs *pr = &ad->port_regs;
+
+        map_page(&ad->lst,
+                 ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
+        map_page(&ad->res_fis,
+                 ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
+        /*
+         * All pending i/o should be flushed out on a migrate. However,
+         * we might not have cleared the busy_slot since this is done
+         * in a bh. Also, issue i/o against any slots that are pending.
+         */
+        if ((ad->busy_slot != -1) &&
+            !(ad->port.ifs[0].status & (BUSY_STAT|DRQ_STAT))) {
+            pr->cmd_issue &= ~(1 << ad->busy_slot);
+            ad->busy_slot = -1;
+        }
+        check_cmd(s, i);
+    }
+
+    return 0;
+}
+
+const VMStateDescription vmstate_ahci = {
+    .name = "ahci",
+    .version_id = 1,
+    .post_load = ahci_state_post_load,
+    .fields = (VMStateField []) {
+        VMSTATE_STRUCT_VARRAY_POINTER_INT32(dev, AHCIState, ports,
+                                     vmstate_ahci_device, AHCIDevice),
+        VMSTATE_UINT32(control_regs.cap, AHCIState),
+        VMSTATE_UINT32(control_regs.ghc, AHCIState),
+        VMSTATE_UINT32(control_regs.irqstatus, AHCIState),
+        VMSTATE_UINT32(control_regs.impl, AHCIState),
+        VMSTATE_UINT32(control_regs.version, AHCIState),
+        VMSTATE_UINT32(idp_index, AHCIState),
+        VMSTATE_INT32(ports, AHCIState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 typedef struct SysbusAHCIState {
     SysBusDevice busdev;
     AHCIState ahci;
@@ -1211,7 +1283,11 @@ typedef struct SysbusAHCIState {
 
 static const VMStateDescription vmstate_sysbus_ahci = {
     .name = "sysbus-ahci",
-    .unmigratable = 1,
+    .unmigratable = 1, /* Still buggy under I/O load */
+    .fields = (VMStateField []) {
+        VMSTATE_AHCI(ahci, AHCIPCIState),
+        VMSTATE_END_OF_LIST()
+    },
 };
 
 static void sysbus_ahci_reset(DeviceState *dev)
@@ -1247,7 +1323,7 @@ static void sysbus_ahci_class_init(ObjectClass *klass, void *data)
     dc->reset = sysbus_ahci_reset;
 }
 
-static TypeInfo sysbus_ahci_info = {
+static const TypeInfo sysbus_ahci_info = {
     .name          = "sysbus-ahci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysbusAHCIState),
diff --git a/hw/ide/ahci.h b/hw/ide/ahci.h
index 1200a56adaa..85f37fe99d3 100644
--- a/hw/ide/ahci.h
+++ b/hw/ide/ahci.h
@@ -281,11 +281,9 @@ struct AHCIDevice {
     QEMUBH *check_bh;
     uint8_t *lst;
     uint8_t *res_fis;
-    int dma_status;
-    int done_atapi_packet;
-    int busy_slot;
-    int init_d2h_sent;
-    BlockDriverCompletionFunc *dma_cb;
+    bool done_atapi_packet;
+    int32_t busy_slot;
+    bool init_d2h_sent;
     AHCICmdHdr *cur_cmd;
     NCQTransferState ncq_tfs[AHCI_MAX_CMDS];
 };
@@ -297,7 +295,7 @@ typedef struct AHCIState {
     MemoryRegion idp;       /* Index-Data Pair I/O port space */
     unsigned idp_offset;    /* Offset of index in I/O port space */
     uint32_t idp_index;     /* Current IDP index */
-    int ports;
+    int32_t ports;
     qemu_irq irq;
     DMAContext *dma;
 } AHCIState;
@@ -307,6 +305,16 @@ typedef struct AHCIPCIState {
     AHCIState ahci;
 } AHCIPCIState;
 
+extern const VMStateDescription vmstate_ahci;
+
+#define VMSTATE_AHCI(_field, _state) {                               \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(AHCIState),                                 \
+    .vmsd       = &vmstate_ahci,                                     \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, AHCIState),   \
+}
+
 typedef struct NCQFrame {
     uint8_t fis_type;
     uint8_t c;
diff --git a/hw/ide/atapi.c b/hw/ide/atapi.c
index f7f714c7263..861fd2bec31 100644
--- a/hw/ide/atapi.c
+++ b/hw/ide/atapi.c
@@ -875,6 +875,12 @@ static void cmd_start_stop_unit(IDEState *s, uint8_t* buf)
     int sense;
     bool start = buf[4] & 1;
     bool loej = buf[4] & 2;     /* load on start, eject on !start */
+    int pwrcnd = buf[4] & 0xf0;
+
+    if (pwrcnd) {
+        /* eject/load only happens for power condition == 0 */
+        return;
+    }
 
     if (loej) {
         if (!start && !s->tray_open && s->tray_locked) {
@@ -1118,12 +1124,17 @@ void ide_atapi_cmd(IDEState *s)
      * GET_EVENT_STATUS_NOTIFICATION to detect such tray open/close
      * states rely on this behavior.
      */
-    if (!s->tray_open && bdrv_is_inserted(s->bs) && s->cdrom_changed) {
-        ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
+    if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
+        !s->tray_open && bdrv_is_inserted(s->bs) && s->cdrom_changed) {
+
+        if (s->cdrom_changed == 1) {
+            ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
+            s->cdrom_changed = 2;
+        } else {
+            ide_atapi_cmd_error(s, UNIT_ATTENTION, ASC_MEDIUM_MAY_HAVE_CHANGED);
+            s->cdrom_changed = 0;
+        }
 
-        s->cdrom_changed = 0;
-        s->sense_key = UNIT_ATTENTION;
-        s->asc = ASC_MEDIUM_MAY_HAVE_CHANGED;
         return;
     }
 
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index e0b94434960..745ef94deb0 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -24,11 +24,11 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -43,7 +43,7 @@
 
 static void cmd646_update_irq(PCIIDEState *d);
 
-static uint64_t cmd646_cmd_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cmd646_cmd_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -54,7 +54,7 @@ static uint64_t cmd646_cmd_read(void *opaque, target_phys_addr_t addr,
     return ide_status_read(cmd646bar->bus, addr + 2);
 }
 
-static void cmd646_cmd_write(void *opaque, target_phys_addr_t addr,
+static void cmd646_cmd_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -71,7 +71,7 @@ static const MemoryRegionOps cmd646_cmd_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t cmd646_data_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cmd646_data_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -88,7 +88,7 @@ static uint64_t cmd646_data_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void cmd646_data_write(void *opaque, target_phys_addr_t addr,
+static void cmd646_data_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -121,7 +121,7 @@ static void setup_cmd646_bar(PCIIDEState *d, int bus_num)
     memory_region_init_io(&bar->data, &cmd646_data_ops, bar, "cmd646-data", 8);
 }
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -159,7 +159,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -342,7 +342,7 @@ static void cmd646_ide_class_init(ObjectClass *klass, void *data)
     dc->props = cmd646_ide_properties;
 }
 
-static TypeInfo cmd646_ide_info = {
+static const TypeInfo cmd646_ide_info = {
     .name          = "cmd646-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/ide/core.c b/hw/ide/core.c
index d65ef3d58d7..3743dc3b55e 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -24,14 +24,14 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "qemu-error.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 #include "hw/block-common.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 #include <hw/ide/internal.h>
 
@@ -53,8 +53,6 @@ static const int smart_attributes[][12] = {
     { 0x0c, 0x03, 0x00, 0x64, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
     /* airflow-temperature-celsius */
     { 190,  0x03, 0x00, 0x45, 0x45, 0x1f, 0x00, 0x1f, 0x1f, 0x00, 0x00, 0x32},
-    /* end of list */
-    { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
 };
 
 static int ide_handle_rw_error(IDEState *s, int error, int op);
@@ -327,18 +325,30 @@ typedef struct TrimAIOCB {
     BlockDriverAIOCB common;
     QEMUBH *bh;
     int ret;
+    QEMUIOVector *qiov;
+    BlockDriverAIOCB *aiocb;
+    int i, j;
 } TrimAIOCB;
 
 static void trim_aio_cancel(BlockDriverAIOCB *acb)
 {
     TrimAIOCB *iocb = container_of(acb, TrimAIOCB, common);
 
+    /* Exit the loop in case bdrv_aio_cancel calls ide_issue_trim_cb again.  */
+    iocb->j = iocb->qiov->niov - 1;
+    iocb->i = (iocb->qiov->iov[iocb->j].iov_len / 8) - 1;
+
+    /* Tell ide_issue_trim_cb not to trigger the completion, too.  */
     qemu_bh_delete(iocb->bh);
     iocb->bh = NULL;
+
+    if (iocb->aiocb) {
+        bdrv_aio_cancel(iocb->aiocb);
+    }
     qemu_aio_release(iocb);
 }
 
-static AIOPool trim_aio_pool = {
+static const AIOCBInfo trim_aiocb_info = {
     .aiocb_size         = sizeof(TrimAIOCB),
     .cancel             = trim_aio_cancel,
 };
@@ -351,43 +361,60 @@ static void ide_trim_bh_cb(void *opaque)
 
     qemu_bh_delete(iocb->bh);
     iocb->bh = NULL;
-
     qemu_aio_release(iocb);
 }
 
+static void ide_issue_trim_cb(void *opaque, int ret)
+{
+    TrimAIOCB *iocb = opaque;
+    if (ret >= 0) {
+        while (iocb->j < iocb->qiov->niov) {
+            int j = iocb->j;
+            while (++iocb->i < iocb->qiov->iov[j].iov_len / 8) {
+                int i = iocb->i;
+                uint64_t *buffer = iocb->qiov->iov[j].iov_base;
+
+                /* 6-byte LBA + 2-byte range per entry */
+                uint64_t entry = le64_to_cpu(buffer[i]);
+                uint64_t sector = entry & 0x0000ffffffffffffULL;
+                uint16_t count = entry >> 48;
+
+                if (count == 0) {
+                    continue;
+                }
+
+                /* Got an entry! Submit and exit.  */
+                iocb->aiocb = bdrv_aio_discard(iocb->common.bs, sector, count,
+                                               ide_issue_trim_cb, opaque);
+                return;
+            }
+
+            iocb->j++;
+            iocb->i = -1;
+        }
+    } else {
+        iocb->ret = ret;
+    }
+
+    iocb->aiocb = NULL;
+    if (iocb->bh) {
+        qemu_bh_schedule(iocb->bh);
+    }
+}
+
 BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque)
 {
     TrimAIOCB *iocb;
-    int i, j, ret;
 
-    iocb = qemu_aio_get(&trim_aio_pool, bs, cb, opaque);
+    iocb = qemu_aio_get(&trim_aiocb_info, bs, cb, opaque);
     iocb->bh = qemu_bh_new(ide_trim_bh_cb, iocb);
     iocb->ret = 0;
-
-    for (j = 0; j < qiov->niov; j++) {
-        uint64_t *buffer = qiov->iov[j].iov_base;
-
-        for (i = 0; i < qiov->iov[j].iov_len / 8; i++) {
-            /* 6-byte LBA + 2-byte range per entry */
-            uint64_t entry = le64_to_cpu(buffer[i]);
-            uint64_t sector = entry & 0x0000ffffffffffffULL;
-            uint16_t count = entry >> 48;
-
-            if (count == 0) {
-                break;
-            }
-
-            ret = bdrv_discard(bs, sector, count);
-            if (!iocb->ret) {
-                iocb->ret = ret;
-            }
-        }
-    }
-
-    qemu_bh_schedule(iocb->bh);
-
+    iocb->qiov = qiov;
+    iocb->i = -1;
+    iocb->j = 0;
+    ide_issue_trim_cb(iocb, 0);
     return &iocb->common;
 }
 
@@ -558,32 +585,22 @@ void ide_dma_error(IDEState *s)
 
 static int ide_handle_rw_error(IDEState *s, int error, int op)
 {
-    int is_read = (op & BM_STATUS_RETRY_READ);
-    BlockErrorAction action = bdrv_get_on_error(s->bs, is_read);
+    bool is_read = (op & BM_STATUS_RETRY_READ) != 0;
+    BlockErrorAction action = bdrv_get_error_action(s->bs, is_read, error);
 
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
+    if (action == BDRV_ACTION_STOP) {
         s->bus->dma->ops->set_unit(s->bus->dma, s->unit);
         s->bus->error_status = op;
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->bs, error);
-    } else {
+    } else if (action == BDRV_ACTION_REPORT) {
         if (op & BM_STATUS_DMA_RETRY) {
             dma_buf_commit(s);
             ide_dma_error(s);
         } else {
             ide_rw_error(s);
         }
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_REPORT, is_read);
     }
-
-    return 1;
+    bdrv_error_action(s->bs, action, is_read, error);
+    return action != BDRV_ACTION_IGNORE;
 }
 
 void ide_dma_cb(void *opaque, int ret)
@@ -591,6 +608,7 @@ void ide_dma_cb(void *opaque, int ret)
     IDEState *s = opaque;
     int n;
     int64_t sector_num;
+    bool stay_active = false;
 
     if (ret < 0) {
         int op = BM_STATUS_DMA_RETRY;
@@ -606,6 +624,14 @@ void ide_dma_cb(void *opaque, int ret)
     }
 
     n = s->io_buffer_size >> 9;
+    if (n > s->nsector) {
+        /* The PRDs were longer than needed for this request. Shorten them so
+         * we don't get a negative remainder. The Active bit must remain set
+         * after the request completes. */
+        n = s->nsector;
+        stay_active = true;
+    }
+
     sector_num = ide_get_sector(s);
     if (n > 0) {
         dma_buf_commit(s);
@@ -628,6 +654,7 @@ void ide_dma_cb(void *opaque, int ret)
     if (s->bus->dma->ops->prepare_buf(s->bus->dma, ide_cmd_is_read(s)) == 0) {
         /* The PRDs were too short. Reset the Active bit, but don't raise an
          * interrupt. */
+        s->status = READY_STAT | SEEK_STAT;
         goto eot;
     }
 
@@ -658,6 +685,9 @@ eot:
         bdrv_acct_done(s->bs, &s->acct);
     }
     ide_set_inactive(s);
+    if (stay_active) {
+        s->bus->dma->ops->add_status(s->bus->dma, BM_STATUS_DMAING);
+    }
 }
 
 static void ide_sector_start_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
@@ -1119,8 +1149,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         }
         ide_set_irq(s->bus);
         break;
+
     case WIN_VERIFY_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_VERIFY:
     case WIN_VERIFY_ONCE:
         /* do sector number check ? */
@@ -1128,8 +1160,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->status = READY_STAT | SEEK_STAT;
         ide_set_irq(s->bus);
         break;
+
     case WIN_READ_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READ:
     case WIN_READ_ONCE:
         if (s->drive_kind == IDE_CD) {
@@ -1143,8 +1177,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->req_nb_sectors = 1;
         ide_sector_read(s);
         break;
+
     case WIN_WRITE_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_WRITE:
     case WIN_WRITE_ONCE:
     case CFA_WRITE_SECT_WO_ERASE:
@@ -1159,8 +1195,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_transfer_start(s, s->io_buffer, 512, ide_sector_write);
         s->media_changed = 1;
         break;
+
     case WIN_MULTREAD_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_MULTREAD:
         if (!s->bs) {
             goto abort_cmd;
@@ -1172,8 +1210,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->req_nb_sectors = s->mult_sectors;
         ide_sector_read(s);
         break;
+
     case WIN_MULTWRITE_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_MULTWRITE:
     case CFA_WRITE_MULTI_WO_ERASE:
         if (!s->bs) {
@@ -1192,8 +1232,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_transfer_start(s, s->io_buffer, 512 * n, ide_sector_write);
         s->media_changed = 1;
         break;
+
     case WIN_READDMA_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READDMA:
     case WIN_READDMA_ONCE:
         if (!s->bs) {
@@ -1202,8 +1244,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	ide_cmd_lba48_transform(s, lba48);
         ide_sector_start_dma(s, IDE_DMA_READ);
         break;
+
     case WIN_WRITEDMA_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_WRITEDMA:
     case WIN_WRITEDMA_ONCE:
         if (!s->bs) {
@@ -1213,14 +1257,17 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_sector_start_dma(s, IDE_DMA_WRITE);
         s->media_changed = 1;
         break;
+
     case WIN_READ_NATIVE_MAX_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READ_NATIVE_MAX:
 	ide_cmd_lba48_transform(s, lba48);
         ide_set_sector(s, s->nb_sectors - 1);
         s->status = READY_STAT | SEEK_STAT;
         ide_set_irq(s->bus);
         break;
+
     case WIN_CHECKPOWERMODE1:
     case WIN_CHECKPOWERMODE2:
         s->error = 0;
@@ -1468,9 +1515,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	case SMART_READ_THRESH:
 		memset(s->io_buffer, 0, 0x200);
 		s->io_buffer[0] = 0x01; /* smart struct version */
-		for (n=0; n<30; n++) {
-		if (smart_attributes[n][0] == 0)
-			break;
+		for (n = 0; n < ARRAY_SIZE(smart_attributes); n++) {
 		s->io_buffer[2+0+(n*12)] = smart_attributes[n][0];
 		s->io_buffer[2+1+(n*12)] = smart_attributes[n][11];
 		}
@@ -1484,10 +1529,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	case SMART_READ_DATA:
 		memset(s->io_buffer, 0, 0x200);
 		s->io_buffer[0] = 0x01; /* smart struct version */
-		for (n=0; n<30; n++) {
-		    if (smart_attributes[n][0] == 0) {
-			break;
-		    }
+		for (n = 0; n < ARRAY_SIZE(smart_attributes); n++) {
 		    int i;
 		    for(i = 0; i < 11; i++) {
 			s->io_buffer[2+i+(n*12)] = smart_attributes[n][i];
@@ -1873,6 +1915,8 @@ static void ide_reset(IDEState *s)
     s->io_buffer_index = 0;
     s->cd_sector_size = 0;
     s->atapi_dma = 0;
+    s->tray_locked = 0;
+    s->tray_open = 0;
     /* ATA DMA state */
     s->io_buffer_size = 0;
     s->req_nb_sectors = 0;
@@ -2164,12 +2208,6 @@ static int ide_drive_post_load(void *opaque, int version_id)
 {
     IDEState *s = opaque;
 
-    if (version_id < 3) {
-        if (s->sense_key == UNIT_ATTENTION &&
-            s->asc == ASC_MEDIUM_MAY_HAVE_CHANGED) {
-            s->cdrom_changed = 1;
-        }
-    }
     if (s->identify_set) {
         bdrv_set_enable_write_cache(s->bs, !!(s->identify_data[85] & (1 << 5)));
     }
diff --git a/hw/ide/ich.c b/hw/ide/ich.c
index 272b7734b55..cc30adc701a 100644
--- a/hw/ide/ich.c
+++ b/hw/ide/ich.c
@@ -61,12 +61,12 @@
  */
 
 #include <hw/hw.h>
-#include <hw/msi.h>
+#include <hw/pci/msi.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
@@ -79,9 +79,15 @@
 #define ICH9_IDP_INDEX          0x10
 #define ICH9_IDP_INDEX_LOG2     0x04
 
-static const VMStateDescription vmstate_ahci = {
-    .name = "ahci",
-    .unmigratable = 1,
+static const VMStateDescription vmstate_ich9_ahci = {
+    .name = "ich9_ahci",
+    .unmigratable = 1, /* Still buggy under I/O load */
+    .version_id = 1,
+    .fields = (VMStateField []) {
+        VMSTATE_PCI_DEVICE(card, AHCIPCIState),
+        VMSTATE_AHCI(ahci, AHCIPCIState),
+        VMSTATE_END_OF_LIST()
+    },
 };
 
 static void pci_ich9_reset(DeviceState *dev)
@@ -152,11 +158,11 @@ static void ich_ahci_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_INTEL_82801IR;
     k->revision = 0x02;
     k->class_id = PCI_CLASS_STORAGE_SATA;
-    dc->vmsd = &vmstate_ahci;
+    dc->vmsd = &vmstate_ich9_ahci;
     dc->reset = pci_ich9_reset;
 }
 
-static TypeInfo ich_ahci_info = {
+static const TypeInfo ich_ahci_info = {
     .name          = "ich9-ahci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(AHCIPCIState),
diff --git a/hw/ide/internal.h b/hw/ide/internal.h
index bf7d313cf40..d80360e85bb 100644
--- a/hw/ide/internal.h
+++ b/hw/ide/internal.h
@@ -8,9 +8,9 @@
  */
 #include <hw/ide.h>
 #include <hw/isa.h>
-#include "iorange.h"
-#include "dma.h"
-#include "sysemu.h"
+#include "exec/iorange.h"
+#include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
 #include "hw/block-common.h"
 #include "hw/scsi-defs.h"
 
diff --git a/hw/ide/isa.c b/hw/ide/isa.c
index 8ab2718eeae..fb7bb8201db 100644
--- a/hw/ide/isa.c
+++ b/hw/ide/isa.c
@@ -25,8 +25,8 @@
 #include <hw/hw.h>
 #include <hw/pc.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
@@ -111,7 +111,7 @@ static void isa_ide_class_initfn(ObjectClass *klass, void *data)
     dc->props = isa_ide_properties;
 }
 
-static TypeInfo isa_ide_info = {
+static const TypeInfo isa_ide_info = {
     .name          = "isa-ide",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAIDEState),
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index 848cb31429c..375c46f9daf 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -22,23 +22,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/ppc_mac.h>
-#include <hw/mac_dbdma.h>
-#include "block.h"
-#include "dma.h"
+#include "hw/hw.h"
+#include "hw/ppc/mac.h"
+#include "hw/mac_dbdma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
 /***********************************************************/
 /* MacIO based PowerPC IDE */
 
-typedef struct MACIOIDEState {
-    MemoryRegion mem;
-    IDEBus bus;
-    BlockDriverAIOCB *aiocb;
-} MACIOIDEState;
-
 #define MACIO_PAGE_SIZE 4096
 
 static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
@@ -76,7 +70,8 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
 
     s->io_buffer_size = io->len;
 
-    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1, NULL);
+    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1,
+                     &dma_context_memory);
     qemu_sglist_add(&s->sg, io->addr, io->len);
     io->addr += io->len;
     io->len = 0;
@@ -89,7 +84,6 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
 done:
     bdrv_acct_done(s->bs, &s->acct);
     io->dma_end(opaque);
-    return;
 }
 
 static void pmac_ide_transfer_cb(void *opaque, int ret)
@@ -133,7 +127,8 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
     s->io_buffer_index = 0;
     s->io_buffer_size = io->len;
 
-    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1, NULL);
+    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1,
+                     &dma_context_memory);
     qemu_sglist_add(&s->sg, io->addr, io->len);
     io->addr += io->len;
     io->len = 0;
@@ -199,7 +194,7 @@ static void pmac_ide_flush(DBDMA_io *io)
 
 /* PowerMac IDE memory IO */
 static void pmac_ide_writeb (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -217,7 +212,7 @@ static void pmac_ide_writeb (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readb (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readb (void *opaque,hwaddr addr)
 {
     uint8_t retval;
     MACIOIDEState *d = opaque;
@@ -239,7 +234,7 @@ static uint32_t pmac_ide_readb (void *opaque,target_phys_addr_t addr)
 }
 
 static void pmac_ide_writew (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -250,7 +245,7 @@ static void pmac_ide_writew (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readw (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readw (void *opaque,hwaddr addr)
 {
     uint16_t retval;
     MACIOIDEState *d = opaque;
@@ -266,7 +261,7 @@ static uint32_t pmac_ide_readw (void *opaque,target_phys_addr_t addr)
 }
 
 static void pmac_ide_writel (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -277,7 +272,7 @@ static void pmac_ide_writel (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readl (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readl (void *opaque,hwaddr addr)
 {
     uint32_t retval;
     MACIOIDEState *d = opaque;
@@ -320,30 +315,70 @@ static const VMStateDescription vmstate_pmac = {
     }
 };
 
-static void pmac_ide_reset(void *opaque)
+static void macio_ide_reset(DeviceState *dev)
 {
-    MACIOIDEState *d = opaque;
+    MACIOIDEState *d = MACIO_IDE(dev);
 
     ide_bus_reset(&d->bus);
 }
 
-/* hd_table must contain 4 block drivers */
-/* PowerMac uses memory mapped registers, not I/O. Return the memory
-   I/O index to access the ide. */
-MemoryRegion *pmac_ide_init (DriveInfo **hd_table, qemu_irq irq,
-                             void *dbdma, int channel, qemu_irq dma_irq)
+static void macio_ide_realizefn(DeviceState *dev, Error **errp)
 {
-    MACIOIDEState *d;
+    MACIOIDEState *s = MACIO_IDE(dev);
 
-    d = g_malloc0(sizeof(MACIOIDEState));
-    ide_init2_with_non_qdev_drives(&d->bus, hd_table[0], hd_table[1], irq);
+    ide_init2(&s->bus, s->irq);
+}
 
-    if (dbdma)
-        DBDMA_register_channel(dbdma, channel, dma_irq, pmac_ide_transfer, pmac_ide_flush, d);
+static void macio_ide_initfn(Object *obj)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(obj);
+    MACIOIDEState *s = MACIO_IDE(obj);
+
+    ide_bus_new(&s->bus, DEVICE(obj), 0);
+    memory_region_init_io(&s->mem, &pmac_ide_ops, s, "pmac-ide", 0x1000);
+    sysbus_init_mmio(d, &s->mem);
+    sysbus_init_irq(d, &s->irq);
+    sysbus_init_irq(d, &s->dma_irq);
+}
 
-    memory_region_init_io(&d->mem, &pmac_ide_ops, d, "pmac-ide", 0x1000);
-    vmstate_register(NULL, 0, &vmstate_pmac, d);
-    qemu_register_reset(pmac_ide_reset, d);
+static void macio_ide_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
 
-    return &d->mem;
+    dc->realize = macio_ide_realizefn;
+    dc->reset = macio_ide_reset;
+    dc->vmsd = &vmstate_pmac;
 }
+
+static const TypeInfo macio_ide_type_info = {
+    .name = TYPE_MACIO_IDE,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MACIOIDEState),
+    .instance_init = macio_ide_initfn,
+    .class_init = macio_ide_class_init,
+};
+
+static void macio_ide_register_types(void)
+{
+    type_register_static(&macio_ide_type_info);
+}
+
+/* hd_table must contain 4 block drivers */
+void macio_ide_init_drives(MACIOIDEState *s, DriveInfo **hd_table)
+{
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        if (hd_table[i]) {
+            ide_create_drive(&s->bus, i, hd_table[i]);
+        }
+    }
+}
+
+void macio_ide_register_dma(MACIOIDEState *s, void *dbdma, int channel)
+{
+    DBDMA_register_channel(dbdma, channel, s->dma_irq,
+                           pmac_ide_transfer, pmac_ide_flush, s);
+}
+
+type_init(macio_ide_register_types)
diff --git a/hw/ide/microdrive.c b/hw/ide/microdrive.c
index 9eee5b50ba9..642774ef98f 100644
--- a/hw/ide/microdrive.c
+++ b/hw/ide/microdrive.c
@@ -25,8 +25,8 @@
 #include <hw/hw.h>
 #include <hw/pc.h>
 #include <hw/pcmcia.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
diff --git a/hw/ide/mmio.c b/hw/ide/mmio.c
index fcfb09eeab7..eb59976eda2 100644
--- a/hw/ide/mmio.c
+++ b/hw/ide/mmio.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 #include <hw/hw.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
@@ -47,7 +47,7 @@ static void mmio_ide_reset(void *opaque)
     ide_bus_reset(&s->bus);
 }
 
-static uint64_t mmio_ide_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mmio_ide_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     MMIOState *s = opaque;
@@ -58,7 +58,7 @@ static uint64_t mmio_ide_read(void *opaque, target_phys_addr_t addr,
         return ide_data_readw(&s->bus, 0);
 }
 
-static void mmio_ide_write(void *opaque, target_phys_addr_t addr,
+static void mmio_ide_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     MMIOState *s = opaque;
@@ -75,14 +75,14 @@ static const MemoryRegionOps mmio_ide_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t mmio_ide_status_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mmio_ide_status_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MMIOState *s= opaque;
     return ide_status_read(&s->bus, 0);
 }
 
-static void mmio_ide_cmd_write(void *opaque, target_phys_addr_t addr,
+static void mmio_ide_cmd_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     MMIOState *s = opaque;
@@ -107,7 +107,7 @@ static const VMStateDescription vmstate_ide_mmio = {
     }
 };
 
-void mmio_ide_init (target_phys_addr_t membase, target_phys_addr_t membase2,
+void mmio_ide_init (hwaddr membase, hwaddr membase2,
                     MemoryRegion *address_space,
                     qemu_irq irq, int shift,
                     DriveInfo *hd0, DriveInfo *hd1)
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index 88c0942e343..59fd53992a3 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -24,10 +24,10 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -188,7 +188,7 @@ static void bmdma_restart_bh(void *opaque)
 {
     BMDMAState *bm = opaque;
     IDEBus *bus = bm->bus;
-    int is_read;
+    bool is_read;
     int error_status;
 
     qemu_bh_delete(bm->bh);
@@ -198,7 +198,7 @@ static void bmdma_restart_bh(void *opaque)
         return;
     }
 
-    is_read = !!(bus->error_status & BM_STATUS_RETRY_READ);
+    is_read = (bus->error_status & BM_STATUS_RETRY_READ) != 0;
 
     /* The error status must be cleared before resubmitting the request: The
      * request may fail again, and this case can only be distinguished if the
@@ -311,7 +311,6 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
             if (bm->bus->dma->aiocb) {
                 bdrv_drain_all();
                 assert(bm->bus->dma->aiocb == NULL);
-                assert((bm->status & BM_STATUS_DMAING) == 0);
             }
         } else {
             bm->cur_addr = bm->addr;
@@ -327,7 +326,7 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
     bm->cmd = val & 0x09;
 }
 
-static uint64_t bmdma_addr_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_addr_read(void *opaque, hwaddr addr,
                                 unsigned width)
 {
     BMDMAState *bm = opaque;
@@ -341,7 +340,7 @@ static uint64_t bmdma_addr_read(void *opaque, target_phys_addr_t addr,
     return data;
 }
 
-static void bmdma_addr_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_addr_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned width)
 {
     BMDMAState *bm = opaque;
diff --git a/hw/ide/piix.c b/hw/ide/piix.c
index 4ded9ee13d5..4d3e82266c3 100644
--- a/hw/ide/piix.c
+++ b/hw/ide/piix.c
@@ -25,15 +25,15 @@
 
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "blockdev.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t bmdma_read(void *opaque, hwaddr addr, unsigned size)
 {
     BMDMAState *bm = opaque;
     uint32_t val;
@@ -59,7 +59,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -251,7 +251,7 @@ static void piix3_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo piix3_ide_info = {
+static const TypeInfo piix3_ide_info = {
     .name          = "piix3-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
@@ -271,7 +271,7 @@ static void piix3_ide_xen_class_init(ObjectClass *klass, void *data)
     dc->unplug = pci_piix3_xen_ide_unplug;
 }
 
-static TypeInfo piix3_ide_xen_info = {
+static const TypeInfo piix3_ide_xen_info = {
     .name          = "piix3-ide-xen",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
@@ -292,7 +292,7 @@ static void piix4_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo piix4_ide_info = {
+static const TypeInfo piix4_ide_info = {
     .name          = "piix4-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 5ea9b8f4b29..c436b38bcbf 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -17,12 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include <hw/hw.h>
-#include "dma.h"
-#include "qemu-error.h"
+#include "sysemu/dma.h"
+#include "qemu/error-report.h"
 #include <hw/ide/internal.h>
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* --------------------------------- */
 
@@ -60,7 +60,7 @@ static char *idebus_get_fw_dev_path(DeviceState *dev)
     snprintf(path, sizeof(path), "%s@%d", qdev_fw_name(dev),
              ((IDEBus*)dev->parent_bus)->bus_id);
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 static int ide_qdev_init(DeviceState *qdev)
@@ -216,7 +216,7 @@ static void ide_hd_class_init(ObjectClass *klass, void *data)
     dc->props = ide_hd_properties;
 }
 
-static TypeInfo ide_hd_info = {
+static const TypeInfo ide_hd_info = {
     .name          = "ide-hd",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -238,7 +238,7 @@ static void ide_cd_class_init(ObjectClass *klass, void *data)
     dc->props = ide_cd_properties;
 }
 
-static TypeInfo ide_cd_info = {
+static const TypeInfo ide_cd_info = {
     .name          = "ide-cd",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -260,7 +260,7 @@ static void ide_drive_class_init(ObjectClass *klass, void *data)
     dc->props = ide_drive_properties;
 }
 
-static TypeInfo ide_drive_info = {
+static const TypeInfo ide_drive_info = {
     .name          = "ide-drive",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -275,7 +275,7 @@ static void ide_device_class_init(ObjectClass *klass, void *data)
     k->props = ide_props;
 }
 
-static TypeInfo ide_device_type_info = {
+static const TypeInfo ide_device_type_info = {
     .name = TYPE_IDE_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(IDEDevice),
diff --git a/hw/ide/via.c b/hw/ide/via.c
index b20e4f094e8..f40c1adc8cf 100644
--- a/hw/ide/via.c
+++ b/hw/ide/via.c
@@ -25,15 +25,15 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -60,7 +60,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -226,7 +226,7 @@ static void via_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo via_ide_info = {
+static const TypeInfo via_ide_info = {
     .name          = "via-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/imx.h b/hw/imx.h
index ccf586fefe7..ea9e093277b 100644
--- a/hw/imx.h
+++ b/hw/imx.h
@@ -11,7 +11,7 @@
 #ifndef IMX_H
 #define IMX_H
 
-void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq);
+void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq);
 
 typedef enum  {
     NOCLK,
@@ -23,10 +23,10 @@ typedef enum  {
 
 uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
 
-void imx_timerp_create(const target_phys_addr_t addr,
+void imx_timerp_create(const hwaddr addr,
                       qemu_irq irq,
                       DeviceState *ccm);
-void imx_timerg_create(const target_phys_addr_t addr,
+void imx_timerg_create(const hwaddr addr,
                       qemu_irq irq,
                       DeviceState *ccm);
 
diff --git a/hw/imx_avic.c b/hw/imx_avic.c
index 4f010e8ee23..f1f066cf9c7 100644
--- a/hw/imx_avic.c
+++ b/hw/imx_avic.c
@@ -6,9 +6,9 @@
  *
  * Copyright (c) 2008 OKL
  * Copyright (c) 2011 NICTA Pty Ltd
- * Originally Written by Hans Jiang
+ * Originally written by Hans Jiang
  *
- * This code is licenced under the GPL version 2 or later.  See
+ * This code is licensed under the GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  * TODO: implement vectors.
@@ -16,7 +16,7 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define DEBUG_INT 1
 #undef DEBUG_INT /* comment out for debugging */
@@ -152,7 +152,7 @@ static void imx_avic_set_irq(void *opaque, int irq, int level)
 
 
 static uint64_t imx_avic_read(void *opaque,
-                             target_phys_addr_t offset, unsigned size)
+                             hwaddr offset, unsigned size)
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
 
@@ -259,7 +259,7 @@ static uint64_t imx_avic_read(void *opaque,
     }
 }
 
-static void imx_avic_write(void *opaque, target_phys_addr_t offset,
+static void imx_avic_write(void *opaque, hwaddr offset,
                           uint64_t val, unsigned size)
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
diff --git a/hw/imx_ccm.c b/hw/imx_ccm.c
index 10952c6ea1a..477903a5467 100644
--- a/hw/imx_ccm.c
+++ b/hw/imx_ccm.c
@@ -12,7 +12,7 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "imx.h"
 
 #define CKIH_FREQ 26000000 /* 26MHz crystal input */
@@ -191,7 +191,7 @@ static void imx_ccm_reset(DeviceState *dev)
     update_clocks(s);
 }
 
-static uint64_t imx_ccm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
@@ -232,7 +232,7 @@ static uint64_t imx_ccm_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void imx_ccm_write(void *opaque, target_phys_addr_t offset,
+static void imx_ccm_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
@@ -306,7 +306,7 @@ static void imx_ccm_class_init(ObjectClass *klass, void *data)
     dc->desc = "i.MX Clock Control Module";
 }
 
-static TypeInfo imx_ccm_info = {
+static const TypeInfo imx_ccm_info = {
     .name = "imx_ccm",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXCCMState),
diff --git a/hw/imx_serial.c b/hw/imx_serial.c
index d4eae430f58..2d8253e0ee0 100644
--- a/hw/imx_serial.c
+++ b/hw/imx_serial.c
@@ -19,8 +19,8 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "imx.h"
 
 //#define DEBUG_SERIAL 1
@@ -183,7 +183,7 @@ static void imx_serial_reset_at_boot(DeviceState *dev)
 
 }
 
-static uint64_t imx_serial_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_serial_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
@@ -244,7 +244,7 @@ static uint64_t imx_serial_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void imx_serial_write(void *opaque, target_phys_addr_t offset,
+static void imx_serial_write(void *opaque, hwaddr offset,
                       uint64_t value, unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
@@ -401,7 +401,7 @@ static int imx_serial_init(SysBusDevice *dev)
     return 0;
 }
 
-void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq)
+void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
 {
     DeviceState *dev;
     SysBusDevice *bus;
@@ -425,9 +425,9 @@ void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq)
     }
 
     qdev_prop_set_chr(dev, "chardev", chr);
-    bus = sysbus_from_qdev(dev);
+    bus = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
     }
     sysbus_connect_irq(bus, 0, irq);
@@ -452,7 +452,7 @@ static void imx_serial_class_init(ObjectClass *klass, void *data)
     dc->props = imx32_serial_properties;
 }
 
-static TypeInfo imx_serial_info = {
+static const TypeInfo imx_serial_info = {
     .name = "imx-serial",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXSerialState),
diff --git a/hw/imx_timer.c b/hw/imx_timer.c
index 16215ccf04e..e924c747c54 100644
--- a/hw/imx_timer.c
+++ b/hw/imx_timer.c
@@ -3,16 +3,16 @@
  *
  * Copyright (c) 2008 OK Labs
  * Copyright (c) 2011 NICTA Pty Ltd
- * Originally Written by Hans Jiang
+ * Originally written by Hans Jiang
  * Updated by Peter Chubb
  *
- * This code is licenced under GPL version 2 or later.  See
+ * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 #include "sysbus.h"
 #include "imx.h"
@@ -194,7 +194,7 @@ static void imx_timerg_reload(IMXTimerGState *s, uint32_t timeout)
     ptimer_set_count(s->timer, diff_cnt);
 }
 
-static uint64_t imx_timerg_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_timerg_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXTimerGState *s = (IMXTimerGState *)opaque;
@@ -251,7 +251,7 @@ static void imx_timerg_reset(DeviceState *dev)
     imx_timerg_set_freq(s);
 }
 
-static void imx_timerg_write(void *opaque, target_phys_addr_t offset,
+static void imx_timerg_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     IMXTimerGState *s = (IMXTimerGState *)opaque;
@@ -468,7 +468,7 @@ static void imx_timerp_reset(DeviceState *dev)
     ptimer_set_count(s->timer, TIMER_MAX);
 }
 
-static uint64_t imx_timerp_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_timerp_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXTimerPState *s = (IMXTimerPState *)opaque;
@@ -517,7 +517,7 @@ static void set_timerp_freq(IMXTimerPState *s)
     }
 }
 
-static void imx_timerp_write(void *opaque, target_phys_addr_t offset,
+static void imx_timerp_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     IMXTimerPState *s = (IMXTimerPState *)opaque;
@@ -580,7 +580,7 @@ static void imx_timerp_tick(void *opaque)
     imx_timerp_update(s);
 }
 
-void imx_timerp_create(const target_phys_addr_t addr,
+void imx_timerp_create(const hwaddr addr,
                               qemu_irq irq,
                               DeviceState *ccm)
 {
@@ -634,7 +634,7 @@ static int imx_timerp_init(SysBusDevice *dev)
 }
 
 
-void imx_timerg_create(const target_phys_addr_t addr,
+void imx_timerg_create(const hwaddr addr,
                               qemu_irq irq,
                               DeviceState *ccm)
 {
diff --git a/hw/integratorcp.c b/hw/integratorcp.c
index d0e2e9068e8..9e3630a43de 100644
--- a/hw/integratorcp.c
+++ b/hw/integratorcp.c
@@ -11,9 +11,9 @@
 #include "devices.h"
 #include "boards.h"
 #include "arm-misc.h"
-#include "net.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -38,7 +38,7 @@ static uint8_t integrator_spd[128] = {
    0xe, 4, 0x1c, 1, 2, 0x20, 0xc0, 0, 0, 0, 0, 0x30, 0x28, 0x30, 0x28, 0x40
 };
 
-static uint64_t integratorcm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t integratorcm_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     integratorcm_state *s = (integratorcm_state *)opaque;
@@ -141,7 +141,7 @@ static void integratorcm_update(integratorcm_state *s)
         hw_error("Core module interrupt\n");
 }
 
-static void integratorcm_write(void *opaque, target_phys_addr_t offset,
+static void integratorcm_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     integratorcm_state *s = (integratorcm_state *)opaque;
@@ -295,7 +295,7 @@ static void icp_pic_set_irq(void *opaque, int irq, int level)
     icp_pic_update(s);
 }
 
-static uint64_t icp_pic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_pic_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     icp_pic_state *s = (icp_pic_state *)opaque;
@@ -324,7 +324,7 @@ static uint64_t icp_pic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void icp_pic_write(void *opaque, target_phys_addr_t offset,
+static void icp_pic_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     icp_pic_state *s = (icp_pic_state *)opaque;
@@ -381,7 +381,7 @@ static int icp_pic_init(SysBusDevice *dev)
 
 /* CP control registers.  */
 
-static uint64_t icp_control_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_control_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     switch (offset >> 2) {
@@ -399,7 +399,7 @@ static uint64_t icp_control_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void icp_control_write(void *opaque, target_phys_addr_t offset,
+static void icp_control_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     switch (offset >> 2) {
@@ -419,7 +419,7 @@ static const MemoryRegionOps icp_control_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void icp_control_init(target_phys_addr_t base)
+static void icp_control_init(hwaddr base)
 {
     MemoryRegion *io;
 
@@ -438,11 +438,13 @@ static struct arm_boot_info integrator_binfo = {
     .board_id = 0x113,
 };
 
-static void integratorcp_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void integratorcp_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -510,6 +512,7 @@ static QEMUMachine integratorcp_machine = {
     .desc = "ARM Integrator/CP (ARM926EJ-S)",
     .init = integratorcp_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void integratorcp_machine_init(void)
@@ -533,7 +536,7 @@ static void core_class_init(ObjectClass *klass, void *data)
     dc->props = core_properties;
 }
 
-static TypeInfo core_info = {
+static const TypeInfo core_info = {
     .name          = "integrator_core",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(integratorcm_state),
@@ -547,7 +550,7 @@ static void icp_pic_class_init(ObjectClass *klass, void *data)
     sdc->init = icp_pic_init;
 }
 
-static TypeInfo icp_pic_info = {
+static const TypeInfo icp_pic_info = {
     .name          = "integrator_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pic_state),
diff --git a/hw/intel-hda.c b/hw/intel-hda.c
index 127e81888b7..784c229d8f6 100644
--- a/hw/intel-hda.c
+++ b/hw/intel-hda.c
@@ -18,13 +18,13 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "msi.h"
-#include "qemu-timer.h"
+#include "pci/pci.h"
+#include "pci/msi.h"
+#include "qemu/timer.h"
 #include "audiodev.h"
 #include "intel-hda.h"
 #include "intel-hda-defs.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 /* --------------------------------------------------------------------- */
 /* hda bus                                                               */
@@ -206,17 +206,11 @@ static void intel_hda_reset(DeviceState *dev);
 
 /* --------------------------------------------------------------------- */
 
-static target_phys_addr_t intel_hda_addr(uint32_t lbase, uint32_t ubase)
+static hwaddr intel_hda_addr(uint32_t lbase, uint32_t ubase)
 {
-    target_phys_addr_t addr;
-
-#if TARGET_PHYS_ADDR_BITS == 32
-    addr = lbase;
-#else
-    addr = ubase;
-    addr <<= 32;
-    addr |= lbase;
-#endif
+    hwaddr addr;
+
+    addr = ((uint64_t)ubase << 32) | lbase;
     return addr;
 }
 
@@ -301,7 +295,7 @@ static int intel_hda_send_command(IntelHDAState *d, uint32_t verb)
 
 static void intel_hda_corb_run(IntelHDAState *d)
 {
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t rp, verb;
 
     if (d->ics & ICH6_IRS_BUSY) {
@@ -338,7 +332,7 @@ static void intel_hda_response(HDACodecDevice *dev, bool solicited, uint32_t res
 {
     HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t wp, ex;
 
     if (d->ics & ICH6_IRS_BUSY) {
@@ -387,7 +381,7 @@ static bool intel_hda_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
 {
     HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t s, copy, left;
     IntelHDAStream *st;
     bool irq = false;
@@ -459,7 +453,7 @@ static bool intel_hda_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
 
 static void intel_hda_parse_bdl(IntelHDAState *d, IntelHDAStream *st)
 {
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint8_t buf[16];
     uint32_t i;
 
@@ -896,7 +890,7 @@ static const struct IntelHDAReg regtab[] = {
 
 };
 
-static const IntelHDAReg *intel_hda_reg_find(IntelHDAState *d, target_phys_addr_t addr)
+static const IntelHDAReg *intel_hda_reg_find(IntelHDAState *d, hwaddr addr)
 {
     const IntelHDAReg *reg;
 
@@ -1039,7 +1033,7 @@ static void intel_hda_regs_reset(IntelHDAState *d)
 
 /* --------------------------------------------------------------------- */
 
-static void intel_hda_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1047,7 +1041,7 @@ static void intel_hda_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xff);
 }
 
-static void intel_hda_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1055,7 +1049,7 @@ static void intel_hda_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xffff);
 }
 
-static void intel_hda_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1063,7 +1057,7 @@ static void intel_hda_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xffffffff);
 }
 
-static uint32_t intel_hda_mmio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readb(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1071,7 +1065,7 @@ static uint32_t intel_hda_mmio_readb(void *opaque, target_phys_addr_t addr)
     return intel_hda_reg_read(d, reg, 0xff);
 }
 
-static uint32_t intel_hda_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readw(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1079,7 +1073,7 @@ static uint32_t intel_hda_mmio_readw(void *opaque, target_phys_addr_t addr)
     return intel_hda_reg_read(d, reg, 0xffff);
 }
 
-static uint32_t intel_hda_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readl(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1238,7 +1232,7 @@ static Property intel_hda_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void intel_hda_class_init(ObjectClass *klass, void *data)
+static void intel_hda_class_init_common(ObjectClass *klass)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
@@ -1246,20 +1240,46 @@ static void intel_hda_class_init(ObjectClass *klass, void *data)
     k->init = intel_hda_init;
     k->exit = intel_hda_exit;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = 0x2668;
-    k->revision = 1;
     k->class_id = PCI_CLASS_MULTIMEDIA_HD_AUDIO;
-    dc->desc = "Intel HD Audio Controller";
     dc->reset = intel_hda_reset;
     dc->vmsd = &vmstate_intel_hda;
     dc->props = intel_hda_properties;
 }
 
-static TypeInfo intel_hda_info = {
+static void intel_hda_class_init_ich6(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    intel_hda_class_init_common(klass);
+    k->device_id = 0x2668;
+    k->revision = 1;
+    dc->desc = "Intel HD Audio Controller (ich6)";
+}
+
+static void intel_hda_class_init_ich9(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    intel_hda_class_init_common(klass);
+    k->device_id = 0x293e;
+    k->revision = 3;
+    dc->desc = "Intel HD Audio Controller (ich9)";
+}
+
+static const TypeInfo intel_hda_info_ich6 = {
     .name          = "intel-hda",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(IntelHDAState),
-    .class_init    = intel_hda_class_init,
+    .class_init    = intel_hda_class_init_ich6,
+};
+
+static const TypeInfo intel_hda_info_ich9 = {
+    .name          = "ich9-intel-hda",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(IntelHDAState),
+    .class_init    = intel_hda_class_init_ich9,
 };
 
 static void hda_codec_device_class_init(ObjectClass *klass, void *data)
@@ -1271,7 +1291,7 @@ static void hda_codec_device_class_init(ObjectClass *klass, void *data)
     k->props = hda_props;
 }
 
-static TypeInfo hda_codec_device_type_info = {
+static const TypeInfo hda_codec_device_type_info = {
     .name = TYPE_HDA_CODEC_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(HDACodecDevice),
@@ -1283,7 +1303,8 @@ static TypeInfo hda_codec_device_type_info = {
 static void intel_hda_register_types(void)
 {
     type_register_static(&hda_codec_bus_info);
-    type_register_static(&intel_hda_info);
+    type_register_static(&intel_hda_info_ich6);
+    type_register_static(&intel_hda_info_ich9);
     type_register_static(&hda_codec_device_type_info);
 }
 
diff --git a/hw/ioapic.c b/hw/ioapic.c
index e2e4796bb55..f06c2dcf2e8 100644
--- a/hw/ioapic.c
+++ b/hw/ioapic.c
@@ -139,7 +139,7 @@ void ioapic_eoi_broadcast(int vector)
 }
 
 static uint64_t
-ioapic_mem_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
 {
     IOAPICCommonState *s = opaque;
     int index;
@@ -181,7 +181,7 @@ ioapic_mem_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-ioapic_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+ioapic_mem_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned int size)
 {
     IOAPICCommonState *s = opaque;
@@ -244,7 +244,7 @@ static void ioapic_class_init(ObjectClass *klass, void *data)
     dc->reset = ioapic_reset_common;
 }
 
-static TypeInfo ioapic_info = {
+static const TypeInfo ioapic_info = {
     .name          = "ioapic",
     .parent        = TYPE_IOAPIC_COMMON,
     .instance_size = sizeof(IOAPICCommonState),
diff --git a/hw/ioapic_common.c b/hw/ioapic_common.c
index 653eef2ce1a..7dc552f033c 100644
--- a/hw/ioapic_common.c
+++ b/hw/ioapic_common.c
@@ -103,7 +103,7 @@ static void ioapic_common_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo ioapic_common_type = {
+static const TypeInfo ioapic_common_type = {
     .name = TYPE_IOAPIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IOAPICCommonState),
diff --git a/hw/ioapic_internal.h b/hw/ioapic_internal.h
index e04c9f3c12c..c8447d7f3ba 100644
--- a/hw/ioapic_internal.h
+++ b/hw/ioapic_internal.h
@@ -23,7 +23,7 @@
 #define QEMU_IOAPIC_INTERNAL_H
 
 #include "hw.h"
-#include "memory.h"
+#include "exec/memory.h"
 #include "sysbus.h"
 
 #define MAX_IOAPICS                     1
diff --git a/hw/ioh3420.c b/hw/ioh3420.c
index 94a537c9b39..95bceb5347a 100644
--- a/hw/ioh3420.c
+++ b/hw/ioh3420.c
@@ -20,9 +20,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "ioh3420.h"
 
 #define PCI_DEVICE_ID_IOH_EPORT         0x3420  /* D0:F0 express mode */
@@ -125,7 +125,6 @@ static int ioh3420_initfn(PCIDevice *d)
     rc = pcie_chassis_add_slot(s);
     if (rc < 0) {
         goto err_pcie_cap;
-        return rc;
     }
     pcie_cap_root_init(d);
     rc = pcie_aer_init(d, IOH_EP_AER_OFFSET);
@@ -227,7 +226,7 @@ static void ioh3420_class_init(ObjectClass *klass, void *data)
     dc->props = ioh3420_properties;
 }
 
-static TypeInfo ioh3420_info = {
+static const TypeInfo ioh3420_info = {
     .name          = "ioh3420",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIESlot),
diff --git a/hw/ioh3420.h b/hw/ioh3420.h
index 68c523ab466..046cf2c281d 100644
--- a/hw/ioh3420.h
+++ b/hw/ioh3420.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_IOH3420_H
 #define QEMU_IOH3420_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIESlot *ioh3420_init(PCIBus *bus, int devfn, bool multifunction,
                        const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/ipack.c b/hw/ipack.c
new file mode 100644
index 00000000000..e15540d5cdb
--- /dev/null
+++ b/hw/ipack.c
@@ -0,0 +1,115 @@
+/*
+ * QEMU IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+
+IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot)
+{
+    BusChild *kid;
+
+    QTAILQ_FOREACH(kid, &BUS(bus)->children, sibling) {
+        DeviceState *qdev = kid->child;
+        IPackDevice *ip = IPACK_DEVICE(qdev);
+        if (ip->slot == slot) {
+            return ip;
+        }
+    }
+    return NULL;
+}
+
+void ipack_bus_new_inplace(IPackBus *bus, DeviceState *parent,
+                           const char *name, uint8_t n_slots,
+                           qemu_irq_handler handler)
+{
+    qbus_create_inplace(&bus->qbus, TYPE_IPACK_BUS, parent, name);
+    bus->n_slots = n_slots;
+    bus->set_irq = handler;
+}
+
+static int ipack_device_dev_init(DeviceState *qdev)
+{
+    IPackBus *bus = IPACK_BUS(qdev_get_parent_bus(qdev));
+    IPackDevice *dev = IPACK_DEVICE(qdev);
+    IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(dev);
+
+    if (dev->slot < 0) {
+        dev->slot = bus->free_slot;
+    }
+    if (dev->slot >= bus->n_slots) {
+        return -1;
+    }
+    bus->free_slot = dev->slot + 1;
+
+    dev->irq = qemu_allocate_irqs(bus->set_irq, dev, 2);
+
+    return k->init(dev);
+}
+
+static int ipack_device_dev_exit(DeviceState *qdev)
+{
+    IPackDevice *dev = IPACK_DEVICE(qdev);
+    IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(dev);
+
+    if (k->exit) {
+        k->exit(dev);
+    }
+
+    qemu_free_irqs(dev->irq);
+
+    return 0;
+}
+
+static Property ipack_device_props[] = {
+    DEFINE_PROP_INT32("slot", IPackDevice, slot, -1),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void ipack_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *k = DEVICE_CLASS(klass);
+    k->bus_type = TYPE_IPACK_BUS;
+    k->init = ipack_device_dev_init;
+    k->exit = ipack_device_dev_exit;
+    k->props = ipack_device_props;
+}
+
+const VMStateDescription vmstate_ipack_device = {
+    .name = "ipack_device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_INT32(slot, IPackDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const TypeInfo ipack_device_info = {
+    .name          = TYPE_IPACK_DEVICE,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(IPackDevice),
+    .class_size    = sizeof(IPackDeviceClass),
+    .class_init    = ipack_device_class_init,
+    .abstract      = true,
+};
+
+static const TypeInfo ipack_bus_info = {
+    .name = TYPE_IPACK_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(IPackBus),
+};
+
+static void ipack_register_types(void)
+{
+    type_register_static(&ipack_device_info);
+    type_register_static(&ipack_bus_info);
+}
+
+type_init(ipack_register_types)
diff --git a/hw/ipack.h b/hw/ipack.h
new file mode 100644
index 00000000000..69e26282d3e
--- /dev/null
+++ b/hw/ipack.h
@@ -0,0 +1,79 @@
+/*
+ * QEMU IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#ifndef QEMU_IPACK_H
+#define QEMU_IPACK_H
+
+#include "qdev.h"
+
+typedef struct IPackBus IPackBus;
+
+#define TYPE_IPACK_BUS "IndustryPack"
+#define IPACK_BUS(obj) OBJECT_CHECK(IPackBus, (obj), TYPE_IPACK_BUS)
+
+struct IPackBus {
+    BusState qbus;
+    /* All fields are private */
+    uint8_t n_slots;
+    uint8_t free_slot;
+    qemu_irq_handler set_irq;
+};
+
+typedef struct IPackDevice IPackDevice;
+typedef struct IPackDeviceClass IPackDeviceClass;
+
+#define TYPE_IPACK_DEVICE "ipack-device"
+#define IPACK_DEVICE(obj) \
+     OBJECT_CHECK(IPackDevice, (obj), TYPE_IPACK_DEVICE)
+#define IPACK_DEVICE_CLASS(klass)                                        \
+     OBJECT_CLASS_CHECK(IPackDeviceClass, (klass), TYPE_IPACK_DEVICE)
+#define IPACK_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IPackDeviceClass, (obj), TYPE_IPACK_DEVICE)
+
+struct IPackDeviceClass {
+    DeviceClass parent_class;
+
+    int (*init)(IPackDevice *dev);
+    int (*exit)(IPackDevice *dev);
+
+    uint16_t (*io_read)(IPackDevice *dev, uint8_t addr);
+    void (*io_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*id_read)(IPackDevice *dev, uint8_t addr);
+    void (*id_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*int_read)(IPackDevice *dev, uint8_t addr);
+    void (*int_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*mem_read16)(IPackDevice *dev, uint32_t addr);
+    void (*mem_write16)(IPackDevice *dev, uint32_t addr, uint16_t val);
+
+    uint8_t (*mem_read8)(IPackDevice *dev, uint32_t addr);
+    void (*mem_write8)(IPackDevice *dev, uint32_t addr, uint8_t val);
+};
+
+struct IPackDevice {
+    DeviceState qdev;
+    int32_t slot;
+    /* IRQ objects for the IndustryPack INT0# and INT1# */
+    qemu_irq *irq;
+};
+
+extern const VMStateDescription vmstate_ipack_device;
+
+#define VMSTATE_IPACK_DEVICE(_field, _state)                            \
+    VMSTATE_STRUCT(_field, _state, 1, vmstate_ipack_device, IPackDevice)
+
+IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot);
+void ipack_bus_new_inplace(IPackBus *bus, DeviceState *parent,
+                           const char *name, uint8_t n_slots,
+                           qemu_irq_handler handler);
+
+#endif
diff --git a/hw/ipoctal232.c b/hw/ipoctal232.c
new file mode 100644
index 00000000000..c1e3b197b57
--- /dev/null
+++ b/hw/ipoctal232.c
@@ -0,0 +1,619 @@
+/*
+ * QEMU GE IP-Octal 232 IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+#include "qemu/bitops.h"
+#include "char/char.h"
+
+/* #define DEBUG_IPOCTAL */
+
+#ifdef DEBUG_IPOCTAL
+#define DPRINTF2(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF2(fmt, ...) do { } while (0)
+#endif
+
+#define DPRINTF(fmt, ...) DPRINTF2("IP-Octal: " fmt, ## __VA_ARGS__)
+
+#define RX_FIFO_SIZE 3
+
+/* The IP-Octal has 8 channels (a-h)
+   divided into 4 blocks (A-D) */
+#define N_CHANNELS 8
+#define N_BLOCKS   4
+
+#define REG_MRa  0x01
+#define REG_MRb  0x11
+#define REG_SRa  0x03
+#define REG_SRb  0x13
+#define REG_CSRa 0x03
+#define REG_CSRb 0x13
+#define REG_CRa  0x05
+#define REG_CRb  0x15
+#define REG_RHRa 0x07
+#define REG_RHRb 0x17
+#define REG_THRa 0x07
+#define REG_THRb 0x17
+#define REG_ACR  0x09
+#define REG_ISR  0x0B
+#define REG_IMR  0x0B
+#define REG_OPCR 0x1B
+
+#define CR_ENABLE_RX    BIT(0)
+#define CR_DISABLE_RX   BIT(1)
+#define CR_ENABLE_TX    BIT(2)
+#define CR_DISABLE_TX   BIT(3)
+#define CR_CMD(cr)      ((cr) >> 4)
+#define CR_NO_OP        0
+#define CR_RESET_MR     1
+#define CR_RESET_RX     2
+#define CR_RESET_TX     3
+#define CR_RESET_ERR    4
+#define CR_RESET_BRKINT 5
+#define CR_START_BRK    6
+#define CR_STOP_BRK     7
+#define CR_ASSERT_RTSN  8
+#define CR_NEGATE_RTSN  9
+#define CR_TIMEOUT_ON   10
+#define CR_TIMEOUT_OFF  12
+
+#define SR_RXRDY   BIT(0)
+#define SR_FFULL   BIT(1)
+#define SR_TXRDY   BIT(2)
+#define SR_TXEMT   BIT(3)
+#define SR_OVERRUN BIT(4)
+#define SR_PARITY  BIT(5)
+#define SR_FRAMING BIT(6)
+#define SR_BREAK   BIT(7)
+
+#define ISR_TXRDYA BIT(0)
+#define ISR_RXRDYA BIT(1)
+#define ISR_BREAKA BIT(2)
+#define ISR_CNTRDY BIT(3)
+#define ISR_TXRDYB BIT(4)
+#define ISR_RXRDYB BIT(5)
+#define ISR_BREAKB BIT(6)
+#define ISR_MPICHG BIT(7)
+#define ISR_TXRDY(CH) (((CH) & 1) ? BIT(4) : BIT(0))
+#define ISR_RXRDY(CH) (((CH) & 1) ? BIT(5) : BIT(1))
+#define ISR_BREAK(CH) (((CH) & 1) ? BIT(6) : BIT(2))
+
+typedef struct IPOctalState IPOctalState;
+typedef struct SCC2698Channel SCC2698Channel;
+typedef struct SCC2698Block SCC2698Block;
+
+struct SCC2698Channel {
+    IPOctalState *ipoctal;
+    CharDriverState *dev;
+    char *devpath;
+    bool rx_enabled;
+    uint8_t mr[2];
+    uint8_t mr_idx;
+    uint8_t sr;
+    uint8_t rhr[RX_FIFO_SIZE];
+    uint8_t rhr_idx;
+    uint8_t rx_pending;
+};
+
+struct SCC2698Block {
+    uint8_t imr;
+    uint8_t isr;
+};
+
+struct IPOctalState {
+    IPackDevice dev;
+    SCC2698Channel ch[N_CHANNELS];
+    SCC2698Block blk[N_BLOCKS];
+    uint8_t irq_vector;
+};
+
+#define TYPE_IPOCTAL "ipoctal232"
+
+#define IPOCTAL(obj) \
+    OBJECT_CHECK(IPOctalState, (obj), TYPE_IPOCTAL)
+
+static const VMStateDescription vmstate_scc2698_channel = {
+    .name = "scc2698_channel",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(rx_enabled, SCC2698Channel),
+        VMSTATE_UINT8_ARRAY(mr, SCC2698Channel, 2),
+        VMSTATE_UINT8(mr_idx, SCC2698Channel),
+        VMSTATE_UINT8(sr, SCC2698Channel),
+        VMSTATE_UINT8_ARRAY(rhr, SCC2698Channel, RX_FIFO_SIZE),
+        VMSTATE_UINT8(rhr_idx, SCC2698Channel),
+        VMSTATE_UINT8(rx_pending, SCC2698Channel),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_scc2698_block = {
+    .name = "scc2698_block",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT8(imr, SCC2698Block),
+        VMSTATE_UINT8(isr, SCC2698Block),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_ipoctal = {
+    .name = "ipoctal232",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_IPACK_DEVICE(dev, IPOctalState),
+        VMSTATE_STRUCT_ARRAY(ch, IPOctalState, N_CHANNELS, 1,
+                             vmstate_scc2698_channel, SCC2698Channel),
+        VMSTATE_STRUCT_ARRAY(blk, IPOctalState, N_BLOCKS, 1,
+                             vmstate_scc2698_block, SCC2698Block),
+        VMSTATE_UINT8(irq_vector, IPOctalState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* data[10] is 0x0C, not 0x0B as the doc says */
+static const uint8_t id_prom_data[] = {
+    0x49, 0x50, 0x41, 0x43, 0xF0, 0x22,
+    0xA1, 0x00, 0x00, 0x00, 0x0C, 0xCC
+};
+
+static void update_irq(IPOctalState *dev, unsigned block)
+{
+    /* Blocks A and B interrupt on INT0#, C and D on INT1#.
+       Thus, to get the status we have to check two blocks. */
+    SCC2698Block *blk0 = &dev->blk[block];
+    SCC2698Block *blk1 = &dev->blk[block^1];
+    unsigned intno = block / 2;
+
+    if ((blk0->isr & blk0->imr) || (blk1->isr & blk1->imr)) {
+        qemu_irq_raise(dev->dev.irq[intno]);
+    } else {
+        qemu_irq_lower(dev->dev.irq[intno]);
+    }
+}
+
+static void write_cr(IPOctalState *dev, unsigned channel, uint8_t val)
+{
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[channel / 2];
+
+    DPRINTF("Write CR%c %u: ", channel + 'a', val);
+
+    /* The lower 4 bits are used to enable and disable Tx and Rx */
+    if (val & CR_ENABLE_RX) {
+        DPRINTF2("Rx on, ");
+        ch->rx_enabled = true;
+    }
+    if (val & CR_DISABLE_RX) {
+        DPRINTF2("Rx off, ");
+        ch->rx_enabled = false;
+    }
+    if (val & CR_ENABLE_TX) {
+        DPRINTF2("Tx on, ");
+        ch->sr |= SR_TXRDY | SR_TXEMT;
+        blk->isr |= ISR_TXRDY(channel);
+    }
+    if (val & CR_DISABLE_TX) {
+        DPRINTF2("Tx off, ");
+        ch->sr &= ~(SR_TXRDY | SR_TXEMT);
+        blk->isr &= ~ISR_TXRDY(channel);
+    }
+
+    DPRINTF2("cmd: ");
+
+    /* The rest of the bits implement different commands */
+    switch (CR_CMD(val)) {
+    case CR_NO_OP:
+        DPRINTF2("none");
+        break;
+    case CR_RESET_MR:
+        DPRINTF2("reset MR");
+        ch->mr_idx = 0;
+        break;
+    case CR_RESET_RX:
+        DPRINTF2("reset Rx");
+        ch->rx_enabled = false;
+        ch->rx_pending = 0;
+        ch->sr &= ~SR_RXRDY;
+        blk->isr &= ~ISR_RXRDY(channel);
+        break;
+    case CR_RESET_TX:
+        DPRINTF2("reset Tx");
+        ch->sr &= ~(SR_TXRDY | SR_TXEMT);
+        blk->isr &= ~ISR_TXRDY(channel);
+        break;
+    case CR_RESET_ERR:
+        DPRINTF2("reset err");
+        ch->sr &= ~(SR_OVERRUN | SR_PARITY | SR_FRAMING | SR_BREAK);
+        break;
+    case CR_RESET_BRKINT:
+        DPRINTF2("reset brk ch int");
+        blk->isr &= ~(ISR_BREAKA | ISR_BREAKB);
+        break;
+    default:
+        DPRINTF2("unsupported 0x%x", CR_CMD(val));
+    }
+
+    DPRINTF2("\n");
+}
+
+static uint16_t io_read(IPackDevice *ip, uint8_t addr)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    uint16_t ret = 0;
+    /* addr[7:6]: block   (A-D)
+       addr[7:5]: channel (a-h)
+       addr[5:0]: register */
+    unsigned block = addr >> 5;
+    unsigned channel = addr >> 4;
+    /* Big endian, accessed using 8-bit bytes at odd locations */
+    unsigned offset = (addr & 0x1F) ^ 1;
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[block];
+    uint8_t old_isr = blk->isr;
+
+    switch (offset) {
+
+    case REG_MRa:
+    case REG_MRb:
+        ret = ch->mr[ch->mr_idx];
+        DPRINTF("Read MR%u%c: 0x%x\n", ch->mr_idx + 1, channel + 'a', ret);
+        ch->mr_idx = 1;
+        break;
+
+    case REG_SRa:
+    case REG_SRb:
+        ret = ch->sr;
+        DPRINTF("Read SR%c: 0x%x\n", channel + 'a', ret);
+        break;
+
+    case REG_RHRa:
+    case REG_RHRb:
+        ret = ch->rhr[ch->rhr_idx];
+        if (ch->rx_pending > 0) {
+            ch->rx_pending--;
+            if (ch->rx_pending == 0) {
+                ch->sr &= ~SR_RXRDY;
+                blk->isr &= ~ISR_RXRDY(channel);
+                if (ch->dev) {
+                    qemu_chr_accept_input(ch->dev);
+                }
+            } else {
+                ch->rhr_idx = (ch->rhr_idx + 1) % RX_FIFO_SIZE;
+            }
+            if (ch->sr & SR_BREAK) {
+                ch->sr &= ~SR_BREAK;
+                blk->isr |= ISR_BREAK(channel);
+            }
+        }
+        DPRINTF("Read RHR%c (0x%x)\n", channel + 'a', ret);
+        break;
+
+    case REG_ISR:
+        ret = blk->isr;
+        DPRINTF("Read ISR%c: 0x%x\n", block + 'A', ret);
+        break;
+
+    default:
+        DPRINTF("Read unknown/unsupported register 0x%02x\n", offset);
+    }
+
+    if (old_isr != blk->isr) {
+        update_irq(dev, block);
+    }
+
+    return ret;
+}
+
+static void io_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    unsigned reg = val & 0xFF;
+    /* addr[7:6]: block   (A-D)
+       addr[7:5]: channel (a-h)
+       addr[5:0]: register */
+    unsigned block = addr >> 5;
+    unsigned channel = addr >> 4;
+    /* Big endian, accessed using 8-bit bytes at odd locations */
+    unsigned offset = (addr & 0x1F) ^ 1;
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[block];
+    uint8_t old_isr = blk->isr;
+    uint8_t old_imr = blk->imr;
+
+    switch (offset) {
+
+    case REG_MRa:
+    case REG_MRb:
+        ch->mr[ch->mr_idx] = reg;
+        DPRINTF("Write MR%u%c 0x%x\n", ch->mr_idx + 1, channel + 'a', reg);
+        ch->mr_idx = 1;
+        break;
+
+    /* Not implemented */
+    case REG_CSRa:
+    case REG_CSRb:
+        DPRINTF("Write CSR%c: 0x%x\n", channel + 'a', reg);
+        break;
+
+    case REG_CRa:
+    case REG_CRb:
+        write_cr(dev, channel, reg);
+        break;
+
+    case REG_THRa:
+    case REG_THRb:
+        if (ch->sr & SR_TXRDY) {
+            DPRINTF("Write THR%c (0x%x)\n", channel + 'a', reg);
+            if (ch->dev) {
+                uint8_t thr = reg;
+                qemu_chr_fe_write(ch->dev, &thr, 1);
+            }
+        } else {
+            DPRINTF("Write THR%c (0x%x), Tx disabled\n", channel + 'a', reg);
+        }
+        break;
+
+    /* Not implemented */
+    case REG_ACR:
+        DPRINTF("Write ACR%c 0x%x\n", block + 'A', val);
+        break;
+
+    case REG_IMR:
+        DPRINTF("Write IMR%c 0x%x\n", block + 'A', val);
+        blk->imr = reg;
+        break;
+
+    /* Not implemented */
+    case REG_OPCR:
+        DPRINTF("Write OPCR%c 0x%x\n", block + 'A', val);
+        break;
+
+    default:
+        DPRINTF("Write unknown/unsupported register 0x%02x %u\n", offset, val);
+    }
+
+    if (old_isr != blk->isr || old_imr != blk->imr) {
+        update_irq(dev, block);
+    }
+}
+
+static uint16_t id_read(IPackDevice *ip, uint8_t addr)
+{
+    uint16_t ret = 0;
+    unsigned pos = addr / 2; /* The ID PROM data is stored every other byte */
+
+    if (pos < ARRAY_SIZE(id_prom_data)) {
+        ret = id_prom_data[pos];
+    } else {
+        DPRINTF("Attempt to read unavailable PROM data at 0x%x\n",  addr);
+    }
+
+    return ret;
+}
+
+static void id_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    if (addr == 1) {
+        DPRINTF("Write IRQ vector: %u\n", (unsigned) val);
+        dev->irq_vector = val; /* Undocumented, but the hw works like that */
+    } else {
+        DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+    }
+}
+
+static uint16_t int_read(IPackDevice *ip, uint8_t addr)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    /* Read address 0 to ACK INT0# and address 2 to ACK INT1# */
+    if (addr != 0 && addr != 2) {
+        DPRINTF("Attempt to read from 0x%x\n", addr);
+        return 0;
+    } else {
+        /* Update interrupts if necessary */
+        update_irq(dev, addr);
+        return dev->irq_vector;
+    }
+}
+
+static void int_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+}
+
+static uint16_t mem_read16(IPackDevice *ip, uint32_t addr)
+{
+    DPRINTF("Attempt to read from 0x%x\n", addr);
+    return 0;
+}
+
+static void mem_write16(IPackDevice *ip, uint32_t addr, uint16_t val)
+{
+    DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+}
+
+static uint8_t mem_read8(IPackDevice *ip, uint32_t addr)
+{
+    DPRINTF("Attempt to read from 0x%x\n", addr);
+    return 0;
+}
+
+static void mem_write8(IPackDevice *ip, uint32_t addr, uint8_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    if (addr == 1) {
+        DPRINTF("Write IRQ vector: %u\n", (unsigned) val);
+        dev->irq_vector = val;
+    } else {
+        DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+    }
+}
+
+static int hostdev_can_receive(void *opaque)
+{
+    SCC2698Channel *ch = opaque;
+    int available_bytes = RX_FIFO_SIZE - ch->rx_pending;
+    return ch->rx_enabled ? available_bytes : 0;
+}
+
+static void hostdev_receive(void *opaque, const uint8_t *buf, int size)
+{
+    SCC2698Channel *ch = opaque;
+    IPOctalState *dev = ch->ipoctal;
+    unsigned pos = ch->rhr_idx + ch->rx_pending;
+    int i;
+
+    assert(size + ch->rx_pending <= RX_FIFO_SIZE);
+
+    /* Copy data to the RxFIFO */
+    for (i = 0; i < size; i++) {
+        pos %= RX_FIFO_SIZE;
+        ch->rhr[pos++] = buf[i];
+    }
+
+    ch->rx_pending += size;
+
+    /* If the RxFIFO was empty raise an interrupt */
+    if (!(ch->sr & SR_RXRDY)) {
+        unsigned block, channel = 0;
+        /* Find channel number to update the ISR register */
+        while (&dev->ch[channel] != ch) {
+            channel++;
+        }
+        block = channel / 2;
+        dev->blk[block].isr |= ISR_RXRDY(channel);
+        ch->sr |= SR_RXRDY;
+        update_irq(dev, block);
+    }
+}
+
+static void hostdev_event(void *opaque, int event)
+{
+    SCC2698Channel *ch = opaque;
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        DPRINTF("Device %s opened\n", ch->dev->label);
+        break;
+    case CHR_EVENT_BREAK: {
+        uint8_t zero = 0;
+        DPRINTF("Device %s received break\n", ch->dev->label);
+
+        if (!(ch->sr & SR_BREAK)) {
+            IPOctalState *dev = ch->ipoctal;
+            unsigned block, channel = 0;
+
+            while (&dev->ch[channel] != ch) {
+                channel++;
+            }
+            block = channel / 2;
+
+            ch->sr |= SR_BREAK;
+            dev->blk[block].isr |= ISR_BREAK(channel);
+        }
+
+        /* Put a zero character in the buffer */
+        hostdev_receive(ch, &zero, 1);
+    }
+        break;
+    default:
+        DPRINTF("Device %s received event %d\n", ch->dev->label, event);
+    }
+}
+
+static int ipoctal_init(IPackDevice *ip)
+{
+    IPOctalState *s = IPOCTAL(ip);
+    unsigned i;
+
+    for (i = 0; i < N_CHANNELS; i++) {
+        SCC2698Channel *ch = &s->ch[i];
+        ch->ipoctal = s;
+
+        /* Redirect IP-Octal channels to host character devices */
+        if (ch->devpath) {
+            const char chr_name[] = "ipoctal";
+            char label[ARRAY_SIZE(chr_name) + 2];
+            static int index;
+
+            snprintf(label, sizeof(label), "%s%d", chr_name, index);
+
+            ch->dev = qemu_chr_new(label, ch->devpath, NULL);
+
+            if (ch->dev) {
+                index++;
+                qemu_chr_add_handlers(ch->dev, hostdev_can_receive,
+                                      hostdev_receive, hostdev_event, ch);
+                DPRINTF("Redirecting channel %u to %s (%s)\n",
+                        i, ch->devpath, label);
+            } else {
+                DPRINTF("Could not redirect channel %u to %s\n",
+                        i, ch->devpath);
+            }
+        }
+    }
+
+    return 0;
+}
+
+static Property ipoctal_properties[] = {
+    DEFINE_PROP_STRING("serial0", IPOctalState, ch[0].devpath),
+    DEFINE_PROP_STRING("serial1", IPOctalState, ch[1].devpath),
+    DEFINE_PROP_STRING("serial2", IPOctalState, ch[2].devpath),
+    DEFINE_PROP_STRING("serial3", IPOctalState, ch[3].devpath),
+    DEFINE_PROP_STRING("serial4", IPOctalState, ch[4].devpath),
+    DEFINE_PROP_STRING("serial5", IPOctalState, ch[5].devpath),
+    DEFINE_PROP_STRING("serial6", IPOctalState, ch[6].devpath),
+    DEFINE_PROP_STRING("serial7", IPOctalState, ch[7].devpath),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ipoctal_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IPackDeviceClass *ic = IPACK_DEVICE_CLASS(klass);
+
+    ic->init        = ipoctal_init;
+    ic->io_read     = io_read;
+    ic->io_write    = io_write;
+    ic->id_read     = id_read;
+    ic->id_write    = id_write;
+    ic->int_read    = int_read;
+    ic->int_write   = int_write;
+    ic->mem_read16  = mem_read16;
+    ic->mem_write16 = mem_write16;
+    ic->mem_read8   = mem_read8;
+    ic->mem_write8  = mem_write8;
+
+    dc->desc    = "GE IP-Octal 232 8-channel RS-232 IndustryPack";
+    dc->props   = ipoctal_properties;
+    dc->vmsd    = &vmstate_ipoctal;
+}
+
+static const TypeInfo ipoctal_info = {
+    .name          = TYPE_IPOCTAL,
+    .parent        = TYPE_IPACK_DEVICE,
+    .instance_size = sizeof(IPOctalState),
+    .class_init    = ipoctal_class_init,
+};
+
+static void ipoctal_register_types(void)
+{
+    type_register_static(&ipoctal_info);
+}
+
+type_init(ipoctal_register_types)
diff --git a/hw/irq.c b/hw/irq.c
index d413a0b2354..f4e2a7804a3 100644
--- a/hw/irq.c
+++ b/hw/irq.c
@@ -38,24 +38,37 @@ void qemu_set_irq(qemu_irq irq, int level)
     irq->handler(irq->opaque, irq->n, level);
 }
 
-qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n)
+qemu_irq *qemu_extend_irqs(qemu_irq *old, int n_old, qemu_irq_handler handler,
+                           void *opaque, int n)
 {
     qemu_irq *s;
     struct IRQState *p;
     int i;
 
-    s = (qemu_irq *)g_malloc0(sizeof(qemu_irq) * n);
-    p = (struct IRQState *)g_malloc0(sizeof(struct IRQState) * n);
-    for (i = 0; i < n; i++) {
-        p->handler = handler;
-        p->opaque = opaque;
-        p->n = i;
+    if (!old) {
+        n_old = 0;
+    }
+    s = old ? g_renew(qemu_irq, old, n + n_old) : g_new(qemu_irq, n);
+    p = old ? g_renew(struct IRQState, s[0], n + n_old) :
+                g_new(struct IRQState, n);
+    for (i = 0; i < n + n_old; i++) {
+        if (i >= n_old) {
+            p->handler = handler;
+            p->opaque = opaque;
+            p->n = i;
+        }
         s[i] = p;
         p++;
     }
     return s;
 }
 
+qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n)
+{
+    return qemu_extend_irqs(NULL, 0, handler, opaque, n);
+}
+
+
 void qemu_free_irqs(qemu_irq *s)
 {
     g_free(s[0]);
diff --git a/hw/irq.h b/hw/irq.h
index 56c55f0c46f..610e6b76236 100644
--- a/hw/irq.h
+++ b/hw/irq.h
@@ -3,6 +3,8 @@
 
 /* Generic IRQ/GPIO pin infrastructure.  */
 
+typedef struct IRQState *qemu_irq;
+
 typedef void (*qemu_irq_handler)(void *opaque, int n, int level);
 
 void qemu_set_irq(qemu_irq irq, int level);
@@ -23,8 +25,17 @@ static inline void qemu_irq_pulse(qemu_irq irq)
     qemu_set_irq(irq, 0);
 }
 
-/* Returns an array of N IRQs.  */
+/* Returns an array of N IRQs. Each IRQ is assigned the argument handler and
+ * opaque data.
+ */
 qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n);
+
+/* Extends an Array of IRQs. Old IRQs have their handlers and opaque data
+ * preserved. New IRQs are assigned the argument handler and opaque data.
+ */
+qemu_irq *qemu_extend_irqs(qemu_irq *old, int n_old, qemu_irq_handler handler,
+                                void *opaque, int n);
+
 void qemu_free_irqs(qemu_irq *s);
 
 /* Returns a new IRQ with opposite polarity.  */
diff --git a/hw/isa-bus.c b/hw/isa-bus.c
index f9b237387ae..fce311bc2aa 100644
--- a/hw/isa-bus.c
+++ b/hw/isa-bus.c
@@ -17,13 +17,14 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
+#include "sysemu/sysemu.h"
 #include "isa.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static ISABus *isabus;
-target_phys_addr_t isa_mem_base = 0;
+hwaddr isa_mem_base = 0;
 
 static void isabus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *isabus_get_fw_dev_path(DeviceState *dev);
@@ -166,6 +167,25 @@ ISADevice *isa_create_simple(ISABus *bus, const char *name)
     return dev;
 }
 
+ISADevice *isa_vga_init(ISABus *bus)
+{
+    switch (vga_interface_type) {
+    case VGA_CIRRUS:
+        return isa_create_simple(bus, "isa-cirrus-vga");
+    case VGA_QXL:
+        fprintf(stderr, "%s: qxl: no PCI bus\n", __func__);
+        return NULL;
+    case VGA_STD:
+        return isa_create_simple(bus, "isa-vga");
+    case VGA_VMWARE:
+        fprintf(stderr, "%s: vmware_vga: no PCI bus\n", __func__);
+        return NULL;
+    case VGA_NONE:
+    default:
+        return NULL;
+    }
+}
+
 static void isabus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 {
     ISADevice *d = ISA_DEVICE(dev);
@@ -195,7 +215,7 @@ static void isabus_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo isabus_bridge_info = {
+static const TypeInfo isabus_bridge_info = {
     .name          = "isabus-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
@@ -209,7 +229,7 @@ static void isa_device_class_init(ObjectClass *klass, void *data)
     k->bus_type = TYPE_ISA_BUS;
 }
 
-static TypeInfo isa_device_type_info = {
+static const TypeInfo isa_device_type_info = {
     .name = TYPE_ISA_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(ISADevice),
@@ -236,7 +256,7 @@ static char *isabus_get_fw_dev_path(DeviceState *dev)
         snprintf(path + off, sizeof(path) - off, "@%04x", d->ioport_id);
     }
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 MemoryRegion *isa_address_space(ISADevice *dev)
@@ -244,4 +264,13 @@ MemoryRegion *isa_address_space(ISADevice *dev)
     return get_system_memory();
 }
 
+MemoryRegion *isa_address_space_io(ISADevice *dev)
+{
+    if (dev) {
+        return isa_bus_from_device(dev)->address_space_io;
+    }
+
+    return isabus->address_space_io;
+}
+
 type_init(isabus_register_types)
diff --git a/hw/isa.h b/hw/isa.h
index dc970527ae6..62e89d3bcdb 100644
--- a/hw/isa.h
+++ b/hw/isa.h
@@ -3,8 +3,8 @@
 
 /* ISA bus */
 
-#include "ioport.h"
-#include "memory.h"
+#include "exec/ioport.h"
+#include "exec/memory.h"
 #include "qdev.h"
 
 #define ISA_NUM_IRQS 16
@@ -43,10 +43,13 @@ void isa_bus_irqs(ISABus *bus, qemu_irq *irqs);
 qemu_irq isa_get_irq(ISADevice *dev, int isairq);
 void isa_init_irq(ISADevice *dev, qemu_irq *p, int isairq);
 MemoryRegion *isa_address_space(ISADevice *dev);
+MemoryRegion *isa_address_space_io(ISADevice *dev);
 ISADevice *isa_create(ISABus *bus, const char *name);
 ISADevice *isa_try_create(ISABus *bus, const char *name);
 ISADevice *isa_create_simple(ISABus *bus, const char *name);
 
+ISADevice *isa_vga_init(ISABus *bus);
+
 /**
  * isa_register_ioport: Install an I/O port region on the ISA bus.
  *
@@ -82,10 +85,10 @@ static inline ISABus *isa_bus_from_device(ISADevice *d)
     return DO_UPCAST(ISABus, qbus, d->qdev.parent_bus);
 }
 
-extern target_phys_addr_t isa_mem_base;
+extern hwaddr isa_mem_base;
 
-void isa_mmio_setup(MemoryRegion *mr, target_phys_addr_t size);
-void isa_mmio_init(target_phys_addr_t base, target_phys_addr_t size);
+void isa_mmio_setup(MemoryRegion *mr, hwaddr size);
+void isa_mmio_init(hwaddr base, hwaddr size);
 
 /* dma.c */
 int DMA_get_channel_mode (int nchan);
diff --git a/hw/isa_mmio.c b/hw/isa_mmio.c
index fd755ab4a88..487cf6a8fb7 100644
--- a/hw/isa_mmio.c
+++ b/hw/isa_mmio.c
@@ -24,37 +24,37 @@
 
 #include "hw.h"
 #include "isa.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
-static void isa_mmio_writeb (void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writeb (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outb(addr & IOPORTS_MASK, val);
 }
 
-static void isa_mmio_writew(void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writew(void *opaque, hwaddr addr,
                                uint32_t val)
 {
     cpu_outw(addr & IOPORTS_MASK, val);
 }
 
-static void isa_mmio_writel(void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writel(void *opaque, hwaddr addr,
                                uint32_t val)
 {
     cpu_outl(addr & IOPORTS_MASK, val);
 }
 
-static uint32_t isa_mmio_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readb (void *opaque, hwaddr addr)
 {
     return cpu_inb(addr & IOPORTS_MASK);
 }
 
-static uint32_t isa_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readw(void *opaque, hwaddr addr)
 {
     return cpu_inw(addr & IOPORTS_MASK);
 }
 
-static uint32_t isa_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readl(void *opaque, hwaddr addr)
 {
     return cpu_inl(addr & IOPORTS_MASK);
 }
@@ -67,12 +67,12 @@ static const MemoryRegionOps isa_mmio_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-void isa_mmio_setup(MemoryRegion *mr, target_phys_addr_t size)
+void isa_mmio_setup(MemoryRegion *mr, hwaddr size)
 {
     memory_region_init_io(mr, &isa_mmio_ops, NULL, "isa-mmio", size);
 }
 
-void isa_mmio_init(target_phys_addr_t base, target_phys_addr_t size)
+void isa_mmio_init(hwaddr base, hwaddr size)
 {
     MemoryRegion *mr = g_malloc(sizeof(*mr));
 
diff --git a/hw/ivshmem.c b/hw/ivshmem.c
index 62fe53ae2b3..afaf9b3bbfc 100644
--- a/hw/ivshmem.c
+++ b/hw/ivshmem.c
@@ -18,16 +18,20 @@
  */
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
-#include "msix.h"
-#include "kvm.h"
-#include "migration.h"
-#include "qerror.h"
-#include "event_notifier.h"
+#include "pci/pci.h"
+#include "pci/msix.h"
+#include "sysemu/kvm.h"
+#include "migration/migration.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/event_notifier.h"
+#include "char/char.h"
 
 #include <sys/mman.h>
 #include <sys/types.h>
 
+#define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
+#define PCI_DEVICE_ID_IVSHMEM   0x1110
+
 #define IVSHMEM_IOEVENTFD   0
 #define IVSHMEM_MSI     1
 
@@ -71,6 +75,8 @@ typedef struct IVShmemState {
     MemoryRegion bar;
     MemoryRegion ivshmem;
     uint64_t ivshmem_size; /* size of shared memory region */
+    uint32_t ivshmem_attr;
+    uint32_t ivshmem_64bit;
     int shm_fd; /* shared memory file descriptor */
 
     Peer *peers;
@@ -147,7 +153,6 @@ static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
     s->intrstatus = val;
 
     ivshmem_update_irq(s, val);
-    return;
 }
 
 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
@@ -162,7 +167,7 @@ static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
     return ret;
 }
 
-static void ivshmem_io_write(void *opaque, target_phys_addr_t addr,
+static void ivshmem_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     IVShmemState *s = opaque;
@@ -201,7 +206,7 @@ static void ivshmem_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t ivshmem_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
 
@@ -339,7 +344,7 @@ static void create_shared_memory_BAR(IVShmemState *s, int fd) {
     memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
     /* region for shared memory */
-    pci_register_bar(&s->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+    pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 }
 
 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
@@ -508,8 +513,6 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
         ivshmem_add_eventfd(s, incoming_posn, guest_max_eventfd);
     }
-
-    return;
 }
 
 /* Select the MSI-X vectors used by device.
@@ -534,7 +537,6 @@ static void ivshmem_reset(DeviceState *d)
 
     s->intrstatus = 0;
     ivshmem_use_msix(s);
-    return;
 }
 
 static uint64_t ivshmem_get_size(IVShmemState * s) {
@@ -690,6 +692,11 @@ static int pci_ivshmem_init(PCIDevice *dev)
                      &s->ivshmem_mmio);
 
     memory_region_init(&s->bar, "ivshmem-bar2-container", s->ivshmem_size);
+    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
+    if (s->ivshmem_64bit) {
+        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+    }
 
     if ((s->server_chr != NULL) &&
                         (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
@@ -715,8 +722,7 @@ static int pci_ivshmem_init(PCIDevice *dev)
         /* allocate/initialize space for interrupt handling */
         s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
 
-        pci_register_bar(&s->dev, 2,
-                         PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+        pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 
         s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
 
@@ -786,6 +792,7 @@ static Property ivshmem_properties[] = {
     DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
     DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
     DEFINE_PROP_STRING("role", IVShmemState, role),
+    DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -796,14 +803,14 @@ static void ivshmem_class_init(ObjectClass *klass, void *data)
 
     k->init = pci_ivshmem_init;
     k->exit = pci_ivshmem_uninit;
-    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
-    k->device_id = 0x1110;
+    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
+    k->device_id = PCI_DEVICE_ID_IVSHMEM;
     k->class_id = PCI_CLASS_MEMORY_RAM;
     dc->reset = ivshmem_reset;
     dc->props = ivshmem_properties;
 }
 
-static TypeInfo ivshmem_info = {
+static const TypeInfo ivshmem_info = {
     .name          = "ivshmem",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(IVShmemState),
diff --git a/hw/jazz_led.c b/hw/jazz_led.c
index 648652302a0..4822c485f20 100644
--- a/hw/jazz_led.c
+++ b/hw/jazz_led.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "console.h"
-#include "pixel_ops.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -39,7 +40,7 @@ typedef struct LedState {
     screen_state_t state;
 } LedState;
 
-static uint64_t jazz_led_read(void *opaque, target_phys_addr_t addr,
+static uint64_t jazz_led_read(void *opaque, hwaddr addr,
                               unsigned int size)
 {
     LedState *s = opaque;
@@ -51,7 +52,7 @@ static uint64_t jazz_led_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void jazz_led_write(void *opaque, target_phys_addr_t addr,
+static void jazz_led_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned int size)
 {
     LedState *s = opaque;
@@ -196,7 +197,7 @@ static void jazz_led_update_display(void *opaque)
     }
 
     s->state = REDRAW_NONE;
-    dpy_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
+    dpy_gfx_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
 }
 
 static void jazz_led_invalidate_display(void *opaque)
@@ -210,7 +211,7 @@ static void jazz_led_text_update(void *opaque, console_ch_t *chardata)
     LedState *s = opaque;
     char buf[2];
 
-    dpy_cursor(s->ds, -1, -1);
+    dpy_text_cursor(s->ds, -1, -1);
     qemu_console_resize(s->ds, 2, 1);
 
     /* TODO: draw the segments */
@@ -218,7 +219,7 @@ static void jazz_led_text_update(void *opaque, console_ch_t *chardata)
     console_write_ch(chardata++, 0x00200100 | buf[0]);
     console_write_ch(chardata++, 0x00200100 | buf[1]);
 
-    dpy_update(s->ds, 0, 0, 2, 1);
+    dpy_text_update(s->ds, 0, 0, 2, 1);
 }
 
 static int jazz_led_post_load(void *opaque, int version_id)
@@ -276,7 +277,7 @@ static void jazz_led_class_init(ObjectClass *klass, void *data)
     dc->reset = jazz_led_reset;
 }
 
-static TypeInfo jazz_led_info = {
+static const TypeInfo jazz_led_info = {
     .name          = "jazz-led",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LedState),
diff --git a/hw/kvm/Makefile.objs b/hw/kvm/Makefile.objs
index 226497a58fd..f620d7ff85e 100644
--- a/hw/kvm/Makefile.objs
+++ b/hw/kvm/Makefile.objs
@@ -1 +1 @@
-obj-$(CONFIG_KVM) += clock.o apic.o i8259.o ioapic.o i8254.o
+obj-$(CONFIG_KVM) += clock.o apic.o i8259.o ioapic.o i8254.o pci-assign.o
diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c
index 80e3e483333..d994ea7c97b 100644
--- a/hw/kvm/apic.c
+++ b/hw/kvm/apic.c
@@ -10,8 +10,8 @@
  * See the COPYING file in the top-level directory.
  */
 #include "hw/apic_internal.h"
-#include "hw/msi.h"
-#include "kvm.h"
+#include "hw/pci/msi.h"
+#include "sysemu/kvm.h"
 
 static inline void kvm_apic_set_reg(struct kvm_lapic_state *kapic,
                                     int reg_id, uint32_t val)
@@ -104,7 +104,7 @@ static void kvm_apic_enable_tpr_reporting(APICCommonState *s, bool enable)
         .enabled = enable
     };
 
-    kvm_vcpu_ioctl(s->cpu_env, KVM_TPR_ACCESS_REPORTING, &ctl);
+    kvm_vcpu_ioctl(CPU(s->cpu), KVM_TPR_ACCESS_REPORTING, &ctl);
 }
 
 static void kvm_apic_vapic_base_update(APICCommonState *s)
@@ -114,7 +114,7 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
     };
     int ret;
 
-    ret = kvm_vcpu_ioctl(s->cpu_env, KVM_SET_VAPIC_ADDR, &vapid_addr);
+    ret = kvm_vcpu_ioctl(CPU(s->cpu), KVM_SET_VAPIC_ADDR, &vapid_addr);
     if (ret < 0) {
         fprintf(stderr, "KVM: setting VAPIC address failed (%s)\n",
                 strerror(-ret));
@@ -125,15 +125,15 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
 static void do_inject_external_nmi(void *data)
 {
     APICCommonState *s = data;
-    CPUX86State *env = s->cpu_env;
+    CPUState *cpu = CPU(s->cpu);
     uint32_t lvt;
     int ret;
 
-    cpu_synchronize_state(env);
+    cpu_synchronize_state(&s->cpu->env);
 
     lvt = s->lvt[APIC_LVT_LINT1];
     if (!(lvt & APIC_LVT_MASKED) && ((lvt >> 8) & 7) == APIC_DM_NMI) {
-        ret = kvm_vcpu_ioctl(env, KVM_NMI);
+        ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
         if (ret < 0) {
             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
                     strerror(-ret));
@@ -143,16 +143,16 @@ static void do_inject_external_nmi(void *data)
 
 static void kvm_apic_external_nmi(APICCommonState *s)
 {
-    run_on_cpu(s->cpu_env, do_inject_external_nmi, s);
+    run_on_cpu(CPU(s->cpu), do_inject_external_nmi, s);
 }
 
-static uint64_t kvm_apic_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t kvm_apic_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return ~(uint64_t)0;
 }
 
-static void kvm_apic_mem_write(void *opaque, target_phys_addr_t addr,
+static void kvm_apic_mem_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     MSIMessage msg = { .address = addr, .data = data };
@@ -194,7 +194,7 @@ static void kvm_apic_class_init(ObjectClass *klass, void *data)
     k->external_nmi = kvm_apic_external_nmi;
 }
 
-static TypeInfo kvm_apic_info = {
+static const TypeInfo kvm_apic_info = {
     .name = "kvm-apic",
     .parent = TYPE_APIC_COMMON,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/kvm/clock.c b/hw/kvm/clock.c
index 824b9783972..fa40e283f7f 100644
--- a/hw/kvm/clock.c
+++ b/hw/kvm/clock.c
@@ -14,8 +14,8 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -76,7 +76,7 @@ static void kvmclock_vm_state_change(void *opaque, int running,
             return;
         }
         for (penv = first_cpu; penv != NULL; penv = penv->next_cpu) {
-            ret = kvm_vcpu_ioctl(penv, KVM_KVMCLOCK_CTRL, 0);
+            ret = kvm_vcpu_ioctl(ENV_GET_CPU(penv), KVM_KVMCLOCK_CTRL, 0);
             if (ret) {
                 if (ret != -EINVAL) {
                     fprintf(stderr, "%s: %s\n", __func__, strerror(-ret));
@@ -118,7 +118,7 @@ static void kvmclock_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &kvmclock_vmsd;
 }
 
-static TypeInfo kvmclock_info = {
+static const TypeInfo kvmclock_info = {
     .name          = "kvmclock",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(KVMClockState),
diff --git a/hw/kvm/i8254.c b/hw/kvm/i8254.c
index 53d13e3123b..04ad649b0ee 100644
--- a/hw/kvm/i8254.c
+++ b/hw/kvm/i8254.c
@@ -22,11 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "hw/i8254.h"
 #include "hw/i8254_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define KVM_PIT_REINJECT_BIT 0
 
@@ -302,7 +302,7 @@ static void kvm_pit_class_init(ObjectClass *klass, void *data)
     dc->props = kvm_pit_properties;
 }
 
-static TypeInfo kvm_pit_info = {
+static const TypeInfo kvm_pit_info = {
     .name          = "kvm-pit",
     .parent        = TYPE_PIT_COMMON,
     .instance_size = sizeof(KVMPITState),
diff --git a/hw/kvm/i8259.c b/hw/kvm/i8259.c
index 1e24cd4f36f..5ae8b6819ba 100644
--- a/hw/kvm/i8259.c
+++ b/hw/kvm/i8259.c
@@ -11,7 +11,7 @@
  */
 #include "hw/i8259_internal.h"
 #include "hw/apic_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static void kvm_pic_get(PICCommonState *s)
 {
@@ -123,7 +123,7 @@ static void kvm_i8259_class_init(ObjectClass *klass, void *data)
     k->post_load  = kvm_pic_put;
 }
 
-static TypeInfo kvm_i8259_info = {
+static const TypeInfo kvm_i8259_info = {
     .name  = "kvm-i8259",
     .parent = TYPE_PIC_COMMON,
     .instance_size = sizeof(PICCommonState),
diff --git a/hw/kvm/ioapic.c b/hw/kvm/ioapic.c
index 6c3b8fe39af..23877d42593 100644
--- a/hw/kvm/ioapic.c
+++ b/hw/kvm/ioapic.c
@@ -13,7 +13,47 @@
 #include "hw/pc.h"
 #include "hw/ioapic_internal.h"
 #include "hw/apic_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
+
+/* PC Utility function */
+void kvm_pc_setup_irq_routing(bool pci_enabled)
+{
+    KVMState *s = kvm_state;
+    int i;
+
+    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
+        for (i = 0; i < 8; ++i) {
+            if (i == 2) {
+                continue;
+            }
+            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_MASTER, i);
+        }
+        for (i = 8; i < 16; ++i) {
+            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_SLAVE, i - 8);
+        }
+        if (pci_enabled) {
+            for (i = 0; i < 24; ++i) {
+                if (i == 0) {
+                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, 2);
+                } else if (i != 2) {
+                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, i);
+                }
+            }
+        }
+    }
+}
+
+void kvm_pc_gsi_handler(void *opaque, int n, int level)
+{
+    GSIState *s = opaque;
+
+    if (n < ISA_NUM_IRQS) {
+        /* Kernel will forward to both PIC and IOAPIC */
+        qemu_set_irq(s->i8259_irq[n], level);
+    } else {
+        qemu_set_irq(s->ioapic_irq[n], level);
+    }
+}
 
 typedef struct KVMIOAPICState KVMIOAPICState;
 
@@ -110,7 +150,7 @@ static void kvm_ioapic_class_init(ObjectClass *klass, void *data)
     dc->props    = kvm_ioapic_properties;
 }
 
-static TypeInfo kvm_ioapic_info = {
+static const TypeInfo kvm_ioapic_info = {
     .name  = "kvm-ioapic",
     .parent = TYPE_IOAPIC_COMMON,
     .instance_size = sizeof(KVMIOAPICState),
diff --git a/hw/kvm/pci-assign.c b/hw/kvm/pci-assign.c
new file mode 100644
index 00000000000..896cfe8a59a
--- /dev/null
+++ b/hw/kvm/pci-assign.c
@@ -0,0 +1,1924 @@
+/*
+ * Copyright (c) 2007, Neocleus Corporation.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ *
+ *  Assign a PCI device from the host to a guest VM.
+ *
+ *  This implementation uses the classic device assignment interface of KVM
+ *  and is only available on x86 hosts. It is expected to be obsoleted by VFIO
+ *  based device assignment.
+ *
+ *  Adapted for KVM (qemu-kvm) by Qumranet. QEMU version was based on qemu-kvm
+ *  revision 4144fe9d48. See its repository for the history.
+ *
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/io.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "hw/hw.h"
+#include "hw/pc.h"
+#include "qemu/error-report.h"
+#include "ui/console.h"
+#include "hw/loader.h"
+#include "monitor/monitor.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
+#include "kvm_i386.h"
+
+#define MSIX_PAGE_SIZE 0x1000
+
+/* From linux/ioport.h */
+#define IORESOURCE_IO       0x00000100  /* Resource type */
+#define IORESOURCE_MEM      0x00000200
+#define IORESOURCE_IRQ      0x00000400
+#define IORESOURCE_DMA      0x00000800
+#define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
+#define IORESOURCE_MEM_64   0x00100000
+
+//#define DEVICE_ASSIGNMENT_DEBUG
+
+#ifdef DEVICE_ASSIGNMENT_DEBUG
+#define DEBUG(fmt, ...)                                       \
+    do {                                                      \
+        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
+    } while (0)
+#else
+#define DEBUG(fmt, ...)
+#endif
+
+typedef struct PCIRegion {
+    int type;           /* Memory or port I/O */
+    int valid;
+    uint64_t base_addr;
+    uint64_t size;    /* size of the region */
+    int resource_fd;
+} PCIRegion;
+
+typedef struct PCIDevRegions {
+    uint8_t bus, dev, func; /* Bus inside domain, device and function */
+    int irq;                /* IRQ number */
+    uint16_t region_number; /* number of active regions */
+
+    /* Port I/O or MMIO Regions */
+    PCIRegion regions[PCI_NUM_REGIONS - 1];
+    int config_fd;
+} PCIDevRegions;
+
+typedef struct AssignedDevRegion {
+    MemoryRegion container;
+    MemoryRegion real_iomem;
+    union {
+        uint8_t *r_virtbase; /* mmapped access address for memory regions */
+        uint32_t r_baseport; /* the base guest port for I/O regions */
+    } u;
+    pcibus_t e_size;    /* emulated size of region in bytes */
+    pcibus_t r_size;    /* real size of region in bytes */
+    PCIRegion *region;
+} AssignedDevRegion;
+
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  0
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  1
+
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
+
+typedef struct MSIXTableEntry {
+    uint32_t addr_lo;
+    uint32_t addr_hi;
+    uint32_t data;
+    uint32_t ctrl;
+} MSIXTableEntry;
+
+typedef enum AssignedIRQType {
+    ASSIGNED_IRQ_NONE = 0,
+    ASSIGNED_IRQ_INTX_HOST_INTX,
+    ASSIGNED_IRQ_INTX_HOST_MSI,
+    ASSIGNED_IRQ_MSI,
+    ASSIGNED_IRQ_MSIX
+} AssignedIRQType;
+
+typedef struct AssignedDevice {
+    PCIDevice dev;
+    PCIHostDeviceAddress host;
+    uint32_t dev_id;
+    uint32_t features;
+    int intpin;
+    AssignedDevRegion v_addrs[PCI_NUM_REGIONS - 1];
+    PCIDevRegions real_device;
+    PCIINTxRoute intx_route;
+    AssignedIRQType assigned_irq_type;
+    struct {
+#define ASSIGNED_DEVICE_CAP_MSI (1 << 0)
+#define ASSIGNED_DEVICE_CAP_MSIX (1 << 1)
+        uint32_t available;
+#define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0)
+#define ASSIGNED_DEVICE_MSIX_ENABLED (1 << 1)
+#define ASSIGNED_DEVICE_MSIX_MASKED (1 << 2)
+        uint32_t state;
+    } cap;
+    uint8_t emulate_config_read[PCI_CONFIG_SPACE_SIZE];
+    uint8_t emulate_config_write[PCI_CONFIG_SPACE_SIZE];
+    int msi_virq_nr;
+    int *msi_virq;
+    MSIXTableEntry *msix_table;
+    hwaddr msix_table_addr;
+    uint16_t msix_max;
+    MemoryRegion mmio;
+    char *configfd_name;
+    int32_t bootindex;
+} AssignedDevice;
+
+static void assigned_dev_update_irq_routing(PCIDevice *dev);
+
+static void assigned_dev_load_option_rom(AssignedDevice *dev);
+
+static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev);
+
+static uint64_t assigned_dev_ioport_rw(AssignedDevRegion *dev_region,
+                                       hwaddr addr, int size,
+                                       uint64_t *data)
+{
+    uint64_t val = 0;
+    int fd = dev_region->region->resource_fd;
+
+    if (fd >= 0) {
+        if (data) {
+            DEBUG("pwrite data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", addr="TARGET_FMT_plx"\n", *data, size, addr, addr);
+            if (pwrite(fd, data, size, addr) != size) {
+                error_report("%s - pwrite failed %s",
+                             __func__, strerror(errno));
+            }
+        } else {
+            if (pread(fd, &val, size, addr) != size) {
+                error_report("%s - pread failed %s",
+                             __func__, strerror(errno));
+                val = (1UL << (size * 8)) - 1;
+            }
+            DEBUG("pread val=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", addr=" TARGET_FMT_plx "\n", val, size, addr, addr);
+        }
+    } else {
+        uint32_t port = addr + dev_region->u.r_baseport;
+
+        if (data) {
+            DEBUG("out data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", host=%x\n", *data, size, addr, port);
+            switch (size) {
+            case 1:
+                outb(*data, port);
+                break;
+            case 2:
+                outw(*data, port);
+                break;
+            case 4:
+                outl(*data, port);
+                break;
+            }
+        } else {
+            switch (size) {
+            case 1:
+                val = inb(port);
+                break;
+            case 2:
+                val = inw(port);
+                break;
+            case 4:
+                val = inl(port);
+                break;
+            }
+            DEBUG("in data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", host=%x\n", val, size, addr, port);
+        }
+    }
+    return val;
+}
+
+static void assigned_dev_ioport_write(void *opaque, hwaddr addr,
+                                      uint64_t data, unsigned size)
+{
+    assigned_dev_ioport_rw(opaque, addr, size, &data);
+}
+
+static uint64_t assigned_dev_ioport_read(void *opaque,
+                                         hwaddr addr, unsigned size)
+{
+    return assigned_dev_ioport_rw(opaque, addr, size, NULL);
+}
+
+static uint32_t slow_bar_readb(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint8_t *in = d->u.r_virtbase + addr;
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static uint32_t slow_bar_readw(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr);
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static uint32_t slow_bar_readl(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr);
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static void slow_bar_writeb(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint8_t *out = d->u.r_virtbase + addr;
+
+    DEBUG("slow_bar_writeb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val);
+    *out = val;
+}
+
+static void slow_bar_writew(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr);
+
+    DEBUG("slow_bar_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val);
+    *out = val;
+}
+
+static void slow_bar_writel(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr);
+
+    DEBUG("slow_bar_writel addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val);
+    *out = val;
+}
+
+static const MemoryRegionOps slow_bar_ops = {
+    .old_mmio = {
+        .read = { slow_bar_readb, slow_bar_readw, slow_bar_readl, },
+        .write = { slow_bar_writeb, slow_bar_writew, slow_bar_writel, },
+    },
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void assigned_dev_iomem_setup(PCIDevice *pci_dev, int region_num,
+                                     pcibus_t e_size)
+{
+    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevRegion *region = &r_dev->v_addrs[region_num];
+    PCIRegion *real_region = &r_dev->real_device.regions[region_num];
+
+    if (e_size > 0) {
+        memory_region_init(&region->container, "assigned-dev-container",
+                           e_size);
+        memory_region_add_subregion(&region->container, 0, &region->real_iomem);
+
+        /* deal with MSI-X MMIO page */
+        if (real_region->base_addr <= r_dev->msix_table_addr &&
+                real_region->base_addr + real_region->size >
+                r_dev->msix_table_addr) {
+            uint64_t offset = r_dev->msix_table_addr - real_region->base_addr;
+
+            memory_region_add_subregion_overlap(&region->container,
+                                                offset,
+                                                &r_dev->mmio,
+                                                1);
+        }
+    }
+}
+
+static const MemoryRegionOps assigned_dev_ioport_ops = {
+    .read = assigned_dev_ioport_read,
+    .write = assigned_dev_ioport_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void assigned_dev_ioport_setup(PCIDevice *pci_dev, int region_num,
+                                      pcibus_t size)
+{
+    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevRegion *region = &r_dev->v_addrs[region_num];
+
+    region->e_size = size;
+    memory_region_init(&region->container, "assigned-dev-container", size);
+    memory_region_init_io(&region->real_iomem, &assigned_dev_ioport_ops,
+                          r_dev->v_addrs + region_num,
+                          "assigned-dev-iomem", size);
+    memory_region_add_subregion(&region->container, 0, &region->real_iomem);
+}
+
+static uint32_t assigned_dev_pci_read(PCIDevice *d, int pos, int len)
+{
+    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    uint32_t val;
+    ssize_t ret;
+    int fd = pci_dev->real_device.config_fd;
+
+again:
+    ret = pread(fd, &val, len, pos);
+    if (ret != len) {
+        if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) {
+            goto again;
+        }
+
+        hw_error("pci read failed, ret = %zd errno = %d\n", ret, errno);
+    }
+
+    return val;
+}
+
+static uint8_t assigned_dev_pci_read_byte(PCIDevice *d, int pos)
+{
+    return (uint8_t)assigned_dev_pci_read(d, pos, 1);
+}
+
+static void assigned_dev_pci_write(PCIDevice *d, int pos, uint32_t val, int len)
+{
+    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    ssize_t ret;
+    int fd = pci_dev->real_device.config_fd;
+
+again:
+    ret = pwrite(fd, &val, len, pos);
+    if (ret != len) {
+        if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) {
+            goto again;
+        }
+
+        hw_error("pci write failed, ret = %zd errno = %d\n", ret, errno);
+    }
+}
+
+static void assigned_dev_emulate_config_read(AssignedDevice *dev,
+                                             uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_read + offset, 0xff, len);
+}
+
+static void assigned_dev_direct_config_read(AssignedDevice *dev,
+                                            uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_read + offset, 0, len);
+}
+
+static void assigned_dev_direct_config_write(AssignedDevice *dev,
+                                             uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_write + offset, 0, len);
+}
+
+static uint8_t pci_find_cap_offset(PCIDevice *d, uint8_t cap, uint8_t start)
+{
+    int id;
+    int max_cap = 48;
+    int pos = start ? start : PCI_CAPABILITY_LIST;
+    int status;
+
+    status = assigned_dev_pci_read_byte(d, PCI_STATUS);
+    if ((status & PCI_STATUS_CAP_LIST) == 0) {
+        return 0;
+    }
+
+    while (max_cap--) {
+        pos = assigned_dev_pci_read_byte(d, pos);
+        if (pos < 0x40) {
+            break;
+        }
+
+        pos &= ~3;
+        id = assigned_dev_pci_read_byte(d, pos + PCI_CAP_LIST_ID);
+
+        if (id == 0xff) {
+            break;
+        }
+        if (id == cap) {
+            return pos;
+        }
+
+        pos += PCI_CAP_LIST_NEXT;
+    }
+    return 0;
+}
+
+static int assigned_dev_register_regions(PCIRegion *io_regions,
+                                         unsigned long regions_num,
+                                         AssignedDevice *pci_dev)
+{
+    uint32_t i;
+    PCIRegion *cur_region = io_regions;
+
+    for (i = 0; i < regions_num; i++, cur_region++) {
+        if (!cur_region->valid) {
+            continue;
+        }
+
+        /* handle memory io regions */
+        if (cur_region->type & IORESOURCE_MEM) {
+            int t = PCI_BASE_ADDRESS_SPACE_MEMORY;
+            if (cur_region->type & IORESOURCE_PREFETCH) {
+                t |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+            }
+            if (cur_region->type & IORESOURCE_MEM_64) {
+                t |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+            }
+
+            /* map physical memory */
+            pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL, cur_region->size,
+                                                    PROT_WRITE | PROT_READ,
+                                                    MAP_SHARED,
+                                                    cur_region->resource_fd,
+                                                    (off_t)0);
+
+            if (pci_dev->v_addrs[i].u.r_virtbase == MAP_FAILED) {
+                pci_dev->v_addrs[i].u.r_virtbase = NULL;
+                error_report("%s: Error: Couldn't mmap 0x%" PRIx64 "!",
+                             __func__, cur_region->base_addr);
+                return -1;
+            }
+
+            pci_dev->v_addrs[i].r_size = cur_region->size;
+            pci_dev->v_addrs[i].e_size = 0;
+
+            /* add offset */
+            pci_dev->v_addrs[i].u.r_virtbase +=
+                (cur_region->base_addr & 0xFFF);
+
+            if (cur_region->size & 0xFFF) {
+                error_report("PCI region %d at address 0x%" PRIx64 " has "
+                             "size 0x%" PRIx64 ", which is not a multiple of "
+                             "4K.  You might experience some performance hit "
+                             "due to that.",
+                             i, cur_region->base_addr, cur_region->size);
+                memory_region_init_io(&pci_dev->v_addrs[i].real_iomem,
+                                      &slow_bar_ops, &pci_dev->v_addrs[i],
+                                      "assigned-dev-slow-bar",
+                                      cur_region->size);
+            } else {
+                void *virtbase = pci_dev->v_addrs[i].u.r_virtbase;
+                char name[32];
+                snprintf(name, sizeof(name), "%s.bar%d",
+                         object_get_typename(OBJECT(pci_dev)), i);
+                memory_region_init_ram_ptr(&pci_dev->v_addrs[i].real_iomem,
+                                           name, cur_region->size,
+                                           virtbase);
+                vmstate_register_ram(&pci_dev->v_addrs[i].real_iomem,
+                                     &pci_dev->dev.qdev);
+            }
+
+            assigned_dev_iomem_setup(&pci_dev->dev, i, cur_region->size);
+            pci_register_bar((PCIDevice *) pci_dev, i, t,
+                             &pci_dev->v_addrs[i].container);
+            continue;
+        } else {
+            /* handle port io regions */
+            uint32_t val;
+            int ret;
+
+            /* Test kernel support for ioport resource read/write.  Old
+             * kernels return EIO.  New kernels only allow 1/2/4 byte reads
+             * so should return EINVAL for a 3 byte read */
+            ret = pread(pci_dev->v_addrs[i].region->resource_fd, &val, 3, 0);
+            if (ret >= 0) {
+                error_report("Unexpected return from I/O port read: %d", ret);
+                abort();
+            } else if (errno != EINVAL) {
+                error_report("Kernel doesn't support ioport resource "
+                             "access, hiding this region.");
+                close(pci_dev->v_addrs[i].region->resource_fd);
+                cur_region->valid = 0;
+                continue;
+            }
+
+            pci_dev->v_addrs[i].u.r_baseport = cur_region->base_addr;
+            pci_dev->v_addrs[i].r_size = cur_region->size;
+            pci_dev->v_addrs[i].e_size = 0;
+
+            assigned_dev_ioport_setup(&pci_dev->dev, i, cur_region->size);
+            pci_register_bar((PCIDevice *) pci_dev, i,
+                             PCI_BASE_ADDRESS_SPACE_IO,
+                             &pci_dev->v_addrs[i].container);
+        }
+    }
+
+    /* success */
+    return 0;
+}
+
+static int get_real_id(const char *devpath, const char *idname, uint16_t *val)
+{
+    FILE *f;
+    char name[128];
+    long id;
+
+    snprintf(name, sizeof(name), "%s%s", devpath, idname);
+    f = fopen(name, "r");
+    if (f == NULL) {
+        error_report("%s: %s: %m", __func__, name);
+        return -1;
+    }
+    if (fscanf(f, "%li\n", &id) == 1) {
+        *val = id;
+    } else {
+        return -1;
+    }
+    fclose(f);
+
+    return 0;
+}
+
+static int get_real_vendor_id(const char *devpath, uint16_t *val)
+{
+    return get_real_id(devpath, "vendor", val);
+}
+
+static int get_real_device_id(const char *devpath, uint16_t *val)
+{
+    return get_real_id(devpath, "device", val);
+}
+
+static int get_real_device(AssignedDevice *pci_dev, uint16_t r_seg,
+                           uint8_t r_bus, uint8_t r_dev, uint8_t r_func)
+{
+    char dir[128], name[128];
+    int fd, r = 0, v;
+    FILE *f;
+    uint64_t start, end, size, flags;
+    uint16_t id;
+    PCIRegion *rp;
+    PCIDevRegions *dev = &pci_dev->real_device;
+
+    dev->region_number = 0;
+
+    snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%x/",
+             r_seg, r_bus, r_dev, r_func);
+
+    snprintf(name, sizeof(name), "%sconfig", dir);
+
+    if (pci_dev->configfd_name && *pci_dev->configfd_name) {
+        dev->config_fd = monitor_handle_fd_param(cur_mon, pci_dev->configfd_name);
+        if (dev->config_fd < 0) {
+            return 1;
+        }
+    } else {
+        dev->config_fd = open(name, O_RDWR);
+
+        if (dev->config_fd == -1) {
+            error_report("%s: %s: %m", __func__, name);
+            return 1;
+        }
+    }
+again:
+    r = read(dev->config_fd, pci_dev->dev.config,
+             pci_config_size(&pci_dev->dev));
+    if (r < 0) {
+        if (errno == EINTR || errno == EAGAIN) {
+            goto again;
+        }
+        error_report("%s: read failed, errno = %d", __func__, errno);
+    }
+
+    /* Restore or clear multifunction, this is always controlled by qemu */
+    if (pci_dev->dev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+        pci_dev->dev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
+    } else {
+        pci_dev->dev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
+    }
+
+    /* Clear host resource mapping info.  If we choose not to register a
+     * BAR, such as might be the case with the option ROM, we can get
+     * confusing, unwritable, residual addresses from the host here. */
+    memset(&pci_dev->dev.config[PCI_BASE_ADDRESS_0], 0, 24);
+    memset(&pci_dev->dev.config[PCI_ROM_ADDRESS], 0, 4);
+
+    snprintf(name, sizeof(name), "%sresource", dir);
+
+    f = fopen(name, "r");
+    if (f == NULL) {
+        error_report("%s: %s: %m", __func__, name);
+        return 1;
+    }
+
+    for (r = 0; r < PCI_ROM_SLOT; r++) {
+        if (fscanf(f, "%" SCNi64 " %" SCNi64 " %" SCNi64 "\n",
+                   &start, &end, &flags) != 3) {
+            break;
+        }
+
+        rp = dev->regions + r;
+        rp->valid = 0;
+        rp->resource_fd = -1;
+        size = end - start + 1;
+        flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH
+                 | IORESOURCE_MEM_64;
+        if (size == 0 || (flags & ~IORESOURCE_PREFETCH) == 0) {
+            continue;
+        }
+        if (flags & IORESOURCE_MEM) {
+            flags &= ~IORESOURCE_IO;
+        } else {
+            flags &= ~IORESOURCE_PREFETCH;
+        }
+        snprintf(name, sizeof(name), "%sresource%d", dir, r);
+        fd = open(name, O_RDWR);
+        if (fd == -1) {
+            continue;
+        }
+        rp->resource_fd = fd;
+
+        rp->type = flags;
+        rp->valid = 1;
+        rp->base_addr = start;
+        rp->size = size;
+        pci_dev->v_addrs[r].region = rp;
+        DEBUG("region %d size %" PRIu64 " start 0x%" PRIx64
+              " type %d resource_fd %d\n",
+              r, rp->size, start, rp->type, rp->resource_fd);
+    }
+
+    fclose(f);
+
+    /* read and fill vendor ID */
+    v = get_real_vendor_id(dir, &id);
+    if (v) {
+        return 1;
+    }
+    pci_dev->dev.config[0] = id & 0xff;
+    pci_dev->dev.config[1] = (id & 0xff00) >> 8;
+
+    /* read and fill device ID */
+    v = get_real_device_id(dir, &id);
+    if (v) {
+        return 1;
+    }
+    pci_dev->dev.config[2] = id & 0xff;
+    pci_dev->dev.config[3] = (id & 0xff00) >> 8;
+
+    pci_word_test_and_clear_mask(pci_dev->emulate_config_write + PCI_COMMAND,
+                                 PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
+
+    dev->region_number = r;
+    return 0;
+}
+
+static void free_msi_virqs(AssignedDevice *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->msi_virq_nr; i++) {
+        if (dev->msi_virq[i] >= 0) {
+            kvm_irqchip_release_virq(kvm_state, dev->msi_virq[i]);
+            dev->msi_virq[i] = -1;
+        }
+    }
+    g_free(dev->msi_virq);
+    dev->msi_virq = NULL;
+    dev->msi_virq_nr = 0;
+}
+
+static void free_assigned_device(AssignedDevice *dev)
+{
+    int i;
+
+    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        assigned_dev_unregister_msix_mmio(dev);
+    }
+    for (i = 0; i < dev->real_device.region_number; i++) {
+        PCIRegion *pci_region = &dev->real_device.regions[i];
+        AssignedDevRegion *region = &dev->v_addrs[i];
+
+        if (!pci_region->valid) {
+            continue;
+        }
+        if (pci_region->type & IORESOURCE_IO) {
+            if (region->u.r_baseport) {
+                memory_region_del_subregion(&region->container,
+                                            &region->real_iomem);
+                memory_region_destroy(&region->real_iomem);
+                memory_region_destroy(&region->container);
+            }
+        } else if (pci_region->type & IORESOURCE_MEM) {
+            if (region->u.r_virtbase) {
+                memory_region_del_subregion(&region->container,
+                                            &region->real_iomem);
+
+                /* Remove MSI-X table subregion */
+                if (pci_region->base_addr <= dev->msix_table_addr &&
+                    pci_region->base_addr + pci_region->size >
+                    dev->msix_table_addr) {
+                    memory_region_del_subregion(&region->container,
+                                                &dev->mmio);
+                }
+
+                memory_region_destroy(&region->real_iomem);
+                memory_region_destroy(&region->container);
+                if (munmap(region->u.r_virtbase,
+                           (pci_region->size + 0xFFF) & 0xFFFFF000)) {
+                    error_report("Failed to unmap assigned device region: %s",
+                                 strerror(errno));
+                }
+            }
+        }
+        if (pci_region->resource_fd >= 0) {
+            close(pci_region->resource_fd);
+        }
+    }
+
+    if (dev->real_device.config_fd >= 0) {
+        close(dev->real_device.config_fd);
+    }
+
+    free_msi_virqs(dev);
+}
+
+static void assign_failed_examine(AssignedDevice *dev)
+{
+    char name[PATH_MAX], dir[PATH_MAX], driver[PATH_MAX] = {}, *ns;
+    uint16_t vendor_id, device_id;
+    int r;
+
+    snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
+            dev->host.domain, dev->host.bus, dev->host.slot,
+            dev->host.function);
+
+    snprintf(name, sizeof(name), "%sdriver", dir);
+
+    r = readlink(name, driver, sizeof(driver));
+    if ((r <= 0) || r >= sizeof(driver)) {
+        goto fail;
+    }
+
+    ns = strrchr(driver, '/');
+    if (!ns) {
+        goto fail;
+    }
+
+    ns++;
+
+    if (get_real_vendor_id(dir, &vendor_id) ||
+        get_real_device_id(dir, &device_id)) {
+        goto fail;
+    }
+
+    error_report("*** The driver '%s' is occupying your device "
+                 "%04x:%02x:%02x.%x.",
+                 ns, dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function);
+    error_report("***");
+    error_report("*** You can try the following commands to free it:");
+    error_report("***");
+    error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/"
+                 "new_id", vendor_id, device_id);
+    error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
+                 "%s/unbind",
+                 dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function, ns);
+    error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
+                 "pci-stub/bind",
+                 dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function);
+    error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub"
+                 "/remove_id", vendor_id, device_id);
+    error_report("***");
+
+    return;
+
+fail:
+    error_report("Couldn't find out why.");
+}
+
+static int assign_device(AssignedDevice *dev)
+{
+    uint32_t flags = KVM_DEV_ASSIGN_ENABLE_IOMMU;
+    int r;
+
+    /* Only pass non-zero PCI segment to capable module */
+    if (!kvm_check_extension(kvm_state, KVM_CAP_PCI_SEGMENT) &&
+        dev->host.domain) {
+        error_report("Can't assign device inside non-zero PCI segment "
+                     "as this KVM module doesn't support it.");
+        return -ENODEV;
+    }
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_IOMMU)) {
+        error_report("No IOMMU found.  Unable to assign device \"%s\"",
+                     dev->dev.qdev.id);
+        return -ENODEV;
+    }
+
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK &&
+        kvm_has_intx_set_mask()) {
+        flags |= KVM_DEV_ASSIGN_PCI_2_3;
+    }
+
+    r = kvm_device_pci_assign(kvm_state, &dev->host, flags, &dev->dev_id);
+    if (r < 0) {
+        error_report("Failed to assign device \"%s\" : %s",
+                     dev->dev.qdev.id, strerror(-r));
+
+        switch (r) {
+        case -EBUSY:
+            assign_failed_examine(dev);
+            break;
+        default:
+            break;
+        }
+    }
+    return r;
+}
+
+static bool check_irqchip_in_kernel(void)
+{
+    if (kvm_irqchip_in_kernel()) {
+        return true;
+    }
+    error_report("pci-assign: error: requires KVM with in-kernel irqchip "
+                 "enabled");
+    return false;
+}
+
+static int assign_intx(AssignedDevice *dev)
+{
+    AssignedIRQType new_type;
+    PCIINTxRoute intx_route;
+    bool intx_host_msi;
+    int r;
+
+    /* Interrupt PIN 0 means don't use INTx */
+    if (assigned_dev_pci_read_byte(&dev->dev, PCI_INTERRUPT_PIN) == 0) {
+        pci_device_set_intx_routing_notifier(&dev->dev, NULL);
+        return 0;
+    }
+
+    if (!check_irqchip_in_kernel()) {
+        return -ENOTSUP;
+    }
+
+    pci_device_set_intx_routing_notifier(&dev->dev,
+                                         assigned_dev_update_irq_routing);
+
+    intx_route = pci_device_route_intx_to_irq(&dev->dev, dev->intpin);
+    assert(intx_route.mode != PCI_INTX_INVERTED);
+
+    if (!pci_intx_route_changed(&dev->intx_route, &intx_route)) {
+        return 0;
+    }
+
+    switch (dev->assigned_irq_type) {
+    case ASSIGNED_IRQ_INTX_HOST_INTX:
+    case ASSIGNED_IRQ_INTX_HOST_MSI:
+        intx_host_msi = dev->assigned_irq_type == ASSIGNED_IRQ_INTX_HOST_MSI;
+        r = kvm_device_intx_deassign(kvm_state, dev->dev_id, intx_host_msi);
+        break;
+    case ASSIGNED_IRQ_MSI:
+        r = kvm_device_msi_deassign(kvm_state, dev->dev_id);
+        break;
+    case ASSIGNED_IRQ_MSIX:
+        r = kvm_device_msix_deassign(kvm_state, dev->dev_id);
+        break;
+    default:
+        r = 0;
+        break;
+    }
+    if (r) {
+        perror("assign_intx: deassignment of previous interrupt failed");
+    }
+    dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+
+    if (intx_route.mode == PCI_INTX_DISABLED) {
+        dev->intx_route = intx_route;
+        return 0;
+    }
+
+retry:
+    if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK &&
+        dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+        intx_host_msi = true;
+        new_type = ASSIGNED_IRQ_INTX_HOST_MSI;
+    } else {
+        intx_host_msi = false;
+        new_type = ASSIGNED_IRQ_INTX_HOST_INTX;
+    }
+
+    r = kvm_device_intx_assign(kvm_state, dev->dev_id, intx_host_msi,
+                               intx_route.irq);
+    if (r < 0) {
+        if (r == -EIO && !(dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK) &&
+            dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+            /* Retry with host-side MSI. There might be an IRQ conflict and
+             * either the kernel or the device doesn't support sharing. */
+            error_report("Host-side INTx sharing not supported, "
+                         "using MSI instead.\n"
+                         "Some devices do not to work properly in this mode.");
+            dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK;
+            goto retry;
+        }
+        error_report("Failed to assign irq for \"%s\": %s",
+                     dev->dev.qdev.id, strerror(-r));
+        error_report("Perhaps you are assigning a device "
+                     "that shares an IRQ with another device?");
+        return r;
+    }
+
+    dev->intx_route = intx_route;
+    dev->assigned_irq_type = new_type;
+    return r;
+}
+
+static void deassign_device(AssignedDevice *dev)
+{
+    int r;
+
+    r = kvm_device_pci_deassign(kvm_state, dev->dev_id);
+    assert(r == 0);
+}
+
+/* The pci config space got updated. Check if irq numbers have changed
+ * for our devices
+ */
+static void assigned_dev_update_irq_routing(PCIDevice *dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, dev);
+    Error *err = NULL;
+    int r;
+
+    r = assign_intx(assigned_dev);
+    if (r < 0) {
+        qdev_unplug(&dev->qdev, &err);
+        assert(!err);
+    }
+}
+
+static void assigned_dev_update_msi(PCIDevice *pci_dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap +
+                                     PCI_MSI_FLAGS);
+    int r;
+
+    /* Some guests gratuitously disable MSI even if they're not using it,
+     * try to catch this by only deassigning irqs if the guest is using
+     * MSI or intends to start. */
+    if (assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSI ||
+        (ctrl_byte & PCI_MSI_FLAGS_ENABLE)) {
+        r = kvm_device_msi_deassign(kvm_state, assigned_dev->dev_id);
+        /* -ENXIO means no assigned irq */
+        if (r && r != -ENXIO) {
+            perror("assigned_dev_update_msi: deassign irq");
+        }
+
+        free_msi_virqs(assigned_dev);
+
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+        pci_device_set_intx_routing_notifier(pci_dev, NULL);
+    }
+
+    if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) {
+        MSIMessage msg = msi_get_message(pci_dev, 0);
+        int virq;
+
+        virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (virq < 0) {
+            perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route");
+            return;
+        }
+
+        assigned_dev->msi_virq = g_malloc(sizeof(*assigned_dev->msi_virq));
+        assigned_dev->msi_virq_nr = 1;
+        assigned_dev->msi_virq[0] = virq;
+        if (kvm_device_msi_assign(kvm_state, assigned_dev->dev_id, virq) < 0) {
+            perror("assigned_dev_update_msi: kvm_device_msi_assign");
+        }
+
+        assigned_dev->intx_route.mode = PCI_INTX_DISABLED;
+        assigned_dev->intx_route.irq = -1;
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSI;
+    } else {
+        assign_intx(assigned_dev);
+    }
+}
+
+static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
+{
+    return (entry->ctrl & cpu_to_le32(0x1)) != 0;
+}
+
+/*
+ * When MSI-X is first enabled the vector table typically has all the
+ * vectors masked, so we can't use that as the obvious test to figure out
+ * how many vectors to initially enable.  Instead we look at the data field
+ * because this is what worked for pci-assign for a long time.  This makes
+ * sure the physical MSI-X state tracks the guest's view, which is important
+ * for some VF/PF and PF/fw communication channels.
+ */
+static bool assigned_dev_msix_skipped(MSIXTableEntry *entry)
+{
+    return !entry->data;
+}
+
+static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
+{
+    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t entries_nr = 0;
+    int i, r = 0;
+    MSIXTableEntry *entry = adev->msix_table;
+    MSIMessage msg;
+
+    /* Get the usable entry number for allocating */
+    for (i = 0; i < adev->msix_max; i++, entry++) {
+        if (assigned_dev_msix_skipped(entry)) {
+            continue;
+        }
+        entries_nr++;
+    }
+
+    DEBUG("MSI-X entries: %d\n", entries_nr);
+
+    /* It's valid to enable MSI-X with all entries masked */
+    if (!entries_nr) {
+        return 0;
+    }
+
+    r = kvm_device_msix_init_vectors(kvm_state, adev->dev_id, entries_nr);
+    if (r != 0) {
+        error_report("fail to set MSI-X entry number for MSIX! %s",
+                     strerror(-r));
+        return r;
+    }
+
+    free_msi_virqs(adev);
+
+    adev->msi_virq_nr = adev->msix_max;
+    adev->msi_virq = g_malloc(adev->msix_max * sizeof(*adev->msi_virq));
+
+    entry = adev->msix_table;
+    for (i = 0; i < adev->msix_max; i++, entry++) {
+        adev->msi_virq[i] = -1;
+
+        if (assigned_dev_msix_skipped(entry)) {
+            continue;
+        }
+
+        msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32);
+        msg.data = entry->data;
+        r = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (r < 0) {
+            return r;
+        }
+        adev->msi_virq[i] = r;
+
+        DEBUG("MSI-X vector %d, gsi %d, addr %08x_%08x, data %08x\n", i,
+              r, entry->addr_hi, entry->addr_lo, entry->data);
+
+        r = kvm_device_msix_set_vector(kvm_state, adev->dev_id, i,
+                                       adev->msi_virq[i]);
+        if (r) {
+            error_report("fail to set MSI-X entry! %s", strerror(-r));
+            break;
+        }
+    }
+
+    return r;
+}
+
+static void assigned_dev_update_msix(PCIDevice *pci_dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t ctrl_word = pci_get_word(pci_dev->config + pci_dev->msix_cap +
+                                      PCI_MSIX_FLAGS);
+    int r;
+
+    /* Some guests gratuitously disable MSIX even if they're not using it,
+     * try to catch this by only deassigning irqs if the guest is using
+     * MSIX or intends to start. */
+    if ((assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSIX) ||
+        (ctrl_word & PCI_MSIX_FLAGS_ENABLE)) {
+        r = kvm_device_msix_deassign(kvm_state, assigned_dev->dev_id);
+        /* -ENXIO means no assigned irq */
+        if (r && r != -ENXIO) {
+            perror("assigned_dev_update_msix: deassign irq");
+        }
+
+        free_msi_virqs(assigned_dev);
+
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+        pci_device_set_intx_routing_notifier(pci_dev, NULL);
+    }
+
+    if (ctrl_word & PCI_MSIX_FLAGS_ENABLE) {
+        if (assigned_dev_update_msix_mmio(pci_dev) < 0) {
+            perror("assigned_dev_update_msix_mmio");
+            return;
+        }
+
+        if (assigned_dev->msi_virq_nr > 0) {
+            if (kvm_device_msix_assign(kvm_state, assigned_dev->dev_id) < 0) {
+                perror("assigned_dev_enable_msix: assign irq");
+                return;
+            }
+        }
+        assigned_dev->intx_route.mode = PCI_INTX_DISABLED;
+        assigned_dev->intx_route.irq = -1;
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSIX;
+    } else {
+        assign_intx(assigned_dev);
+    }
+}
+
+static uint32_t assigned_dev_pci_read_config(PCIDevice *pci_dev,
+                                             uint32_t address, int len)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint32_t virt_val = pci_default_read_config(pci_dev, address, len);
+    uint32_t real_val, emulate_mask, full_emulation_mask;
+
+    emulate_mask = 0;
+    memcpy(&emulate_mask, assigned_dev->emulate_config_read + address, len);
+    emulate_mask = le32_to_cpu(emulate_mask);
+
+    full_emulation_mask = 0xffffffff >> (32 - len * 8);
+
+    if (emulate_mask != full_emulation_mask) {
+        real_val = assigned_dev_pci_read(pci_dev, address, len);
+        return (virt_val & emulate_mask) | (real_val & ~emulate_mask);
+    } else {
+        return virt_val;
+    }
+}
+
+static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
+                                          uint32_t val, int len)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
+    uint32_t emulate_mask, full_emulation_mask;
+    int ret;
+
+    pci_default_write_config(pci_dev, address, val, len);
+
+    if (kvm_has_intx_set_mask() &&
+        range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state, assigned_dev->dev_id,
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+        if (range_covers_byte(address, len,
+                              pci_dev->msi_cap + PCI_MSI_FLAGS)) {
+            assigned_dev_update_msi(pci_dev);
+        }
+    }
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        if (range_covers_byte(address, len,
+                              pci_dev->msix_cap + PCI_MSIX_FLAGS + 1)) {
+            assigned_dev_update_msix(pci_dev);
+        }
+    }
+
+    emulate_mask = 0;
+    memcpy(&emulate_mask, assigned_dev->emulate_config_write + address, len);
+    emulate_mask = le32_to_cpu(emulate_mask);
+
+    full_emulation_mask = 0xffffffff >> (32 - len * 8);
+
+    if (emulate_mask != full_emulation_mask) {
+        if (emulate_mask) {
+            val &= ~emulate_mask;
+            val |= assigned_dev_pci_read(pci_dev, address, len) & emulate_mask;
+        }
+        assigned_dev_pci_write(pci_dev, address, val, len);
+    }
+}
+
+static void assigned_dev_setup_cap_read(AssignedDevice *dev, uint32_t offset,
+                                        uint32_t len)
+{
+    assigned_dev_direct_config_read(dev, offset, len);
+    assigned_dev_emulate_config_read(dev, offset + PCI_CAP_LIST_NEXT, 1);
+}
+
+static int assigned_device_pci_cap_init(PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    PCIRegion *pci_region = dev->real_device.regions;
+    int ret, pos;
+
+    /* Clear initial capabilities pointer and status copied from hw */
+    pci_set_byte(pci_dev->config + PCI_CAPABILITY_LIST, 0);
+    pci_set_word(pci_dev->config + PCI_STATUS,
+                 pci_get_word(pci_dev->config + PCI_STATUS) &
+                 ~PCI_STATUS_CAP_LIST);
+
+    /* Expose MSI capability
+     * MSI capability is the 1st capability in capability config */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSI, 0);
+    if (pos != 0 && kvm_check_extension(kvm_state, KVM_CAP_ASSIGN_DEV_IRQ)) {
+        if (!check_irqchip_in_kernel()) {
+            return -ENOTSUP;
+        }
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSI;
+        /* Only 32-bit/no-mask currently supported */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSI, pos, 10);
+        if (ret < 0) {
+            return ret;
+        }
+        pci_dev->msi_cap = pos;
+
+        pci_set_word(pci_dev->config + pos + PCI_MSI_FLAGS,
+                     pci_get_word(pci_dev->config + pos + PCI_MSI_FLAGS) &
+                     PCI_MSI_FLAGS_QMASK);
+        pci_set_long(pci_dev->config + pos + PCI_MSI_ADDRESS_LO, 0);
+        pci_set_word(pci_dev->config + pos + PCI_MSI_DATA_32, 0);
+
+        /* Set writable fields */
+        pci_set_word(pci_dev->wmask + pos + PCI_MSI_FLAGS,
+                     PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
+        pci_set_long(pci_dev->wmask + pos + PCI_MSI_ADDRESS_LO, 0xfffffffc);
+        pci_set_word(pci_dev->wmask + pos + PCI_MSI_DATA_32, 0xffff);
+    }
+    /* Expose MSI-X capability */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSIX, 0);
+    if (pos != 0 && kvm_device_msix_supported(kvm_state)) {
+        int bar_nr;
+        uint32_t msix_table_entry;
+
+        if (!check_irqchip_in_kernel()) {
+            return -ENOTSUP;
+        }
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSIX;
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSIX, pos, 12);
+        if (ret < 0) {
+            return ret;
+        }
+        pci_dev->msix_cap = pos;
+
+        pci_set_word(pci_dev->config + pos + PCI_MSIX_FLAGS,
+                     pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS) &
+                     PCI_MSIX_FLAGS_QSIZE);
+
+        /* Only enable and function mask bits are writable */
+        pci_set_word(pci_dev->wmask + pos + PCI_MSIX_FLAGS,
+                     PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
+
+        msix_table_entry = pci_get_long(pci_dev->config + pos + PCI_MSIX_TABLE);
+        bar_nr = msix_table_entry & PCI_MSIX_FLAGS_BIRMASK;
+        msix_table_entry &= ~PCI_MSIX_FLAGS_BIRMASK;
+        dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry;
+        dev->msix_max = pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS);
+        dev->msix_max &= PCI_MSIX_FLAGS_QSIZE;
+        dev->msix_max += 1;
+    }
+
+    /* Minimal PM support, nothing writable, device appears to NAK changes */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PM, 0);
+    if (pos) {
+        uint16_t pmc;
+
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, pos, PCI_PM_SIZEOF);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, PCI_PM_SIZEOF);
+
+        pmc = pci_get_word(pci_dev->config + pos + PCI_CAP_FLAGS);
+        pmc &= (PCI_PM_CAP_VER_MASK | PCI_PM_CAP_DSI);
+        pci_set_word(pci_dev->config + pos + PCI_CAP_FLAGS, pmc);
+
+        /* assign_device will bring the device up to D0, so we don't need
+         * to worry about doing that ourselves here. */
+        pci_set_word(pci_dev->config + pos + PCI_PM_CTRL,
+                     PCI_PM_CTRL_NO_SOFT_RESET);
+
+        pci_set_byte(pci_dev->config + pos + PCI_PM_PPB_EXTENSIONS, 0);
+        pci_set_byte(pci_dev->config + pos + PCI_PM_DATA_REGISTER, 0);
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_EXP, 0);
+    if (pos) {
+        uint8_t version, size = 0;
+        uint16_t type, devctl, lnksta;
+        uint32_t devcap, lnkcap;
+
+        version = pci_get_byte(pci_dev->config + pos + PCI_EXP_FLAGS);
+        version &= PCI_EXP_FLAGS_VERS;
+        if (version == 1) {
+            size = 0x14;
+        } else if (version == 2) {
+            /*
+             * Check for non-std size, accept reduced size to 0x34,
+             * which is what bcm5761 implemented, violating the
+             * PCIe v3.0 spec that regs should exist and be read as 0,
+             * not optionally provided and shorten the struct size.
+             */
+            size = MIN(0x3c, PCI_CONFIG_SPACE_SIZE - pos);
+            if (size < 0x34) {
+                error_report("%s: Invalid size PCIe cap-id 0x%x",
+                             __func__, PCI_CAP_ID_EXP);
+                return -EINVAL;
+            } else if (size != 0x3c) {
+                error_report("WARNING, %s: PCIe cap-id 0x%x has "
+                             "non-standard size 0x%x; std size should be 0x3c",
+                             __func__, PCI_CAP_ID_EXP, size);
+            }
+        } else if (version == 0) {
+            uint16_t vid, did;
+            vid = pci_get_word(pci_dev->config + PCI_VENDOR_ID);
+            did = pci_get_word(pci_dev->config + PCI_DEVICE_ID);
+            if (vid == PCI_VENDOR_ID_INTEL && did == 0x10ed) {
+                /*
+                 * quirk for Intel 82599 VF with invalid PCIe capability
+                 * version, should really be version 2 (same as PF)
+                 */
+                size = 0x3c;
+            }
+        }
+
+        if (size == 0) {
+            error_report("%s: Unsupported PCI express capability version %d",
+                         __func__, version);
+            return -EINVAL;
+        }
+
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_EXP, pos, size);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, size);
+
+        type = pci_get_word(pci_dev->config + pos + PCI_EXP_FLAGS);
+        type = (type & PCI_EXP_FLAGS_TYPE) >> 4;
+        if (type != PCI_EXP_TYPE_ENDPOINT &&
+            type != PCI_EXP_TYPE_LEG_END && type != PCI_EXP_TYPE_RC_END) {
+            error_report("Device assignment only supports endpoint assignment,"
+                         " device type %d", type);
+            return -EINVAL;
+        }
+
+        /* capabilities, pass existing read-only copy
+         * PCI_EXP_FLAGS_IRQ: updated by hardware, should be direct read */
+
+        /* device capabilities: hide FLR */
+        devcap = pci_get_long(pci_dev->config + pos + PCI_EXP_DEVCAP);
+        devcap &= ~PCI_EXP_DEVCAP_FLR;
+        pci_set_long(pci_dev->config + pos + PCI_EXP_DEVCAP, devcap);
+
+        /* device control: clear all error reporting enable bits, leaving
+         *                 only a few host values.  Note, these are
+         *                 all writable, but not passed to hw.
+         */
+        devctl = pci_get_word(pci_dev->config + pos + PCI_EXP_DEVCTL);
+        devctl = (devctl & (PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_PAYLOAD)) |
+                  PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN;
+        pci_set_word(pci_dev->config + pos + PCI_EXP_DEVCTL, devctl);
+        devctl = PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_AUX_PME;
+        pci_set_word(pci_dev->wmask + pos + PCI_EXP_DEVCTL, ~devctl);
+
+        /* Clear device status */
+        pci_set_word(pci_dev->config + pos + PCI_EXP_DEVSTA, 0);
+
+        /* Link capabilities, expose links and latencues, clear reporting */
+        lnkcap = pci_get_long(pci_dev->config + pos + PCI_EXP_LNKCAP);
+        lnkcap &= (PCI_EXP_LNKCAP_SLS | PCI_EXP_LNKCAP_MLW |
+                   PCI_EXP_LNKCAP_ASPMS | PCI_EXP_LNKCAP_L0SEL |
+                   PCI_EXP_LNKCAP_L1EL);
+        pci_set_long(pci_dev->config + pos + PCI_EXP_LNKCAP, lnkcap);
+
+        /* Link control, pass existing read-only copy.  Should be writable? */
+
+        /* Link status, only expose current speed and width */
+        lnksta = pci_get_word(pci_dev->config + pos + PCI_EXP_LNKSTA);
+        lnksta &= (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW);
+        pci_set_word(pci_dev->config + pos + PCI_EXP_LNKSTA, lnksta);
+
+        if (version >= 2) {
+            /* Slot capabilities, control, status - not needed for endpoints */
+            pci_set_long(pci_dev->config + pos + PCI_EXP_SLTCAP, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_SLTCTL, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_SLTSTA, 0);
+
+            /* Root control, capabilities, status - not needed for endpoints */
+            pci_set_word(pci_dev->config + pos + PCI_EXP_RTCTL, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_RTCAP, 0);
+            pci_set_long(pci_dev->config + pos + PCI_EXP_RTSTA, 0);
+
+            /* Device capabilities/control 2, pass existing read-only copy */
+            /* Link control 2, pass existing read-only copy */
+        }
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PCIX, 0);
+    if (pos) {
+        uint16_t cmd;
+        uint32_t status;
+
+        /* Only expose the minimum, 8 byte capability */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_PCIX, pos, 8);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, 8);
+
+        /* Command register, clear upper bits, including extended modes */
+        cmd = pci_get_word(pci_dev->config + pos + PCI_X_CMD);
+        cmd &= (PCI_X_CMD_DPERR_E | PCI_X_CMD_ERO | PCI_X_CMD_MAX_READ |
+                PCI_X_CMD_MAX_SPLIT);
+        pci_set_word(pci_dev->config + pos + PCI_X_CMD, cmd);
+
+        /* Status register, update with emulated PCI bus location, clear
+         * error bits, leave the rest. */
+        status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS);
+        status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN);
+        status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn;
+        status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL |
+                    PCI_X_STATUS_SPL_ERR);
+        pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status);
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VPD, 0);
+    if (pos) {
+        /* Direct R/W passthrough */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_VPD, pos, 8);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, 8);
+
+        /* direct write for cap content */
+        assigned_dev_direct_config_write(dev, pos + 2, 6);
+    }
+
+    /* Devices can have multiple vendor capabilities, get them all */
+    for (pos = 0; (pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VNDR, pos));
+        pos += PCI_CAP_LIST_NEXT) {
+        uint8_t len = pci_get_byte(pci_dev->config + pos + PCI_CAP_FLAGS);
+        /* Direct R/W passthrough */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_VNDR, pos, len);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, len);
+
+        /* direct write for cap content */
+        assigned_dev_direct_config_write(dev, pos + 2, len - 2);
+    }
+
+    /* If real and virtual capability list status bits differ, virtualize the
+     * access. */
+    if ((pci_get_word(pci_dev->config + PCI_STATUS) & PCI_STATUS_CAP_LIST) !=
+        (assigned_dev_pci_read_byte(pci_dev, PCI_STATUS) &
+         PCI_STATUS_CAP_LIST)) {
+        dev->emulate_config_read[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
+    }
+
+    return 0;
+}
+
+static uint64_t
+assigned_dev_msix_mmio_read(void *opaque, hwaddr addr,
+                            unsigned size)
+{
+    AssignedDevice *adev = opaque;
+    uint64_t val;
+
+    memcpy(&val, (void *)((uint8_t *)adev->msix_table + addr), size);
+
+    return val;
+}
+
+static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr,
+                                         uint64_t val, unsigned size)
+{
+    AssignedDevice *adev = opaque;
+    PCIDevice *pdev = &adev->dev;
+    uint16_t ctrl;
+    MSIXTableEntry orig;
+    int i = addr >> 4;
+
+    if (i >= adev->msix_max) {
+        return; /* Drop write */
+    }
+
+    ctrl = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
+
+    DEBUG("write to MSI-X table offset 0x%lx, val 0x%lx\n", addr, val);
+
+    if (ctrl & PCI_MSIX_FLAGS_ENABLE) {
+        orig = adev->msix_table[i];
+    }
+
+    memcpy((uint8_t *)adev->msix_table + addr, &val, size);
+
+    if (ctrl & PCI_MSIX_FLAGS_ENABLE) {
+        MSIXTableEntry *entry = &adev->msix_table[i];
+
+        if (!assigned_dev_msix_masked(&orig) &&
+            assigned_dev_msix_masked(entry)) {
+            /*
+             * Vector masked, disable it
+             *
+             * XXX It's not clear if we can or should actually attempt
+             * to mask or disable the interrupt.  KVM doesn't have
+             * support for pending bits and kvm_assign_set_msix_entry
+             * doesn't modify the device hardware mask.  Interrupts
+             * while masked are simply not injected to the guest, so
+             * are lost.  Can we get away with always injecting an
+             * interrupt on unmask?
+             */
+        } else if (assigned_dev_msix_masked(&orig) &&
+                   !assigned_dev_msix_masked(entry)) {
+            /* Vector unmasked */
+            if (i >= adev->msi_virq_nr || adev->msi_virq[i] < 0) {
+                /* Previously unassigned vector, start from scratch */
+                assigned_dev_update_msix(pdev);
+                return;
+            } else {
+                /* Update an existing, previously masked vector */
+                MSIMessage msg;
+                int ret;
+
+                msg.address = entry->addr_lo |
+                    ((uint64_t)entry->addr_hi << 32);
+                msg.data = entry->data;
+
+                ret = kvm_irqchip_update_msi_route(kvm_state,
+                                                   adev->msi_virq[i], msg);
+                if (ret) {
+                    error_report("Error updating irq routing entry (%d)", ret);
+                }
+            }
+        }
+    }
+}
+
+static const MemoryRegionOps assigned_dev_msix_mmio_ops = {
+    .read = assigned_dev_msix_mmio_read,
+    .write = assigned_dev_msix_mmio_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+};
+
+static void assigned_dev_msix_reset(AssignedDevice *dev)
+{
+    MSIXTableEntry *entry;
+    int i;
+
+    if (!dev->msix_table) {
+        return;
+    }
+
+    memset(dev->msix_table, 0, MSIX_PAGE_SIZE);
+
+    for (i = 0, entry = dev->msix_table; i < dev->msix_max; i++, entry++) {
+        entry->ctrl = cpu_to_le32(0x1); /* Masked */
+    }
+}
+
+static int assigned_dev_register_msix_mmio(AssignedDevice *dev)
+{
+    dev->msix_table = mmap(NULL, MSIX_PAGE_SIZE, PROT_READ|PROT_WRITE,
+                           MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+    if (dev->msix_table == MAP_FAILED) {
+        error_report("fail allocate msix_table! %s", strerror(errno));
+        return -EFAULT;
+    }
+
+    assigned_dev_msix_reset(dev);
+
+    memory_region_init_io(&dev->mmio, &assigned_dev_msix_mmio_ops, dev,
+                          "assigned-dev-msix", MSIX_PAGE_SIZE);
+    return 0;
+}
+
+static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev)
+{
+    if (!dev->msix_table) {
+        return;
+    }
+
+    memory_region_destroy(&dev->mmio);
+
+    if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) {
+        error_report("error unmapping msix_table! %s", strerror(errno));
+    }
+    dev->msix_table = NULL;
+}
+
+static const VMStateDescription vmstate_assigned_device = {
+    .name = "pci-assign",
+    .unmigratable = 1,
+};
+
+static void reset_assigned_device(DeviceState *dev)
+{
+    PCIDevice *pci_dev = DO_UPCAST(PCIDevice, qdev, dev);
+    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    char reset_file[64];
+    const char reset[] = "1";
+    int fd, ret;
+
+    /*
+     * If a guest is reset without being shutdown, MSI/MSI-X can still
+     * be running.  We want to return the device to a known state on
+     * reset, so disable those here.  We especially do not want MSI-X
+     * enabled since it lives in MMIO space, which is about to get
+     * disabled.
+     */
+    if (adev->assigned_irq_type == ASSIGNED_IRQ_MSIX) {
+        uint16_t ctrl = pci_get_word(pci_dev->config +
+                                     pci_dev->msix_cap + PCI_MSIX_FLAGS);
+
+        pci_set_word(pci_dev->config + pci_dev->msix_cap + PCI_MSIX_FLAGS,
+                     ctrl & ~PCI_MSIX_FLAGS_ENABLE);
+        assigned_dev_update_msix(pci_dev);
+    } else if (adev->assigned_irq_type == ASSIGNED_IRQ_MSI) {
+        uint8_t ctrl = pci_get_byte(pci_dev->config +
+                                    pci_dev->msi_cap + PCI_MSI_FLAGS);
+
+        pci_set_byte(pci_dev->config + pci_dev->msi_cap + PCI_MSI_FLAGS,
+                     ctrl & ~PCI_MSI_FLAGS_ENABLE);
+        assigned_dev_update_msi(pci_dev);
+    }
+
+    snprintf(reset_file, sizeof(reset_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/reset",
+             adev->host.domain, adev->host.bus, adev->host.slot,
+             adev->host.function);
+
+    /*
+     * Issue a device reset via pci-sysfs.  Note that we use write(2) here
+     * and ignore the return value because some kernels have a bug that
+     * returns 0 rather than bytes written on success, sending us into an
+     * infinite retry loop using other write mechanisms.
+     */
+    fd = open(reset_file, O_WRONLY);
+    if (fd != -1) {
+        ret = write(fd, reset, strlen(reset));
+        (void)ret;
+        close(fd);
+    }
+
+    /*
+     * When a 0 is written to the bus master register, the device is logically
+     * disconnected from the PCI bus. This avoids further DMA transfers.
+     */
+    assigned_dev_pci_write_config(pci_dev, PCI_COMMAND, 0, 1);
+}
+
+static int assigned_initfn(struct PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint8_t e_intx;
+    int r;
+
+    if (!kvm_enabled()) {
+        error_report("pci-assign: error: requires KVM support");
+        return -1;
+    }
+
+    if (!dev->host.domain && !dev->host.bus && !dev->host.slot &&
+        !dev->host.function) {
+        error_report("pci-assign: error: no host device specified");
+        return -1;
+    }
+
+    /*
+     * Set up basic config space access control. Will be further refined during
+     * device initialization.
+     */
+    assigned_dev_emulate_config_read(dev, 0, PCI_CONFIG_SPACE_SIZE);
+    assigned_dev_direct_config_read(dev, PCI_STATUS, 2);
+    assigned_dev_direct_config_read(dev, PCI_REVISION_ID, 1);
+    assigned_dev_direct_config_read(dev, PCI_CLASS_PROG, 3);
+    assigned_dev_direct_config_read(dev, PCI_CACHE_LINE_SIZE, 1);
+    assigned_dev_direct_config_read(dev, PCI_LATENCY_TIMER, 1);
+    assigned_dev_direct_config_read(dev, PCI_BIST, 1);
+    assigned_dev_direct_config_read(dev, PCI_CARDBUS_CIS, 4);
+    assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_VENDOR_ID, 2);
+    assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_ID, 2);
+    assigned_dev_direct_config_read(dev, PCI_CAPABILITY_LIST + 1, 7);
+    assigned_dev_direct_config_read(dev, PCI_MIN_GNT, 1);
+    assigned_dev_direct_config_read(dev, PCI_MAX_LAT, 1);
+    memcpy(dev->emulate_config_write, dev->emulate_config_read,
+           sizeof(dev->emulate_config_read));
+
+    if (get_real_device(dev, dev->host.domain, dev->host.bus,
+                        dev->host.slot, dev->host.function)) {
+        error_report("pci-assign: Error: Couldn't get real device (%s)!",
+                     dev->dev.qdev.id);
+        goto out;
+    }
+
+    if (assigned_device_pci_cap_init(pci_dev) < 0) {
+        goto out;
+    }
+
+    /* intercept MSI-X entry page in the MMIO */
+    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        if (assigned_dev_register_msix_mmio(dev)) {
+            goto out;
+        }
+    }
+
+    /* handle real device's MMIO/PIO BARs */
+    if (assigned_dev_register_regions(dev->real_device.regions,
+                                      dev->real_device.region_number,
+                                      dev)) {
+        goto out;
+    }
+
+    /* handle interrupt routing */
+    e_intx = dev->dev.config[PCI_INTERRUPT_PIN] - 1;
+    dev->intpin = e_intx;
+    dev->intx_route.mode = PCI_INTX_DISABLED;
+    dev->intx_route.irq = -1;
+
+    /* assign device to guest */
+    r = assign_device(dev);
+    if (r < 0) {
+        goto out;
+    }
+
+    /* assign legacy INTx to the device */
+    r = assign_intx(dev);
+    if (r < 0) {
+        goto assigned_out;
+    }
+
+    assigned_dev_load_option_rom(dev);
+
+    add_boot_device_path(dev->bootindex, &pci_dev->qdev, NULL);
+
+    return 0;
+
+assigned_out:
+    deassign_device(dev);
+out:
+    free_assigned_device(dev);
+    return -1;
+}
+
+static void assigned_exitfn(struct PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+
+    deassign_device(dev);
+    free_assigned_device(dev);
+}
+
+static Property assigned_dev_properties[] = {
+    DEFINE_PROP_PCI_HOST_DEVADDR("host", AssignedDevice, host),
+    DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
+                    ASSIGNED_DEVICE_PREFER_MSI_BIT, false),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, true),
+    DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
+    DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void assign_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    k->init         = assigned_initfn;
+    k->exit         = assigned_exitfn;
+    k->config_read  = assigned_dev_pci_read_config;
+    k->config_write = assigned_dev_pci_write_config;
+    dc->props       = assigned_dev_properties;
+    dc->vmsd        = &vmstate_assigned_device;
+    dc->reset       = reset_assigned_device;
+    dc->desc        = "KVM-based PCI passthrough";
+}
+
+static const TypeInfo assign_info = {
+    .name               = "kvm-pci-assign",
+    .parent             = TYPE_PCI_DEVICE,
+    .instance_size      = sizeof(AssignedDevice),
+    .class_init         = assign_class_init,
+};
+
+static void assign_register_types(void)
+{
+    type_register_static(&assign_info);
+}
+
+type_init(assign_register_types)
+
+/*
+ * Scan the assigned devices for the devices that have an option ROM, and then
+ * load the corresponding ROM data to RAM. If an error occurs while loading an
+ * option ROM, we just ignore that option ROM and continue with the next one.
+ */
+static void assigned_dev_load_option_rom(AssignedDevice *dev)
+{
+    char name[32], rom_file[64];
+    FILE *fp;
+    uint8_t val;
+    struct stat st;
+    void *ptr;
+
+    /* If loading ROM from file, pci handles it */
+    if (dev->dev.romfile || !dev->dev.rom_bar) {
+        return;
+    }
+
+    snprintf(rom_file, sizeof(rom_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
+             dev->host.domain, dev->host.bus, dev->host.slot,
+             dev->host.function);
+
+    if (stat(rom_file, &st)) {
+        return;
+    }
+
+    if (access(rom_file, F_OK)) {
+        error_report("pci-assign: Insufficient privileges for %s", rom_file);
+        return;
+    }
+
+    /* Write "1" to the ROM file to enable it */
+    fp = fopen(rom_file, "r+");
+    if (fp == NULL) {
+        return;
+    }
+    val = 1;
+    if (fwrite(&val, 1, 1, fp) != 1) {
+        goto close_rom;
+    }
+    fseek(fp, 0, SEEK_SET);
+
+    snprintf(name, sizeof(name), "%s.rom",
+            object_get_typename(OBJECT(dev)));
+    memory_region_init_ram(&dev->dev.rom, name, st.st_size);
+    vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
+    ptr = memory_region_get_ram_ptr(&dev->dev.rom);
+    memset(ptr, 0xff, st.st_size);
+
+    if (!fread(ptr, 1, st.st_size, fp)) {
+        error_report("pci-assign: Cannot read from host %s\n"
+                     "\tDevice option ROM contents are probably invalid "
+                     "(check dmesg).\n\tSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=", rom_file);
+        memory_region_destroy(&dev->dev.rom);
+        goto close_rom;
+    }
+
+    pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom);
+    dev->dev.has_rom = true;
+close_rom:
+    /* Write "0" to disable ROM */
+    fseek(fp, 0, SEEK_SET);
+    val = 0;
+    if (!fwrite(&val, 1, 1, fp)) {
+        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
+    }
+    fclose(fp);
+}
diff --git a/hw/kvmvapic.c b/hw/kvmvapic.c
index 5d83625f4a6..1b5f416a782 100644
--- a/hw/kvmvapic.c
+++ b/hw/kvmvapic.c
@@ -8,9 +8,9 @@
  * (at your option) any later version. See the COPYING file in the
  * top-level directory.
  */
-#include "sysemu.h"
-#include "cpus.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/cpus.h"
+#include "sysemu/kvm.h"
 #include "apic_internal.h"
 
 #define APIC_DEFAULT_ADDRESS    0xfee00000
@@ -144,7 +144,7 @@ static void update_guest_rom_state(VAPICROMState *s)
 
 static int find_real_tpr_addr(VAPICROMState *s, CPUX86State *env)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong addr;
 
     if (s->state == VAPIC_ACTIVE) {
@@ -269,7 +269,7 @@ instruction_ok:
 
 static int update_rom_mapping(VAPICROMState *s, CPUX86State *env, target_ulong ip)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t rom_state_vaddr;
     uint32_t pos, patch, offset;
 
@@ -350,14 +350,14 @@ static int get_kpcr_number(CPUX86State *env)
 static int vapic_enable(VAPICROMState *s, CPUX86State *env)
 {
     int cpu_number = get_kpcr_number(env);
-    target_phys_addr_t vapic_paddr;
+    hwaddr vapic_paddr;
     static const uint8_t enabled = 1;
 
     if (cpu_number < 0) {
         return -1;
     }
     vapic_paddr = s->vapic_paddr +
-        (((target_phys_addr_t)cpu_number) << VAPIC_CPU_SHIFT);
+        (((hwaddr)cpu_number) << VAPIC_CPU_SHIFT);
     cpu_physical_memory_rw(vapic_paddr + offsetof(VAPICState, enabled),
                            (void *)&enabled, sizeof(enabled), 1);
     apic_enable_vapic(env->apic_state, vapic_paddr);
@@ -384,10 +384,12 @@ static void patch_call(VAPICROMState *s, CPUX86State *env, target_ulong ip,
 
 static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong ip)
 {
-    target_phys_addr_t paddr;
     VAPICHandlers *handlers;
     uint8_t opcode[2];
     uint32_t imm32;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
 
     if (smp_cpus == 1) {
         handlers = &s->rom_state.up;
@@ -395,6 +397,12 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
         handlers = &s->rom_state.mp;
     }
 
+    if (!kvm_enabled()) {
+        cpu_restore_state(env, env->mem_io_pc);
+        cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                             &current_flags);
+    }
+
     pause_all_vcpus();
 
     cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0);
@@ -430,9 +438,11 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
 
     resume_all_vcpus();
 
-    paddr = cpu_get_phys_page_debug(env, ip);
-    paddr += ip & ~TARGET_PAGE_MASK;
-    tb_invalidate_phys_page_range(paddr, paddr + 1, 1);
+    if (!kvm_enabled()) {
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, NULL);
+    }
 }
 
 void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
@@ -475,11 +485,13 @@ static void vapic_enable_tpr_reporting(bool enable)
     VAPICEnableTPRReporting info = {
         .enable = enable,
     };
+    X86CPU *cpu;
     CPUX86State *env;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = x86_env_get_cpu(env);
         info.apic = env->apic_state;
-        run_on_cpu(env, vapic_do_enable_tpr_reporting, &info);
+        run_on_cpu(CPU(cpu), vapic_do_enable_tpr_reporting, &info);
     }
 }
 
@@ -500,7 +512,7 @@ static void vapic_reset(DeviceState *dev)
  */
 static int patch_hypercalls(VAPICROMState *s)
 {
-    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    hwaddr rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
     static const uint8_t vmcall_pattern[] = { /* vmcall */
         0xb8, 0x1, 0, 0, 0, 0xf, 0x1, 0xc1
     };
@@ -557,7 +569,7 @@ static int patch_hypercalls(VAPICROMState *s)
  */
 static void vapic_map_rom_writable(VAPICROMState *s)
 {
-    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    hwaddr rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
     MemoryRegionSection section;
     MemoryRegion *as;
     size_t rom_size;
@@ -603,11 +615,11 @@ static int vapic_prepare(VAPICROMState *s)
     return 0;
 }
 
-static void vapic_write(void *opaque, target_phys_addr_t addr, uint64_t data,
+static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
                         unsigned int size)
 {
     CPUX86State *env = cpu_single_env;
-    target_phys_addr_t rom_paddr;
+    hwaddr rom_paddr;
     VAPICROMState *s = opaque;
 
     cpu_synchronize_state(env);
@@ -717,7 +729,7 @@ static int vapic_post_load(void *opaque, int version_id)
     }
     if (s->state == VAPIC_ACTIVE) {
         if (smp_cpus == 1) {
-            run_on_cpu(first_cpu, do_vapic_enable, s);
+            run_on_cpu(ENV_GET_CPU(first_cpu), do_vapic_enable, s);
         } else {
             zero = g_malloc0(s->rom_state.vapic_size);
             cpu_physical_memory_rw(s->vapic_paddr, zero,
@@ -792,7 +804,7 @@ static void vapic_class_init(ObjectClass *klass, void *data)
     sc->init    = vapic_init;
 }
 
-static TypeInfo vapic_type = {
+static const TypeInfo vapic_type = {
     .name          = "kvmvapic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(VAPICROMState),
diff --git a/hw/kzm.c b/hw/kzm.c
index 6a5e9dfacaa..fb3316551dd 100644
--- a/hw/kzm.c
+++ b/hw/kzm.c
@@ -5,7 +5,7 @@
  * Written by Hans at OK-Labs
  * Updated by Peter Chubb.
  *
- * This code is licenced under the GPL, version 2 or later.
+ * This code is licensed under the GPL, version 2 or later.
  * See the file `COPYING' in the top level directory.
  *
  * It (partially) emulates a Kyoto Microcomputer
@@ -14,14 +14,14 @@
  */
 
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "pc.h" /* for the FPGA UART that emulates a 16550 */
+#include "serial.h"
 #include "imx.h"
 
     /* Memory map for Kzm Emulation Baseboard:
@@ -70,11 +70,13 @@ static struct arm_boot_info kzm_binfo = {
     .board_id = 1722,
 };
 
-static void kzm_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void kzm_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -144,6 +146,7 @@ static QEMUMachine kzm_machine = {
     .name = "kzm",
     .desc = "ARM KZM Emulation Baseboard (ARM1136)",
     .init = kzm_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void kzm_machine_init(void)
diff --git a/hw/lan9118.c b/hw/lan9118.c
index ff0a50be196..6596979d8b0 100644
--- a/hw/lan9118.c
+++ b/hw/lan9118.c
@@ -11,9 +11,9 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "ptimer.h"
 /* For crc32 */
 #include <zlib.h>
@@ -401,7 +401,7 @@ static void phy_reset(lan9118_state *s)
 
 static void lan9118_reset(DeviceState *d)
 {
-    lan9118_state *s = FROM_SYSBUS(lan9118_state, sysbus_from_qdev(d));
+    lan9118_state *s = FROM_SYSBUS(lan9118_state, SYS_BUS_DEVICE(d));
     s->irq_cfg &= (IRQ_TYPE | IRQ_POL);
     s->int_sts = 0;
     s->int_en = 0;
@@ -500,7 +500,7 @@ static int lan9118_filter(lan9118_state *s, const uint8_t *addr)
         }
     } else {
         /* Hash matching  */
-        hash = (crc32(~0, addr, 6) >> 26);
+        hash = compute_mcast_idx(addr);
         if (hash & 0x20) {
             return (s->mac_hashh >> (hash & 0x1f)) & 1;
         } else {
@@ -1000,7 +1000,7 @@ static void lan9118_tick(void *opaque)
     lan9118_update(s);
 }
 
-static void lan9118_writel(void *opaque, target_phys_addr_t offset,
+static void lan9118_writel(void *opaque, hwaddr offset,
                            uint64_t val, unsigned size)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1134,7 +1134,7 @@ static void lan9118_writel(void *opaque, target_phys_addr_t offset,
     lan9118_update(s);
 }
 
-static void lan9118_writew(void *opaque, target_phys_addr_t offset,
+static void lan9118_writew(void *opaque, hwaddr offset,
                            uint32_t val)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1161,7 +1161,7 @@ static void lan9118_writew(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void lan9118_16bit_mode_write(void *opaque, target_phys_addr_t offset,
+static void lan9118_16bit_mode_write(void *opaque, hwaddr offset,
                                      uint64_t val, unsigned size)
 {
     switch (size) {
@@ -1176,7 +1176,7 @@ static void lan9118_16bit_mode_write(void *opaque, target_phys_addr_t offset,
     hw_error("lan9118_write: Bad size 0x%x\n", size);
 }
 
-static uint64_t lan9118_readl(void *opaque, target_phys_addr_t offset,
+static uint64_t lan9118_readl(void *opaque, hwaddr offset,
                               unsigned size)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1250,7 +1250,7 @@ static uint64_t lan9118_readl(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static uint32_t lan9118_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t lan9118_readw(void *opaque, hwaddr offset)
 {
     lan9118_state *s = (lan9118_state *)opaque;
     uint32_t val;
@@ -1278,7 +1278,7 @@ static uint32_t lan9118_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint64_t lan9118_16bit_mode_read(void *opaque, target_phys_addr_t offset,
+static uint64_t lan9118_16bit_mode_read(void *opaque, hwaddr offset,
                                         unsigned size)
 {
     switch (size) {
@@ -1368,7 +1368,7 @@ static void lan9118_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lan9118;
 }
 
-static TypeInfo lan9118_info = {
+static const TypeInfo lan9118_info = {
     .name          = "lan9118",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(lan9118_state),
@@ -1391,7 +1391,7 @@ void lan9118_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "lan9118");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
diff --git a/hw/lance.c b/hw/lance.c
index 9b98bb849a7..a5997fd64ec 100644
--- a/hw/lance.c
+++ b/hw/lance.c
@@ -36,9 +36,9 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "qemu_socket.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "qemu/sockets.h"
 #include "sun4m.h"
 #include "pcnet.h"
 #include "trace.h"
@@ -55,7 +55,7 @@ static void parent_lance_reset(void *opaque, int irq, int level)
         pcnet_h_reset(&d->state);
 }
 
-static void lance_mem_write(void *opaque, target_phys_addr_t addr,
+static void lance_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     SysBusPCNetState *d = opaque;
@@ -64,7 +64,7 @@ static void lance_mem_write(void *opaque, target_phys_addr_t addr,
     pcnet_ioport_writew(&d->state, addr, val & 0xffff);
 }
 
-static uint64_t lance_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lance_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     SysBusPCNetState *d = opaque;
@@ -155,7 +155,7 @@ static void lance_class_init(ObjectClass *klass, void *data)
     dc->props = lance_properties;
 }
 
-static TypeInfo lance_info = {
+static const TypeInfo lance_info = {
     .name          = "lance",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusPCNetState),
diff --git a/hw/leon3.c b/hw/leon3.c
index 878d3aa5577..f16a8bb4ece 100644
--- a/hw/leon3.c
+++ b/hw/leon3.c
@@ -22,15 +22,15 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-char.h"
-#include "sysemu.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include "grlib.h"
 
@@ -94,13 +94,11 @@ static void leon3_set_pil_in(void *opaque, uint32_t pil_in)
     }
 }
 
-static void leon3_generic_hw_init(ram_addr_t  ram_size,
-                                  const char *boot_device,
-                                  const char *kernel_filename,
-                                  const char *kernel_cmdline,
-                                  const char *initrd_filename,
-                                  const char *cpu_model)
+static void leon3_generic_hw_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     SPARCCPU *cpu;
     CPUSPARCState   *env;
     MemoryRegion *address_space_mem = get_system_memory();
@@ -210,11 +208,11 @@ static void leon3_generic_hw_init(ram_addr_t  ram_size,
     }
 }
 
-QEMUMachine leon3_generic_machine = {
+static QEMUMachine leon3_generic_machine = {
     .name     = "leon3_generic",
     .desc     = "Leon-3 generic",
     .init     = leon3_generic_hw_init,
-    .use_scsi = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void leon3_machine_init(void)
diff --git a/hw/lm32.h b/hw/lm32.h
index 0a676329fdc..236686ef2b5 100644
--- a/hw/lm32.h
+++ b/hw/lm32.h
@@ -1,3 +1,6 @@
+#ifndef HW_LM32_H
+#define HW_LM32_H 1
+
 
 #include "qemu-common.h"
 
@@ -8,7 +11,7 @@ static inline DeviceState *lm32_pic_init(qemu_irq cpu_irq)
 
     dev = qdev_create(NULL, "lm32-pic");
     qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
+    d = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(d, 0, cpu_irq);
 
     return dev;
@@ -23,3 +26,5 @@ static inline DeviceState *lm32_juart_init(void)
 
     return dev;
 }
+
+#endif
diff --git a/hw/lm32_boards.c b/hw/lm32_boards.c
index b76d8008be3..2bc06d7b7b3 100644
--- a/hw/lm32_boards.c
+++ b/hw/lm32_boards.c
@@ -19,25 +19,24 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
 #include "flash.h"
 #include "devices.h"
 #include "boards.h"
 #include "loader.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "elf.h"
 #include "lm32_hwsetup.h"
 #include "lm32.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     LM32CPU *cpu;
-    target_phys_addr_t bootstrap_pc;
-    target_phys_addr_t flash_base;
-    target_phys_addr_t hwsetup_base;
-    target_phys_addr_t initrd_base;
+    hwaddr bootstrap_pc;
+    hwaddr flash_base;
+    hwaddr hwsetup_base;
+    hwaddr initrd_base;
     size_t initrd_size;
-    target_phys_addr_t cmdline_base;
+    hwaddr cmdline_base;
 } ResetInfo;
 
 static void cpu_irq_handler(void *opaque, int irq, int level)
@@ -69,12 +68,10 @@ static void main_cpu_reset(void *opaque)
     env->deba = reset_info->flash_base;
 }
 
-static void lm32_evr_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+static void lm32_evr_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
@@ -85,14 +82,14 @@ static void lm32_evr_init(ram_addr_t ram_size_not_used,
     int i;
 
     /* memory map */
-    target_phys_addr_t flash_base  = 0x04000000;
+    hwaddr flash_base  = 0x04000000;
     size_t flash_sector_size       = 256 * 1024;
     size_t flash_size              = 32 * 1024 * 1024;
-    target_phys_addr_t ram_base    = 0x08000000;
+    hwaddr ram_base    = 0x08000000;
     size_t ram_size                = 64 * 1024 * 1024;
-    target_phys_addr_t timer0_base = 0x80002000;
-    target_phys_addr_t uart0_base  = 0x80006000;
-    target_phys_addr_t timer1_base = 0x8000a000;
+    hwaddr timer0_base = 0x80002000;
+    hwaddr uart0_base  = 0x80006000;
+    hwaddr timer1_base = 0x8000a000;
     int uart0_irq                  = 0;
     int timer0_irq                 = 1;
     int timer1_irq                 = 3;
@@ -159,12 +156,12 @@ static void lm32_evr_init(ram_addr_t ram_size_not_used,
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static void lm32_uclinux_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+static void lm32_uclinux_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
@@ -176,22 +173,22 @@ static void lm32_uclinux_init(ram_addr_t ram_size_not_used,
     int i;
 
     /* memory map */
-    target_phys_addr_t flash_base   = 0x04000000;
+    hwaddr flash_base   = 0x04000000;
     size_t flash_sector_size        = 256 * 1024;
     size_t flash_size               = 32 * 1024 * 1024;
-    target_phys_addr_t ram_base     = 0x08000000;
+    hwaddr ram_base     = 0x08000000;
     size_t ram_size                 = 64 * 1024 * 1024;
-    target_phys_addr_t uart0_base   = 0x80000000;
-    target_phys_addr_t timer0_base  = 0x80002000;
-    target_phys_addr_t timer1_base  = 0x80010000;
-    target_phys_addr_t timer2_base  = 0x80012000;
+    hwaddr uart0_base   = 0x80000000;
+    hwaddr timer0_base  = 0x80002000;
+    hwaddr timer1_base  = 0x80010000;
+    hwaddr timer2_base  = 0x80012000;
     int uart0_irq                   = 0;
     int timer0_irq                  = 1;
     int timer1_irq                  = 20;
     int timer2_irq                  = 21;
-    target_phys_addr_t hwsetup_base = 0x0bffe000;
-    target_phys_addr_t cmdline_base = 0x0bfff000;
-    target_phys_addr_t initrd_base  = 0x08400000;
+    hwaddr hwsetup_base = 0x0bffe000;
+    hwaddr cmdline_base = 0x0bfff000;
+    hwaddr initrd_base  = 0x08400000;
     size_t initrd_max               = 0x01000000;
 
     reset_info = g_malloc0(sizeof(ResetInfo));
@@ -290,14 +287,16 @@ static QEMUMachine lm32_evr_machine = {
     .name = "lm32-evr",
     .desc = "LatticeMico32 EVR32 eval system",
     .init = lm32_evr_init,
-    .is_default = 1
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine lm32_uclinux_machine = {
     .name = "lm32-uclinux",
     .desc = "lm32 platform for uClinux and u-boot by Theobroma Systems",
     .init = lm32_uclinux_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void lm32_machine_init(void)
diff --git a/hw/lm32_hwsetup.h b/hw/lm32_hwsetup.h
index 8fc285efc28..853e9abc7bd 100644
--- a/hw/lm32_hwsetup.h
+++ b/hw/lm32_hwsetup.h
@@ -71,7 +71,7 @@ static inline void hwsetup_free(HWSetup *hw)
 }
 
 static inline void hwsetup_create_rom(HWSetup *hw,
-        target_phys_addr_t base)
+        hwaddr base)
 {
     rom_add_blob("hwsetup", hw->data, TARGET_PAGE_SIZE, base);
 }
@@ -96,7 +96,7 @@ static inline void hwsetup_add_tag(HWSetup *hw, enum hwsetup_tag t)
 
 static inline void hwsetup_add_str(HWSetup *hw, const char *str)
 {
-    strncpy(hw->ptr, str, 31); /* make sure last byte is zero */
+    pstrcpy(hw->ptr, 32, str);
     hw->ptr += 32;
 }
 
diff --git a/hw/lm32_juart.c b/hw/lm32_juart.c
index f07ed3977f5..8c82c85f6dc 100644
--- a/hw/lm32_juart.c
+++ b/hw/lm32_juart.c
@@ -20,7 +20,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #include "lm32_juart.h"
 
@@ -144,7 +144,7 @@ static void lm32_juart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_juart;
 }
 
-static TypeInfo lm32_juart_info = {
+static const TypeInfo lm32_juart_info = {
     .name          = "lm32-juart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32JuartState),
diff --git a/hw/lm32_pic.c b/hw/lm32_pic.c
index 32f65db7f13..42f298ad510 100644
--- a/hw/lm32_pic.c
+++ b/hw/lm32_pic.c
@@ -21,7 +21,7 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 #include "trace.h"
 #include "lm32_pic.h"
@@ -39,7 +39,7 @@ struct LM32PicState {
 typedef struct LM32PicState LM32PicState;
 
 static LM32PicState *pic;
-void lm32_do_pic_info(Monitor *mon)
+void lm32_do_pic_info(Monitor *mon, const QDict *qdict)
 {
     if (pic == NULL) {
         return;
@@ -49,7 +49,7 @@ void lm32_do_pic_info(Monitor *mon)
             pic->im, pic->ip, pic->irq_state);
 }
 
-void lm32_irq_info(Monitor *mon)
+void lm32_irq_info(Monitor *mon, const QDict *qdict)
 {
     int i;
     uint32_t count;
@@ -184,7 +184,7 @@ static void lm32_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_pic;
 }
 
-static TypeInfo lm32_pic_info = {
+static const TypeInfo lm32_pic_info = {
     .name          = "lm32-pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32PicState),
diff --git a/hw/lm32_pic.h b/hw/lm32_pic.h
index 14456f37cb4..555680304ed 100644
--- a/hw/lm32_pic.h
+++ b/hw/lm32_pic.h
@@ -8,7 +8,7 @@ uint32_t lm32_pic_get_im(DeviceState *d);
 void lm32_pic_set_ip(DeviceState *d, uint32_t ip);
 void lm32_pic_set_im(DeviceState *d, uint32_t im);
 
-void lm32_do_pic_info(Monitor *mon);
-void lm32_irq_info(Monitor *mon);
+void lm32_do_pic_info(Monitor *mon, const QDict *qdict);
+void lm32_irq_info(Monitor *mon, const QDict *qdict);
 
 #endif /* QEMU_HW_LM32_PIC_H */
diff --git a/hw/lm32_sys.c b/hw/lm32_sys.c
index bbe03c41d56..187ef6d0d6f 100644
--- a/hw/lm32_sys.c
+++ b/hw/lm32_sys.c
@@ -31,10 +31,10 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-log.h"
-#include "qemu-error.h"
-#include "sysemu.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 enum {
     R_CTRL = 0,
@@ -61,7 +61,7 @@ static void copy_testname(LM32SysState *s)
     s->testname[MAX_TESTNAME_LEN - 1] = '\0';
 }
 
-static void sys_write(void *opaque, target_phys_addr_t addr,
+static void sys_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned size)
 {
     LM32SysState *s = opaque;
@@ -91,7 +91,7 @@ static void sys_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static bool sys_ops_accepts(void *opaque, target_phys_addr_t addr,
+static bool sys_ops_accepts(void *opaque, hwaddr addr,
                             unsigned size, bool is_write)
 {
     return is_write && size == 4;
@@ -157,7 +157,7 @@ static void lm32_sys_class_init(ObjectClass *klass, void *data)
     dc->props = lm32_sys_properties;
 }
 
-static TypeInfo lm32_sys_info = {
+static const TypeInfo lm32_sys_info = {
     .name          = "lm32-sys",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32SysState),
diff --git a/hw/lm32_timer.c b/hw/lm32_timer.c
index e9450a0ce17..db527e9dc6f 100644
--- a/hw/lm32_timer.c
+++ b/hw/lm32_timer.c
@@ -24,9 +24,9 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #define DEFAULT_FREQUENCY (50*1000000)
 
@@ -72,7 +72,7 @@ static void timer_update_irq(LM32TimerState *s)
     qemu_set_irq(s->irq, state);
 }
 
-static uint64_t timer_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t timer_read(void *opaque, hwaddr addr, unsigned size)
 {
     LM32TimerState *s = opaque;
     uint32_t r = 0;
@@ -97,7 +97,7 @@ static uint64_t timer_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void timer_write(void *opaque, target_phys_addr_t addr,
+static void timer_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     LM32TimerState *s = opaque;
@@ -215,7 +215,7 @@ static void lm32_timer_class_init(ObjectClass *klass, void *data)
     dc->props = lm32_timer_properties;
 }
 
-static TypeInfo lm32_timer_info = {
+static const TypeInfo lm32_timer_info = {
     .name          = "lm32-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32TimerState),
diff --git a/hw/lm32_uart.c b/hw/lm32_uart.c
index 57066e28c66..9c89cca49b5 100644
--- a/hw/lm32_uart.c
+++ b/hw/lm32_uart.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 
 enum {
     R_RXTX = 0,
@@ -125,7 +125,7 @@ static void uart_update_irq(LM32UartState *s)
     qemu_set_irq(s->irq, irq);
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t uart_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     LM32UartState *s = opaque;
@@ -160,7 +160,7 @@ static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void uart_write(void *opaque, target_phys_addr_t addr,
+static void uart_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     LM32UartState *s = opaque;
@@ -281,7 +281,7 @@ static void lm32_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_uart;
 }
 
-static TypeInfo lm32_uart_info = {
+static const TypeInfo lm32_uart_info = {
     .name          = "lm32-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32UartState),
diff --git a/hw/lm4549.c b/hw/lm4549.c
index 80b3ec4a5d0..b3c2d5f25d1 100644
--- a/hw/lm4549.c
+++ b/hw/lm4549.c
@@ -150,7 +150,7 @@ static void lm4549_audio_out_callback(void *opaque, int free)
     }
 }
 
-uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset)
+uint32_t lm4549_read(lm4549_state *s, hwaddr offset)
 {
     uint16_t *regfile = s->regfile;
     uint32_t value = 0;
@@ -165,7 +165,7 @@ uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset)
 }
 
 void lm4549_write(lm4549_state *s,
-                  target_phys_addr_t offset, uint32_t value)
+                  hwaddr offset, uint32_t value)
 {
     uint16_t *regfile = s->regfile;
 
@@ -224,7 +224,7 @@ uint32_t lm4549_write_samples(lm4549_state *s, uint32_t left, uint32_t right)
        This model supports 16-bit playback.
     */
 
-    if (s->buffer_level >= LM4549_BUFFER_SIZE) {
+    if (s->buffer_level > LM4549_BUFFER_SIZE - 2) {
         DPRINTF("write_sample Buffer full\n");
         return 0;
     }
diff --git a/hw/lm4549.h b/hw/lm4549.h
index 5948780e009..812a7a44404 100644
--- a/hw/lm4549.h
+++ b/hw/lm4549.h
@@ -36,8 +36,8 @@ extern const VMStateDescription vmstate_lm4549_state;
 
 
 void lm4549_init(lm4549_state *s, lm4549_callback data_req, void *opaque);
-uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset);
-void lm4549_write(lm4549_state *s, target_phys_addr_t offset, uint32_t value);
+uint32_t lm4549_read(lm4549_state *s, hwaddr offset);
+void lm4549_write(lm4549_state *s, hwaddr offset, uint32_t value);
 uint32_t lm4549_write_samples(lm4549_state *s, uint32_t left, uint32_t right);
 
 #endif /* #ifndef HW_LM4549_H */
diff --git a/hw/lm832x.c b/hw/lm832x.c
index 8e09f9bcc96..af49dd68bfd 100644
--- a/hw/lm832x.c
+++ b/hw/lm832x.c
@@ -20,8 +20,8 @@
 
 #include "hw.h"
 #include "i2c.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 
 typedef struct {
     I2CSlave i2c;
@@ -506,7 +506,7 @@ static void lm8323_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm_kbd;
 }
 
-static TypeInfo lm8323_info = {
+static const TypeInfo lm8323_info = {
     .name          = "lm8323",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(LM823KbdState),
diff --git a/hw/loader.c b/hw/loader.c
index 33acc2fdab4..995edc3f981 100644
--- a/hw/loader.c
+++ b/hw/loader.c
@@ -43,14 +43,14 @@
  */
 
 #include "hw.h"
-#include "disas.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "disas/disas.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "uboot_image.h"
 #include "loader.h"
 #include "fw_cfg.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 #include <zlib.h>
 
@@ -88,7 +88,7 @@ int load_image(const char *filename, uint8_t *addr)
 
 /* read()-like version */
 ssize_t read_targphys(const char *name,
-                      int fd, target_phys_addr_t dst_addr, size_t nbytes)
+                      int fd, hwaddr dst_addr, size_t nbytes)
 {
     uint8_t *buf;
     ssize_t did;
@@ -103,7 +103,7 @@ ssize_t read_targphys(const char *name,
 
 /* return the size or -1 if error */
 int load_image_targphys(const char *filename,
-                        target_phys_addr_t addr, uint64_t max_sz)
+                        hwaddr addr, uint64_t max_sz)
 {
     int size;
 
@@ -117,7 +117,7 @@ int load_image_targphys(const char *filename,
     return size;
 }
 
-void pstrcpy_targphys(const char *name, target_phys_addr_t dest, int buf_size,
+void pstrcpy_targphys(const char *name, hwaddr dest, int buf_size,
                       const char *source)
 {
     const char *nulp;
@@ -179,8 +179,8 @@ static void bswap_ahdr(struct exec *e)
      : (_N_SEGMENT_ROUND (_N_TXTENDADDR(x, target_page_size), target_page_size)))
 
 
-int load_aout(const char *filename, target_phys_addr_t addr, int max_sz,
-              int bswap_needed, target_phys_addr_t target_page_size)
+int load_aout(const char *filename, hwaddr addr, int max_sz,
+              int bswap_needed, hwaddr target_page_size)
 {
     int fd;
     ssize_t size, ret;
@@ -434,8 +434,8 @@ static ssize_t gunzip(void *dst, size_t dstlen, uint8_t *src,
 }
 
 /* Load a U-Boot image.  */
-int load_uimage(const char *filename, target_phys_addr_t *ep,
-                target_phys_addr_t *loadaddr, int *is_linux)
+int load_uimage(const char *filename, hwaddr *ep,
+                hwaddr *loadaddr, int *is_linux)
 {
     int fd;
     int size;
@@ -539,7 +539,7 @@ struct Rom {
     char *fw_dir;
     char *fw_file;
 
-    target_phys_addr_t addr;
+    hwaddr addr;
     QTAILQ_ENTRY(Rom) next;
 };
 
@@ -565,7 +565,7 @@ static void rom_insert(Rom *rom)
 }
 
 int rom_add_file(const char *file, const char *fw_dir,
-                 target_phys_addr_t addr, int32_t bootindex)
+                 hwaddr addr, int32_t bootindex)
 {
     Rom *rom;
     int rc, fd = -1;
@@ -633,7 +633,7 @@ err:
 }
 
 int rom_add_blob(const char *name, const void *blob, size_t len,
-                 target_phys_addr_t addr)
+                 hwaddr addr)
 {
     Rom *rom;
 
@@ -679,7 +679,7 @@ static void rom_reset(void *unused)
 
 int rom_load_all(void)
 {
-    target_phys_addr_t addr = 0;
+    hwaddr addr = 0;
     MemoryRegionSection section;
     Rom *rom;
 
@@ -709,7 +709,7 @@ void rom_set_fw(void *f)
     fw_cfg = f;
 }
 
-static Rom *find_rom(target_phys_addr_t addr)
+static Rom *find_rom(hwaddr addr)
 {
     Rom *rom;
 
@@ -733,9 +733,9 @@ static Rom *find_rom(target_phys_addr_t addr)
  * a ROM between addr and addr + size is copied. Note that this can involve
  * multiple ROMs, which need not start at addr and need not end at addr + size.
  */
-int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size)
+int rom_copy(uint8_t *dest, hwaddr addr, size_t size)
 {
-    target_phys_addr_t end = addr + size;
+    hwaddr end = addr + size;
     uint8_t *s, *d = dest;
     size_t l = 0;
     Rom *rom;
@@ -768,7 +768,7 @@ int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size)
     return (d + l) - dest;
 }
 
-void *rom_ptr(target_phys_addr_t addr)
+void *rom_ptr(hwaddr addr)
 {
     Rom *rom;
 
@@ -778,7 +778,7 @@ void *rom_ptr(target_phys_addr_t addr)
     return rom->data + (addr - rom->addr);
 }
 
-void do_info_roms(Monitor *mon)
+void do_info_roms(Monitor *mon, const QDict *qdict)
 {
     Rom *rom;
 
diff --git a/hw/loader.h b/hw/loader.h
index 6da291e31f5..5e61c95b847 100644
--- a/hw/loader.h
+++ b/hw/loader.h
@@ -1,36 +1,37 @@
 #ifndef LOADER_H
 #define LOADER_H
+#include "qapi/qmp/qdict.h"
 
 /* loader.c */
 int get_image_size(const char *filename);
 int load_image(const char *filename, uint8_t *addr); /* deprecated */
-int load_image_targphys(const char *filename, target_phys_addr_t,
+int load_image_targphys(const char *filename, hwaddr,
                         uint64_t max_sz);
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
              uint64_t *highaddr, int big_endian, int elf_machine,
              int clear_lsb);
-int load_aout(const char *filename, target_phys_addr_t addr, int max_sz,
-              int bswap_needed, target_phys_addr_t target_page_size);
-int load_uimage(const char *filename, target_phys_addr_t *ep,
-                target_phys_addr_t *loadaddr, int *is_linux);
+int load_aout(const char *filename, hwaddr addr, int max_sz,
+              int bswap_needed, hwaddr target_page_size);
+int load_uimage(const char *filename, hwaddr *ep,
+                hwaddr *loadaddr, int *is_linux);
 
 ssize_t read_targphys(const char *name,
-                      int fd, target_phys_addr_t dst_addr, size_t nbytes);
+                      int fd, hwaddr dst_addr, size_t nbytes);
 void pstrcpy_targphys(const char *name,
-                      target_phys_addr_t dest, int buf_size,
+                      hwaddr dest, int buf_size,
                       const char *source);
 
 
 int rom_add_file(const char *file, const char *fw_dir,
-                 target_phys_addr_t addr, int32_t bootindex);
+                 hwaddr addr, int32_t bootindex);
 int rom_add_blob(const char *name, const void *blob, size_t len,
-                 target_phys_addr_t addr);
+                 hwaddr addr);
 int rom_load_all(void);
 void rom_set_fw(void *f);
-int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size);
-void *rom_ptr(target_phys_addr_t addr);
-void do_info_roms(Monitor *mon);
+int rom_copy(uint8_t *dest, hwaddr addr, size_t size);
+void *rom_ptr(hwaddr addr);
+void do_info_roms(Monitor *mon, const QDict *qdict);
 
 #define rom_add_file_fixed(_f, _a, _i)          \
     rom_add_file(_f, NULL, _a, _i)
diff --git a/hw/lpc_ich9.c b/hw/lpc_ich9.c
new file mode 100644
index 00000000000..16843d76bc1
--- /dev/null
+++ b/hw/lpc_ich9.c
@@ -0,0 +1,538 @@
+/*
+ * QEMU ICH9 Emulation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009, 2010, 2011
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on piix_pci.c, but heavily modified.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "hw.h"
+#include "qemu/range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "ioapic.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
+#include "pci/pci_bridge.h"
+#include "ich9.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+#include "pci/pci_bus.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
+
+static int ich9_lpc_sci_irq(ICH9LPCState *lpc);
+
+/*****************************************************************************/
+/* ICH9 LPC PCI to ISA bridge */
+
+static void ich9_lpc_reset(DeviceState *qdev);
+
+/* chipset configuration register
+ * to access chipset configuration registers, pci_[sg]et_{byte, word, long}
+ * are used.
+ * Although it's not pci configuration space, it's little endian as Intel.
+ */
+
+static void ich9_cc_update_ir(uint8_t irr[PCI_NUM_PINS], uint16_t ir)
+{
+    int intx;
+    for (intx = 0; intx < PCI_NUM_PINS; intx++) {
+        irr[intx] = (ir >> (intx * ICH9_CC_DIR_SHIFT)) & ICH9_CC_DIR_MASK;
+    }
+}
+
+static void ich9_cc_update(ICH9LPCState *lpc)
+{
+    int slot;
+    int pci_intx;
+
+    const int reg_offsets[] = {
+        ICH9_CC_D25IR,
+        ICH9_CC_D26IR,
+        ICH9_CC_D27IR,
+        ICH9_CC_D28IR,
+        ICH9_CC_D29IR,
+        ICH9_CC_D30IR,
+        ICH9_CC_D31IR,
+    };
+    const int *offset;
+
+    /* D{25 - 31}IR, but D30IR is read only to 0. */
+    for (slot = 25, offset = reg_offsets; slot < 32; slot++, offset++) {
+        if (slot == 30) {
+            continue;
+        }
+        ich9_cc_update_ir(lpc->irr[slot],
+                          pci_get_word(lpc->chip_config + *offset));
+    }
+
+    /*
+     * D30: DMI2PCI bridge
+     * It is arbitrarily decided how INTx lines of PCI devicesbehind the bridge
+     * are connected to pirq lines. Our choice is PIRQ[E-H].
+     * INT[A-D] are connected to PIRQ[E-H]
+     */
+    for (pci_intx = 0; pci_intx < PCI_NUM_PINS; pci_intx++) {
+        lpc->irr[30][pci_intx] = pci_intx + 4;
+    }
+}
+
+static void ich9_cc_init(ICH9LPCState *lpc)
+{
+    int slot;
+    int intx;
+
+    /* the default irq routing is arbitrary as long as it matches with
+     * acpi irq routing table.
+     * The one that is incompatible with piix_pci(= bochs) one is
+     * intentionally chosen to let the users know that the different
+     * board is used.
+     *
+     * int[A-D] -> pirq[E-F]
+     * avoid pirq A-D because they are used for pci express port
+     */
+    for (slot = 0; slot < PCI_SLOT_MAX; slot++) {
+        for (intx = 0; intx < PCI_NUM_PINS; intx++) {
+            lpc->irr[slot][intx] = (slot + intx) % 4 + 4;
+        }
+    }
+    ich9_cc_update(lpc);
+}
+
+static void ich9_cc_reset(ICH9LPCState *lpc)
+{
+    uint8_t *c = lpc->chip_config;
+
+    memset(lpc->chip_config, 0, sizeof(lpc->chip_config));
+
+    pci_set_long(c + ICH9_CC_D31IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D30IR, ICH9_CC_D30IR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D29IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D28IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D27IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D26IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D25IR, ICH9_CC_DIR_DEFAULT);
+
+    ich9_cc_update(lpc);
+}
+
+static void ich9_cc_addr_len(uint64_t *addr, unsigned *len)
+{
+    *addr &= ICH9_CC_ADDR_MASK;
+    if (*addr + *len >= ICH9_CC_SIZE) {
+        *len = ICH9_CC_SIZE - *addr;
+    }
+}
+
+/* val: little endian */
+static void ich9_cc_write(void *opaque, hwaddr addr,
+                          uint64_t val, unsigned len)
+{
+    ICH9LPCState *lpc = (ICH9LPCState *)opaque;
+
+    ich9_cc_addr_len(&addr, &len);
+    memcpy(lpc->chip_config + addr, &val, len);
+    ich9_cc_update(lpc);
+}
+
+/* return value: little endian */
+static uint64_t ich9_cc_read(void *opaque, hwaddr addr,
+                              unsigned len)
+{
+    ICH9LPCState *lpc = (ICH9LPCState *)opaque;
+
+    uint32_t val = 0;
+    ich9_cc_addr_len(&addr, &len);
+    memcpy(&val, lpc->chip_config + addr, len);
+    return val;
+}
+
+/* IRQ routing */
+/* */
+static void ich9_lpc_rout(uint8_t pirq_rout, int *pic_irq, int *pic_dis)
+{
+    *pic_irq = pirq_rout & ICH9_LPC_PIRQ_ROUT_MASK;
+    *pic_dis = pirq_rout & ICH9_LPC_PIRQ_ROUT_IRQEN;
+}
+
+static void ich9_lpc_pic_irq(ICH9LPCState *lpc, int pirq_num,
+                             int *pic_irq, int *pic_dis)
+{
+    switch (pirq_num) {
+    case 0 ... 3: /* A-D */
+        ich9_lpc_rout(lpc->d.config[ICH9_LPC_PIRQA_ROUT + pirq_num],
+                      pic_irq, pic_dis);
+        return;
+    case 4 ... 7: /* E-H */
+        ich9_lpc_rout(lpc->d.config[ICH9_LPC_PIRQE_ROUT + (pirq_num - 4)],
+                      pic_irq, pic_dis);
+        return;
+    default:
+        break;
+    }
+    abort();
+}
+
+/* pic_irq: i8254 irq 0-15 */
+static void ich9_lpc_update_pic(ICH9LPCState *lpc, int pic_irq)
+{
+    int i, pic_level;
+
+    /* The pic level is the logical OR of all the PCI irqs mapped to it */
+    pic_level = 0;
+    for (i = 0; i < ICH9_LPC_NB_PIRQS; i++) {
+        int tmp_irq;
+        int tmp_dis;
+        ich9_lpc_pic_irq(lpc, i, &tmp_irq, &tmp_dis);
+        if (!tmp_dis && pic_irq == tmp_irq) {
+            pic_level |= pci_bus_get_irq_level(lpc->d.bus, i);
+        }
+    }
+    if (pic_irq == ich9_lpc_sci_irq(lpc)) {
+        pic_level |= lpc->sci_level;
+    }
+
+    qemu_set_irq(lpc->pic[pic_irq], pic_level);
+}
+
+/* pirq: pirq[A-H] 0-7*/
+static void ich9_lpc_update_by_pirq(ICH9LPCState *lpc, int pirq)
+{
+    int pic_irq;
+    int pic_dis;
+
+    ich9_lpc_pic_irq(lpc, pirq, &pic_irq, &pic_dis);
+    assert(pic_irq < ICH9_LPC_PIC_NUM_PINS);
+    if (pic_dis) {
+        return;
+    }
+
+    ich9_lpc_update_pic(lpc, pic_irq);
+}
+
+/* APIC mode: GSIx: PIRQ[A-H] -> GSI 16, ... no pirq shares same APIC pins. */
+static int ich9_pirq_to_gsi(int pirq)
+{
+    return pirq + ICH9_LPC_PIC_NUM_PINS;
+}
+
+static int ich9_gsi_to_pirq(int gsi)
+{
+    return gsi - ICH9_LPC_PIC_NUM_PINS;
+}
+
+static void ich9_lpc_update_apic(ICH9LPCState *lpc, int gsi)
+{
+    int level = 0;
+
+    if (gsi >= ICH9_LPC_PIC_NUM_PINS) {
+        level |= pci_bus_get_irq_level(lpc->d.bus, ich9_gsi_to_pirq(gsi));
+    }
+    if (gsi == ich9_lpc_sci_irq(lpc)) {
+        level |= lpc->sci_level;
+    }
+
+    qemu_set_irq(lpc->ioapic[gsi], level);
+}
+
+void ich9_lpc_set_irq(void *opaque, int pirq, int level)
+{
+    ICH9LPCState *lpc = opaque;
+
+    assert(0 <= pirq);
+    assert(pirq < ICH9_LPC_NB_PIRQS);
+
+    ich9_lpc_update_apic(lpc, ich9_pirq_to_gsi(pirq));
+    ich9_lpc_update_by_pirq(lpc, pirq);
+}
+
+/* return the pirq number (PIRQ[A-H]:0-7) corresponding to
+ * a given device irq pin.
+ */
+int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx)
+{
+    BusState *bus = qdev_get_parent_bus(&pci_dev->qdev);
+    PCIBus *pci_bus = PCI_BUS(bus);
+    PCIDevice *lpc_pdev =
+            pci_bus->devices[PCI_DEVFN(ICH9_LPC_DEV, ICH9_LPC_FUNC)];
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pdev);
+
+    return lpc->irr[PCI_SLOT(pci_dev->devfn)][intx];
+}
+
+static int ich9_lpc_sci_irq(ICH9LPCState *lpc)
+{
+    switch (lpc->d.config[ICH9_LPC_ACPI_CTRL] &
+            ICH9_LPC_ACPI_CTRL_SCI_IRQ_SEL_MASK) {
+    case ICH9_LPC_ACPI_CTRL_9:
+        return 9;
+    case ICH9_LPC_ACPI_CTRL_10:
+        return 10;
+    case ICH9_LPC_ACPI_CTRL_11:
+        return 11;
+    case ICH9_LPC_ACPI_CTRL_20:
+        return 20;
+    case ICH9_LPC_ACPI_CTRL_21:
+        return 21;
+    default:
+        /* reserved */
+        break;
+    }
+    return -1;
+}
+
+static void ich9_set_sci(void *opaque, int irq_num, int level)
+{
+    ICH9LPCState *lpc = opaque;
+    int irq;
+
+    assert(irq_num == 0);
+    level = !!level;
+    if (level == lpc->sci_level) {
+        return;
+    }
+    lpc->sci_level = level;
+
+    irq = ich9_lpc_sci_irq(lpc);
+    if (irq < 0) {
+        return;
+    }
+
+    ich9_lpc_update_apic(lpc, irq);
+    if (irq < ICH9_LPC_PIC_NUM_PINS) {
+        ich9_lpc_update_pic(lpc, irq);
+    }
+}
+
+void ich9_lpc_pm_init(PCIDevice *lpc_pci, qemu_irq cmos_s3)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pci);
+    qemu_irq *sci_irq;
+
+    sci_irq = qemu_allocate_irqs(ich9_set_sci, lpc, 1);
+    ich9_pm_init(lpc_pci, &lpc->pm, sci_irq[0], cmos_s3);
+
+    ich9_lpc_reset(&lpc->d.qdev);
+}
+
+/* APM */
+
+static void ich9_apm_ctrl_changed(uint32_t val, void *arg)
+{
+    ICH9LPCState *lpc = arg;
+
+    /* ACPI specs 3.0, 4.7.2.5 */
+    acpi_pm1_cnt_update(&lpc->pm.acpi_regs,
+                        val == ICH9_APM_ACPI_ENABLE,
+                        val == ICH9_APM_ACPI_DISABLE);
+
+    /* SMI_EN = PMBASE + 30. SMI control and enable register */
+    if (lpc->pm.smi_en & ICH9_PMIO_SMI_EN_APMC_EN) {
+        cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+    }
+}
+
+/* config:PMBASE */
+static void
+ich9_lpc_pmbase_update(ICH9LPCState *lpc)
+{
+    uint32_t pm_io_base = pci_get_long(lpc->d.config + ICH9_LPC_PMBASE);
+    pm_io_base &= ICH9_LPC_PMBASE_BASE_ADDRESS_MASK;
+
+    ich9_pm_iospace_update(&lpc->pm, pm_io_base);
+}
+
+/* config:RBCA */
+static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rbca_old)
+{
+    uint32_t rbca = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
+
+    if (rbca_old & ICH9_LPC_RCBA_EN) {
+            memory_region_del_subregion(get_system_memory(), &lpc->rbca_mem);
+    }
+    if (rbca & ICH9_LPC_RCBA_EN) {
+            memory_region_add_subregion_overlap(get_system_memory(),
+                                                rbca & ICH9_LPC_RCBA_BA_MASK,
+                                                &lpc->rbca_mem, 1);
+    }
+}
+
+static int ich9_lpc_post_load(void *opaque, int version_id)
+{
+    ICH9LPCState *lpc = opaque;
+
+    ich9_lpc_pmbase_update(lpc);
+    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RBCA_EN */);
+    return 0;
+}
+
+static void ich9_lpc_config_write(PCIDevice *d,
+                                  uint32_t addr, uint32_t val, int len)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+
+    pci_default_write_config(d, addr, val, len);
+    if (ranges_overlap(addr, len, ICH9_LPC_PMBASE, 4)) {
+        ich9_lpc_pmbase_update(lpc);
+    }
+    if (ranges_overlap(addr, len, ICH9_LPC_RCBA, 4)) {
+        ich9_lpc_rcba_update(lpc, rbca_old);
+    }
+}
+
+static void ich9_lpc_reset(DeviceState *qdev)
+{
+    PCIDevice *d = PCI_DEVICE(qdev);
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        pci_set_byte(d->config + ICH9_LPC_PIRQA_ROUT + i,
+                     ICH9_LPC_PIRQ_ROUT_DEFAULT);
+    }
+    for (i = 0; i < 4; i++) {
+        pci_set_byte(d->config + ICH9_LPC_PIRQE_ROUT + i,
+                     ICH9_LPC_PIRQ_ROUT_DEFAULT);
+    }
+    pci_set_byte(d->config + ICH9_LPC_ACPI_CTRL, ICH9_LPC_ACPI_CTRL_DEFAULT);
+
+    pci_set_long(d->config + ICH9_LPC_PMBASE, ICH9_LPC_PMBASE_DEFAULT);
+    pci_set_long(d->config + ICH9_LPC_RCBA, ICH9_LPC_RCBA_DEFAULT);
+
+    ich9_cc_reset(lpc);
+
+    ich9_lpc_pmbase_update(lpc);
+    ich9_lpc_rcba_update(lpc, rbca_old);
+
+    lpc->sci_level = 0;
+}
+
+static const MemoryRegionOps rbca_mmio_ops = {
+    .read = ich9_cc_read,
+    .write = ich9_cc_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ich9_lpc_machine_ready(Notifier *n, void *opaque)
+{
+    ICH9LPCState *s = container_of(n, ICH9LPCState, machine_ready);
+    uint8_t *pci_conf;
+
+    pci_conf = s->d.config;
+    if (isa_is_ioport_assigned(0x3f8)) {
+        /* com1 */
+        pci_conf[0x82] |= 0x01;
+    }
+    if (isa_is_ioport_assigned(0x2f8)) {
+        /* com2 */
+        pci_conf[0x82] |= 0x02;
+    }
+    if (isa_is_ioport_assigned(0x378)) {
+        /* lpt */
+        pci_conf[0x82] |= 0x04;
+    }
+    if (isa_is_ioport_assigned(0x3f0)) {
+        /* floppy */
+        pci_conf[0x82] |= 0x08;
+    }
+}
+
+static int ich9_lpc_initfn(PCIDevice *d)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    ISABus *isa_bus;
+
+    isa_bus = isa_bus_new(&d->qdev, get_system_io());
+
+    pci_set_long(d->wmask + ICH9_LPC_PMBASE,
+                 ICH9_LPC_PMBASE_BASE_ADDRESS_MASK);
+
+    memory_region_init_io(&lpc->rbca_mem, &rbca_mmio_ops, lpc,
+                            "lpc-rbca-mmio", ICH9_CC_SIZE);
+
+    lpc->isa_bus = isa_bus;
+
+    ich9_cc_init(lpc);
+    apm_init(d, &lpc->apm, ich9_apm_ctrl_changed, lpc);
+
+    lpc->machine_ready.notify = ich9_lpc_machine_ready;
+    qemu_add_machine_init_done_notifier(&lpc->machine_ready);
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_ich9_lpc = {
+    .name = "ICH9LPC",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = ich9_lpc_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(d, ICH9LPCState),
+        VMSTATE_STRUCT(apm, ICH9LPCState, 0, vmstate_apm, APMState),
+        VMSTATE_STRUCT(pm, ICH9LPCState, 0, vmstate_ich9_pm, ICH9LPCPMRegs),
+        VMSTATE_UINT8_ARRAY(chip_config, ICH9LPCState, ICH9_CC_SIZE),
+        VMSTATE_UINT32(sci_level, ICH9LPCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ich9_lpc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->reset = ich9_lpc_reset;
+    k->init = ich9_lpc_initfn;
+    dc->vmsd = &vmstate_ich9_lpc;
+    dc->no_user = 1;
+    k->config_write = ich9_lpc_config_write;
+    dc->desc = "ICH9 LPC bridge";
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_ICH9_8;
+    k->revision = ICH9_A2_LPC_REVISION;
+    k->class_id = PCI_CLASS_BRIDGE_ISA;
+
+}
+
+static const TypeInfo ich9_lpc_info = {
+    .name       = TYPE_ICH9_LPC_DEVICE,
+    .parent     = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(struct ICH9LPCState),
+    .class_init  = ich9_lpc_class_init,
+};
+
+static void ich9_lpc_register(void)
+{
+    type_register_static(&ich9_lpc_info);
+}
+
+type_init(ich9_lpc_register);
diff --git a/hw/lsi53c895a.c b/hw/lsi53c895a.c
index 34afe96742c..860df328e59 100644
--- a/hw/lsi53c895a.c
+++ b/hw/lsi53c895a.c
@@ -13,9 +13,9 @@
 #include <assert.h>
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "scsi.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 //#define DEBUG_LSI
 //#define DEBUG_LSI_REG
@@ -1670,12 +1670,7 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
         }
         if (val & LSI_SCNTL1_RST) {
             if (!(s->sstat0 & LSI_SSTAT0_RST)) {
-                BusChild *kid;
-
-                QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-                    DeviceState *dev = kid->child;
-                    device_reset(dev);
-                }
+                qbus_reset_all(&s->bus.qbus);
                 s->sstat0 |= LSI_SSTAT0_RST;
                 lsi_script_scsi_interrupt(s, LSI_SIST0_RST, 0);
             }
@@ -1878,7 +1873,7 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
 #undef CASE_SET_REG32
 }
 
-static void lsi_mmio_write(void *opaque, target_phys_addr_t addr,
+static void lsi_mmio_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
@@ -1886,7 +1881,7 @@ static void lsi_mmio_write(void *opaque, target_phys_addr_t addr,
     lsi_reg_writeb(s, addr & 0xff, val);
 }
 
-static uint64_t lsi_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_mmio_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     LSIState *s = opaque;
@@ -1904,7 +1899,7 @@ static const MemoryRegionOps lsi_mmio_ops = {
     },
 };
 
-static void lsi_ram_write(void *opaque, target_phys_addr_t addr,
+static void lsi_ram_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
@@ -1920,7 +1915,7 @@ static void lsi_ram_write(void *opaque, target_phys_addr_t addr,
     s->script_ram[addr >> 2] = newval;
 }
 
-static uint64_t lsi_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_ram_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     LSIState *s = opaque;
@@ -1939,14 +1934,14 @@ static const MemoryRegionOps lsi_ram_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t lsi_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_io_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     LSIState *s = opaque;
     return lsi_reg_readb(s, addr & 0xff);
 }
 
-static void lsi_io_write(void *opaque, target_phys_addr_t addr,
+static void lsi_io_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
@@ -2126,7 +2121,7 @@ static void lsi_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lsi_scsi;
 }
 
-static TypeInfo lsi_info = {
+static const TypeInfo lsi_info = {
     .name          = "lsi53c895a",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(LSIState),
diff --git a/hw/m25p80.c b/hw/m25p80.c
new file mode 100644
index 00000000000..788c19608c1
--- /dev/null
+++ b/hw/m25p80.c
@@ -0,0 +1,653 @@
+/*
+ * ST M25P80 emulator. Emulate all SPI flash devices based on the m25p80 command
+ * set. Known devices table current as of Jun/2012 and taken from linux.
+ * See drivers/mtd/devices/m25p80.c.
+ *
+ * Copyright (C) 2011 Edgar E. Iglesias <edgar.iglesias@gmail.com>
+ * Copyright (C) 2012 Peter A. G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ * Copyright (C) 2012 PetaLogix
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) a later version of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw.h"
+#include "sysemu/blockdev.h"
+#include "ssi.h"
+#include "devices.h"
+
+#ifdef M25P80_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+/* Fields for FlashPartInfo->flags */
+
+/* erase capabilities */
+#define ER_4K 1
+#define ER_32K 2
+/* set to allow the page program command to write 0s back to 1. Useful for
+ * modelling EEPROM with SPI flash command set
+ */
+#define WR_1 0x100
+
+typedef struct FlashPartInfo {
+    const char *part_name;
+    /* jedec code. (jedec >> 16) & 0xff is the 1st byte, >> 8 the 2nd etc */
+    uint32_t jedec;
+    /* extended jedec code */
+    uint16_t ext_jedec;
+    /* there is confusion between manufacturers as to what a sector is. In this
+     * device model, a "sector" is the size that is erased by the ERASE_SECTOR
+     * command (opcode 0xd8).
+     */
+    uint32_t sector_size;
+    uint32_t n_sectors;
+    uint32_t page_size;
+    uint8_t flags;
+} FlashPartInfo;
+
+/* adapted from linux */
+
+#define INFO(_part_name, _jedec, _ext_jedec, _sector_size, _n_sectors, _flags)\
+    .part_name = (_part_name),\
+    .jedec = (_jedec),\
+    .ext_jedec = (_ext_jedec),\
+    .sector_size = (_sector_size),\
+    .n_sectors = (_n_sectors),\
+    .page_size = 256,\
+    .flags = (_flags),\
+
+#define JEDEC_NUMONYX 0x20
+#define JEDEC_WINBOND 0xEF
+#define JEDEC_SPANSION 0x01
+
+static const FlashPartInfo known_devices[] = {
+    /* Atmel -- some are (confusingly) marketed as "DataFlash" */
+    { INFO("at25fs010",   0x1f6601,      0,  32 << 10,   4, ER_4K) },
+    { INFO("at25fs040",   0x1f6604,      0,  64 << 10,   8, ER_4K) },
+
+    { INFO("at25df041a",  0x1f4401,      0,  64 << 10,   8, ER_4K) },
+    { INFO("at25df321a",  0x1f4701,      0,  64 << 10,  64, ER_4K) },
+    { INFO("at25df641",   0x1f4800,      0,  64 << 10, 128, ER_4K) },
+
+    { INFO("at26f004",    0x1f0400,      0,  64 << 10,   8, ER_4K) },
+    { INFO("at26df081a",  0x1f4501,      0,  64 << 10,  16, ER_4K) },
+    { INFO("at26df161a",  0x1f4601,      0,  64 << 10,  32, ER_4K) },
+    { INFO("at26df321",   0x1f4700,      0,  64 << 10,  64, ER_4K) },
+
+    /* EON -- en25xxx */
+    { INFO("en25f32",     0x1c3116,      0,  64 << 10,  64, ER_4K) },
+    { INFO("en25p32",     0x1c2016,      0,  64 << 10,  64, 0) },
+    { INFO("en25q32b",    0x1c3016,      0,  64 << 10,  64, 0) },
+    { INFO("en25p64",     0x1c2017,      0,  64 << 10, 128, 0) },
+
+    /* Intel/Numonyx -- xxxs33b */
+    { INFO("160s33b",     0x898911,      0,  64 << 10,  32, 0) },
+    { INFO("320s33b",     0x898912,      0,  64 << 10,  64, 0) },
+    { INFO("640s33b",     0x898913,      0,  64 << 10, 128, 0) },
+
+    /* Macronix */
+    { INFO("mx25l4005a",  0xc22013,      0,  64 << 10,   8, ER_4K) },
+    { INFO("mx25l8005",   0xc22014,      0,  64 << 10,  16, 0) },
+    { INFO("mx25l1606e",  0xc22015,      0,  64 << 10,  32, ER_4K) },
+    { INFO("mx25l3205d",  0xc22016,      0,  64 << 10,  64, 0) },
+    { INFO("mx25l6405d",  0xc22017,      0,  64 << 10, 128, 0) },
+    { INFO("mx25l12805d", 0xc22018,      0,  64 << 10, 256, 0) },
+    { INFO("mx25l12855e", 0xc22618,      0,  64 << 10, 256, 0) },
+    { INFO("mx25l25635e", 0xc22019,      0,  64 << 10, 512, 0) },
+    { INFO("mx25l25655e", 0xc22619,      0,  64 << 10, 512, 0) },
+
+    /* Spansion -- single (large) sector size only, at least
+     * for the chips listed here (without boot sectors).
+     */
+    { INFO("s25sl004a",   0x010212,      0,  64 << 10,   8, 0) },
+    { INFO("s25sl008a",   0x010213,      0,  64 << 10,  16, 0) },
+    { INFO("s25sl016a",   0x010214,      0,  64 << 10,  32, 0) },
+    { INFO("s25sl032a",   0x010215,      0,  64 << 10,  64, 0) },
+    { INFO("s25sl032p",   0x010215, 0x4d00,  64 << 10,  64, ER_4K) },
+    { INFO("s25sl064a",   0x010216,      0,  64 << 10, 128, 0) },
+    { INFO("s25fl256s0",  0x010219, 0x4d00, 256 << 10, 128, 0) },
+    { INFO("s25fl256s1",  0x010219, 0x4d01,  64 << 10, 512, 0) },
+    { INFO("s25fl512s",   0x010220, 0x4d00, 256 << 10, 256, 0) },
+    { INFO("s70fl01gs",   0x010221, 0x4d00, 256 << 10, 256, 0) },
+    { INFO("s25sl12800",  0x012018, 0x0300, 256 << 10,  64, 0) },
+    { INFO("s25sl12801",  0x012018, 0x0301,  64 << 10, 256, 0) },
+    { INFO("s25fl129p0",  0x012018, 0x4d00, 256 << 10,  64, 0) },
+    { INFO("s25fl129p1",  0x012018, 0x4d01,  64 << 10, 256, 0) },
+    { INFO("s25fl016k",   0xef4015,      0,  64 << 10,  32, ER_4K | ER_32K) },
+    { INFO("s25fl064k",   0xef4017,      0,  64 << 10, 128, ER_4K | ER_32K) },
+
+    /* SST -- large erase sizes are "overlays", "sectors" are 4<< 10 */
+    { INFO("sst25vf040b", 0xbf258d,      0,  64 << 10,   8, ER_4K) },
+    { INFO("sst25vf080b", 0xbf258e,      0,  64 << 10,  16, ER_4K) },
+    { INFO("sst25vf016b", 0xbf2541,      0,  64 << 10,  32, ER_4K) },
+    { INFO("sst25vf032b", 0xbf254a,      0,  64 << 10,  64, ER_4K) },
+    { INFO("sst25wf512",  0xbf2501,      0,  64 << 10,   1, ER_4K) },
+    { INFO("sst25wf010",  0xbf2502,      0,  64 << 10,   2, ER_4K) },
+    { INFO("sst25wf020",  0xbf2503,      0,  64 << 10,   4, ER_4K) },
+    { INFO("sst25wf040",  0xbf2504,      0,  64 << 10,   8, ER_4K) },
+
+    /* ST Microelectronics -- newer production may have feature updates */
+    { INFO("m25p05",      0x202010,      0,  32 << 10,   2, 0) },
+    { INFO("m25p10",      0x202011,      0,  32 << 10,   4, 0) },
+    { INFO("m25p20",      0x202012,      0,  64 << 10,   4, 0) },
+    { INFO("m25p40",      0x202013,      0,  64 << 10,   8, 0) },
+    { INFO("m25p80",      0x202014,      0,  64 << 10,  16, 0) },
+    { INFO("m25p16",      0x202015,      0,  64 << 10,  32, 0) },
+    { INFO("m25p32",      0x202016,      0,  64 << 10,  64, 0) },
+    { INFO("m25p64",      0x202017,      0,  64 << 10, 128, 0) },
+    { INFO("m25p128",     0x202018,      0, 256 << 10,  64, 0) },
+
+    { INFO("m45pe10",     0x204011,      0,  64 << 10,   2, 0) },
+    { INFO("m45pe80",     0x204014,      0,  64 << 10,  16, 0) },
+    { INFO("m45pe16",     0x204015,      0,  64 << 10,  32, 0) },
+
+    { INFO("m25pe80",     0x208014,      0,  64 << 10,  16, 0) },
+    { INFO("m25pe16",     0x208015,      0,  64 << 10,  32, ER_4K) },
+
+    { INFO("m25px32",     0x207116,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px32-s0",  0x207316,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px32-s1",  0x206316,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px64",     0x207117,      0,  64 << 10, 128, 0) },
+
+    /* Winbond -- w25x "blocks" are 64k, "sectors" are 4KiB */
+    { INFO("w25x10",      0xef3011,      0,  64 << 10,   2, ER_4K) },
+    { INFO("w25x20",      0xef3012,      0,  64 << 10,   4, ER_4K) },
+    { INFO("w25x40",      0xef3013,      0,  64 << 10,   8, ER_4K) },
+    { INFO("w25x80",      0xef3014,      0,  64 << 10,  16, ER_4K) },
+    { INFO("w25x16",      0xef3015,      0,  64 << 10,  32, ER_4K) },
+    { INFO("w25x32",      0xef3016,      0,  64 << 10,  64, ER_4K) },
+    { INFO("w25q32",      0xef4016,      0,  64 << 10,  64, ER_4K) },
+    { INFO("w25x64",      0xef3017,      0,  64 << 10, 128, ER_4K) },
+    { INFO("w25q64",      0xef4017,      0,  64 << 10, 128, ER_4K) },
+
+    /* Numonyx -- n25q128 */
+    { INFO("n25q128",      0x20ba18,      0,  64 << 10, 256, 0) },
+
+    { },
+};
+
+typedef enum {
+    NOP = 0,
+    WRDI = 0x4,
+    RDSR = 0x5,
+    WREN = 0x6,
+    JEDEC_READ = 0x9f,
+    BULK_ERASE = 0xc7,
+
+    READ = 0x3,
+    FAST_READ = 0xb,
+    DOR = 0x3b,
+    QOR = 0x6b,
+    DIOR = 0xbb,
+    QIOR = 0xeb,
+
+    PP = 0x2,
+    DPP = 0xa2,
+    QPP = 0x32,
+
+    ERASE_4K = 0x20,
+    ERASE_32K = 0x52,
+    ERASE_SECTOR = 0xd8,
+} FlashCMD;
+
+typedef enum {
+    STATE_IDLE,
+    STATE_PAGE_PROGRAM,
+    STATE_READ,
+    STATE_COLLECTING_DATA,
+    STATE_READING_DATA,
+} CMDState;
+
+typedef struct Flash {
+    SSISlave ssidev;
+    uint32_t r;
+
+    BlockDriverState *bdrv;
+
+    uint8_t *storage;
+    uint32_t size;
+    int page_size;
+
+    uint8_t state;
+    uint8_t data[16];
+    uint32_t len;
+    uint32_t pos;
+    uint8_t needed_bytes;
+    uint8_t cmd_in_progress;
+    uint64_t cur_addr;
+    bool write_enable;
+
+    int64_t dirty_page;
+
+    char *part_name;
+    const FlashPartInfo *pi;
+
+} Flash;
+
+static void bdrv_sync_complete(void *opaque, int ret)
+{
+    /* do nothing. Masters do not directly interact with the backing store,
+     * only the working copy so no mutexing required.
+     */
+}
+
+static void flash_sync_page(Flash *s, int page)
+{
+    if (s->bdrv) {
+        int bdrv_sector, nb_sectors;
+        QEMUIOVector iov;
+
+        bdrv_sector = (page * s->pi->page_size) / BDRV_SECTOR_SIZE;
+        nb_sectors = DIV_ROUND_UP(s->pi->page_size, BDRV_SECTOR_SIZE);
+        qemu_iovec_init(&iov, 1);
+        qemu_iovec_add(&iov, s->storage + bdrv_sector * BDRV_SECTOR_SIZE,
+                                                nb_sectors * BDRV_SECTOR_SIZE);
+        bdrv_aio_writev(s->bdrv, bdrv_sector, &iov, nb_sectors,
+                                                bdrv_sync_complete, NULL);
+    }
+}
+
+static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
+{
+    int64_t start, end, nb_sectors;
+    QEMUIOVector iov;
+
+    if (!s->bdrv) {
+        return;
+    }
+
+    assert(!(len % BDRV_SECTOR_SIZE));
+    start = off / BDRV_SECTOR_SIZE;
+    end = (off + len) / BDRV_SECTOR_SIZE;
+    nb_sectors = end - start;
+    qemu_iovec_init(&iov, 1);
+    qemu_iovec_add(&iov, s->storage + (start * BDRV_SECTOR_SIZE),
+                                        nb_sectors * BDRV_SECTOR_SIZE);
+    bdrv_aio_writev(s->bdrv, start, &iov, nb_sectors, bdrv_sync_complete, NULL);
+}
+
+static void flash_erase(Flash *s, int offset, FlashCMD cmd)
+{
+    uint32_t len;
+    uint8_t capa_to_assert = 0;
+
+    switch (cmd) {
+    case ERASE_4K:
+        len = 4 << 10;
+        capa_to_assert = ER_4K;
+        break;
+    case ERASE_32K:
+        len = 32 << 10;
+        capa_to_assert = ER_32K;
+        break;
+    case ERASE_SECTOR:
+        len = s->pi->sector_size;
+        break;
+    case BULK_ERASE:
+        len = s->size;
+        break;
+    default:
+        abort();
+    }
+
+    DB_PRINT("offset = %#x, len = %d\n", offset, len);
+    if ((s->pi->flags & capa_to_assert) != capa_to_assert) {
+        hw_error("m25p80: %dk erase size not supported by device\n", len);
+    }
+
+    if (!s->write_enable) {
+        DB_PRINT("erase with write protect!\n");
+        return;
+    }
+    memset(s->storage + offset, 0xff, len);
+    flash_sync_area(s, offset, len);
+}
+
+static inline void flash_sync_dirty(Flash *s, int64_t newpage)
+{
+    if (s->dirty_page >= 0 && s->dirty_page != newpage) {
+        flash_sync_page(s, s->dirty_page);
+        s->dirty_page = newpage;
+    }
+}
+
+static inline
+void flash_write8(Flash *s, uint64_t addr, uint8_t data)
+{
+    int64_t page = addr / s->pi->page_size;
+    uint8_t prev = s->storage[s->cur_addr];
+
+    if (!s->write_enable) {
+        DB_PRINT("write with write protect!\n");
+    }
+
+    if ((prev ^ data) & data) {
+        DB_PRINT("programming zero to one! addr=%lx  %x -> %x\n",
+                  addr, prev, data);
+    }
+
+    if (s->pi->flags & WR_1) {
+        s->storage[s->cur_addr] = data;
+    } else {
+        s->storage[s->cur_addr] &= data;
+    }
+
+    flash_sync_dirty(s, page);
+    s->dirty_page = page;
+}
+
+static void complete_collecting_data(Flash *s)
+{
+    s->cur_addr = s->data[0] << 16;
+    s->cur_addr |= s->data[1] << 8;
+    s->cur_addr |= s->data[2];
+
+    s->state = STATE_IDLE;
+
+    switch (s->cmd_in_progress) {
+    case DPP:
+    case QPP:
+    case PP:
+        s->state = STATE_PAGE_PROGRAM;
+        break;
+    case READ:
+    case FAST_READ:
+    case DOR:
+    case QOR:
+    case DIOR:
+    case QIOR:
+        s->state = STATE_READ;
+        break;
+    case ERASE_4K:
+    case ERASE_32K:
+    case ERASE_SECTOR:
+        flash_erase(s, s->cur_addr, s->cmd_in_progress);
+        break;
+    default:
+        break;
+    }
+}
+
+static void decode_new_cmd(Flash *s, uint32_t value)
+{
+    s->cmd_in_progress = value;
+    DB_PRINT("decoded new command:%x\n", value);
+
+    switch (value) {
+
+    case ERASE_4K:
+    case ERASE_32K:
+    case ERASE_SECTOR:
+    case READ:
+    case DPP:
+    case QPP:
+    case PP:
+        s->needed_bytes = 3;
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case FAST_READ:
+    case DOR:
+    case QOR:
+        s->needed_bytes = 4;
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case DIOR:
+        switch ((s->pi->jedec >> 16) & 0xFF) {
+        case JEDEC_WINBOND:
+        case JEDEC_SPANSION:
+            s->needed_bytes = 4;
+            break;
+        case JEDEC_NUMONYX:
+        default:
+            s->needed_bytes = 5;
+        }
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case QIOR:
+        switch ((s->pi->jedec >> 16) & 0xFF) {
+        case JEDEC_WINBOND:
+        case JEDEC_SPANSION:
+            s->needed_bytes = 6;
+            break;
+        case JEDEC_NUMONYX:
+        default:
+            s->needed_bytes = 8;
+        }
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case WRDI:
+        s->write_enable = false;
+        break;
+    case WREN:
+        s->write_enable = true;
+        break;
+
+    case RDSR:
+        s->data[0] = (!!s->write_enable) << 1;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+
+    case JEDEC_READ:
+        DB_PRINT("populated jedec code\n");
+        s->data[0] = (s->pi->jedec >> 16) & 0xff;
+        s->data[1] = (s->pi->jedec >> 8) & 0xff;
+        s->data[2] = s->pi->jedec & 0xff;
+        if (s->pi->ext_jedec) {
+            s->data[3] = (s->pi->ext_jedec >> 8) & 0xff;
+            s->data[4] = s->pi->ext_jedec & 0xff;
+            s->len = 5;
+        } else {
+            s->len = 3;
+        }
+        s->pos = 0;
+        s->state = STATE_READING_DATA;
+        break;
+
+    case BULK_ERASE:
+        if (s->write_enable) {
+            DB_PRINT("chip erase\n");
+            flash_erase(s, 0, BULK_ERASE);
+        } else {
+            DB_PRINT("chip erase with write protect!\n");
+        }
+        break;
+    case NOP:
+        break;
+    default:
+        DB_PRINT("Unknown cmd %x\n", value);
+        break;
+    }
+}
+
+static int m25p80_cs(SSISlave *ss, bool select)
+{
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+
+    if (select) {
+        s->len = 0;
+        s->pos = 0;
+        s->state = STATE_IDLE;
+        flash_sync_dirty(s, -1);
+    }
+
+    DB_PRINT("%sselect\n", select ? "de" : "");
+
+    return 0;
+}
+
+static uint32_t m25p80_transfer8(SSISlave *ss, uint32_t tx)
+{
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+    uint32_t r = 0;
+
+    switch (s->state) {
+
+    case STATE_PAGE_PROGRAM:
+        DB_PRINT("page program cur_addr=%lx data=%x\n", s->cur_addr,
+                 (uint8_t)tx);
+        flash_write8(s, s->cur_addr, (uint8_t)tx);
+        s->cur_addr++;
+        break;
+
+    case STATE_READ:
+        r = s->storage[s->cur_addr];
+        DB_PRINT("READ 0x%lx=%x\n", s->cur_addr, r);
+        s->cur_addr = (s->cur_addr + 1) % s->size;
+        break;
+
+    case STATE_COLLECTING_DATA:
+        s->data[s->len] = (uint8_t)tx;
+        s->len++;
+
+        if (s->len == s->needed_bytes) {
+            complete_collecting_data(s);
+        }
+        break;
+
+    case STATE_READING_DATA:
+        r = s->data[s->pos];
+        s->pos++;
+        if (s->pos == s->len) {
+            s->pos = 0;
+            s->state = STATE_IDLE;
+        }
+        break;
+
+    default:
+    case STATE_IDLE:
+        decode_new_cmd(s, (uint8_t)tx);
+        break;
+    }
+
+    return r;
+}
+
+static int m25p80_init(SSISlave *ss)
+{
+    DriveInfo *dinfo;
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+    const FlashPartInfo *i;
+
+    if (!s->part_name) { /* default to actual m25p80 if no partname given */
+        s->part_name = (char *)"m25p80";
+    }
+
+    i = known_devices;
+    for (i = known_devices;; i++) {
+        assert(i);
+        if (!i->part_name) {
+            fprintf(stderr, "Unknown SPI flash part: \"%s\"\n", s->part_name);
+            return 1;
+        } else if (!strcmp(i->part_name, s->part_name)) {
+            s->pi = i;
+            break;
+        }
+    }
+
+    s->size = s->pi->sector_size * s->pi->n_sectors;
+    s->dirty_page = -1;
+    s->storage = qemu_blockalign(s->bdrv, s->size);
+
+    dinfo = drive_get_next(IF_MTD);
+
+    if (dinfo && dinfo->bdrv) {
+        DB_PRINT("Binding to IF_MTD drive\n");
+        s->bdrv = dinfo->bdrv;
+        /* FIXME: Move to late init */
+        if (bdrv_read(s->bdrv, 0, s->storage, DIV_ROUND_UP(s->size,
+                                                    BDRV_SECTOR_SIZE))) {
+            fprintf(stderr, "Failed to initialize SPI flash!\n");
+            return 1;
+        }
+    } else {
+        memset(s->storage, 0xFF, s->size);
+    }
+
+    return 0;
+}
+
+static void m25p80_pre_save(void *opaque)
+{
+    flash_sync_dirty((Flash *)opaque, -1);
+}
+
+static const VMStateDescription vmstate_m25p80 = {
+    .name = "xilinx_spi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .pre_save = m25p80_pre_save,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(state, Flash),
+        VMSTATE_UINT8_ARRAY(data, Flash, 16),
+        VMSTATE_UINT32(len, Flash),
+        VMSTATE_UINT32(pos, Flash),
+        VMSTATE_UINT8(needed_bytes, Flash),
+        VMSTATE_UINT8(cmd_in_progress, Flash),
+        VMSTATE_UINT64(cur_addr, Flash),
+        VMSTATE_BOOL(write_enable, Flash),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property m25p80_properties[] = {
+    DEFINE_PROP_STRING("partname", Flash, part_name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void m25p80_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
+
+    k->init = m25p80_init;
+    k->transfer = m25p80_transfer8;
+    k->set_cs = m25p80_cs;
+    k->cs_polarity = SSI_CS_LOW;
+    dc->props = m25p80_properties;
+    dc->vmsd = &vmstate_m25p80;
+}
+
+static const TypeInfo m25p80_info = {
+    .name           = "m25p80",
+    .parent         = TYPE_SSI_SLAVE,
+    .instance_size  = sizeof(Flash),
+    .class_init     = m25p80_class_init,
+};
+
+static void m25p80_register_types(void)
+{
+    type_register_static(&m25p80_info);
+}
+
+type_init(m25p80_register_types)
diff --git a/hw/m48t59.c b/hw/m48t59.c
index dd6cb37ba6f..427d95b5a60 100644
--- a/hw/m48t59.c
+++ b/hw/m48t59.c
@@ -23,10 +23,11 @@
  */
 #include "hw.h"
 #include "nvram.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "isa.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_NVRAM
 
@@ -80,6 +81,7 @@ typedef struct M48t59ISAState {
 typedef struct M48t59SysBusState {
     SysBusDevice busdev;
     M48t59State state;
+    MemoryRegion io;
 } M48t59SysBusState;
 
 /* Fake timer functions */
@@ -466,13 +468,6 @@ uint32_t m48t59_read (void *opaque, uint32_t addr)
     return retval;
 }
 
-void m48t59_set_addr (void *opaque, uint32_t addr)
-{
-    M48t59State *NVRAM = opaque;
-
-    NVRAM->addr = addr;
-}
-
 void m48t59_toggle_lock (void *opaque, int lock)
 {
     M48t59State *NVRAM = opaque;
@@ -481,7 +476,8 @@ void m48t59_toggle_lock (void *opaque, int lock)
 }
 
 /* IO access to NVRAM */
-static void NVRAM_writeb (void *opaque, uint32_t addr, uint32_t val)
+static void NVRAM_writeb(void *opaque, hwaddr addr, uint64_t val,
+                         unsigned size)
 {
     M48t59State *NVRAM = opaque;
 
@@ -504,7 +500,7 @@ static void NVRAM_writeb (void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t NVRAM_readb (void *opaque, uint32_t addr)
+static uint64_t NVRAM_readb(void *opaque, hwaddr addr, unsigned size)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -522,14 +518,14 @@ static uint32_t NVRAM_readb (void *opaque, uint32_t addr)
     return retval;
 }
 
-static void nvram_writeb (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writeb (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
     m48t59_write(NVRAM, addr, value & 0xff);
 }
 
-static void nvram_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writew (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
@@ -537,7 +533,7 @@ static void nvram_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
     m48t59_write(NVRAM, addr + 1, value & 0xff);
 }
 
-static void nvram_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writel (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
@@ -547,7 +543,7 @@ static void nvram_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
     m48t59_write(NVRAM, addr + 3, value & 0xff);
 }
 
-static uint32_t nvram_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readb (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -556,7 +552,7 @@ static uint32_t nvram_readb (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t nvram_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readw (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -566,7 +562,7 @@ static uint32_t nvram_readw (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t nvram_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readl (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -626,17 +622,18 @@ static void m48t59_reset_sysbus(DeviceState *d)
     m48t59_reset_common(NVRAM);
 }
 
-static const MemoryRegionPortio m48t59_portio[] = {
-    {0, 4, 1, .read = NVRAM_readb, .write = NVRAM_writeb },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps m48t59_io_ops = {
-    .old_portio = m48t59_portio,
+    .read = NVRAM_readb,
+    .write = NVRAM_writeb,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 /* Initialisation routine */
-M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
+M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
                          uint32_t io_base, uint16_t size, int model)
 {
     DeviceState *dev;
@@ -649,13 +646,13 @@ M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
     qdev_prop_set_uint32(dev, "size", size);
     qdev_prop_set_uint32(dev, "io_base", io_base);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     d = FROM_SYSBUS(M48t59SysBusState, s);
     state = &d->state;
     sysbus_connect_irq(s, 0, IRQ);
+    memory_region_init_io(&d->io, &m48t59_io_ops, state, "m48t59", 4);
     if (io_base != 0) {
-        register_ioport_read(io_base, 0x04, 1, NVRAM_readb, state);
-        register_ioport_write(io_base, 0x04, 1, NVRAM_writeb, state);
+        memory_region_add_subregion(get_system_io(), io_base, &d->io);
     }
     if (mem_base != 0) {
         sysbus_mmio_map(s, 0, mem_base);
@@ -741,7 +738,7 @@ static void m48t59_init_class_isa1(ObjectClass *klass, void *data)
     dc->props = m48t59_isa_properties;
 }
 
-static TypeInfo m48t59_isa_info = {
+static const TypeInfo m48t59_isa_info = {
     .name          = "m48t59_isa",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(M48t59ISAState),
@@ -765,7 +762,7 @@ static void m48t59_class_init(ObjectClass *klass, void *data)
     dc->props = m48t59_properties;
 }
 
-static TypeInfo m48t59_info = {
+static const TypeInfo m48t59_info = {
     .name          = "m48t59",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(M48t59SysBusState),
diff --git a/hw/mac_dbdma.c b/hw/mac_dbdma.c
index 1791ec12e16..b894ab21aa9 100644
--- a/hw/mac_dbdma.c
+++ b/hw/mac_dbdma.c
@@ -39,6 +39,7 @@
 #include "hw.h"
 #include "isa.h"
 #include "mac_dbdma.h"
+#include "qemu/main-loop.h"
 
 /* debug DBDMA */
 //#define DEBUG_DBDMA
@@ -699,7 +700,7 @@ dbdma_control_write(DBDMA_channel *ch)
         ch->flush(&ch->io);
 }
 
-static void dbdma_write(void *opaque, target_phys_addr_t addr,
+static void dbdma_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     int channel = addr >> DBDMA_CHANNEL_SHIFT;
@@ -749,7 +750,7 @@ static void dbdma_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t dbdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dbdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     uint32_t value;
diff --git a/hw/mac_dbdma.h b/hw/mac_dbdma.h
index 6d1abe6aae9..691263eedef 100644
--- a/hw/mac_dbdma.h
+++ b/hw/mac_dbdma.h
@@ -19,8 +19,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_MAC_DBDMA_H
+#define HW_MAC_DBDMA_H 1
 
-#include "memory.h"
+#include "exec/memory.h"
 
 typedef struct DBDMA_io DBDMA_io;
 
@@ -30,7 +32,7 @@ typedef void (*DBDMA_end)(DBDMA_io *io);
 struct DBDMA_io {
     void *opaque;
     void *channel;
-    target_phys_addr_t addr;
+    hwaddr addr;
     int len;
     int is_last;
     int is_dma_out;
@@ -42,3 +44,5 @@ void DBDMA_register_channel(void *dbdma, int nchan, qemu_irq irq,
                             DBDMA_rw rw, DBDMA_flush flush,
                             void *opaque);
 void* DBDMA_init (MemoryRegion **dbdma_mem);
+
+#endif
diff --git a/hw/mac_nvram.c b/hw/mac_nvram.c
index ed0a2b7ef25..25121fa482c 100644
--- a/hw/mac_nvram.c
+++ b/hw/mac_nvram.c
@@ -24,8 +24,8 @@
  */
 #include "hw.h"
 #include "firmware_abi.h"
-#include "sysemu.h"
-#include "ppc_mac.h"
+#include "sysemu/sysemu.h"
+#include "ppc/mac.h"
 
 /* debug NVR */
 //#define DEBUG_NVR
@@ -37,51 +37,43 @@
 #define NVR_DPRINTF(fmt, ...)
 #endif
 
-struct MacIONVRAMState {
-    uint32_t size;
-    MemoryRegion mem;
-    unsigned int it_shift;
-    uint8_t *data;
-};
-
 #define DEF_SYSTEM_SIZE 0xc10
 
 /* Direct access to NVRAM */
-uint32_t macio_nvram_read (void *opaque, uint32_t addr)
+uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr)
 {
-    MacIONVRAMState *s = opaque;
     uint32_t ret;
 
-    if (addr < s->size)
+    if (addr < s->size) {
         ret = s->data[addr];
-    else
+    } else {
         ret = -1;
-    NVR_DPRINTF("read addr %04x val %x\n", addr, ret);
+    }
+    NVR_DPRINTF("read addr %04" PRIx32 " val %" PRIx8 "\n", addr, ret);
 
     return ret;
 }
 
-void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val)
+void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val)
 {
-    MacIONVRAMState *s = opaque;
-
-    NVR_DPRINTF("write addr %04x val %x\n", addr, val);
-    if (addr < s->size)
+    NVR_DPRINTF("write addr %04" PRIx32 " val %" PRIx8 "\n", addr, val);
+    if (addr < s->size) {
         s->data[addr] = val;
+    }
 }
 
 /* macio style NVRAM device */
-static void macio_nvram_writeb(void *opaque, target_phys_addr_t addr,
+static void macio_nvram_writeb(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     MacIONVRAMState *s = opaque;
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     s->data[addr] = value;
-    NVR_DPRINTF("writeb addr %04x val %x\n", (int)addr, value);
+    NVR_DPRINTF("writeb addr %04" PHYS_PRIx " val %" PRIx64 "\n", addr, value);
 }
 
-static uint64_t macio_nvram_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     MacIONVRAMState *s = opaque;
@@ -97,7 +89,7 @@ static uint64_t macio_nvram_readb(void *opaque, target_phys_addr_t addr,
 static const MemoryRegionOps macio_nvram_ops = {
     .read = macio_nvram_readb,
     .write = macio_nvram_writeb,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_BIG_ENDIAN,
 };
 
 static const VMStateDescription vmstate_macio_nvram = {
@@ -112,32 +104,56 @@ static const VMStateDescription vmstate_macio_nvram = {
 };
 
 
-static void macio_nvram_reset(void *opaque)
+static void macio_nvram_reset(DeviceState *dev)
 {
 }
 
-MacIONVRAMState *macio_nvram_init (target_phys_addr_t size,
-                                   unsigned int it_shift)
+static void macio_nvram_realizefn(DeviceState *dev, Error **errp)
 {
-    MacIONVRAMState *s;
+    SysBusDevice *d = SYS_BUS_DEVICE(dev);
+    MacIONVRAMState *s = MACIO_NVRAM(dev);
 
-    s = g_malloc0(sizeof(MacIONVRAMState));
-    s->data = g_malloc0(size);
-    s->size = size;
-    s->it_shift = it_shift;
+    s->data = g_malloc0(s->size);
 
     memory_region_init_io(&s->mem, &macio_nvram_ops, s, "macio-nvram",
-                          size << it_shift);
-    vmstate_register(NULL, -1, &vmstate_macio_nvram, s);
-    qemu_register_reset(macio_nvram_reset, s);
+                          s->size << s->it_shift);
+    sysbus_init_mmio(d, &s->mem);
+}
+
+static void macio_nvram_unrealizefn(DeviceState *dev, Error **errp)
+{
+    MacIONVRAMState *s = MACIO_NVRAM(dev);
 
-    return s;
+    g_free(s->data);
 }
 
-void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           target_phys_addr_t mem_base)
+static Property macio_nvram_properties[] = {
+    DEFINE_PROP_UINT32("size", MacIONVRAMState, size, 0),
+    DEFINE_PROP_UINT32("it_shift", MacIONVRAMState, it_shift, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void macio_nvram_class_init(ObjectClass *oc, void *data)
 {
-    memory_region_add_subregion(bar, mem_base, &s->mem);
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = macio_nvram_realizefn;
+    dc->unrealize = macio_nvram_unrealizefn;
+    dc->reset = macio_nvram_reset;
+    dc->vmsd = &vmstate_macio_nvram;
+    dc->props = macio_nvram_properties;
+}
+
+static const TypeInfo macio_nvram_type_info = {
+    .name = TYPE_MACIO_NVRAM,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MacIONVRAMState),
+    .class_init = macio_nvram_class_init,
+};
+
+static void macio_nvram_register_types(void)
+{
+    type_register_static(&macio_nvram_type_info);
 }
 
 /* Set up a system OpenBIOS NVRAM partition */
@@ -176,3 +192,5 @@ void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len)
     end = len;
     OpenBIOS_finish_partition(part_header, end - start);
 }
+
+type_init(macio_nvram_register_types)
diff --git a/hw/macio.c b/hw/macio.c
index eb15b890b19..74bdcd10397 100644
--- a/hw/macio.c
+++ b/hw/macio.c
@@ -23,118 +23,283 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
+#include "mac_dbdma.h"
 #include "escc.h"
 
+#define TYPE_MACIO "macio"
+#define MACIO(obj) OBJECT_CHECK(MacIOState, (obj), TYPE_MACIO)
+
 typedef struct MacIOState
 {
+    /*< private >*/
     PCIDevice parent;
-    int is_oldworld;
+    /*< public >*/
+
     MemoryRegion bar;
+    CUDAState cuda;
+    void *dbdma;
     MemoryRegion *pic_mem;
-    MemoryRegion *dbdma_mem;
-    MemoryRegion *cuda_mem;
     MemoryRegion *escc_mem;
-    void *nvram;
-    int nb_ide;
-    MemoryRegion *ide_mem[4];
 } MacIOState;
 
+#define OLDWORLD_MACIO(obj) \
+    OBJECT_CHECK(OldWorldMacIOState, (obj), TYPE_OLDWORLD_MACIO)
+
+typedef struct OldWorldMacIOState {
+    /*< private >*/
+    MacIOState parent_obj;
+    /*< public >*/
+
+    qemu_irq irqs[3];
+
+    MacIONVRAMState nvram;
+    MACIOIDEState ide;
+} OldWorldMacIOState;
+
+#define NEWWORLD_MACIO(obj) \
+    OBJECT_CHECK(NewWorldMacIOState, (obj), TYPE_NEWWORLD_MACIO)
+
+typedef struct NewWorldMacIOState {
+    /*< private >*/
+    MacIOState parent_obj;
+    /*< public >*/
+    qemu_irq irqs[5];
+    MACIOIDEState ide[2];
+} NewWorldMacIOState;
+
 static void macio_bar_setup(MacIOState *macio_state)
 {
-    int i;
     MemoryRegion *bar = &macio_state->bar;
 
-    memory_region_init(bar, "macio", 0x80000);
-    if (macio_state->pic_mem) {
-        if (macio_state->is_oldworld) {
-            /* Heathrow PIC */
-            memory_region_add_subregion(bar, 0x00000, macio_state->pic_mem);
-        } else {
-            /* OpenPIC */
-            memory_region_add_subregion(bar, 0x40000, macio_state->pic_mem);
-        }
-    }
-    if (macio_state->dbdma_mem) {
-        memory_region_add_subregion(bar, 0x08000, macio_state->dbdma_mem);
-    }
     if (macio_state->escc_mem) {
         memory_region_add_subregion(bar, 0x13000, macio_state->escc_mem);
     }
-    if (macio_state->cuda_mem) {
-        memory_region_add_subregion(bar, 0x16000, macio_state->cuda_mem);
+}
+
+static int macio_common_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret;
+
+    d->config[0x3d] = 0x01; // interrupt on pin 1
+
+    ret = qdev_init(DEVICE(&s->cuda));
+    if (ret < 0) {
+        return ret;
     }
-    for (i = 0; i < macio_state->nb_ide; i++) {
-        if (macio_state->ide_mem[i]) {
-            memory_region_add_subregion(bar, 0x1f000 + (i * 0x1000),
-                                        macio_state->ide_mem[i]);
-        }
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    memory_region_add_subregion(&s->bar, 0x16000,
+                                sysbus_mmio_get_region(sysbus_dev, 0));
+
+    macio_bar_setup(s);
+    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+
+    return 0;
+}
+
+static int macio_oldworld_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    OldWorldMacIOState *os = OLDWORLD_MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret = macio_common_initfn(d);
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    sysbus_connect_irq(sysbus_dev, 0, os->irqs[0]);
+
+    ret = qdev_init(DEVICE(&os->nvram));
+    if (ret < 0) {
+        return ret;
     }
-    if (macio_state->nvram != NULL)
-        macio_nvram_setup_bar(macio_state->nvram, bar, 0x60000);
+    sysbus_dev = SYS_BUS_DEVICE(&os->nvram);
+    memory_region_add_subregion(&s->bar, 0x60000,
+                                sysbus_mmio_get_region(sysbus_dev, 0));
+    pmac_format_nvram_partition(&os->nvram, os->nvram.size);
+
+    if (s->pic_mem) {
+        /* Heathrow PIC */
+        memory_region_add_subregion(&s->bar, 0x00000, s->pic_mem);
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&os->ide);
+    sysbus_connect_irq(sysbus_dev, 0, os->irqs[1]);
+    sysbus_connect_irq(sysbus_dev, 1, os->irqs[2]);
+    macio_ide_register_dma(&os->ide, s->dbdma, 0x16);
+    ret = qdev_init(DEVICE(&os->ide));
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
 }
 
-static int macio_initfn(PCIDevice *d)
+static void macio_oldworld_init(Object *obj)
 {
-    d->config[0x3d] = 0x01; // interrupt on pin 1
+    MacIOState *s = MACIO(obj);
+    OldWorldMacIOState *os = OLDWORLD_MACIO(obj);
+    DeviceState *dev;
+
+    qdev_init_gpio_out(DEVICE(obj), os->irqs, ARRAY_SIZE(os->irqs));
+
+    object_initialize(&os->nvram, TYPE_MACIO_NVRAM);
+    dev = DEVICE(&os->nvram);
+    qdev_prop_set_uint32(dev, "size", 0x2000);
+    qdev_prop_set_uint32(dev, "it_shift", 4);
+
+    object_initialize(&os->ide, TYPE_MACIO_IDE);
+    qdev_set_parent_bus(DEVICE(&os->ide), sysbus_get_default());
+    memory_region_add_subregion(&s->bar, 0x1f000 + (1 * 0x1000), &os->ide.mem);
+    object_property_add_child(obj, "ide", OBJECT(&os->ide), NULL);
+}
+
+static int macio_newworld_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    NewWorldMacIOState *ns = NEWWORLD_MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret = macio_common_initfn(d);
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[0]);
+
+    if (s->pic_mem) {
+        /* OpenPIC */
+        memory_region_add_subregion(&s->bar, 0x40000, s->pic_mem);
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&ns->ide[0]);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[1]);
+    sysbus_connect_irq(sysbus_dev, 1, ns->irqs[2]);
+    macio_ide_register_dma(&ns->ide[0], s->dbdma, 0x16);
+    ret = qdev_init(DEVICE(&ns->ide[0]));
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&ns->ide[1]);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[3]);
+    sysbus_connect_irq(sysbus_dev, 1, ns->irqs[4]);
+    macio_ide_register_dma(&ns->ide[0], s->dbdma, 0x1a);
+    ret = qdev_init(DEVICE(&ns->ide[1]));
+    if (ret < 0) {
+        return ret;
+    }
+
     return 0;
 }
 
+static void macio_newworld_init(Object *obj)
+{
+    MacIOState *s = MACIO(obj);
+    NewWorldMacIOState *ns = NEWWORLD_MACIO(obj);
+    int i;
+    gchar *name;
+
+    qdev_init_gpio_out(DEVICE(obj), ns->irqs, ARRAY_SIZE(ns->irqs));
+
+    for (i = 0; i < 2; i++) {
+        object_initialize(&ns->ide[i], TYPE_MACIO_IDE);
+        qdev_set_parent_bus(DEVICE(&ns->ide[i]), sysbus_get_default());
+        memory_region_add_subregion(&s->bar, 0x1f000 + ((i + 1) * 0x1000),
+                                    &ns->ide[i].mem);
+        name = g_strdup_printf("ide[%i]", i);
+        object_property_add_child(obj, name, OBJECT(&ns->ide[i]), NULL);
+        g_free(name);
+    }
+}
+
+static void macio_instance_init(Object *obj)
+{
+    MacIOState *s = MACIO(obj);
+    MemoryRegion *dbdma_mem;
+
+    memory_region_init(&s->bar, "macio", 0x80000);
+
+    object_initialize(&s->cuda, TYPE_CUDA);
+    qdev_set_parent_bus(DEVICE(&s->cuda), sysbus_get_default());
+    object_property_add_child(obj, "cuda", OBJECT(&s->cuda), NULL);
+
+    s->dbdma = DBDMA_init(&dbdma_mem);
+    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
+}
+
+static void macio_oldworld_class_init(ObjectClass *oc, void *data)
+{
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+
+    pdc->init = macio_oldworld_initfn;
+    pdc->device_id = PCI_DEVICE_ID_APPLE_343S1201;
+}
+
+static void macio_newworld_class_init(ObjectClass *oc, void *data)
+{
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+
+    pdc->init = macio_newworld_initfn;
+    pdc->device_id = PCI_DEVICE_ID_APPLE_UNI_N_KEYL;
+}
+
 static void macio_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = macio_initfn;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
 }
 
-static TypeInfo macio_info = {
-    .name          = "macio",
+static const TypeInfo macio_oldworld_type_info = {
+    .name          = TYPE_OLDWORLD_MACIO,
+    .parent        = TYPE_MACIO,
+    .instance_size = sizeof(OldWorldMacIOState),
+    .instance_init = macio_oldworld_init,
+    .class_init    = macio_oldworld_class_init,
+};
+
+static const TypeInfo macio_newworld_type_info = {
+    .name          = TYPE_NEWWORLD_MACIO,
+    .parent        = TYPE_MACIO,
+    .instance_size = sizeof(NewWorldMacIOState),
+    .instance_init = macio_newworld_init,
+    .class_init    = macio_newworld_class_init,
+};
+
+static const TypeInfo macio_type_info = {
+    .name          = TYPE_MACIO,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(MacIOState),
+    .instance_init = macio_instance_init,
+    .abstract      = true,
     .class_init    = macio_class_init,
 };
 
 static void macio_register_types(void)
 {
-    type_register_static(&macio_info);
+    type_register_static(&macio_type_info);
+    type_register_static(&macio_oldworld_type_info);
+    type_register_static(&macio_newworld_type_info);
 }
 
 type_init(macio_register_types)
 
-void macio_init (PCIBus *bus, int device_id, int is_oldworld,
-                 MemoryRegion *pic_mem, MemoryRegion *dbdma_mem,
-                 MemoryRegion *cuda_mem, void *nvram,
-                 int nb_ide, MemoryRegion **ide_mem,
-                 MemoryRegion *escc_mem)
+void macio_init(PCIDevice *d,
+                MemoryRegion *pic_mem,
+                MemoryRegion *escc_mem)
 {
-    PCIDevice *d;
-    MacIOState *macio_state;
-    int i;
+    MacIOState *macio_state = MACIO(d);
 
-    d = pci_create_simple(bus, -1, "macio");
-
-    macio_state = DO_UPCAST(MacIOState, parent, d);
-    macio_state->is_oldworld = is_oldworld;
     macio_state->pic_mem = pic_mem;
-    macio_state->dbdma_mem = dbdma_mem;
-    macio_state->cuda_mem = cuda_mem;
     macio_state->escc_mem = escc_mem;
-    macio_state->nvram = nvram;
-    if (nb_ide > 4)
-        nb_ide = 4;
-    macio_state->nb_ide = nb_ide;
-    for (i = 0; i < nb_ide; i++)
-        macio_state->ide_mem[i] = ide_mem[i];
-    for (; i < 4; i++)
-        macio_state->ide_mem[i] = NULL;
     /* Note: this code is strongly inspirated from the corresponding code
        in PearPC */
 
-    pci_config_set_device_id(d->config, device_id);
-
-    macio_bar_setup(macio_state);
-    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &macio_state->bar);
+    qdev_init_nofail(DEVICE(d));
 }
diff --git a/hw/mainstone.c b/hw/mainstone.c
index 97687b6eebd..d1ff6e76d6d 100644
--- a/hw/mainstone.c
+++ b/hw/mainstone.c
@@ -14,13 +14,13 @@
 #include "hw.h"
 #include "pxa.h"
 #include "arm-misc.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 #include "boards.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 /* Device addresses */
 #define MST_FPGA_PHYS	0x08000000
@@ -95,19 +95,18 @@ static struct arm_boot_info mainstone_binfo = {
 };
 
 static void mainstone_common_init(MemoryRegion *address_space_mem,
-                ram_addr_t ram_size,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, enum mainstone_model_e model, int arm_id)
+                                  QEMUMachineInitArgs *args,
+                                  enum mainstone_model_e model, int arm_id)
 {
     uint32_t sector_len = 256 * 1024;
-    target_phys_addr_t mainstone_flash_base[] = { MST_FLASH_0, MST_FLASH_1 };
+    hwaddr mainstone_flash_base[] = { MST_FLASH_0, MST_FLASH_1 };
     PXA2xxState *mpu;
     DeviceState *mst_irq;
     DriveInfo *dinfo;
     int i;
     int be;
     MemoryRegion *rom = g_new(MemoryRegion, 1);
+    const char *cpu_model = args->cpu_model;
 
     if (!cpu_model)
         cpu_model = "pxa270-c5";
@@ -164,26 +163,23 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
     smc91c111_init(&nd_table[0], MST_ETH_PHYS,
                     qdev_get_gpio_in(mst_irq, ETHERNET_IRQ));
 
-    mainstone_binfo.kernel_filename = kernel_filename;
-    mainstone_binfo.kernel_cmdline = kernel_cmdline;
-    mainstone_binfo.initrd_filename = initrd_filename;
+    mainstone_binfo.kernel_filename = args->kernel_filename;
+    mainstone_binfo.kernel_cmdline = args->kernel_cmdline;
+    mainstone_binfo.initrd_filename = args->initrd_filename;
     mainstone_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &mainstone_binfo);
 }
 
-static void mainstone_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void mainstone_init(QEMUMachineInitArgs *args)
 {
-    mainstone_common_init(get_system_memory(), ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, mainstone, 0x196);
+    mainstone_common_init(get_system_memory(), args, mainstone, 0x196);
 }
 
 static QEMUMachine mainstone2_machine = {
     .name = "mainstone",
     .desc = "Mainstone II (PXA27x)",
     .init = mainstone_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mainstone_machine_init(void)
diff --git a/hw/marvell_88w8618_audio.c b/hw/marvell_88w8618_audio.c
index f6f1937442b..c792caf2717 100644
--- a/hw/marvell_88w8618_audio.c
+++ b/hw/marvell_88w8618_audio.c
@@ -138,7 +138,7 @@ static void mv88w8618_audio_clock_update(mv88w8618_audio_state *s)
     wm8750_set_bclk_in(s->wm, rate);
 }
 
-static uint64_t mv88w8618_audio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_audio_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     mv88w8618_audio_state *s = opaque;
@@ -164,7 +164,7 @@ static uint64_t mv88w8618_audio_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_audio_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_audio_write(void *opaque, hwaddr offset,
                                   uint64_t value, unsigned size)
 {
     mv88w8618_audio_state *s = opaque;
@@ -222,7 +222,7 @@ static void mv88w8618_audio_write(void *opaque, target_phys_addr_t offset,
 static void mv88w8618_audio_reset(DeviceState *d)
 {
     mv88w8618_audio_state *s = FROM_SYSBUS(mv88w8618_audio_state,
-                                           sysbus_from_qdev(d));
+                                           SYS_BUS_DEVICE(d));
 
     s->playback_mode = 0;
     s->status = 0;
@@ -288,7 +288,7 @@ static void mv88w8618_audio_class_init(ObjectClass *klass, void *data)
     dc->props = mv88w8618_audio_properties;
 }
 
-static TypeInfo mv88w8618_audio_info = {
+static const TypeInfo mv88w8618_audio_info = {
     .name          = "mv88w8618_audio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_audio_state),
diff --git a/hw/max111x.c b/hw/max111x.c
index 706d89f4fd3..de1be4ddd6c 100644
--- a/hw/max111x.c
+++ b/hw/max111x.c
@@ -99,10 +99,11 @@ static uint32_t max111x_transfer(SSISlave *dev, uint32_t value)
 
 static const VMStateDescription vmstate_max111x = {
     .name = "max111x",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .minimum_version_id_old = 0,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
     .fields      = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, MAX111xState),
         VMSTATE_UINT8(tb1, MAX111xState),
         VMSTATE_UINT8(rb2, MAX111xState),
         VMSTATE_UINT8(rb3, MAX111xState),
@@ -161,7 +162,7 @@ static void max1110_class_init(ObjectClass *klass, void *data)
     k->transfer = max111x_transfer;
 }
 
-static TypeInfo max1110_info = {
+static const TypeInfo max1110_info = {
     .name          = "max1110",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(MAX111xState),
@@ -176,7 +177,7 @@ static void max1111_class_init(ObjectClass *klass, void *data)
     k->transfer = max111x_transfer;
 }
 
-static TypeInfo max1111_info = {
+static const TypeInfo max1111_info = {
     .name          = "max1111",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(MAX111xState),
diff --git a/hw/max7310.c b/hw/max7310.c
index 1ed18ba8764..de2221ba012 100644
--- a/hw/max7310.c
+++ b/hw/max7310.c
@@ -198,7 +198,7 @@ static void max7310_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_max7310;
 }
 
-static TypeInfo max7310_info = {
+static const TypeInfo max7310_info = {
     .name          = "max7310",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(MAX7310State),
diff --git a/hw/mc146818rtc.c b/hw/mc146818rtc.c
index 3777f858a1d..2fb11f69a31 100644
--- a/hw/mc146818rtc.c
+++ b/hw/mc146818rtc.c
@@ -22,9 +22,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "mc146818rtc.h"
+#include "qapi/visitor.h"
 
 #ifdef TARGET_I386
 #include "apic.h"
@@ -45,36 +46,65 @@
 # define DPRINTF_C(format, ...)      do { } while (0)
 #endif
 
+#define NSEC_PER_SEC    1000000000LL
+#define SEC_PER_MIN     60
+#define MIN_PER_HOUR    60
+#define SEC_PER_HOUR    3600
+#define HOUR_PER_DAY    24
+#define SEC_PER_DAY     86400
+
 #define RTC_REINJECT_ON_ACK_COUNT 20
+#define RTC_CLOCK_RATE            32768
+#define UIP_HOLD_LENGTH           (8 * NSEC_PER_SEC / 32768)
 
 typedef struct RTCState {
     ISADevice dev;
     MemoryRegion io;
     uint8_t cmos_data[128];
     uint8_t cmos_index;
-    struct tm current_tm;
     int32_t base_year;
+    uint64_t base_rtc;
+    uint64_t last_update;
+    int64_t offset;
     qemu_irq irq;
     qemu_irq sqw_irq;
     int it_shift;
     /* periodic timer */
     QEMUTimer *periodic_timer;
     int64_t next_periodic_time;
-    /* second update */
-    int64_t next_second_time;
+    /* update-ended timer */
+    QEMUTimer *update_timer;
+    uint64_t next_alarm_time;
     uint16_t irq_reinject_on_ack_count;
     uint32_t irq_coalesced;
     uint32_t period;
     QEMUTimer *coalesced_timer;
-    QEMUTimer *second_timer;
-    QEMUTimer *second_timer2;
     Notifier clock_reset_notifier;
     LostTickPolicy lost_tick_policy;
     Notifier suspend_notifier;
 } RTCState;
 
 static void rtc_set_time(RTCState *s);
-static void rtc_copy_date(RTCState *s);
+static void rtc_update_time(RTCState *s);
+static void rtc_set_cmos(RTCState *s, const struct tm *tm);
+static inline int rtc_from_bcd(RTCState *s, int a);
+static uint64_t get_next_alarm(RTCState *s);
+
+static inline bool rtc_running(RTCState *s)
+{
+    return (!(s->cmos_data[RTC_REG_B] & REG_B_SET) &&
+            (s->cmos_data[RTC_REG_A] & 0x70) <= 0x20);
+}
+
+static uint64_t get_guest_rtc_ns(RTCState *s)
+{
+    uint64_t guest_rtc;
+    uint64_t guest_clock = qemu_get_clock_ns(rtc_clock);
+
+    guest_rtc = s->base_rtc * NSEC_PER_SEC
+                 + guest_clock - s->last_update + s->offset;
+    return guest_rtc;
+}
 
 #ifdef TARGET_I386
 static void rtc_coalesced_timer_update(RTCState *s)
@@ -85,7 +115,7 @@ static void rtc_coalesced_timer_update(RTCState *s)
         /* divide each RTC interval to 2 - 8 smaller intervals */
         int c = MIN(s->irq_coalesced, 7) + 1; 
         int64_t next_clock = qemu_get_clock_ns(rtc_clock) +
-            muldiv64(s->period / c, get_ticks_per_sec(), 32768);
+            muldiv64(s->period / c, get_ticks_per_sec(), RTC_CLOCK_RATE);
         qemu_mod_timer(s->coalesced_timer, next_clock);
     }
 }
@@ -110,7 +140,8 @@ static void rtc_coalesced_timer(void *opaque)
 }
 #endif
 
-static void rtc_timer_update(RTCState *s, int64_t current_time)
+/* handle periodic timer */
+static void periodic_timer_update(RTCState *s, int64_t current_time)
 {
     int period_code, period;
     int64_t cur_clock, next_irq_clock;
@@ -131,10 +162,10 @@ static void rtc_timer_update(RTCState *s, int64_t current_time)
         s->period = period;
 #endif
         /* compute 32 khz clock */
-        cur_clock = muldiv64(current_time, 32768, get_ticks_per_sec());
+        cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, get_ticks_per_sec());
         next_irq_clock = (cur_clock & ~(period - 1)) + period;
         s->next_periodic_time =
-            muldiv64(next_irq_clock, get_ticks_per_sec(), 32768) + 1;
+            muldiv64(next_irq_clock, get_ticks_per_sec(), RTC_CLOCK_RATE) + 1;
         qemu_mod_timer(s->periodic_timer, s->next_periodic_time);
     } else {
 #ifdef TARGET_I386
@@ -148,7 +179,7 @@ static void rtc_periodic_timer(void *opaque)
 {
     RTCState *s = opaque;
 
-    rtc_timer_update(s, s->next_periodic_time);
+    periodic_timer_update(s, s->next_periodic_time);
     s->cmos_data[RTC_REG_C] |= REG_C_PF;
     if (s->cmos_data[RTC_REG_B] & REG_B_PIE) {
         s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
@@ -175,7 +206,186 @@ static void rtc_periodic_timer(void *opaque)
     }
 }
 
-static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
+/* handle update-ended timer */
+static void check_update_timer(RTCState *s)
+{
+    uint64_t next_update_time;
+    uint64_t guest_nsec;
+    int next_alarm_sec;
+
+    /* From the data sheet: "Holding the dividers in reset prevents
+     * interrupts from operating, while setting the SET bit allows"
+     * them to occur.  However, it will prevent an alarm interrupt
+     * from occurring, because the time of day is not updated.
+     */
+    if ((s->cmos_data[RTC_REG_A] & 0x60) == 0x60) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+    if ((s->cmos_data[RTC_REG_C] & REG_C_UF) &&
+        (s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+    if ((s->cmos_data[RTC_REG_C] & REG_C_UF) &&
+        (s->cmos_data[RTC_REG_C] & REG_C_AF)) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+
+    guest_nsec = get_guest_rtc_ns(s) % NSEC_PER_SEC;
+    /* if UF is clear, reprogram to next second */
+    next_update_time = qemu_get_clock_ns(rtc_clock)
+        + NSEC_PER_SEC - guest_nsec;
+
+    /* Compute time of next alarm.  One second is already accounted
+     * for in next_update_time.
+     */
+    next_alarm_sec = get_next_alarm(s);
+    s->next_alarm_time = next_update_time + (next_alarm_sec - 1) * NSEC_PER_SEC;
+
+    if (s->cmos_data[RTC_REG_C] & REG_C_UF) {
+        /* UF is set, but AF is clear.  Program the timer to target
+         * the alarm time.  */
+        next_update_time = s->next_alarm_time;
+    }
+    if (next_update_time != qemu_timer_expire_time_ns(s->update_timer)) {
+        qemu_mod_timer(s->update_timer, next_update_time);
+    }
+}
+
+static inline uint8_t convert_hour(RTCState *s, uint8_t hour)
+{
+    if (!(s->cmos_data[RTC_REG_B] & REG_B_24H)) {
+        hour %= 12;
+        if (s->cmos_data[RTC_HOURS] & 0x80) {
+            hour += 12;
+        }
+    }
+    return hour;
+}
+
+static uint64_t get_next_alarm(RTCState *s)
+{
+    int32_t alarm_sec, alarm_min, alarm_hour, cur_hour, cur_min, cur_sec;
+    int32_t hour, min, sec;
+
+    rtc_update_time(s);
+
+    alarm_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS_ALARM]);
+    alarm_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES_ALARM]);
+    alarm_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS_ALARM]);
+    alarm_hour = alarm_hour == -1 ? -1 : convert_hour(s, alarm_hour);
+
+    cur_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS]);
+    cur_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES]);
+    cur_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS]);
+    cur_hour = convert_hour(s, cur_hour);
+
+    if (alarm_hour == -1) {
+        alarm_hour = cur_hour;
+        if (alarm_min == -1) {
+            alarm_min = cur_min;
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else if (cur_sec > alarm_sec) {
+                alarm_min++;
+            }
+        } else if (cur_min == alarm_min) {
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else {
+                if (cur_sec > alarm_sec) {
+                    alarm_hour++;
+                }
+            }
+            if (alarm_sec == SEC_PER_MIN) {
+                /* wrap to next hour, minutes is not in don't care mode */
+                alarm_sec = 0;
+                alarm_hour++;
+            }
+        } else if (cur_min > alarm_min) {
+            alarm_hour++;
+        }
+    } else if (cur_hour == alarm_hour) {
+        if (alarm_min == -1) {
+            alarm_min = cur_min;
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else if (cur_sec > alarm_sec) {
+                alarm_min++;
+            }
+
+            if (alarm_sec == SEC_PER_MIN) {
+                alarm_sec = 0;
+                alarm_min++;
+            }
+            /* wrap to next day, hour is not in don't care mode */
+            alarm_min %= MIN_PER_HOUR;
+        } else if (cur_min == alarm_min) {
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            }
+            /* wrap to next day, hours+minutes not in don't care mode */
+            alarm_sec %= SEC_PER_MIN;
+        }
+    }
+
+    /* values that are still don't care fire at the next min/sec */
+    if (alarm_min == -1) {
+        alarm_min = 0;
+    }
+    if (alarm_sec == -1) {
+        alarm_sec = 0;
+    }
+
+    /* keep values in range */
+    if (alarm_sec == SEC_PER_MIN) {
+        alarm_sec = 0;
+        alarm_min++;
+    }
+    if (alarm_min == MIN_PER_HOUR) {
+        alarm_min = 0;
+        alarm_hour++;
+    }
+    alarm_hour %= HOUR_PER_DAY;
+
+    hour = alarm_hour - cur_hour;
+    min = hour * MIN_PER_HOUR + alarm_min - cur_min;
+    sec = min * SEC_PER_MIN + alarm_sec - cur_sec;
+    return sec <= 0 ? sec + SEC_PER_DAY : sec;
+}
+
+static void rtc_update_timer(void *opaque)
+{
+    RTCState *s = opaque;
+    int32_t irqs = REG_C_UF;
+    int32_t new_irqs;
+
+    assert((s->cmos_data[RTC_REG_A] & 0x60) != 0x60);
+
+    /* UIP might have been latched, update time and clear it.  */
+    rtc_update_time(s);
+    s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+
+    if (qemu_get_clock_ns(rtc_clock) >= s->next_alarm_time) {
+        irqs |= REG_C_AF;
+        if (s->cmos_data[RTC_REG_B] & REG_B_AIE) {
+            qemu_system_wakeup_request(QEMU_WAKEUP_REASON_RTC);
+        }
+    }
+
+    new_irqs = irqs & ~s->cmos_data[RTC_REG_C];
+    s->cmos_data[RTC_REG_C] |= irqs;
+    if ((new_irqs & s->cmos_data[RTC_REG_B]) != 0) {
+        s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+        qemu_irq_raise(s->irq);
+    }
+    check_update_timer(s);
+}
+
+static void cmos_ioport_write(void *opaque, hwaddr addr,
+                              uint64_t data, unsigned size)
 {
     RTCState *s = opaque;
 
@@ -189,7 +399,12 @@ static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
         case RTC_MINUTES_ALARM:
         case RTC_HOURS_ALARM:
             s->cmos_data[s->cmos_index] = data;
+            check_update_timer(s);
             break;
+	case RTC_IBM_PS2_CENTURY_BYTE:
+            s->cmos_index = RTC_CENTURY;
+            /* fall through */
+        case RTC_CENTURY:
         case RTC_SECONDS:
         case RTC_MINUTES:
         case RTC_HOURS:
@@ -199,37 +414,66 @@ static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
         case RTC_YEAR:
             s->cmos_data[s->cmos_index] = data;
             /* if in set mode, do not update the time */
-            if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+            if (rtc_running(s)) {
                 rtc_set_time(s);
+                check_update_timer(s);
             }
             break;
         case RTC_REG_A:
+            if ((data & 0x60) == 0x60) {
+                if (rtc_running(s)) {
+                    rtc_update_time(s);
+                }
+                /* What happens to UIP when divider reset is enabled is
+                 * unclear from the datasheet.  Shouldn't matter much
+                 * though.
+                 */
+                s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+            } else if (((s->cmos_data[RTC_REG_A] & 0x60) == 0x60) &&
+                    (data & 0x70)  <= 0x20) {
+                /* when the divider reset is removed, the first update cycle
+                 * begins one-half second later*/
+                if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+                    s->offset = 500000000;
+                    rtc_set_time(s);
+                }
+                s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+            }
             /* UIP bit is read only */
             s->cmos_data[RTC_REG_A] = (data & ~REG_A_UIP) |
                 (s->cmos_data[RTC_REG_A] & REG_A_UIP);
-            rtc_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            periodic_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            check_update_timer(s);
             break;
         case RTC_REG_B:
             if (data & REG_B_SET) {
+                /* update cmos to when the rtc was stopping */
+                if (rtc_running(s)) {
+                    rtc_update_time(s);
+                }
                 /* set mode: reset UIP mode */
                 s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
                 data &= ~REG_B_UIE;
             } else {
                 /* if disabling set mode, update the time */
-                if (s->cmos_data[RTC_REG_B] & REG_B_SET) {
+                if ((s->cmos_data[RTC_REG_B] & REG_B_SET) &&
+                    (s->cmos_data[RTC_REG_A] & 0x70) <= 0x20) {
+                    s->offset = get_guest_rtc_ns(s) % NSEC_PER_SEC;
                     rtc_set_time(s);
                 }
             }
-            if (((s->cmos_data[RTC_REG_B] ^ data) & (REG_B_DM | REG_B_24H)) &&
-                !(data & REG_B_SET)) {
-                /* If the time format has changed and not in set mode,
-                   update the registers immediately. */
-                s->cmos_data[RTC_REG_B] = data;
-                rtc_copy_date(s);
+            /* if an interrupt flag is already set when the interrupt
+             * becomes enabled, raise an interrupt immediately.  */
+            if (data & s->cmos_data[RTC_REG_C] & REG_C_MASK) {
+                s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+                qemu_irq_raise(s->irq);
             } else {
-                s->cmos_data[RTC_REG_B] = data;
+                s->cmos_data[RTC_REG_C] &= ~REG_C_IRQF;
+                qemu_irq_lower(s->irq);
             }
-            rtc_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            s->cmos_data[RTC_REG_B] = data;
+            periodic_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            check_update_timer(s);
             break;
         case RTC_REG_C:
         case RTC_REG_D:
@@ -253,6 +497,9 @@ static inline int rtc_to_bcd(RTCState *s, int a)
 
 static inline int rtc_from_bcd(RTCState *s, int a)
 {
+    if ((a & 0xc0) == 0xc0) {
+        return -1;
+    }
     if (s->cmos_data[RTC_REG_B] & REG_B_DM) {
         return a;
     } else {
@@ -260,10 +507,8 @@ static inline int rtc_from_bcd(RTCState *s, int a)
     }
 }
 
-static void rtc_set_time(RTCState *s)
+static void rtc_get_time(RTCState *s, struct tm *tm)
 {
-    struct tm *tm = &s->current_tm;
-
     tm->tm_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS]);
     tm->tm_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES]);
     tm->tm_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS] & 0x7f);
@@ -276,14 +521,24 @@ static void rtc_set_time(RTCState *s)
     tm->tm_wday = rtc_from_bcd(s, s->cmos_data[RTC_DAY_OF_WEEK]) - 1;
     tm->tm_mday = rtc_from_bcd(s, s->cmos_data[RTC_DAY_OF_MONTH]);
     tm->tm_mon = rtc_from_bcd(s, s->cmos_data[RTC_MONTH]) - 1;
-    tm->tm_year = rtc_from_bcd(s, s->cmos_data[RTC_YEAR]) + s->base_year - 1900;
+    tm->tm_year =
+        rtc_from_bcd(s, s->cmos_data[RTC_YEAR]) + s->base_year +
+        rtc_from_bcd(s, s->cmos_data[RTC_CENTURY]) * 100 - 1900;
+}
+
+static void rtc_set_time(RTCState *s)
+{
+    struct tm tm;
+
+    rtc_get_time(s, &tm);
+    s->base_rtc = mktimegm(&tm);
+    s->last_update = qemu_get_clock_ns(rtc_clock);
 
-    rtc_change_mon_event(tm);
+    rtc_change_mon_event(&tm);
 }
 
-static void rtc_copy_date(RTCState *s)
+static void rtc_set_cmos(RTCState *s, const struct tm *tm)
 {
-    const struct tm *tm = &s->current_tm;
     int year;
 
     s->cmos_data[RTC_SECONDS] = rtc_to_bcd(s, tm->tm_sec);
@@ -301,131 +556,53 @@ static void rtc_copy_date(RTCState *s)
     s->cmos_data[RTC_DAY_OF_WEEK] = rtc_to_bcd(s, tm->tm_wday + 1);
     s->cmos_data[RTC_DAY_OF_MONTH] = rtc_to_bcd(s, tm->tm_mday);
     s->cmos_data[RTC_MONTH] = rtc_to_bcd(s, tm->tm_mon + 1);
-    year = (tm->tm_year - s->base_year) % 100;
-    if (year < 0)
-        year += 100;
-    s->cmos_data[RTC_YEAR] = rtc_to_bcd(s, year);
+    year = tm->tm_year + 1900 - s->base_year;
+    s->cmos_data[RTC_YEAR] = rtc_to_bcd(s, year % 100);
+    s->cmos_data[RTC_CENTURY] = rtc_to_bcd(s, year / 100);
 }
 
-/* month is between 0 and 11. */
-static int get_days_in_month(int month, int year)
+static void rtc_update_time(RTCState *s)
 {
-    static const int days_tab[12] = {
-        31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
-    };
-    int d;
-    if ((unsigned )month >= 12)
-        return 31;
-    d = days_tab[month];
-    if (month == 1) {
-        if ((year % 4) == 0 && ((year % 100) != 0 || (year % 400) == 0))
-            d++;
-    }
-    return d;
-}
+    struct tm ret;
+    time_t guest_sec;
+    int64_t guest_nsec;
 
-/* update 'tm' to the next second */
-static void rtc_next_second(struct tm *tm)
-{
-    int days_in_month;
-
-    tm->tm_sec++;
-    if ((unsigned)tm->tm_sec >= 60) {
-        tm->tm_sec = 0;
-        tm->tm_min++;
-        if ((unsigned)tm->tm_min >= 60) {
-            tm->tm_min = 0;
-            tm->tm_hour++;
-            if ((unsigned)tm->tm_hour >= 24) {
-                tm->tm_hour = 0;
-                /* next day */
-                tm->tm_wday++;
-                if ((unsigned)tm->tm_wday >= 7)
-                    tm->tm_wday = 0;
-                days_in_month = get_days_in_month(tm->tm_mon,
-                                                  tm->tm_year + 1900);
-                tm->tm_mday++;
-                if (tm->tm_mday < 1) {
-                    tm->tm_mday = 1;
-                } else if (tm->tm_mday > days_in_month) {
-                    tm->tm_mday = 1;
-                    tm->tm_mon++;
-                    if (tm->tm_mon >= 12) {
-                        tm->tm_mon = 0;
-                        tm->tm_year++;
-                    }
-                }
-            }
-        }
-    }
-}
-
-
-static void rtc_update_second(void *opaque)
-{
-    RTCState *s = opaque;
-    int64_t delay;
-
-    /* if the oscillator is not in normal operation, we do not update */
-    if ((s->cmos_data[RTC_REG_A] & 0x70) != 0x20) {
-        s->next_second_time += get_ticks_per_sec();
-        qemu_mod_timer(s->second_timer, s->next_second_time);
-    } else {
-        rtc_next_second(&s->current_tm);
+    guest_nsec = get_guest_rtc_ns(s);
+    guest_sec = guest_nsec / NSEC_PER_SEC;
+    gmtime_r(&guest_sec, &ret);
 
-        if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
-            /* update in progress bit */
-            s->cmos_data[RTC_REG_A] |= REG_A_UIP;
-        }
-        /* should be 244 us = 8 / 32768 seconds, but currently the
-           timers do not have the necessary resolution. */
-        delay = (get_ticks_per_sec() * 1) / 100;
-        if (delay < 1)
-            delay = 1;
-        qemu_mod_timer(s->second_timer2,
-                       s->next_second_time + delay);
+    /* Is SET flag of Register B disabled? */
+    if ((s->cmos_data[RTC_REG_B] & REG_B_SET) == 0) {
+        rtc_set_cmos(s, &ret);
     }
 }
 
-static void rtc_update_second2(void *opaque)
+static int update_in_progress(RTCState *s)
 {
-    RTCState *s = opaque;
+    int64_t guest_nsec;
 
-    if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
-        rtc_copy_date(s);
+    if (!rtc_running(s)) {
+        return 0;
     }
-
-    /* check alarm */
-    if (((s->cmos_data[RTC_SECONDS_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_SECONDS_ALARM]) == s->current_tm.tm_sec) &&
-        ((s->cmos_data[RTC_MINUTES_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_MINUTES_ALARM]) == s->current_tm.tm_min) &&
-        ((s->cmos_data[RTC_HOURS_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_HOURS_ALARM]) == s->current_tm.tm_hour)) {
-
-        s->cmos_data[RTC_REG_C] |= REG_C_AF;
-        if (s->cmos_data[RTC_REG_B] & REG_B_AIE) {
-            qemu_system_wakeup_request(QEMU_WAKEUP_REASON_RTC);
-            qemu_irq_raise(s->irq);
-            s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+    if (qemu_timer_pending(s->update_timer)) {
+        int64_t next_update_time = qemu_timer_expire_time_ns(s->update_timer);
+        /* Latch UIP until the timer expires.  */
+        if (qemu_get_clock_ns(rtc_clock) >= (next_update_time - UIP_HOLD_LENGTH)) {
+            s->cmos_data[RTC_REG_A] |= REG_A_UIP;
+            return 1;
         }
     }
 
-    /* update ended interrupt */
-    s->cmos_data[RTC_REG_C] |= REG_C_UF;
-    if (s->cmos_data[RTC_REG_B] & REG_B_UIE) {
-        s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
-        qemu_irq_raise(s->irq);
+    guest_nsec = get_guest_rtc_ns(s);
+    /* UIP bit will be set at last 244us of every second. */
+    if ((guest_nsec % NSEC_PER_SEC) >= (NSEC_PER_SEC - UIP_HOLD_LENGTH)) {
+        return 1;
     }
-
-    /* clear update in progress bit */
-    s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
-
-    s->next_second_time += get_ticks_per_sec();
-    qemu_mod_timer(s->second_timer, s->next_second_time);
+    return 0;
 }
 
-static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
+static uint64_t cmos_ioport_read(void *opaque, hwaddr addr,
+                                 unsigned size)
 {
     RTCState *s = opaque;
     int ret;
@@ -433,6 +610,10 @@ static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
         return 0xff;
     } else {
         switch(s->cmos_index) {
+	case RTC_IBM_PS2_CENTURY_BYTE:
+            s->cmos_index = RTC_CENTURY;
+            /* fall through */
+        case RTC_CENTURY:
         case RTC_SECONDS:
         case RTC_MINUTES:
         case RTC_HOURS:
@@ -440,15 +621,28 @@ static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
         case RTC_DAY_OF_MONTH:
         case RTC_MONTH:
         case RTC_YEAR:
+            /* if not in set mode, calibrate cmos before
+             * reading*/
+            if (rtc_running(s)) {
+                rtc_update_time(s);
+            }
             ret = s->cmos_data[s->cmos_index];
             break;
         case RTC_REG_A:
+            if (update_in_progress(s)) {
+                s->cmos_data[s->cmos_index] |= REG_A_UIP;
+            } else {
+                s->cmos_data[s->cmos_index] &= ~REG_A_UIP;
+            }
             ret = s->cmos_data[s->cmos_index];
             break;
         case RTC_REG_C:
             ret = s->cmos_data[s->cmos_index];
             qemu_irq_lower(s->irq);
             s->cmos_data[RTC_REG_C] = 0x00;
+            if (ret & (REG_C_UF | REG_C_AF)) {
+                check_update_timer(s);
+            }
 #ifdef TARGET_I386
             if(s->irq_coalesced &&
                     (s->cmos_data[RTC_REG_B] & REG_B_PIE) &&
@@ -483,37 +677,32 @@ void rtc_set_memory(ISADevice *dev, int addr, int val)
         s->cmos_data[addr] = val;
 }
 
-void rtc_set_date(ISADevice *dev, const struct tm *tm)
-{
-    RTCState *s = DO_UPCAST(RTCState, dev, dev);
-    s->current_tm = *tm;
-    rtc_copy_date(s);
-}
-
-/* PC cmos mappings */
-#define REG_IBM_CENTURY_BYTE        0x32
-#define REG_IBM_PS2_CENTURY_BYTE    0x37
-
 static void rtc_set_date_from_host(ISADevice *dev)
 {
     RTCState *s = DO_UPCAST(RTCState, dev, dev);
     struct tm tm;
-    int val;
 
-    /* set the CMOS date */
     qemu_get_timedate(&tm, 0);
-    rtc_set_date(dev, &tm);
 
-    val = rtc_to_bcd(s, (tm.tm_year / 100) + 19);
-    rtc_set_memory(dev, REG_IBM_CENTURY_BYTE, val);
-    rtc_set_memory(dev, REG_IBM_PS2_CENTURY_BYTE, val);
+    s->base_rtc = mktimegm(&tm);
+    s->last_update = qemu_get_clock_ns(rtc_clock);
+    s->offset = 0;
+
+    /* set the CMOS date */
+    rtc_set_cmos(s, &tm);
 }
 
 static int rtc_post_load(void *opaque, int version_id)
 {
-#ifdef TARGET_I386
     RTCState *s = opaque;
 
+    if (version_id <= 2) {
+        rtc_set_time(s);
+        s->offset = 0;
+        check_update_timer(s);
+    }
+
+#ifdef TARGET_I386
     if (version_id >= 2) {
         if (s->lost_tick_policy == LOST_TICK_SLEW) {
             rtc_coalesced_timer_update(s);
@@ -525,27 +714,24 @@ static int rtc_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_rtc = {
     .name = "mc146818rtc",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .post_load = rtc_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_BUFFER(cmos_data, RTCState),
         VMSTATE_UINT8(cmos_index, RTCState),
-        VMSTATE_INT32(current_tm.tm_sec, RTCState),
-        VMSTATE_INT32(current_tm.tm_min, RTCState),
-        VMSTATE_INT32(current_tm.tm_hour, RTCState),
-        VMSTATE_INT32(current_tm.tm_wday, RTCState),
-        VMSTATE_INT32(current_tm.tm_mday, RTCState),
-        VMSTATE_INT32(current_tm.tm_mon, RTCState),
-        VMSTATE_INT32(current_tm.tm_year, RTCState),
+        VMSTATE_UNUSED(7*4),
         VMSTATE_TIMER(periodic_timer, RTCState),
         VMSTATE_INT64(next_periodic_time, RTCState),
-        VMSTATE_INT64(next_second_time, RTCState),
-        VMSTATE_TIMER(second_timer, RTCState),
-        VMSTATE_TIMER(second_timer2, RTCState),
+        VMSTATE_UNUSED(3*8),
         VMSTATE_UINT32_V(irq_coalesced, RTCState, 2),
         VMSTATE_UINT32_V(period, RTCState, 2),
+        VMSTATE_UINT64_V(base_rtc, RTCState, 3),
+        VMSTATE_UINT64_V(last_update, RTCState, 3),
+        VMSTATE_INT64_V(offset, RTCState, 3),
+        VMSTATE_TIMER_V(update_timer, RTCState, 3),
+        VMSTATE_UINT64_V(next_alarm_time, RTCState, 3),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -556,9 +742,8 @@ static void rtc_notify_clock_reset(Notifier *notifier, void *data)
     int64_t now = *(int64_t *)data;
 
     rtc_set_date_from_host(&s->dev);
-    s->next_second_time = now + (get_ticks_per_sec() * 99) / 100;
-    qemu_mod_timer(s->second_timer2, s->next_second_time);
-    rtc_timer_update(s, now);
+    periodic_timer_update(s, now);
+    check_update_timer(s);
 #ifdef TARGET_I386
     if (s->lost_tick_policy == LOST_TICK_SLEW) {
         rtc_coalesced_timer_update(s);
@@ -580,6 +765,7 @@ static void rtc_reset(void *opaque)
 
     s->cmos_data[RTC_REG_B] &= ~(REG_B_PIE | REG_B_AIE | REG_B_SQWE);
     s->cmos_data[RTC_REG_C] &= ~(REG_C_UF | REG_C_IRQF | REG_C_PF | REG_C_AF);
+    check_update_timer(s);
 
     qemu_irq_lower(s->irq);
 
@@ -590,13 +776,14 @@ static void rtc_reset(void *opaque)
 #endif
 }
 
-static const MemoryRegionPortio cmos_portio[] = {
-    {0, 2, 1, .read = cmos_ioport_read, .write = cmos_ioport_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps cmos_ops = {
-    .old_portio = cmos_portio
+    .read = cmos_ioport_read,
+    .write = cmos_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void rtc_get_date(Object *obj, Visitor *v, void *opaque,
@@ -604,14 +791,17 @@ static void rtc_get_date(Object *obj, Visitor *v, void *opaque,
 {
     ISADevice *isa = ISA_DEVICE(obj);
     RTCState *s = DO_UPCAST(RTCState, dev, isa);
+    struct tm current_tm;
 
+    rtc_update_time(s);
+    rtc_get_time(s, &current_tm);
     visit_start_struct(v, NULL, "struct tm", name, 0, errp);
-    visit_type_int32(v, &s->current_tm.tm_year, "tm_year", errp);
-    visit_type_int32(v, &s->current_tm.tm_mon, "tm_mon", errp);
-    visit_type_int32(v, &s->current_tm.tm_mday, "tm_mday", errp);
-    visit_type_int32(v, &s->current_tm.tm_hour, "tm_hour", errp);
-    visit_type_int32(v, &s->current_tm.tm_min, "tm_min", errp);
-    visit_type_int32(v, &s->current_tm.tm_sec, "tm_sec", errp);
+    visit_type_int32(v, &current_tm.tm_year, "tm_year", errp);
+    visit_type_int32(v, &current_tm.tm_mon, "tm_mon", errp);
+    visit_type_int32(v, &current_tm.tm_mday, "tm_mday", errp);
+    visit_type_int32(v, &current_tm.tm_hour, "tm_hour", errp);
+    visit_type_int32(v, &current_tm.tm_min, "tm_min", errp);
+    visit_type_int32(v, &current_tm.tm_sec, "tm_sec", errp);
     visit_end_struct(v, errp);
 }
 
@@ -625,6 +815,18 @@ static int rtc_initfn(ISADevice *dev)
     s->cmos_data[RTC_REG_C] = 0x00;
     s->cmos_data[RTC_REG_D] = 0x80;
 
+    /* This is for historical reasons.  The default base year qdev property
+     * was set to 2000 for most machine types before the century byte was
+     * implemented.
+     *
+     * This if statement means that the century byte will be always 0
+     * (at least until 2079...) for base_year = 1980, but will be set
+     * correctly for base_year = 2000.
+     */
+    if (s->base_year == 2000) {
+        s->base_year = 0;
+    }
+
     rtc_set_date_from_host(dev);
 
 #ifdef TARGET_I386
@@ -641,8 +843,8 @@ static int rtc_initfn(ISADevice *dev)
 #endif
 
     s->periodic_timer = qemu_new_timer_ns(rtc_clock, rtc_periodic_timer, s);
-    s->second_timer = qemu_new_timer_ns(rtc_clock, rtc_update_second, s);
-    s->second_timer2 = qemu_new_timer_ns(rtc_clock, rtc_update_second2, s);
+    s->update_timer = qemu_new_timer_ns(rtc_clock, rtc_update_timer, s);
+    check_update_timer(s);
 
     s->clock_reset_notifier.notify = rtc_notify_clock_reset;
     qemu_register_clock_reset_notifier(rtc_clock, &s->clock_reset_notifier);
@@ -650,14 +852,10 @@ static int rtc_initfn(ISADevice *dev)
     s->suspend_notifier.notify = rtc_notify_suspend;
     qemu_register_suspend_notifier(&s->suspend_notifier);
 
-    s->next_second_time =
-        qemu_get_clock_ns(rtc_clock) + (get_ticks_per_sec() * 99) / 100;
-    qemu_mod_timer(s->second_timer2, s->next_second_time);
-
     memory_region_init_io(&s->io, &cmos_ops, s, "rtc", 2);
     isa_register_ioport(dev, &s->io, base);
 
-    qdev_set_legacy_instance_id(&dev->qdev, base, 2);
+    qdev_set_legacy_instance_id(&dev->qdev, base, 3);
     qemu_register_reset(rtc_reset, s);
 
     object_property_add(OBJECT(s), "date", "struct tm",
@@ -700,7 +898,7 @@ static void rtc_class_initfn(ObjectClass *klass, void *data)
     dc->props = mc146818rtc_properties;
 }
 
-static TypeInfo mc146818rtc_info = {
+static const TypeInfo mc146818rtc_info = {
     .name          = "mc146818rtc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(RTCState),
diff --git a/hw/mc146818rtc_regs.h b/hw/mc146818rtc_regs.h
index 3ab37709f0d..ccdee42b3ca 100644
--- a/hw/mc146818rtc_regs.h
+++ b/hw/mc146818rtc_regs.h
@@ -44,6 +44,10 @@
 #define RTC_REG_C               12
 #define RTC_REG_D               13
 
+/* PC cmos mappings */
+#define RTC_CENTURY              0x32
+#define RTC_IBM_PS2_CENTURY_BYTE 0x37
+
 #define REG_A_UIP 0x80
 
 #define REG_B_SET  0x80
@@ -58,5 +62,6 @@
 #define REG_C_IRQF 0x80
 #define REG_C_PF   0x40
 #define REG_C_AF   0x20
+#define REG_C_MASK 0x70
 
 #endif
diff --git a/hw/mcf.h b/hw/mcf.h
index 19a8b547785..f929910f027 100644
--- a/hw/mcf.h
+++ b/hw/mcf.h
@@ -5,23 +5,23 @@
 struct MemoryRegion;
 
 /* mcf_uart.c */
-uint64_t mcf_uart_read(void *opaque, target_phys_addr_t addr,
+uint64_t mcf_uart_read(void *opaque, hwaddr addr,
                        unsigned size);
-void mcf_uart_write(void *opaque, target_phys_addr_t addr,
+void mcf_uart_write(void *opaque, hwaddr addr,
                     uint64_t val, unsigned size);
 void *mcf_uart_init(qemu_irq irq, CharDriverState *chr);
 void mcf_uart_mm_init(struct MemoryRegion *sysmem,
-                      target_phys_addr_t base,
+                      hwaddr base,
                       qemu_irq irq, CharDriverState *chr);
 
 /* mcf_intc.c */
 qemu_irq *mcf_intc_init(struct MemoryRegion *sysmem,
-                        target_phys_addr_t base,
+                        hwaddr base,
                         CPUM68KState *env);
 
 /* mcf_fec.c */
 void mcf_fec_init(struct MemoryRegion *sysmem, NICInfo *nd,
-                  target_phys_addr_t base, qemu_irq *irq);
+                  hwaddr base, qemu_irq *irq);
 
 /* mcf5206.c */
 qemu_irq *mcf5206_init(struct MemoryRegion *sysmem,
diff --git a/hw/mcf5206.c b/hw/mcf5206.c
index 539b3913380..d8c0059ed62 100644
--- a/hw/mcf5206.c
+++ b/hw/mcf5206.c
@@ -7,10 +7,10 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 /* General purpose timer module.  */
 typedef struct {
@@ -359,7 +359,7 @@ static void m5206_mbar_write(m5206_mbar_state *s, uint32_t offset,
 
 /* Internal peripherals use a variety of register widths.
    This lookup table allows a single routine to handle all of them.  */
-static const int m5206_mbar_width[] =
+static const uint8_t m5206_mbar_width[] =
 {
   /* 000-040 */ 1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  2, 2, 2, 2,
   /* 040-080 */ 1, 2, 2, 2,  4, 1, 2, 4,  1, 2, 4, 2,  2, 4, 2, 2,
@@ -371,14 +371,14 @@ static const int m5206_mbar_width[] =
   /* 1c0-200 */ 1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
 };
 
-static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset);
-static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset);
+static uint32_t m5206_mbar_readw(void *opaque, hwaddr offset);
+static uint32_t m5206_mbar_readl(void *opaque, hwaddr offset);
 
-static uint32_t m5206_mbar_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readb(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     if (m5206_mbar_width[offset >> 2] > 1) {
@@ -392,12 +392,12 @@ static uint32_t m5206_mbar_readb(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 1);
 }
 
-static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readw(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -416,12 +416,12 @@ static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 2);
 }
 
-static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readl(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -434,18 +434,18 @@ static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 4);
 }
 
-static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writew(void *opaque, hwaddr offset,
                               uint32_t value);
-static void m5206_mbar_writel(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writel(void *opaque, hwaddr offset,
                               uint32_t value);
 
-static void m5206_mbar_writeb(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writeb(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -463,13 +463,13 @@ static void m5206_mbar_writeb(void *opaque, target_phys_addr_t offset,
     m5206_mbar_write(s, offset, value, 1);
 }
 
-static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writew(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -491,13 +491,13 @@ static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
     m5206_mbar_write(s, offset, value, 2);
 }
 
-static void m5206_mbar_writel(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writel(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
diff --git a/hw/mcf5208.c b/hw/mcf5208.c
index ee25b1b3876..2c9a5dc98a1 100644
--- a/hw/mcf5208.c
+++ b/hw/mcf5208.c
@@ -7,14 +7,14 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "sysemu.h"
-#include "net.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define SYS_FREQ 66000000
 
@@ -45,7 +45,7 @@ static void m5208_timer_update(m5208_timer_state *s)
         qemu_irq_lower(s->irq);
 }
 
-static void m5208_timer_write(void *opaque, target_phys_addr_t offset,
+static void m5208_timer_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     m5208_timer_state *s = (m5208_timer_state *)opaque;
@@ -107,7 +107,7 @@ static void m5208_timer_trigger(void *opaque)
     m5208_timer_update(s);
 }
 
-static uint64_t m5208_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t m5208_timer_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     m5208_timer_state *s = (m5208_timer_state *)opaque;
@@ -130,7 +130,7 @@ static const MemoryRegionOps m5208_timer_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t m5208_sys_read(void *opaque, target_phys_addr_t addr,
+static uint64_t m5208_sys_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     switch (addr) {
@@ -152,7 +152,7 @@ static uint64_t m5208_sys_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void m5208_sys_write(void *opaque, target_phys_addr_t addr,
+static void m5208_sys_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     hw_error("m5208_sys_write: Bad offset 0x%x\n", (int)addr);
@@ -187,15 +187,15 @@ static void mcf5208_sys_init(MemoryRegion *address_space, qemu_irq *pic)
     }
 }
 
-static void mcf5208evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void mcf5208evb_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     qemu_irq *pic;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -292,6 +292,7 @@ static QEMUMachine mcf5208evb_machine = {
     .desc = "MCF5206EVB",
     .init = mcf5208evb_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mcf5208evb_machine_init(void)
diff --git a/hw/mcf_fec.c b/hw/mcf_fec.c
index 2fec5bc73ee..2423f64bf64 100644
--- a/hw/mcf_fec.c
+++ b/hw/mcf_fec.c
@@ -6,11 +6,11 @@
  * This code is licensed under the GPL
  */
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "mcf.h"
 /* For crc32 */
 #include <zlib.h>
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_FEC 1
 
@@ -216,7 +216,7 @@ static void mcf_fec_reset(mcf_fec_state *s)
     s->rfsr = 0x500;
 }
 
-static uint64_t mcf_fec_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mcf_fec_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     mcf_fec_state *s = (mcf_fec_state *)opaque;
@@ -254,7 +254,7 @@ static uint64_t mcf_fec_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void mcf_fec_write(void *opaque, target_phys_addr_t addr,
+static void mcf_fec_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     mcf_fec_state *s = (mcf_fec_state *)opaque;
@@ -458,7 +458,7 @@ static NetClientInfo net_mcf_fec_info = {
 };
 
 void mcf_fec_init(MemoryRegion *sysmem, NICInfo *nd,
-                  target_phys_addr_t base, qemu_irq *irq)
+                  hwaddr base, qemu_irq *irq)
 {
     mcf_fec_state *s;
 
diff --git a/hw/mcf_intc.c b/hw/mcf_intc.c
index cc1a5f37630..3bed3a2e4c0 100644
--- a/hw/mcf_intc.c
+++ b/hw/mcf_intc.c
@@ -7,7 +7,7 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     MemoryRegion iomem;
@@ -43,7 +43,7 @@ static void mcf_intc_update(mcf_intc_state *s)
     m68k_set_irq_level(s->env, best_level, s->active_vector);
 }
 
-static uint64_t mcf_intc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mcf_intc_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     int offset;
@@ -76,7 +76,7 @@ static uint64_t mcf_intc_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void mcf_intc_write(void *opaque, target_phys_addr_t addr,
+static void mcf_intc_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     int offset;
@@ -138,7 +138,7 @@ static const MemoryRegionOps mcf_intc_ops = {
 };
 
 qemu_irq *mcf_intc_init(MemoryRegion *sysmem,
-                        target_phys_addr_t base,
+                        hwaddr base,
                         CPUM68KState *env)
 {
     mcf_intc_state *s;
diff --git a/hw/mcf_uart.c b/hw/mcf_uart.c
index ec6a87f238f..c44344317a9 100644
--- a/hw/mcf_uart.c
+++ b/hw/mcf_uart.c
@@ -7,8 +7,8 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-char.h"
-#include "exec-memory.h"
+#include "char/char.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     MemoryRegion iomem;
@@ -66,7 +66,7 @@ static void mcf_uart_update(mcf_uart_state *s)
     qemu_set_irq(s->irq, (s->isr & s->imr) != 0);
 }
 
-uint64_t mcf_uart_read(void *opaque, target_phys_addr_t addr,
+uint64_t mcf_uart_read(void *opaque, hwaddr addr,
                        unsigned size)
 {
     mcf_uart_state *s = (mcf_uart_state *)opaque;
@@ -185,7 +185,7 @@ static void mcf_do_command(mcf_uart_state *s, uint8_t cmd)
     }
 }
 
-void mcf_uart_write(void *opaque, target_phys_addr_t addr,
+void mcf_uart_write(void *opaque, hwaddr addr,
                     uint64_t val, unsigned size)
 {
     mcf_uart_state *s = (mcf_uart_state *)opaque;
@@ -294,7 +294,7 @@ static const MemoryRegionOps mcf_uart_ops = {
 };
 
 void mcf_uart_mm_init(MemoryRegion *sysmem,
-                      target_phys_addr_t base,
+                      hwaddr base,
                       qemu_irq irq,
                       CharDriverState *chr)
 {
diff --git a/hw/megasas.c b/hw/megasas.c
index c728aea6999..eb191f5e12c 100644
--- a/hw/megasas.c
+++ b/hw/megasas.c
@@ -19,13 +19,12 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "dma.h"
-#include "msix.h"
-#include "iov.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
+#include "pci/msix.h"
+#include "qemu/iov.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "block_int.h"
 #include "trace.h"
 
 #include "mfi.h"
@@ -59,8 +58,8 @@ typedef struct MegasasCmd {
     uint16_t count;
     uint64_t context;
 
-    target_phys_addr_t pa;
-    target_phys_addr_t pa_size;
+    hwaddr pa;
+    hwaddr pa_size;
     union mfi_frame *frame;
     SCSIRequest *req;
     QEMUSGList qsg;
@@ -277,7 +276,7 @@ static int megasas_build_sense(MegasasCmd *cmd, uint8_t *sense_ptr,
     uint8_t sense_len)
 {
     uint32_t pa_hi = 0, pa_lo;
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     if (sense_len > cmd->frame->header.sense_len) {
         sense_len = cmd->frame->header.sense_len;
@@ -404,7 +403,7 @@ static int megasas_next_index(MegasasState *s, int index, int limit)
 }
 
 static MegasasCmd *megasas_lookup_frame(MegasasState *s,
-    target_phys_addr_t frame)
+    hwaddr frame)
 {
     MegasasCmd *cmd = NULL;
     int num = 0, index;
@@ -424,7 +423,7 @@ static MegasasCmd *megasas_lookup_frame(MegasasState *s,
 }
 
 static MegasasCmd *megasas_next_frame(MegasasState *s,
-    target_phys_addr_t frame)
+    hwaddr frame)
 {
     MegasasCmd *cmd = NULL;
     int num = 0, index;
@@ -452,11 +451,11 @@ static MegasasCmd *megasas_next_frame(MegasasState *s,
 }
 
 static MegasasCmd *megasas_enqueue_frame(MegasasState *s,
-    target_phys_addr_t frame, uint64_t context, int count)
+    hwaddr frame, uint64_t context, int count)
 {
     MegasasCmd *cmd = NULL;
     int frame_size = MFI_FRAME_SIZE * 16;
-    target_phys_addr_t frame_size_p = frame_size;
+    hwaddr frame_size_p = frame_size;
 
     cmd = megasas_next_frame(s, frame);
     /* All frames busy */
@@ -561,7 +560,7 @@ static void megasas_abort_command(MegasasCmd *cmd)
 static int megasas_init_firmware(MegasasState *s, MegasasCmd *cmd)
 {
     uint32_t pa_hi, pa_lo;
-    target_phys_addr_t iq_pa, initq_size;
+    hwaddr iq_pa, initq_size;
     struct mfi_init_qinfo *initq;
     uint32_t flags;
     int ret = MFI_STAT_OK;
@@ -652,7 +651,6 @@ static void megasas_finish_dcmd(MegasasCmd *cmd, uint32_t iov_size)
         }
     }
     cmd->iov_size = 0;
-    return;
 }
 
 static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
@@ -1081,6 +1079,7 @@ static int megasas_dcmd_ld_get_list(MegasasState *s, MegasasCmd *cmd)
         /* Logical device size is in blocks */
         bdrv_get_geometry(conf->bs, &ld_size);
         info.ld_list[num_ld_disks].ld.v.target_id = sdev->id;
+        info.ld_list[num_ld_disks].ld.v.lun_id = sdev->lun;
         info.ld_list[num_ld_disks].state = MFI_LD_STATE_OPTIMAL;
         info.ld_list[num_ld_disks].size = cpu_to_le64(ld_size);
         num_ld_disks++;
@@ -1297,7 +1296,7 @@ static int megasas_dcmd_get_properties(MegasasState *s, MegasasCmd *cmd)
 
 static int megasas_cache_flush(MegasasState *s, MegasasCmd *cmd)
 {
-    qemu_aio_flush();
+    bdrv_drain_all();
     return MFI_STAT_OK;
 }
 
@@ -1772,7 +1771,7 @@ static void megasas_command_cancel(SCSIRequest *req)
 static int megasas_handle_abort(MegasasState *s, MegasasCmd *cmd)
 {
     uint64_t abort_ctx = le64_to_cpu(cmd->frame->abort.abort_context);
-    target_phys_addr_t abort_addr, addr_hi, addr_lo;
+    hwaddr abort_addr, addr_hi, addr_lo;
     MegasasCmd *abort_cmd;
 
     addr_hi = le32_to_cpu(cmd->frame->abort.abort_mfi_addr_hi);
@@ -1862,7 +1861,7 @@ static void megasas_handle_frame(MegasasState *s, uint64_t frame_addr,
     }
 }
 
-static uint64_t megasas_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_mmio_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     MegasasState *s = opaque;
@@ -1898,7 +1897,7 @@ static uint64_t megasas_mmio_read(void *opaque, target_phys_addr_t addr,
     return retval;
 }
 
-static void megasas_mmio_write(void *opaque, target_phys_addr_t addr,
+static void megasas_mmio_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     MegasasState *s = opaque;
@@ -1978,13 +1977,13 @@ static const MemoryRegionOps megasas_mmio_ops = {
     }
 };
 
-static uint64_t megasas_port_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_port_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return megasas_mmio_read(opaque, addr & 0xff, size);
 }
 
-static void megasas_port_write(void *opaque, target_phys_addr_t addr,
+static void megasas_port_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     megasas_mmio_write(opaque, addr & 0xff, val, size);
@@ -2000,7 +1999,7 @@ static const MemoryRegionOps megasas_port_ops = {
     }
 };
 
-static uint64_t megasas_queue_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_queue_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     return 0;
diff --git a/hw/mfi.h b/hw/mfi.h
index 436b6906b15..cd8355badfc 100644
--- a/hw/mfi.h
+++ b/hw/mfi.h
@@ -1085,7 +1085,7 @@ struct mfi_pd_list {
 union mfi_ld_ref {
     struct {
         uint8_t target_id;
-        uint8_t reserved;
+        uint8_t lun_id;
         uint16_t seq;
     } v;
     uint32_t ref;
diff --git a/hw/microblaze/Makefile.objs b/hw/microblaze/Makefile.objs
index 274d2c543e0..3028e651c8f 100644
--- a/hw/microblaze/Makefile.objs
+++ b/hw/microblaze/Makefile.objs
@@ -1,6 +1,7 @@
 obj-y = petalogix_s3adsp1800_mmu.o
 obj-y += petalogix_ml605_mmu.o
 obj-y += microblaze_boot.o
+obj-y += xilinx_spi.o
 
 obj-y += microblaze_pic_cpu.o
 obj-y += xilinx_ethlite.o
diff --git a/hw/microblaze_boot.c b/hw/microblaze_boot.c
index 1030e9c8ed8..3ec5c0f7dd6 100644
--- a/hw/microblaze_boot.c
+++ b/hw/microblaze_boot.c
@@ -24,10 +24,10 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "qemu-common.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
 
@@ -55,7 +55,7 @@ static void main_cpu_reset(void *opaque)
     }
 }
 
-static int microblaze_load_dtb(target_phys_addr_t addr,
+static int microblaze_load_dtb(hwaddr addr,
                                       uint32_t ramsize,
                                       const char *kernel_cmdline,
                                       const char *dtb_filename)
@@ -100,7 +100,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return addr - 0x30000000LL;
 }
 
-void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
+void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
                             uint32_t ramsize, const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *))
 {
@@ -149,7 +149,7 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
 
         /* If it wasn't an ELF image, try an u-boot image.  */
         if (kernel_size < 0) {
-            target_phys_addr_t uentry, loadaddr;
+            hwaddr uentry, loadaddr;
 
             kernel_size = load_uimage(kernel_filename, &uentry, &loadaddr, 0);
             boot_info.bootstrap_pc = uentry;
diff --git a/hw/microblaze_boot.h b/hw/microblaze_boot.h
index c9a3064d272..c1cf836b993 100644
--- a/hw/microblaze_boot.h
+++ b/hw/microblaze_boot.h
@@ -3,7 +3,7 @@
 
 #include "hw.h"
 
-void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
+void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
                             uint32_t ramsize, const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *));
 
diff --git a/hw/milkymist-ac97.c b/hw/milkymist-ac97.c
index 4414f397340..d51d1ac9937 100644
--- a/hw/milkymist-ac97.c
+++ b/hw/milkymist-ac97.c
@@ -25,7 +25,7 @@
 #include "sysbus.h"
 #include "trace.h"
 #include "audio/audio.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_AC97_CTRL = 0,
@@ -83,7 +83,7 @@ static void update_voices(MilkymistAC97State *s)
     }
 }
 
-static uint64_t ac97_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ac97_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistAC97State *s = opaque;
@@ -115,7 +115,7 @@ static uint64_t ac97_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void ac97_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void ac97_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistAC97State *s = opaque;
@@ -329,7 +329,7 @@ static void milkymist_ac97_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_ac97;
 }
 
-static TypeInfo milkymist_ac97_info = {
+static const TypeInfo milkymist_ac97_info = {
     .name          = "milkymist-ac97",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistAC97State),
diff --git a/hw/milkymist-hpdmc.c b/hw/milkymist-hpdmc.c
index 2da0293683a..ea4d210685d 100644
--- a/hw/milkymist-hpdmc.c
+++ b/hw/milkymist-hpdmc.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_SYSTEM = 0,
@@ -48,7 +48,7 @@ struct MilkymistHpdmcState {
 };
 typedef struct MilkymistHpdmcState MilkymistHpdmcState;
 
-static uint64_t hpdmc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t hpdmc_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     MilkymistHpdmcState *s = opaque;
@@ -74,7 +74,7 @@ static uint64_t hpdmc_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void hpdmc_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void hpdmc_write(void *opaque, hwaddr addr, uint64_t value,
                         unsigned size)
 {
     MilkymistHpdmcState *s = opaque;
@@ -155,7 +155,7 @@ static void milkymist_hpdmc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_hpdmc;
 }
 
-static TypeInfo milkymist_hpdmc_info = {
+static const TypeInfo milkymist_hpdmc_info = {
     .name          = "milkymist-hpdmc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistHpdmcState),
diff --git a/hw/milkymist-hw.h b/hw/milkymist-hw.h
index 9f358a7d691..c8bd7e93dd1 100644
--- a/hw/milkymist-hw.h
+++ b/hw/milkymist-hw.h
@@ -3,43 +3,44 @@
 
 #include "qdev.h"
 #include "qdev-addr.h"
+#include "net/net.h"
 
-static inline DeviceState *milkymist_uart_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_uart_create(hwaddr base,
         qemu_irq irq)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "milkymist-uart");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_hpdmc_create(target_phys_addr_t base)
+static inline DeviceState *milkymist_hpdmc_create(hwaddr base)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "milkymist-hpdmc");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_memcard_create(target_phys_addr_t base)
+static inline DeviceState *milkymist_memcard_create(hwaddr base)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "milkymist-memcard");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_vgafb_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_vgafb_create(hwaddr base,
         uint32_t fb_offset, uint32_t fb_mask)
 {
     DeviceState *dev;
@@ -48,12 +49,12 @@ static inline DeviceState *milkymist_vgafb_create(target_phys_addr_t base,
     qdev_prop_set_uint32(dev, "fb_offset", fb_offset);
     qdev_prop_set_uint32(dev, "fb_mask", fb_mask);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_sysctl_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_sysctl_create(hwaddr base,
         qemu_irq gpio_irq, qemu_irq timer0_irq, qemu_irq timer1_irq,
         uint32_t freq_hz, uint32_t system_id, uint32_t capabilities,
         uint32_t gpio_strappings)
@@ -66,23 +67,23 @@ static inline DeviceState *milkymist_sysctl_create(target_phys_addr_t base,
     qdev_prop_set_uint32(dev, "capabilities", capabilities);
     qdev_prop_set_uint32(dev, "gpio_strappings", gpio_strappings);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, gpio_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, timer0_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2, timer1_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, gpio_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, timer0_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, timer1_irq);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_pfpu_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_pfpu_create(hwaddr base,
         qemu_irq irq)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "milkymist-pfpu");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
@@ -97,7 +98,7 @@ static const int glx_fbconfig_attr[] = {
 };
 #endif
 
-static inline DeviceState *milkymist_tmu2_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_tmu2_create(hwaddr base,
         qemu_irq irq)
 {
 #ifdef CONFIG_OPENGL
@@ -143,8 +144,8 @@ static inline DeviceState *milkymist_tmu2_create(target_phys_addr_t base,
 
     dev = qdev_create(NULL, "milkymist-tmu2");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 #else
@@ -152,7 +153,7 @@ static inline DeviceState *milkymist_tmu2_create(target_phys_addr_t base,
 #endif
 }
 
-static inline DeviceState *milkymist_ac97_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_ac97_create(hwaddr base,
         qemu_irq crrequest_irq, qemu_irq crreply_irq, qemu_irq dmar_irq,
         qemu_irq dmaw_irq)
 {
@@ -160,16 +161,16 @@ static inline DeviceState *milkymist_ac97_create(target_phys_addr_t base,
 
     dev = qdev_create(NULL, "milkymist-ac97");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, crrequest_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, crreply_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2, dmar_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 3, dmaw_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, crrequest_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, crreply_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, dmar_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 3, dmaw_irq);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_minimac_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_minimac_create(hwaddr base,
         qemu_irq rx_irq, qemu_irq tx_irq)
 {
     DeviceState *dev;
@@ -178,15 +179,15 @@ static inline DeviceState *milkymist_minimac_create(target_phys_addr_t base,
     dev = qdev_create(NULL, "milkymist-minimac");
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, rx_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, tx_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, rx_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, tx_irq);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_minimac2_create(target_phys_addr_t base,
-        target_phys_addr_t buffers_base, qemu_irq rx_irq, qemu_irq tx_irq)
+static inline DeviceState *milkymist_minimac2_create(hwaddr base,
+        hwaddr buffers_base, qemu_irq rx_irq, qemu_irq tx_irq)
 {
     DeviceState *dev;
 
@@ -195,14 +196,14 @@ static inline DeviceState *milkymist_minimac2_create(target_phys_addr_t base,
     qdev_prop_set_taddr(dev, "buffers_base", buffers_base);
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, rx_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, tx_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, rx_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, tx_irq);
 
     return dev;
 }
 
-static inline DeviceState *milkymist_softusb_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_softusb_create(hwaddr base,
         qemu_irq irq, uint32_t pmem_base, uint32_t pmem_size,
         uint32_t dmem_base, uint32_t dmem_size)
 {
@@ -214,8 +215,8 @@ static inline DeviceState *milkymist_softusb_create(target_phys_addr_t base,
     qdev_prop_set_uint32(dev, "dmem_base", dmem_base);
     qdev_prop_set_uint32(dev, "dmem_size", dmem_size);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
diff --git a/hw/milkymist-memcard.c b/hw/milkymist-memcard.c
index 3515c3cd9a4..9d15309ab79 100644
--- a/hw/milkymist-memcard.c
+++ b/hw/milkymist-memcard.c
@@ -23,10 +23,10 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu-error.h"
-#include "blockdev.h"
+#include "qemu/error-report.h"
+#include "sysemu/blockdev.h"
 #include "sd.h"
 
 enum {
@@ -117,7 +117,7 @@ static void memcard_sd_command(MilkymistMemcardState *s)
     }
 }
 
-static uint64_t memcard_read(void *opaque, target_phys_addr_t addr,
+static uint64_t memcard_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MilkymistMemcardState *s = opaque;
@@ -166,7 +166,7 @@ static uint64_t memcard_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void memcard_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void memcard_write(void *opaque, hwaddr addr, uint64_t value,
                           unsigned size)
 {
     MilkymistMemcardState *s = opaque;
@@ -288,7 +288,7 @@ static void milkymist_memcard_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_memcard;
 }
 
-static TypeInfo milkymist_memcard_info = {
+static const TypeInfo milkymist_memcard_info = {
     .name          = "milkymist-memcard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistMemcardState),
diff --git a/hw/milkymist-minimac2.c b/hw/milkymist-minimac2.c
index b483a02f21b..43d6c195ebc 100644
--- a/hw/milkymist-minimac2.c
+++ b/hw/milkymist-minimac2.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "net.h"
-#include "qemu-error.h"
+#include "net/net.h"
+#include "qemu/error-report.h"
 #include "qdev-addr.h"
 
 #include <zlib.h>
@@ -96,7 +96,7 @@ struct MilkymistMinimac2State {
     NICState *nic;
     NICConf conf;
     char *phy_model;
-    target_phys_addr_t buffers_base;
+    hwaddr buffers_base;
     MemoryRegion buffers;
     MemoryRegion regs_region;
 
@@ -323,7 +323,7 @@ static ssize_t minimac2_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 }
 
 static uint64_t
-minimac2_read(void *opaque, target_phys_addr_t addr, unsigned size)
+minimac2_read(void *opaque, hwaddr addr, unsigned size)
 {
     MilkymistMinimac2State *s = opaque;
     uint32_t r = 0;
@@ -352,7 +352,7 @@ minimac2_read(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-minimac2_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+minimac2_write(void *opaque, hwaddr addr, uint64_t value,
                unsigned size)
 {
     MilkymistMinimac2State *s = opaque;
@@ -535,7 +535,7 @@ static void milkymist_minimac2_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_minimac2_properties;
 }
 
-static TypeInfo milkymist_minimac2_info = {
+static const TypeInfo milkymist_minimac2_info = {
     .name          = "milkymist-minimac2",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistMinimac2State),
diff --git a/hw/milkymist-pfpu.c b/hw/milkymist-pfpu.c
index 0f9ff4a13d4..c347680ad74 100644
--- a/hw/milkymist-pfpu.c
+++ b/hw/milkymist-pfpu.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-log.h"
-#include "qemu-error.h"
+#include "qemu/log.h"
+#include "qemu/error-report.h"
 #include <math.h>
 
 /* #define TRACE_EXEC */
@@ -131,7 +131,7 @@ struct MilkymistPFPUState {
 };
 typedef struct MilkymistPFPUState MilkymistPFPUState;
 
-static inline target_phys_addr_t
+static inline hwaddr
 get_dma_address(uint32_t base, uint32_t x, uint32_t y)
 {
     return base + 8 * (128 * y + x);
@@ -225,7 +225,7 @@ static int pfpu_decode_insn(MilkymistPFPUState *s)
     {
         uint32_t a = cpu_to_be32(s->gp_regs[reg_a]);
         uint32_t b = cpu_to_be32(s->gp_regs[reg_b]);
-        target_phys_addr_t dma_ptr =
+        hwaddr dma_ptr =
             get_dma_address(s->regs[R_MESHBASE],
                     s->gp_regs[GPR_X], s->gp_regs[GPR_Y]);
         cpu_physical_memory_write(dma_ptr, (uint8_t *)&a, 4);
@@ -380,7 +380,7 @@ static inline int get_microcode_address(MilkymistPFPUState *s, uint32_t addr)
     return (512 * s->regs[R_CODEPAGE]) + addr - MICROCODE_BEGIN;
 }
 
-static uint64_t pfpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pfpu_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistPFPUState *s = opaque;
@@ -420,7 +420,7 @@ static uint64_t pfpu_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void pfpu_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void pfpu_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistPFPUState *s = opaque;
@@ -529,7 +529,7 @@ static void milkymist_pfpu_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_pfpu;
 }
 
-static TypeInfo milkymist_pfpu_info = {
+static const TypeInfo milkymist_pfpu_info = {
     .name          = "milkymist-pfpu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistPFPUState),
diff --git a/hw/milkymist-softusb.c b/hw/milkymist-softusb.c
index ecc2be92251..01660bebf05 100644
--- a/hw/milkymist-softusb.c
+++ b/hw/milkymist-softusb.c
@@ -24,9 +24,9 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hid.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_CTRL = 0,
@@ -71,7 +71,7 @@ struct MilkymistSoftUsbState {
 };
 typedef struct MilkymistSoftUsbState MilkymistSoftUsbState;
 
-static uint64_t softusb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t softusb_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MilkymistSoftUsbState *s = opaque;
@@ -95,7 +95,7 @@ static uint64_t softusb_read(void *opaque, target_phys_addr_t addr,
 }
 
 static void
-softusb_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+softusb_write(void *opaque, hwaddr addr, uint64_t value,
               unsigned size)
 {
     MilkymistSoftUsbState *s = opaque;
@@ -316,7 +316,7 @@ static void milkymist_softusb_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_softusb_properties;
 }
 
-static TypeInfo milkymist_softusb_info = {
+static const TypeInfo milkymist_softusb_info = {
     .name          = "milkymist-softusb",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistSoftUsbState),
diff --git a/hw/milkymist-sysctl.c b/hw/milkymist-sysctl.c
index 8878d2bd17f..e69ac6f0479 100644
--- a/hw/milkymist-sysctl.c
+++ b/hw/milkymist-sysctl.c
@@ -23,11 +23,11 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     CTRL_ENABLE      = (1<<0),
@@ -89,7 +89,7 @@ static void sysctl_icap_write(MilkymistSysctlState *s, uint32_t value)
     }
 }
 
-static uint64_t sysctl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sysctl_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     MilkymistSysctlState *s = opaque;
@@ -134,7 +134,7 @@ static uint64_t sysctl_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void sysctl_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void sysctl_write(void *opaque, hwaddr addr, uint64_t value,
                          unsigned size)
 {
     MilkymistSysctlState *s = opaque;
@@ -323,7 +323,7 @@ static void milkymist_sysctl_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_sysctl_properties;
 }
 
-static TypeInfo milkymist_sysctl_info = {
+static const TypeInfo milkymist_sysctl_info = {
     .name          = "milkymist-sysctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistSysctlState),
diff --git a/hw/milkymist-tmu2.c b/hw/milkymist-tmu2.c
index 210ceede2a9..42de10aafde 100644
--- a/hw/milkymist-tmu2.c
+++ b/hw/milkymist-tmu2.c
@@ -27,7 +27,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #include <X11/Xlib.h>
 #include <GL/gl.h>
@@ -182,9 +182,9 @@ static void tmu2_start(MilkymistTMU2State *s)
     GLXPbuffer pbuffer;
     GLuint texture;
     void *fb;
-    target_phys_addr_t fb_len;
+    hwaddr fb_len;
     void *mesh;
-    target_phys_addr_t mesh_len;
+    hwaddr mesh_len;
     float m;
 
     trace_milkymist_tmu2_start();
@@ -310,7 +310,7 @@ static void tmu2_start(MilkymistTMU2State *s)
     qemu_irq_pulse(s->irq);
 }
 
-static uint64_t tmu2_read(void *opaque, target_phys_addr_t addr,
+static uint64_t tmu2_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistTMU2State *s = opaque;
@@ -372,7 +372,7 @@ static void tmu2_check_registers(MilkymistTMU2State *s)
     }
 }
 
-static void tmu2_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void tmu2_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistTMU2State *s = opaque;
@@ -475,7 +475,7 @@ static void milkymist_tmu2_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_tmu2;
 }
 
-static TypeInfo milkymist_tmu2_info = {
+static const TypeInfo milkymist_tmu2_info = {
     .name          = "milkymist-tmu2",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistTMU2State),
diff --git a/hw/milkymist-uart.c b/hw/milkymist-uart.c
index 291fe3c57be..e73eb8476c0 100644
--- a/hw/milkymist-uart.c
+++ b/hw/milkymist-uart.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 
 enum {
     R_RXTX = 0,
@@ -78,7 +78,7 @@ static void uart_update_irq(MilkymistUartState *s)
     }
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t uart_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistUartState *s = opaque;
@@ -107,7 +107,7 @@ static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void uart_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void uart_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistUartState *s = opaque;
@@ -228,7 +228,7 @@ static void milkymist_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_uart;
 }
 
-static TypeInfo milkymist_uart_info = {
+static const TypeInfo milkymist_uart_info = {
     .name          = "milkymist-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistUartState),
diff --git a/hw/milkymist-vgafb.c b/hw/milkymist-vgafb.c
index cd4365d64b0..4d0a5dfb78d 100644
--- a/hw/milkymist-vgafb.c
+++ b/hw/milkymist-vgafb.c
@@ -25,10 +25,10 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
-#include "pixel_ops.h"
-#include "qemu-error.h"
+#include "ui/pixel_ops.h"
+#include "qemu/error-report.h"
 
 #define BITS 8
 #include "milkymist-vgafb_template.h"
@@ -134,7 +134,7 @@ static void vgafb_update_display(void *opaque)
                                &first, &last);
 
     if (first >= 0) {
-        dpy_update(s->ds, 0, first, s->regs[R_HRES], last - first + 1);
+        dpy_gfx_update(s->ds, 0, first, s->regs[R_HRES], last - first + 1);
     }
     s->invalidate = 0;
 }
@@ -155,7 +155,7 @@ static void vgafb_resize(MilkymistVgafbState *s)
     s->invalidate = 1;
 }
 
-static uint64_t vgafb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t vgafb_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     MilkymistVgafbState *s = opaque;
@@ -193,7 +193,7 @@ static uint64_t vgafb_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void vgafb_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void vgafb_write(void *opaque, hwaddr addr, uint64_t value,
                         unsigned size)
 {
     MilkymistVgafbState *s = opaque;
@@ -319,7 +319,7 @@ static void milkymist_vgafb_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_vgafb_properties;
 }
 
-static TypeInfo milkymist_vgafb_info = {
+static const TypeInfo milkymist_vgafb_info = {
     .name          = "milkymist-vgafb",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistVgafbState),
diff --git a/hw/milkymist.c b/hw/milkymist.c
index 2e7235b4b35..c04eb35fddb 100644
--- a/hw/milkymist.c
+++ b/hw/milkymist.c
@@ -19,17 +19,16 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "milkymist-hw.h"
 #include "lm32.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME    "mmone-bios.bin"
 #define BIOS_OFFSET      0x00860000
@@ -38,11 +37,11 @@
 
 typedef struct {
     LM32CPU *cpu;
-    target_phys_addr_t bootstrap_pc;
-    target_phys_addr_t flash_base;
-    target_phys_addr_t initrd_base;
+    hwaddr bootstrap_pc;
+    hwaddr flash_base;
+    hwaddr initrd_base;
     size_t initrd_size;
-    target_phys_addr_t cmdline_base;
+    hwaddr cmdline_base;
 } ResetInfo;
 
 static void cpu_irq_handler(void *opaque, int irq, int level)
@@ -73,12 +72,12 @@ static void main_cpu_reset(void *opaque)
 }
 
 static void
-milkymist_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+milkymist_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     int kernel_size;
@@ -91,14 +90,14 @@ milkymist_init(ram_addr_t ram_size_not_used,
     ResetInfo *reset_info;
 
     /* memory map */
-    target_phys_addr_t flash_base   = 0x00000000;
+    hwaddr flash_base   = 0x00000000;
     size_t flash_sector_size        = 128 * 1024;
     size_t flash_size               = 32 * 1024 * 1024;
-    target_phys_addr_t sdram_base   = 0x40000000;
+    hwaddr sdram_base   = 0x40000000;
     size_t sdram_size               = 128 * 1024 * 1024;
 
-    target_phys_addr_t initrd_base  = sdram_base + 0x1002000;
-    target_phys_addr_t cmdline_base = sdram_base + 0x1000000;
+    hwaddr initrd_base  = sdram_base + 0x1002000;
+    hwaddr cmdline_base = sdram_base + 0x1000000;
     size_t initrd_max = sdram_size - 0x1002000;
 
     reset_info = g_malloc0(sizeof(ResetInfo));
@@ -207,7 +206,8 @@ static QEMUMachine milkymist_machine = {
     .name = "milkymist",
     .desc = "Milkymist One",
     .init = milkymist_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void milkymist_machine_init(void)
diff --git a/hw/mips.h b/hw/mips.h
index a7e6d4cc625..291e85f6b92 100644
--- a/hw/mips.h
+++ b/hw/mips.h
@@ -2,7 +2,7 @@
 #define HW_MIPS_H
 /* Definitions for mips board emulation.  */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* gt64xxx.c */
 PCIBus *gt64120_register(qemu_irq *pic);
@@ -12,7 +12,7 @@ PCIBus *bonito_init(qemu_irq *pic);
 
 /* rc4030.c */
 typedef struct rc4030DMAState *rc4030_dma;
-void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write);
+void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
 void rc4030_dma_read(void *dma, uint8_t *buf, int len);
 void rc4030_dma_write(void *dma, uint8_t *buf, int len);
 
@@ -21,9 +21,9 @@ void *rc4030_init(qemu_irq timer, qemu_irq jazz_bus,
                   MemoryRegion *sysmem);
 
 /* dp8393x.c */
-void dp83932_init(NICInfo *nd, target_phys_addr_t base, int it_shift,
+void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
                   MemoryRegion *address_space,
                   qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write));
+                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write));
 
 #endif
diff --git a/hw/mips_fulong2e.c b/hw/mips_fulong2e.c
index 38e4b861504..8b532e1e0dc 100644
--- a/hw/mips_fulong2e.c
+++ b/hw/mips_fulong2e.c
@@ -20,19 +20,20 @@
 
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "smbus.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pci.h"
-#include "qemu-char.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "audio/audio.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
 #include "mips-bios.h"
 #include "ide.h"
@@ -40,8 +41,8 @@
 #include "vt82c686.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define DEBUG_FULONG2E_INIT
 
@@ -256,10 +257,13 @@ static void cpu_request_exit(void *opaque, int irq, int level)
     }
 }
 
-static void mips_fulong2e_init(ram_addr_t ram_size, const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void mips_fulong2e_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -392,10 +396,11 @@ static void mips_fulong2e_init(ram_addr_t ram_size, const char *boot_device,
     network_init();
 }
 
-QEMUMachine mips_fulong2e_machine = {
+static QEMUMachine mips_fulong2e_machine = {
     .name = "fulong2e",
     .desc = "Fulong 2e mini pc",
     .init = mips_fulong2e_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_fulong2e_machine_init(void)
diff --git a/hw/mips_jazz.c b/hw/mips_jazz.c
index db927f14d02..17fbdde0630 100644
--- a/hw/mips_jazz.c
+++ b/hw/mips_jazz.c
@@ -26,21 +26,22 @@
 #include "mips.h"
 #include "mips_cpudevs.h"
 #include "pc.h"
+#include "serial.h"
 #include "isa.h"
 #include "fdc.h"
-#include "sysemu.h"
-#include "arch_init.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/arch_init.h"
 #include "boards.h"
-#include "net.h"
+#include "net/net.h"
 #include "esp.h"
 #include "mips-bios.h"
 #include "loader.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
 #include "pcspk.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 enum jazz_model_e
 {
@@ -55,12 +56,12 @@ static void main_cpu_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static uint64_t rtc_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t rtc_read(void *opaque, hwaddr addr, unsigned size)
 {
     return cpu_inw(0x71);
 }
 
-static void rtc_write(void *opaque, target_phys_addr_t addr,
+static void rtc_write(void *opaque, hwaddr addr,
                       uint64_t val, unsigned size)
 {
     cpu_outw(0x71, val & 0xff);
@@ -72,7 +73,7 @@ static const MemoryRegionOps rtc_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t dma_dummy_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dma_dummy_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     /* Nothing to do. That is only to ensure that
@@ -80,7 +81,7 @@ static uint64_t dma_dummy_read(void *opaque, target_phys_addr_t addr,
     return 0xff;
 }
 
-static void dma_dummy_write(void *opaque, target_phys_addr_t addr,
+static void dma_dummy_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     /* Nothing to do. That is only to ensure that
@@ -208,7 +209,7 @@ static void mips_jazz_init(MemoryRegion *address_space,
     case JAZZ_MAGNUM:
         dev = qdev_create(NULL, "sysbus-g364");
         qdev_init_nofail(dev);
-        sysbus = sysbus_from_qdev(dev);
+        sysbus = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(sysbus, 0, 0x60080000);
         sysbus_mmio_map(sysbus, 1, 0x40000000);
         sysbus_connect_irq(sysbus, 0, rc4030[3]);
@@ -294,7 +295,7 @@ static void mips_jazz_init(MemoryRegion *address_space,
     /* NVRAM */
     dev = qdev_create(NULL, "ds1225y");
     qdev_init_nofail(dev);
-    sysbus = sysbus_from_qdev(dev);
+    sysbus = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(sysbus, 0, 0x80009000);
 
     /* LED indicator */
@@ -302,21 +303,19 @@ static void mips_jazz_init(MemoryRegion *address_space,
 }
 
 static
-void mips_magnum_init (ram_addr_t ram_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+void mips_magnum_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
         mips_jazz_init(get_system_memory(), get_system_io(),
                        ram_size, cpu_model, JAZZ_MAGNUM);
 }
 
 static
-void mips_pica61_init (ram_addr_t ram_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+void mips_pica61_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     mips_jazz_init(get_system_memory(), get_system_io(),
                    ram_size, cpu_model, JAZZ_PICA61);
 }
@@ -325,14 +324,16 @@ static QEMUMachine mips_magnum_machine = {
     .name = "magnum",
     .desc = "MIPS Magnum",
     .init = mips_magnum_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine mips_pica61_machine = {
     .name = "pica61",
     .desc = "Acer Pica 61",
     .init = mips_pica61_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_jazz_machine_init(void)
diff --git a/hw/mips_malta.c b/hw/mips_malta.c
index ad23f26e597..2a150dfb84d 100644
--- a/hw/mips_malta.c
+++ b/hw/mips_malta.c
@@ -24,31 +24,30 @@
 
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "smbus.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pci.h"
-#include "vmware_vga.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "arch_init.h"
+#include "pci/pci.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/arch_init.h"
 #include "boards.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "mips-bios.h"
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"             /* SysBusDevice */
-#include "vga-pci.h"
 
 //#define DEBUG_BOARD_INIT
 
@@ -232,7 +231,7 @@ static void eeprom24c0x_write(int scl, int sda)
     eeprom.sda = sda;
 }
 
-static uint64_t malta_fpga_read(void *opaque, target_phys_addr_t addr,
+static uint64_t malta_fpga_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     MaltaFPGAState *s = opaque;
@@ -320,7 +319,7 @@ static uint64_t malta_fpga_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void malta_fpga_write(void *opaque, target_phys_addr_t addr,
+static void malta_fpga_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     MaltaFPGAState *s = opaque;
@@ -442,7 +441,7 @@ static void malta_fpga_led_init(CharDriverState *chr)
 }
 
 static MaltaFPGAState *malta_fpga_init(MemoryRegion *address_space,
-         target_phys_addr_t base, qemu_irq uart_irq, CharDriverState *uart_chr)
+         hwaddr base, qemu_irq uart_irq, CharDriverState *uart_chr)
 {
     MaltaFPGAState *s;
 
@@ -744,10 +743,13 @@ static int64_t load_kernel (void)
     return kernel_entry;
 }
 
-static void malta_mips_config(CPUMIPSState *env)
+static void malta_mips_config(MIPSCPU *cpu)
 {
+    CPUMIPSState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
     env->mvp->CP0_MVPConf0 |= ((smp_cpus - 1) << CP0MVPC0_PVPE) |
-                         ((smp_cpus * env->nr_threads - 1) << CP0MVPC0_PTC);
+                         ((smp_cpus * cs->nr_threads - 1) << CP0MVPC0_PTC);
 }
 
 static void main_cpu_reset(void *opaque)
@@ -764,7 +766,7 @@ static void main_cpu_reset(void *opaque)
         env->CP0_Status &= ~((1 << CP0St_BEV) | (1 << CP0St_ERL));
     }
 
-    malta_mips_config(env);
+    malta_mips_config(cpu);
 }
 
 static void cpu_request_exit(void *opaque, int irq, int level)
@@ -777,11 +779,13 @@ static void cpu_request_exit(void *opaque, int irq, int level)
 }
 
 static
-void mips_malta_init (ram_addr_t ram_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+void mips_malta_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     pflash_t *fl;
     MemoryRegion *system_memory = get_system_memory();
@@ -860,7 +864,8 @@ void mips_malta_init (ram_addr_t ram_size,
     be = 0;
 #endif
     /* FPGA */
-    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[2], serial_hds[2]);
+    /* The CBUS UART is attached to the MIPS CPU INT2 pin, ie interrupt 4 */
+    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[4], serial_hds[2]);
 
     /* Load firmware in flash / BIOS. */
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
@@ -987,13 +992,7 @@ void mips_malta_init (ram_addr_t ram_size,
     network_init();
 
     /* Optional PCI video card */
-    if (cirrus_vga_enabled) {
-        pci_cirrus_vga_init(pci_bus);
-    } else if (vmsvga_enabled) {
-        pci_vmsvga_init(pci_bus);
-    } else if (std_vga_enabled) {
-        pci_vga_init(pci_bus);
-    }
+    pci_vga_init(pci_bus);
 }
 
 static int mips_malta_sysbus_device_init(SysBusDevice *sysbusdev)
@@ -1008,7 +1007,7 @@ static void mips_malta_class_init(ObjectClass *klass, void *data)
     k->init = mips_malta_sysbus_device_init;
 }
 
-static TypeInfo mips_malta_device = {
+static const TypeInfo mips_malta_device = {
     .name          = "mips-malta",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MaltaState),
@@ -1021,6 +1020,7 @@ static QEMUMachine mips_malta_machine = {
     .init = mips_malta_init,
     .max_cpus = 16,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_malta_register_types(void)
diff --git a/hw/mips_mipssim.c b/hw/mips_mipssim.c
index 830f635597b..b0ab8f69e2e 100644
--- a/hw/mips_mipssim.c
+++ b/hw/mips_mipssim.c
@@ -27,16 +27,16 @@
 #include "hw.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pc.h"
+#include "serial.h"
 #include "isa.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "mips-bios.h"
 #include "loader.h"
 #include "elf.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static struct _loaderparams {
     int ram_size;
@@ -123,7 +123,7 @@ static void mipsnet_init(int base, qemu_irq irq, NICInfo *nd)
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(get_system_io(),
                                 base,
@@ -131,11 +131,13 @@ static void mipsnet_init(int base, qemu_irq irq, NICInfo *nd)
 }
 
 static void
-mips_mipssim_init (ram_addr_t ram_size,
-                   const char *boot_device,
-                   const char *kernel_filename, const char *kernel_cmdline,
-                   const char *initrd_filename, const char *cpu_model)
+mips_mipssim_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -215,7 +217,8 @@ mips_mipssim_init (ram_addr_t ram_size,
     /* A single 16450 sits at offset 0x3f8. It is attached to
        MIPS CPU INT2, which is interrupt 4. */
     if (serial_hds[0])
-        serial_init(0x3f8, env->irq[4], 115200, serial_hds[0]);
+        serial_init(0x3f8, env->irq[4], 115200, serial_hds[0],
+                    get_system_io());
 
     if (nd_table[0].used)
         /* MIPSnet uses the MIPS CPU INT0, which is interrupt 2. */
@@ -226,6 +229,7 @@ static QEMUMachine mips_mipssim_machine = {
     .name = "mipssim",
     .desc = "MIPS MIPSsim platform",
     .init = mips_mipssim_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_mipssim_machine_init(void)
diff --git a/hw/mips_r4k.c b/hw/mips_r4k.c
index 967a76e5330..5df7eb44696 100644
--- a/hw/mips_r4k.c
+++ b/hw/mips_r4k.c
@@ -11,20 +11,21 @@
 #include "mips.h"
 #include "mips_cpudevs.h"
 #include "pc.h"
+#include "serial.h"
 #include "isa.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "flash.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "mips-bios.h"
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define MAX_IDE_BUS 2
 
@@ -43,7 +44,7 @@ static struct _loaderparams {
     const char *initrd_filename;
 } loaderparams;
 
-static void mips_qemu_write (void *opaque, target_phys_addr_t addr,
+static void mips_qemu_write (void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     if ((addr & 0xffff) == 0 && val == 42)
@@ -52,7 +53,7 @@ static void mips_qemu_write (void *opaque, target_phys_addr_t addr,
         qemu_system_shutdown_request ();
 }
 
-static uint64_t mips_qemu_read (void *opaque, target_phys_addr_t addr,
+static uint64_t mips_qemu_read (void *opaque, hwaddr addr,
                                 unsigned size)
 {
     return 0;
@@ -151,11 +152,13 @@ static void main_cpu_reset(void *opaque)
 
 static const int sector_len = 32 * 1024;
 static
-void mips_r4k_init (ram_addr_t ram_size,
-                    const char *boot_device,
-                    const char *kernel_filename, const char *kernel_cmdline,
-                    const char *initrd_filename, const char *cpu_model)
+void mips_r4k_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -299,6 +302,7 @@ static QEMUMachine mips_machine = {
     .name = "mips",
     .desc = "mips r4k platform",
     .init = mips_r4k_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_machine_init(void)
diff --git a/hw/mips_timer.c b/hw/mips_timer.c
index 7aa9004a0e4..83c400c158c 100644
--- a/hw/mips_timer.c
+++ b/hw/mips_timer.c
@@ -22,7 +22,7 @@
 
 #include "hw.h"
 #include "mips_cpudevs.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define TIMER_FREQ	100 * 1000 * 1000
 
diff --git a/hw/mipsnet.c b/hw/mipsnet.c
index 28063b1106a..feac8159ee8 100644
--- a/hw/mipsnet.c
+++ b/hw/mipsnet.c
@@ -1,5 +1,5 @@
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -96,7 +96,7 @@ static ssize_t mipsnet_receive(NetClientState *nc, const uint8_t *buf, size_t si
     return size;
 }
 
-static uint64_t mipsnet_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mipsnet_ioport_read(void *opaque, hwaddr addr,
                                     unsigned int size)
 {
     MIPSnetState *s = opaque;
@@ -142,7 +142,7 @@ static uint64_t mipsnet_ioport_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void mipsnet_ioport_write(void *opaque, target_phys_addr_t addr,
+static void mipsnet_ioport_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned int size)
 {
     MIPSnetState *s = opaque;
@@ -269,7 +269,7 @@ static void mipsnet_class_init(ObjectClass *klass, void *data)
     dc->props = mipsnet_properties;
 }
 
-static TypeInfo mipsnet_info = {
+static const TypeInfo mipsnet_info = {
     .name          = "mipsnet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MIPSnetState),
diff --git a/hw/mpc8544_guts.c b/hw/mpc8544_guts.c
index 13b0dddc1e4..728723c946c 100644
--- a/hw/mpc8544_guts.c
+++ b/hw/mpc8544_guts.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 
 #define MPC8544_GUTS_MMIO_SIZE        0x1000
@@ -58,7 +58,7 @@ struct GutsState {
 
 typedef struct GutsState GutsState;
 
-static uint64_t mpc8544_guts_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mpc8544_guts_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     uint32_t value = 0;
@@ -80,7 +80,7 @@ static uint64_t mpc8544_guts_read(void *opaque, target_phys_addr_t addr,
     return value;
 }
 
-static void mpc8544_guts_write(void *opaque, target_phys_addr_t addr,
+static void mpc8544_guts_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     addr &= MPC8544_GUTS_MMIO_SIZE - 1;
@@ -112,7 +112,7 @@ static int mpc8544_guts_initfn(SysBusDevice *dev)
 {
     GutsState *s;
 
-    s = FROM_SYSBUS(GutsState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(GutsState, SYS_BUS_DEVICE(dev));
 
     memory_region_init_io(&s->iomem, &mpc8544_guts_ops, s,
                           "mpc6544.guts", MPC8544_GUTS_MMIO_SIZE);
@@ -128,7 +128,7 @@ static void mpc8544_guts_class_init(ObjectClass *klass, void *data)
     k->init = mpc8544_guts_initfn;
 }
 
-static TypeInfo mpc8544_guts_info = {
+static const TypeInfo mpc8544_guts_info = {
     .name          = "mpc8544-guts",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GutsState),
diff --git a/hw/msmouse.c b/hw/msmouse.c
index 9c492a46371..ef47aed4e92 100644
--- a/hw/msmouse.c
+++ b/hw/msmouse.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include <stdlib.h>
-#include "../qemu-common.h"
-#include "../qemu-char.h"
-#include "../console.h"
+#include "qemu-common.h"
+#include "char/char.h"
+#include "ui/console.h"
 #include "msmouse.h"
 
 #define MSMOUSE_LO6(n) ((n) & 0x3f)
diff --git a/hw/msmouse.h b/hw/msmouse.h
index 456cb214241..8cff3a71c33 100644
--- a/hw/msmouse.h
+++ b/hw/msmouse.h
@@ -1,2 +1,7 @@
+#ifndef HW_MSMOUSE_H
+#define HW_MSMOUSE_H 1
+
 /* msmouse.c */
 CharDriverState *qemu_chr_open_msmouse(QemuOpts *opts);
+
+#endif
diff --git a/hw/mst_fpga.c b/hw/mst_fpga.c
index 024192d135a..7ae05e389f8 100644
--- a/hw/mst_fpga.c
+++ b/hw/mst_fpga.c
@@ -91,7 +91,7 @@ mst_fpga_set_irq(void *opaque, int irq, int level)
 
 
 static uint64_t
-mst_fpga_readb(void *opaque, target_phys_addr_t addr, unsigned size)
+mst_fpga_readb(void *opaque, hwaddr addr, unsigned size)
 {
 	mst_irq_state *s = (mst_irq_state *) opaque;
 
@@ -128,7 +128,7 @@ mst_fpga_readb(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-mst_fpga_writeb(void *opaque, target_phys_addr_t addr, uint64_t value,
+mst_fpga_writeb(void *opaque, hwaddr addr, uint64_t value,
 		unsigned size)
 {
 	mst_irq_state *s = (mst_irq_state *) opaque;
@@ -248,7 +248,7 @@ static void mst_fpga_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_mst_fpga_regs;
 }
 
-static TypeInfo mst_fpga_info = {
+static const TypeInfo mst_fpga_info = {
     .name          = "mainstone-fpga",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mst_irq_state),
diff --git a/hw/multiboot.c b/hw/multiboot.c
index b1e04c57180..c4ec2e34a7c 100644
--- a/hw/multiboot.c
+++ b/hw/multiboot.c
@@ -27,7 +27,7 @@
 #include "multiboot.h"
 #include "loader.h"
 #include "elf.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* Show multiboot debug output */
 //#define DEBUG_MULTIBOOT
@@ -80,15 +80,15 @@ typedef struct {
     /* buffer holding kernel, cmdlines and mb_infos */
     void *mb_buf;
     /* address in target */
-    target_phys_addr_t mb_buf_phys;
+    hwaddr mb_buf_phys;
     /* size of mb_buf in bytes */
     unsigned mb_buf_size;
     /* offset of mb-info's in bytes */
-    target_phys_addr_t offset_mbinfo;
+    hwaddr offset_mbinfo;
     /* offset in buffer for cmdlines in bytes */
-    target_phys_addr_t offset_cmdlines;
+    hwaddr offset_cmdlines;
     /* offset of modules in bytes */
-    target_phys_addr_t offset_mods;
+    hwaddr offset_mods;
     /* available slots for mb modules infos */
     int mb_mods_avail;
     /* currently used slots of mb modules */
@@ -97,7 +97,7 @@ typedef struct {
 
 static uint32_t mb_add_cmdline(MultibootState *s, const char *cmdline)
 {
-    target_phys_addr_t p = s->offset_cmdlines;
+    hwaddr p = s->offset_cmdlines;
     char *b = (char *)s->mb_buf + p;
 
     get_opt_value(b, strlen(cmdline) + 1, cmdline);
@@ -106,8 +106,8 @@ static uint32_t mb_add_cmdline(MultibootState *s, const char *cmdline)
 }
 
 static void mb_add_mod(MultibootState *s,
-                       target_phys_addr_t start, target_phys_addr_t end,
-                       target_phys_addr_t cmdline_phys)
+                       hwaddr start, hwaddr end,
+                       hwaddr cmdline_phys)
 {
     char *p;
     assert(s->mb_mods_count < s->mb_mods_avail);
@@ -276,7 +276,7 @@ int load_multiboot(void *fw_cfg,
             *next_initrd = '\0';
             /* if a space comes after the module filename, treat everything
                after that as parameters */
-            target_phys_addr_t c = mb_add_cmdline(&mbs, initrd_filename);
+            hwaddr c = mb_add_cmdline(&mbs, initrd_filename);
             if ((next_space = strchr(initrd_filename, ' ')))
                 *next_space = '\0';
             mb_debug("multiboot loading module: %s\n", initrd_filename);
diff --git a/hw/musicpal.c b/hw/musicpal.c
index ad725b55998..7ac0a918fbd 100644
--- a/hw/musicpal.c
+++ b/hw/musicpal.c
@@ -12,18 +12,19 @@
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "pc.h"
-#include "qemu-timer.h"
+#include "serial.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
-#include "console.h"
+#include "ui/console.h"
 #include "i2c.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
+#include "ui/pixel_ops.h"
 
 #define MP_MISC_BASE            0x80002000
 #define MP_MISC_SIZE            0x00001000
@@ -266,7 +267,7 @@ static void eth_send(mv88w8618_eth_state *s, int queue_index)
     } while (desc_addr != s->tx_queue[queue_index]);
 }
 
-static uint64_t mv88w8618_eth_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_eth_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_eth_state *s = opaque;
@@ -308,7 +309,7 @@ static uint64_t mv88w8618_eth_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_eth_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_eth_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_eth_state *s = opaque;
@@ -427,7 +428,7 @@ static void mv88w8618_eth_class_init(ObjectClass *klass, void *data)
     dc->props = mv88w8618_eth_properties;
 }
 
-static TypeInfo mv88w8618_eth_info = {
+static const TypeInfo mv88w8618_eth_info = {
     .name          = "mv88w8618_eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_eth_state),
@@ -492,8 +493,6 @@ SET_LCD_PIXEL(8, uint8_t)
 SET_LCD_PIXEL(16, uint16_t)
 SET_LCD_PIXEL(32, uint32_t)
 
-#include "pixel_ops.h"
-
 static void lcd_refresh(void *opaque)
 {
     musicpal_lcd_state *s = opaque;
@@ -526,7 +525,7 @@ static void lcd_refresh(void *opaque)
                   ds_get_bits_per_pixel(s->ds));
     }
 
-    dpy_update(s->ds, 0, 0, 128*3, 64*3);
+    dpy_gfx_update(s->ds, 0, 0, 128*3, 64*3);
 }
 
 static void lcd_invalidate(void *opaque)
@@ -540,7 +539,7 @@ static void musicpal_lcd_gpio_brigthness_in(void *opaque, int irq, int level)
     s->brightness |= level << irq;
 }
 
-static uint64_t musicpal_lcd_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_lcd_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     musicpal_lcd_state *s = opaque;
@@ -554,7 +553,7 @@ static uint64_t musicpal_lcd_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_lcd_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_lcd_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     musicpal_lcd_state *s = opaque;
@@ -644,7 +643,7 @@ static void musicpal_lcd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_lcd_vmsd;
 }
 
-static TypeInfo musicpal_lcd_info = {
+static const TypeInfo musicpal_lcd_info = {
     .name          = "musicpal_lcd",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_lcd_state),
@@ -682,7 +681,7 @@ static void mv88w8618_pic_set_irq(void *opaque, int irq, int level)
     mv88w8618_pic_update(s);
 }
 
-static uint64_t mv88w8618_pic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_pic_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_pic_state *s = opaque;
@@ -696,7 +695,7 @@ static uint64_t mv88w8618_pic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_pic_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_pic_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_pic_state *s = opaque;
@@ -717,7 +716,7 @@ static void mv88w8618_pic_write(void *opaque, target_phys_addr_t offset,
 static void mv88w8618_pic_reset(DeviceState *d)
 {
     mv88w8618_pic_state *s = FROM_SYSBUS(mv88w8618_pic_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
 
     s->level = 0;
     s->enabled = 0;
@@ -763,7 +762,7 @@ static void mv88w8618_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_pic_vmsd;
 }
 
-static TypeInfo mv88w8618_pic_info = {
+static const TypeInfo mv88w8618_pic_info = {
     .name          = "mv88w8618_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_pic_state),
@@ -815,7 +814,7 @@ static void mv88w8618_timer_init(SysBusDevice *dev, mv88w8618_timer_state *s,
     s->ptimer = ptimer_init(bh);
 }
 
-static uint64_t mv88w8618_pit_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_pit_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_pit_state *s = opaque;
@@ -831,7 +830,7 @@ static uint64_t mv88w8618_pit_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_pit_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_pit_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_pit_state *s = opaque;
@@ -874,7 +873,7 @@ static void mv88w8618_pit_write(void *opaque, target_phys_addr_t offset,
 static void mv88w8618_pit_reset(DeviceState *d)
 {
     mv88w8618_pit_state *s = FROM_SYSBUS(mv88w8618_pit_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
     int i;
 
     for (i = 0; i < 4; i++) {
@@ -940,7 +939,7 @@ static void mv88w8618_pit_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_pit_vmsd;
 }
 
-static TypeInfo mv88w8618_pit_info = {
+static const TypeInfo mv88w8618_pit_info = {
     .name          = "mv88w8618_pit",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_pit_state),
@@ -957,7 +956,7 @@ typedef struct mv88w8618_flashcfg_state {
 } mv88w8618_flashcfg_state;
 
 static uint64_t mv88w8618_flashcfg_read(void *opaque,
-                                        target_phys_addr_t offset,
+                                        hwaddr offset,
                                         unsigned size)
 {
     mv88w8618_flashcfg_state *s = opaque;
@@ -971,7 +970,7 @@ static uint64_t mv88w8618_flashcfg_read(void *opaque,
     }
 }
 
-static void mv88w8618_flashcfg_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_flashcfg_write(void *opaque, hwaddr offset,
                                      uint64_t value, unsigned size)
 {
     mv88w8618_flashcfg_state *s = opaque;
@@ -1020,7 +1019,7 @@ static void mv88w8618_flashcfg_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_flashcfg_vmsd;
 }
 
-static TypeInfo mv88w8618_flashcfg_info = {
+static const TypeInfo mv88w8618_flashcfg_info = {
     .name          = "mv88w8618_flashcfg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_flashcfg_state),
@@ -1032,7 +1031,7 @@ static TypeInfo mv88w8618_flashcfg_info = {
 
 #define MP_BOARD_REVISION       0x31
 
-static uint64_t musicpal_misc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_misc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     switch (offset) {
@@ -1044,7 +1043,7 @@ static uint64_t musicpal_misc_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_misc_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_misc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
 }
@@ -1068,7 +1067,7 @@ static void musicpal_misc_init(SysBusDevice *dev)
 #define MP_WLAN_MAGIC1          0x11c
 #define MP_WLAN_MAGIC2          0x124
 
-static uint64_t mv88w8618_wlan_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_wlan_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     switch (offset) {
@@ -1084,7 +1083,7 @@ static uint64_t mv88w8618_wlan_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_wlan_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_wlan_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
 }
@@ -1202,7 +1201,7 @@ static void musicpal_gpio_pin_event(void *opaque, int pin, int level)
     }
 }
 
-static uint64_t musicpal_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_gpio_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     musicpal_gpio_state *s = opaque;
@@ -1241,7 +1240,7 @@ static uint64_t musicpal_gpio_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_gpio_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_gpio_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     musicpal_gpio_state *s = opaque;
@@ -1289,7 +1288,7 @@ static const MemoryRegionOps musicpal_gpio_ops = {
 static void musicpal_gpio_reset(DeviceState *d)
 {
     musicpal_gpio_state *s = FROM_SYSBUS(musicpal_gpio_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
 
     s->lcd_brightness = 0;
     s->out_state = 0;
@@ -1342,7 +1341,7 @@ static void musicpal_gpio_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_gpio_vmsd;
 }
 
-static TypeInfo musicpal_gpio_info = {
+static const TypeInfo musicpal_gpio_info = {
     .name          = "musicpal_gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_gpio_state),
@@ -1496,7 +1495,7 @@ static void musicpal_key_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_key_vmsd;
 }
 
-static TypeInfo musicpal_key_info = {
+static const TypeInfo musicpal_key_info = {
     .name          = "musicpal_key",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_key_state),
@@ -1508,11 +1507,12 @@ static struct arm_boot_info musicpal_binfo = {
     .board_id = 0x20e,
 };
 
-static void musicpal_init(ram_addr_t ram_size,
-               const char *boot_device,
-               const char *kernel_filename, const char *kernel_cmdline,
-               const char *initrd_filename, const char *cpu_model)
+static void musicpal_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     qemu_irq *cpu_pic;
     qemu_irq pic[32];
@@ -1583,7 +1583,7 @@ static void musicpal_init(ram_addr_t ram_size,
          * image is smaller than 32 MB.
          */
 #ifdef TARGET_WORDS_BIGENDIAN
-        pflash_cfi02_register(0-MP_FLASH_SIZE_MAX, NULL,
+        pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
                               dinfo->bdrv, 0x10000,
                               (flash_size + 0xffff) >> 16,
@@ -1591,7 +1591,7 @@ static void musicpal_init(ram_addr_t ram_size,
                               2, 0x00BF, 0x236D, 0x0000, 0x0000,
                               0x5555, 0x2AAA, 1);
 #else
-        pflash_cfi02_register(0-MP_FLASH_SIZE_MAX, NULL,
+        pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
                               dinfo->bdrv, 0x10000,
                               (flash_size + 0xffff) >> 16,
@@ -1607,12 +1607,12 @@ static void musicpal_init(ram_addr_t ram_size,
     dev = qdev_create(NULL, "mv88w8618_eth");
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, MP_ETH_BASE);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[MP_ETH_IRQ]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, MP_ETH_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[MP_ETH_IRQ]);
 
     sysbus_create_simple("mv88w8618_wlan", MP_WLAN_BASE, NULL);
 
-    musicpal_misc_init(sysbus_from_qdev(dev));
+    musicpal_misc_init(SYS_BUS_DEVICE(dev));
 
     dev = sysbus_create_simple("musicpal_gpio", MP_GPIO_BASE, pic[MP_GPIO_IRQ]);
     i2c_dev = sysbus_create_simple("gpio_i2c", -1, NULL);
@@ -1641,7 +1641,7 @@ static void musicpal_init(ram_addr_t ram_size,
 
     wm8750_dev = i2c_create_slave(i2c, "wm8750", MP_WM_ADDR);
     dev = qdev_create(NULL, "mv88w8618_audio");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_prop_set_ptr(dev, "wm8750", wm8750_dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(s, 0, MP_AUDIO_BASE);
@@ -1658,6 +1658,7 @@ static QEMUMachine musicpal_machine = {
     .name = "musicpal",
     .desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)",
     .init = musicpal_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void musicpal_machine_init(void)
@@ -1674,7 +1675,7 @@ static void mv88w8618_wlan_class_init(ObjectClass *klass, void *data)
     sdc->init = mv88w8618_wlan_init;
 }
 
-static TypeInfo mv88w8618_wlan_info = {
+static const TypeInfo mv88w8618_wlan_info = {
     .name          = "mv88w8618_wlan",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
diff --git a/hw/nand.c b/hw/nand.c
index e9501ae038c..4a71265ed3c 100644
--- a/hw/nand.c
+++ b/hw/nand.c
@@ -20,9 +20,9 @@
 
 # include "hw.h"
 # include "flash.h"
-# include "blockdev.h"
+# include "sysemu/blockdev.h"
 # include "sysbus.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 # define NAND_CMD_READ0		0x00
 # define NAND_CMD_READ1		0x01
@@ -224,7 +224,7 @@ static const struct {
 
 static void nand_reset(DeviceState *dev)
 {
-    NANDFlashState *s = FROM_SYSBUS(NANDFlashState, sysbus_from_qdev(dev));
+    NANDFlashState *s = FROM_SYSBUS(NANDFlashState, SYS_BUS_DEVICE(dev));
     s->cmd = NAND_CMD_READ0;
     s->addr = 0;
     s->addrlen = 0;
@@ -435,7 +435,7 @@ static void nand_class_init(ObjectClass *klass, void *data)
     dc->props = nand_properties;
 }
 
-static TypeInfo nand_info = {
+static const TypeInfo nand_info = {
     .name          = "nand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(NANDFlashState),
@@ -654,7 +654,7 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
         sector = SECTOR(s->addr);
         off = (s->addr & PAGE_MASK) + s->offset;
         soff = SECTOR_OFFSET(s->addr);
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS) == -1) {
+        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
@@ -666,21 +666,23 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
                             MIN(OOB_SIZE, off + s->iolen - PAGE_SIZE));
         }
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS) == -1)
+        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
+        }
     } else {
         off = PAGE_START(s->addr) + (s->addr & PAGE_MASK) + s->offset;
         sector = off >> 9;
         soff = off & 0x1ff;
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) == -1) {
+        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
 
         mem_and(iobuf + soff, s->io, s->iolen);
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) == -1)
+        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
+        }
     }
     s->offset = 0;
 }
@@ -704,31 +706,37 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         i = SECTOR(addr);
         page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
         for (; i < page; i ++)
-            if (bdrv_write(s->bdrv, i, iobuf, 1) == -1)
+            if (bdrv_write(s->bdrv, i, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
+            }
     } else {
         addr = PAGE_START(addr);
         page = addr >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
+        }
         memset(iobuf + (addr & 0x1ff), 0xff, (~addr & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
+        }
 
         memset(iobuf, 0xff, 0x200);
         i = (addr & ~0x1ff) + 0x200;
         for (addr += ((PAGE_SIZE + OOB_SIZE) << s->erase_shift) - 0x200;
                         i < addr; i += 0x200)
-            if (bdrv_write(s->bdrv, i >> 9, iobuf, 1) == -1)
+            if (bdrv_write(s->bdrv, i >> 9, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n",
                        __func__, i >> 9);
+            }
 
         page = i >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
+        }
         memset(iobuf, 0xff, ((addr - 1) & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
+        }
     }
 }
 
@@ -740,18 +748,20 @@ static void glue(nand_blk_load_, PAGE_SIZE)(NANDFlashState *s,
 
     if (s->bdrv) {
         if (s->mem_oob) {
-            if (bdrv_read(s->bdrv, SECTOR(addr), s->io, PAGE_SECTORS) == -1)
+            if (bdrv_read(s->bdrv, SECTOR(addr), s->io, PAGE_SECTORS) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, SECTOR(addr));
+            }
             memcpy(s->io + SECTOR_OFFSET(s->addr) + PAGE_SIZE,
                             s->storage + (PAGE(s->addr) << OOB_SHIFT),
                             OOB_SIZE);
             s->ioaddr = s->io + SECTOR_OFFSET(s->addr) + offset;
         } else {
             if (bdrv_read(s->bdrv, PAGE_START(addr) >> 9,
-                                    s->io, (PAGE_SECTORS + 2)) == -1)
+                                    s->io, (PAGE_SECTORS + 2)) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, PAGE_START(addr) >> 9);
+            }
             s->ioaddr = s->io + (PAGE_START(addr) & 0x1ff) + offset;
         }
     } else {
diff --git a/hw/ne2000-isa.c b/hw/ne2000-isa.c
index 69982a9abbb..7c11229f1a7 100644
--- a/hw/ne2000-isa.c
+++ b/hw/ne2000-isa.c
@@ -25,9 +25,9 @@
 #include "pc.h"
 #include "isa.h"
 #include "qdev.h"
-#include "net.h"
+#include "net/net.h"
 #include "ne2000.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct ISANE2000State {
     ISADevice dev;
@@ -97,7 +97,7 @@ static void isa_ne2000_class_initfn(ObjectClass *klass, void *data)
     dc->props = ne2000_isa_properties;
 }
 
-static TypeInfo ne2000_isa_info = {
+static const TypeInfo ne2000_isa_info = {
     .name          = "ne2k_isa",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISANE2000State),
diff --git a/hw/ne2000.c b/hw/ne2000.c
index 15605c478f6..872115c4546 100644
--- a/hw/ne2000.c
+++ b/hw/ne2000.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "ne2000.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* debug NE2000 card */
 //#define DEBUG_NE2000
@@ -652,7 +652,7 @@ static const VMStateDescription vmstate_pci_ne2000 = {
     }
 };
 
-static uint64_t ne2000_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ne2000_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     NE2000State *s = opaque;
@@ -671,7 +671,7 @@ static uint64_t ne2000_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void ne2000_write(void *opaque, target_phys_addr_t addr,
+static void ne2000_write(void *opaque, hwaddr addr,
                          uint64_t data, unsigned size)
 {
     NE2000State *s = opaque;
@@ -773,7 +773,7 @@ static void ne2000_class_init(ObjectClass *klass, void *data)
     dc->props = ne2000_properties;
 }
 
-static TypeInfo ne2000_info = {
+static const TypeInfo ne2000_info = {
     .name          = "ne2k_pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCINE2000State),
diff --git a/hw/ne2000.h b/hw/ne2000.h
index 1e7ab073e35..b31ae030f9a 100644
--- a/hw/ne2000.h
+++ b/hw/ne2000.h
@@ -1,3 +1,6 @@
+#ifndef HW_NE2000_H
+#define HW_NE2000_H 1
+
 #define NE2000_PMEM_SIZE    (32*1024)
 #define NE2000_PMEM_START   (16*1024)
 #define NE2000_PMEM_END     (NE2000_PMEM_SIZE+NE2000_PMEM_START)
@@ -33,3 +36,5 @@ extern const VMStateDescription vmstate_ne2000;
 void ne2000_reset(NE2000State *s);
 int ne2000_can_receive(NetClientState *nc);
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
+
+#endif
diff --git a/hw/nseries.c b/hw/nseries.c
index 4df2670327a..99d353aaa97 100644
--- a/hw/nseries.c
+++ b/hw/nseries.c
@@ -19,11 +19,11 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "omap.h"
 #include "arm-misc.h"
 #include "irq.h"
-#include "console.h"
+#include "ui/console.h"
 #include "boards.h"
 #include "i2c.h"
 #include "devices.h"
@@ -31,9 +31,9 @@
 #include "hw.h"
 #include "bt.h"
 #include "loader.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 /* Nokia N8x0 support */
 struct n800_s {
@@ -178,10 +178,10 @@ static void n8x0_nand_setup(struct n800_s *s)
         qdev_prop_set_drive_nofail(s->nand, "drive", dinfo->bdrv);
     }
     qdev_init_nofail(s->nand);
-    sysbus_connect_irq(sysbus_from_qdev(s->nand), 0,
+    sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
                        qdev_get_gpio_in(s->mpu->gpio, N8X0_ONENAND_GPIO));
     omap_gpmc_attach(s->mpu->gpmc, N8X0_ONENAND_CS,
-                     sysbus_mmio_get_region(sysbus_from_qdev(s->nand), 0));
+                     sysbus_mmio_get_region(SYS_BUS_DEVICE(s->nand), 0));
     otp_region = onenand_raw_otp(s->nand);
 
     memcpy(otp_region + 0x000, n8x0_cal_wlan_mac, sizeof(n8x0_cal_wlan_mac));
@@ -189,6 +189,17 @@ static void n8x0_nand_setup(struct n800_s *s)
     /* XXX: in theory should also update the OOB for both pages */
 }
 
+static qemu_irq n8x0_system_powerdown;
+
+static void n8x0_powerdown_req(Notifier *n, void *opaque)
+{
+    qemu_irq_raise(n8x0_system_powerdown);
+}
+
+static Notifier n8x0_system_powerdown_notifier = {
+    .notify = n8x0_powerdown_req
+};
+
 static void n8x0_i2c_setup(struct n800_s *s)
 {
     DeviceState *dev;
@@ -201,7 +212,8 @@ static void n8x0_i2c_setup(struct n800_s *s)
                           qdev_get_gpio_in(s->mpu->ih[0],
                                            OMAP_INT_24XX_SYS_NIRQ));
 
-    qemu_system_powerdown = qdev_get_gpio_in(dev, 3);
+    n8x0_system_powerdown = qdev_get_gpio_in(dev, 3);
+    qemu_register_powerdown_notifier(&n8x0_system_powerdown_notifier);
 
     /* Attach a TMP105 PM chip (A0 wired to ground) */
     dev = i2c_create_slave(i2c, "tmp105", N8X0_TMP105_ADDR);
@@ -771,7 +783,7 @@ static void n8x0_usb_setup(struct n800_s *s)
 {
     SysBusDevice *dev;
     s->usb = qdev_create(NULL, "tusb6010");
-    dev = sysbus_from_qdev(s->usb);
+    dev = SYS_BUS_DEVICE(s->usb);
     qdev_init_nofail(s->usb);
     sysbus_connect_irq(dev, 0,
                        qdev_get_gpio_in(s->mpu->gpio, N8X0_TUSB_INT_GPIO));
@@ -1272,17 +1284,15 @@ static int n810_atag_setup(const struct arm_boot_info *info, void *p)
     return n8x0_atag_setup(p, 810);
 }
 
-static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, struct arm_boot_info *binfo, int model)
+static void n8x0_init(QEMUMachineInitArgs *args,
+                      struct arm_boot_info *binfo, int model)
 {
     MemoryRegion *sysmem = get_system_memory();
     struct n800_s *s = (struct n800_s *) g_malloc0(sizeof(*s));
     int sdram_size = binfo->ram_size;
     DisplayState *ds;
 
-    s->mpu = omap2420_mpu_init(sysmem, sdram_size, cpu_model);
+    s->mpu = omap2420_mpu_init(sysmem, sdram_size, args->cpu_model);
 
     /* Setup peripherals
      *
@@ -1322,20 +1332,22 @@ static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
     n8x0_dss_setup(s);
     n8x0_cbus_setup(s);
     n8x0_uart_setup(s);
-    if (usb_enabled)
+    if (usb_enabled(false)) {
         n8x0_usb_setup(s);
+    }
 
-    if (kernel_filename) {
+    if (args->kernel_filename) {
         /* Or at the linux loader.  */
-        binfo->kernel_filename = kernel_filename;
-        binfo->kernel_cmdline = kernel_cmdline;
-        binfo->initrd_filename = initrd_filename;
+        binfo->kernel_filename = args->kernel_filename;
+        binfo->kernel_cmdline = args->kernel_cmdline;
+        binfo->initrd_filename = args->initrd_filename;
         arm_load_kernel(s->mpu->cpu, binfo);
 
         qemu_register_reset(n8x0_boot_init, s);
     }
 
-    if (option_rom[0].name && (boot_device[0] == 'n' || !kernel_filename)) {
+    if (option_rom[0].name &&
+        (args->boot_device[0] == 'n' || !args->kernel_filename)) {
         int rom_size;
         uint8_t nolo_tags[0x10000];
         /* No, wait, better start at the ROM.  */
@@ -1363,7 +1375,7 @@ static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
        size until the guest activates the display.  */
     ds = get_displaystate();
     ds->surface = qemu_resize_displaysurface(ds, 800, 480);
-    dpy_resize(ds);
+    dpy_gfx_resize(ds);
 }
 
 static struct arm_boot_info n800_binfo = {
@@ -1385,36 +1397,28 @@ static struct arm_boot_info n810_binfo = {
     .atag_board = n810_atag_setup,
 };
 
-static void n800_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void n800_init(QEMUMachineInitArgs *args)
 {
-    return n8x0_init(ram_size, boot_device,
-                    kernel_filename, kernel_cmdline, initrd_filename,
-                    cpu_model, &n800_binfo, 800);
+    return n8x0_init(args, &n800_binfo, 800);
 }
 
-static void n810_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void n810_init(QEMUMachineInitArgs *args)
 {
-    return n8x0_init(ram_size, boot_device,
-                    kernel_filename, kernel_cmdline, initrd_filename,
-                    cpu_model, &n810_binfo, 810);
+    return n8x0_init(args, &n810_binfo, 810);
 }
 
 static QEMUMachine n800_machine = {
     .name = "n800",
     .desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)",
     .init = n800_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine n810_machine = {
     .name = "n810",
     .desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)",
     .init = n810_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void nseries_machine_init(void)
diff --git a/hw/null-machine.c b/hw/null-machine.c
index 69910d399ce..bdf109fef1e 100644
--- a/hw/null-machine.c
+++ b/hw/null-machine.c
@@ -15,12 +15,7 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
 
-static void machine_none_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void machine_none_init(QEMUMachineInitArgs *args)
 {
 }
 
@@ -29,6 +24,7 @@ static QEMUMachine machine_none = {
     .desc = "empty machine",
     .init = machine_none_init,
     .max_cpus = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void register_machines(void)
diff --git a/hw/nvram.h b/hw/nvram.h
index 8924da47d81..59337faaadd 100644
--- a/hw/nvram.h
+++ b/hw/nvram.h
@@ -10,17 +10,9 @@ typedef struct nvram_t {
     nvram_write_t write_fn;
 } nvram_t;
 
-void NVRAM_set_byte (nvram_t *nvram, uint32_t addr, uint8_t value);
-uint8_t NVRAM_get_byte (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_word (nvram_t *nvram, uint32_t addr, uint16_t value);
-uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_lword (nvram_t *nvram, uint32_t addr, uint32_t value);
 uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_string (nvram_t *nvram, uint32_t addr,
-                       const char *str, uint32_t max);
 int NVRAM_get_string (nvram_t *nvram, uint8_t *dst, uint16_t addr, int max);
-void NVRAM_set_crc (nvram_t *nvram, uint32_t addr,
-                    uint32_t start, uint32_t count);
+
 int PPC_NVRAM_set_params (nvram_t *nvram, uint16_t NVRAM_size,
                           const char *arch,
                           uint32_t RAM_size, int boot_device,
@@ -36,8 +28,7 @@ uint32_t m48t59_read (void *private, uint32_t addr);
 void m48t59_toggle_lock (void *private, int lock);
 M48t59State *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
                              int type);
-M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
+M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
                          uint32_t io_base, uint16_t size, int type);
-void m48t59_set_addr (void *opaque, uint32_t addr);
 
 #endif /* !NVRAM_H */
diff --git a/hw/omap.h b/hw/omap.h
index 413851bc342..188cda8771e 100644
--- a/hw/omap.h
+++ b/hw/omap.h
@@ -17,8 +17,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef hw_omap_h
-#include "memory.h"
+#include "exec/memory.h"
 # define hw_omap_h		"omap.h"
+#include "hw/irq.h"
 
 # define OMAP_EMIFS_BASE	0x00000000
 # define OMAP2_Q0_BASE		0x00000000
@@ -65,7 +66,7 @@ void omap_clk_reparent(omap_clk clk, omap_clk parent);
 /* OMAP2 l4 Interconnect */
 struct omap_l4_s;
 struct omap_l4_region_s {
-    target_phys_addr_t offset;
+    hwaddr offset;
     size_t size;
     int access;
 };
@@ -80,13 +81,13 @@ struct omap_target_agent_s {
     struct omap_l4_s *bus;
     int regions;
     const struct omap_l4_region_s *start;
-    target_phys_addr_t base;
+    hwaddr base;
     uint32_t component;
     uint32_t control;
     uint32_t status;
 };
 struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
-                               target_phys_addr_t base, int ta_num);
+                               hwaddr base, int ta_num);
 
 struct omap_target_agent_s;
 struct omap_target_agent_s *omap_l4ta_get(
@@ -94,23 +95,23 @@ struct omap_target_agent_s *omap_l4ta_get(
     const struct omap_l4_region_s *regions,
     const struct omap_l4_agent_info_s *agents,
     int cs);
-target_phys_addr_t omap_l4_attach(struct omap_target_agent_s *ta,
+hwaddr omap_l4_attach(struct omap_target_agent_s *ta,
                                          int region, MemoryRegion *mr);
-target_phys_addr_t omap_l4_region_base(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_base(struct omap_target_agent_s *ta,
                                        int region);
-target_phys_addr_t omap_l4_region_size(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_size(struct omap_target_agent_s *ta,
                                        int region);
 
 /* OMAP2 SDRAM controller */
 struct omap_sdrc_s;
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
-                                   target_phys_addr_t base);
+                                   hwaddr base);
 void omap_sdrc_reset(struct omap_sdrc_s *s);
 
 /* OMAP2 general purpose memory controller */
 struct omap_gpmc_s;
 struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
-                                   target_phys_addr_t base,
+                                   hwaddr base,
                                    qemu_irq irq, qemu_irq drq);
 void omap_gpmc_reset(struct omap_gpmc_s *s);
 void omap_gpmc_attach(struct omap_gpmc_s *s, int cs, MemoryRegion *iomem);
@@ -433,11 +434,11 @@ enum omap_dma_model {
 };
 
 struct soc_dma_s;
-struct soc_dma_s *omap_dma_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 qemu_irq lcd_irq, struct omap_mpu_state_s *mpu, omap_clk clk,
                 enum omap_dma_model model);
-struct soc_dma_s *omap_dma4_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 struct omap_mpu_state_s *mpu, int fifo,
                 int chans, omap_clk iclk, omap_clk fclk);
@@ -469,10 +470,10 @@ typedef enum {
 /* Only used in OMAP DMA 3.x gigacells */
 struct omap_dma_lcd_channel_s {
     enum omap_dma_port src;
-    target_phys_addr_t src_f1_top;
-    target_phys_addr_t src_f1_bottom;
-    target_phys_addr_t src_f2_top;
-    target_phys_addr_t src_f2_bottom;
+    hwaddr src_f1_top;
+    hwaddr src_f1_bottom;
+    hwaddr src_f2_top;
+    hwaddr src_f2_bottom;
 
     /* Used in OMAP DMA 3.2 gigacell */
     unsigned char brust_f1;
@@ -508,7 +509,7 @@ struct omap_dma_lcd_channel_s {
     int dual;
 
     int current_frame;
-    target_phys_addr_t phys_framebuffer[2];
+    hwaddr phys_framebuffer[2];
     qemu_irq irq;
     struct omap_mpu_state_s *mpu;
 } *omap_dma_get_lcdch(struct soc_dma_s *s);
@@ -659,7 +660,7 @@ struct omap_synctimer_s *omap_synctimer_init(struct omap_target_agent_s *ta,
 void omap_synctimer_reset(struct omap_synctimer_s *s);
 
 struct omap_uart_s;
-struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
+struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr);
@@ -728,7 +729,7 @@ void omap_tap_init(struct omap_target_agent_s *ta,
 struct omap_lcd_panel_s;
 void omap_lcdc_reset(struct omap_lcd_panel_s *s);
 struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq irq,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk);
@@ -744,7 +745,7 @@ struct omap_dss_s;
 void omap_dss_reset(struct omap_dss_s *s);
 struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t l3_base,
+                hwaddr l3_base,
                 qemu_irq irq, qemu_irq drq,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2);
@@ -752,7 +753,7 @@ void omap_rfbi_attach(struct omap_dss_s *s, int cs, struct rfbi_chip_s *chip);
 
 /* omap_mmc.c */
 struct omap_mmc_s;
-struct omap_mmc_s *omap_mmc_init(target_phys_addr_t base,
+struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
                 BlockDriverState *bd,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk);
@@ -829,11 +830,11 @@ struct omap_mpu_state_s {
 
     struct omap_dma_port_if_s {
         uint32_t (*read[3])(struct omap_mpu_state_s *s,
-                        target_phys_addr_t offset);
+                        hwaddr offset);
         void (*write[3])(struct omap_mpu_state_s *s,
-                        target_phys_addr_t offset, uint32_t value);
+                        hwaddr offset, uint32_t value);
         int (*addr_valid)(struct omap_mpu_state_s *s,
-                        target_phys_addr_t addr);
+                        hwaddr addr);
     } port[__omap_dma_port_last];
 
     unsigned long sdram_size;
@@ -942,16 +943,16 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                 unsigned long sdram_size,
                 const char *core);
 
-#define OMAP_FMT_plx "%#08" TARGET_PRIxPHYS
+#define OMAP_FMT_plx "%#08" HWADDR_PRIx
 
-uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read8(void *opaque, hwaddr addr);
+void omap_badwidth_write8(void *opaque, hwaddr addr,
                 uint32_t value);
-uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read16(void *opaque, hwaddr addr);
+void omap_badwidth_write16(void *opaque, hwaddr addr,
                 uint32_t value);
-uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write32(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read32(void *opaque, hwaddr addr);
+void omap_badwidth_write32(void *opaque, hwaddr addr,
                 uint32_t value);
 
 void omap_mpu_wakeup(void *opaque, int irq, int req);
diff --git a/hw/omap1.c b/hw/omap1.c
index ad60cc49195..623b101f801 100644
--- a/hw/omap1.c
+++ b/hw/omap1.c
@@ -19,14 +19,14 @@
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "soc_dma.h"
-#include "blockdev.h"
-#include "range.h"
+#include "sysemu/blockdev.h"
+#include "qemu/range.h"
 #include "sysbus.h"
 
 /* Should signal the TCMI/GPMC */
-uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read8(void *opaque, hwaddr addr)
 {
     uint8_t ret;
 
@@ -35,7 +35,7 @@ uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write8(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint8_t val8 = value;
@@ -44,7 +44,7 @@ void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
     cpu_physical_memory_write(addr, (void *) &val8, 1);
 }
 
-uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read16(void *opaque, hwaddr addr)
 {
     uint16_t ret;
 
@@ -53,7 +53,7 @@ uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write16(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint16_t val16 = value;
@@ -62,7 +62,7 @@ void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
     cpu_physical_memory_write(addr, (void *) &val16, 2);
 }
 
-uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read32(void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -71,7 +71,7 @@ uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write32(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write32(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAP_32B_REG(addr);
@@ -176,7 +176,7 @@ static void omap_timer_clk_setup(struct omap_mpu_timer_s *timer)
     timer->rate = omap_clk_getrate(timer->clk);
 }
 
-static uint64_t omap_mpu_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpu_timer_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *) opaque;
@@ -200,7 +200,7 @@ static uint64_t omap_mpu_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpu_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpu_timer_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *) opaque;
@@ -251,7 +251,7 @@ static void omap_mpu_timer_reset(struct omap_mpu_timer_s *s)
 }
 
 static struct omap_mpu_timer_s *omap_mpu_timer_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *)
@@ -282,7 +282,7 @@ struct omap_watchdog_timer_s {
     int reset;
 };
 
-static uint64_t omap_wd_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_wd_timer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *) opaque;
@@ -307,7 +307,7 @@ static uint64_t omap_wd_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_wd_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_wd_timer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *) opaque;
@@ -380,7 +380,7 @@ static void omap_wd_timer_reset(struct omap_watchdog_timer_s *s)
 }
 
 static struct omap_watchdog_timer_s *omap_wd_timer_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *)
@@ -405,7 +405,7 @@ struct omap_32khz_timer_s {
     MemoryRegion iomem;
 };
 
-static uint64_t omap_os_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_os_timer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *) opaque;
@@ -432,7 +432,7 @@ static uint64_t omap_os_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_os_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_os_timer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *) opaque;
@@ -486,7 +486,7 @@ static void omap_os_timer_reset(struct omap_32khz_timer_s *s)
 }
 
 static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *)
@@ -506,7 +506,7 @@ static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
 }
 
 /* Ultra Low-Power Device Module */
-static uint64_t omap_ulpd_pm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_ulpd_pm_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -529,6 +529,7 @@ static uint64_t omap_ulpd_pm_read(void *opaque, target_phys_addr_t addr,
     case 0x28:	/* Reserved */
     case 0x2c:	/* Reserved */
         OMAP_BAD_REG(addr);
+        /* fall through */
     case 0x00:	/* COUNTER_32_LSB */
     case 0x04:	/* COUNTER_32_MSB */
     case 0x08:	/* COUNTER_HIGH_FREQ_LSB */
@@ -573,7 +574,7 @@ static inline void omap_ulpd_req_update(struct omap_mpu_state_s *s,
         omap_clk_canidle(omap_findclk(s, "usb_clk0"), (~value >> 3) & 1);
 }
 
-static void omap_ulpd_pm_write(void *opaque, target_phys_addr_t addr,
+static void omap_ulpd_pm_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -633,6 +634,7 @@ static void omap_ulpd_pm_write(void *opaque, target_phys_addr_t addr,
     case 0x28:	/* Reserved */
     case 0x2c:	/* Reserved */
         OMAP_BAD_REG(addr);
+        /* fall through */
     case 0x24:	/* SETUP_ANALOG_CELL3_ULPD1 */
     case 0x38:	/* COUNTER_32_FIQ */
     case 0x48:	/* LOCL_TIME */
@@ -726,7 +728,7 @@ static void omap_ulpd_pm_reset(struct omap_mpu_state_s *mpu)
 }
 
 static void omap_ulpd_pm_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->ulpd_pm_iomem, &omap_ulpd_pm_ops, mpu,
@@ -736,7 +738,7 @@ static void omap_ulpd_pm_init(MemoryRegion *system_memory,
 }
 
 /* OMAP Pin Configuration */
-static uint64_t omap_pin_cfg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pin_cfg_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -843,7 +845,7 @@ static inline void omap_pin_modconf1_update(struct omap_mpu_state_s *s,
          omap_clk_onoff(omap_findclk(s, "usb_hhc_ck"), (value >> 9) & 1);
 }
 
-static void omap_pin_cfg_write(void *opaque, target_phys_addr_t addr,
+static void omap_pin_cfg_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -944,7 +946,7 @@ static void omap_pin_cfg_reset(struct omap_mpu_state_s *mpu)
 }
 
 static void omap_pin_cfg_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->pin_cfg_iomem, &omap_pin_cfg_ops, mpu,
@@ -954,7 +956,7 @@ static void omap_pin_cfg_init(MemoryRegion *system_memory,
 }
 
 /* Device Identification, Die Identification */
-static uint64_t omap_id_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_id_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1001,7 +1003,7 @@ static uint64_t omap_id_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_id_write(void *opaque, target_phys_addr_t addr,
+static void omap_id_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -1035,7 +1037,7 @@ static void omap_id_init(MemoryRegion *memory, struct omap_mpu_state_s *mpu)
 }
 
 /* MPUI Control (Dummy) */
-static uint64_t omap_mpui_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpui_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1068,7 +1070,7 @@ static uint64_t omap_mpui_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpui_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpui_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1089,6 +1091,7 @@ static void omap_mpui_write(void *opaque, target_phys_addr_t addr,
     /* Not in OMAP310 */
     case 0x14:	/* DSP_STATUS */
         OMAP_RO_REG(addr);
+        break;
     case 0x18:	/* DSP_BOOT_CONFIG */
     case 0x1c:	/* DSP_MPUI_CONFIG */
         break;
@@ -1109,7 +1112,7 @@ static void omap_mpui_reset(struct omap_mpu_state_s *s)
     s->mpui_ctrl = 0x0003ff1b;
 }
 
-static void omap_mpui_init(MemoryRegion *memory, target_phys_addr_t base,
+static void omap_mpui_init(MemoryRegion *memory, hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->mpui_iomem, &omap_mpui_ops, mpu,
@@ -1131,7 +1134,7 @@ struct omap_tipb_bridge_s {
     uint16_t enh_control;
 };
 
-static uint64_t omap_tipb_bridge_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tipb_bridge_read(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *) opaque;
@@ -1161,7 +1164,7 @@ static uint64_t omap_tipb_bridge_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tipb_bridge_write(void *opaque, target_phys_addr_t addr,
+static void omap_tipb_bridge_write(void *opaque, hwaddr addr,
                                    uint64_t value, unsigned size)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *) opaque;
@@ -1215,7 +1218,7 @@ static void omap_tipb_bridge_reset(struct omap_tipb_bridge_s *s)
 }
 
 static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
-    MemoryRegion *memory, target_phys_addr_t base,
+    MemoryRegion *memory, hwaddr base,
     qemu_irq abort_irq, omap_clk clk)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *)
@@ -1232,7 +1235,7 @@ static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
 }
 
 /* Dummy Traffic Controller's Memory Interface */
-static uint64_t omap_tcmi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tcmi_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1270,7 +1273,7 @@ static uint64_t omap_tcmi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tcmi_write(void *opaque, target_phys_addr_t addr,
+static void omap_tcmi_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1330,7 +1333,7 @@ static void omap_tcmi_reset(struct omap_mpu_state_s *mpu)
     mpu->tcmi_regs[0x40 >> 2] = 0x00000000;
 }
 
-static void omap_tcmi_init(MemoryRegion *memory, target_phys_addr_t base,
+static void omap_tcmi_init(MemoryRegion *memory, hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->tcmi_iomem, &omap_tcmi_ops, mpu,
@@ -1346,7 +1349,7 @@ struct dpll_ctl_s {
     omap_clk dpll;
 };
 
-static uint64_t omap_dpll_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dpll_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct dpll_ctl_s *s = (struct dpll_ctl_s *) opaque;
@@ -1362,7 +1365,7 @@ static uint64_t omap_dpll_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_dpll_write(void *opaque, target_phys_addr_t addr,
+static void omap_dpll_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct dpll_ctl_s *s = (struct dpll_ctl_s *) opaque;
@@ -1412,7 +1415,7 @@ static void omap_dpll_reset(struct dpll_ctl_s *s)
 }
 
 static struct dpll_ctl_s  *omap_dpll_init(MemoryRegion *memory,
-                           target_phys_addr_t base, omap_clk clk)
+                           hwaddr base, omap_clk clk)
 {
     struct dpll_ctl_s *s = g_malloc0(sizeof(*s));
     memory_region_init_io(&s->iomem, &omap_dpll_ops, s, "omap-dpll", 0x100);
@@ -1425,7 +1428,7 @@ static struct dpll_ctl_s  *omap_dpll_init(MemoryRegion *memory,
 }
 
 /* MPU Clock/Reset/Power Mode Control */
-static uint64_t omap_clkm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_clkm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1627,7 +1630,7 @@ static inline void omap_clkm_ckout1_update(struct omap_mpu_state_s *s,
     }
 }
 
-static void omap_clkm_write(void *opaque, target_phys_addr_t addr,
+static void omap_clkm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1714,7 +1717,7 @@ static const MemoryRegionOps omap_clkm_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_clkdsp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_clkdsp_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1758,7 +1761,7 @@ static inline void omap_clkdsp_idlect2_update(struct omap_mpu_state_s *s,
     SET_ONOFF("dspxor_ck", 1);				/* EN_XORPCK */
 }
 
-static void omap_clkdsp_write(void *opaque, target_phys_addr_t addr,
+static void omap_clkdsp_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1823,8 +1826,8 @@ static void omap_clkm_reset(struct omap_mpu_state_s *s)
     s->clkm.dsp_rstct2 = 0x0000;
 }
 
-static void omap_clkm_init(MemoryRegion *memory, target_phys_addr_t mpu_base,
-                target_phys_addr_t dsp_base, struct omap_mpu_state_s *s)
+static void omap_clkm_init(MemoryRegion *memory, hwaddr mpu_base,
+                hwaddr dsp_base, struct omap_mpu_state_s *s)
 {
     memory_region_init_io(&s->clkm_iomem, &omap_clkm_ops, s,
                           "omap-clkm", 0x100);
@@ -1903,7 +1906,7 @@ static void omap_mpuio_kbd_update(struct omap_mpuio_s *s)
     s->row_latch = ~rows;
 }
 
-static uint64_t omap_mpuio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpuio_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mpuio_s *s = (struct omap_mpuio_s *) opaque;
@@ -1963,7 +1966,7 @@ static uint64_t omap_mpuio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpuio_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpuio_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_mpuio_s *s = (struct omap_mpuio_s *) opaque;
@@ -2072,7 +2075,7 @@ static void omap_mpuio_onoff(void *opaque, int line, int on)
 }
 
 static struct omap_mpuio_s *omap_mpuio_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq kbd_int, qemu_irq gpio_int, qemu_irq wakeup,
                 omap_clk clk)
 {
@@ -2159,7 +2162,7 @@ static void omap_uwire_transfer_start(struct omap_uwire_s *s)
     }
 }
 
-static uint64_t omap_uwire_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_uwire_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_uwire_s *s = (struct omap_uwire_s *) opaque;
@@ -2193,7 +2196,7 @@ static uint64_t omap_uwire_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_uwire_write(void *opaque, target_phys_addr_t addr,
+static void omap_uwire_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_uwire_s *s = (struct omap_uwire_s *) opaque;
@@ -2263,7 +2266,7 @@ static void omap_uwire_reset(struct omap_uwire_s *s)
 }
 
 static struct omap_uwire_s *omap_uwire_init(MemoryRegion *system_memory,
-                                            target_phys_addr_t base,
+                                            hwaddr base,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq dma,
                                             omap_clk clk)
@@ -2312,7 +2315,7 @@ static void omap_pwl_update(struct omap_pwl_s *s)
     }
 }
 
-static uint64_t omap_pwl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pwl_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_pwl_s *s = (struct omap_pwl_s *) opaque;
@@ -2332,7 +2335,7 @@ static uint64_t omap_pwl_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_pwl_write(void *opaque, target_phys_addr_t addr,
+static void omap_pwl_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_pwl_s *s = (struct omap_pwl_s *) opaque;
@@ -2381,7 +2384,7 @@ static void omap_pwl_clk_update(void *opaque, int line, int on)
 }
 
 static struct omap_pwl_s *omap_pwl_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         omap_clk clk)
 {
     struct omap_pwl_s *s = g_malloc0(sizeof(*s));
@@ -2405,7 +2408,7 @@ struct omap_pwt_s {
     omap_clk clk;
 };
 
-static uint64_t omap_pwt_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pwt_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_pwt_s *s = (struct omap_pwt_s *) opaque;
@@ -2427,7 +2430,7 @@ static uint64_t omap_pwt_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_pwt_write(void *opaque, target_phys_addr_t addr,
+static void omap_pwt_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_pwt_s *s = (struct omap_pwt_s *) opaque;
@@ -2488,7 +2491,7 @@ static void omap_pwt_reset(struct omap_pwt_s *s)
 }
 
 static struct omap_pwt_s *omap_pwt_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         omap_clk clk)
 {
     struct omap_pwt_s *s = g_malloc0(sizeof(*s));
@@ -2536,7 +2539,7 @@ static void omap_rtc_alarm_update(struct omap_rtc_s *s)
         printf("%s: conversion failed\n", __FUNCTION__);
 }
 
-static uint64_t omap_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_rtc_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_rtc_s *s = (struct omap_rtc_s *) opaque;
@@ -2618,7 +2621,7 @@ static uint64_t omap_rtc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_rtc_write(void *opaque, target_phys_addr_t addr,
+static void omap_rtc_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_rtc_s *s = (struct omap_rtc_s *) opaque;
@@ -2830,7 +2833,7 @@ static void omap_rtc_tick(void *opaque)
         s->round = 0;
     }
 
-    memcpy(&s->current_tm, localtime(&s->ti), sizeof(s->current_tm));
+    localtime_r(&s->ti, &s->current_tm);
 
     if ((s->interrupts & 0x08) && s->ti == s->alarm_ti) {
         s->status |= 0x40;
@@ -2901,7 +2904,7 @@ static void omap_rtc_reset(struct omap_rtc_s *s)
 }
 
 static struct omap_rtc_s *omap_rtc_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq timerirq, qemu_irq alarmirq,
                                         omap_clk clk)
 {
@@ -3129,7 +3132,7 @@ static void omap_mcbsp_req_update(struct omap_mcbsp_s *s)
         omap_mcbsp_rx_stop(s);
 }
 
-static uint64_t omap_mcbsp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mcbsp_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3227,7 +3230,7 @@ static uint64_t omap_mcbsp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mcbsp_writeh(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3365,7 +3368,7 @@ static void omap_mcbsp_writeh(void *opaque, target_phys_addr_t addr,
     OMAP_BAD_REG(addr);
 }
 
-static void omap_mcbsp_writew(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_writew(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3396,7 +3399,7 @@ static void omap_mcbsp_writew(void *opaque, target_phys_addr_t addr,
     omap_badwidth_write16(opaque, addr, value);
 }
 
-static void omap_mcbsp_write(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     switch (size) {
@@ -3432,7 +3435,7 @@ static void omap_mcbsp_reset(struct omap_mcbsp_s *s)
 }
 
 static struct omap_mcbsp_s *omap_mcbsp_init(MemoryRegion *system_memory,
-                                            target_phys_addr_t base,
+                                            hwaddr base,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq *dma, omap_clk clk)
 {
@@ -3547,7 +3550,7 @@ static void omap_lpg_reset(struct omap_lpg_s *s)
     omap_lpg_update(s);
 }
 
-static uint64_t omap_lpg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_lpg_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *) opaque;
@@ -3569,7 +3572,7 @@ static uint64_t omap_lpg_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_lpg_write(void *opaque, target_phys_addr_t addr,
+static void omap_lpg_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *) opaque;
@@ -3613,7 +3616,7 @@ static void omap_lpg_clk_update(void *opaque, int line, int on)
 }
 
 static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base, omap_clk clk)
+                                        hwaddr base, omap_clk clk)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *)
             g_malloc0(sizeof(struct omap_lpg_s));
@@ -3631,7 +3634,7 @@ static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
 }
 
 /* MPUI Peripheral Bridge configuration */
-static uint64_t omap_mpui_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpui_io_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     if (size != 2) {
@@ -3645,7 +3648,7 @@ static uint64_t omap_mpui_io_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpui_io_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpui_io_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     /* FIXME: infinite loop */
@@ -3706,8 +3709,8 @@ static void omap1_mpu_reset(void *opaque)
 }
 
 static const struct omap_map_s {
-    target_phys_addr_t phys_dsp;
-    target_phys_addr_t phys_mpu;
+    hwaddr phys_dsp;
+    hwaddr phys_mpu;
     uint32_t size;
     const char *name;
 } omap15xx_dsp_mm[] = {
@@ -3778,38 +3781,38 @@ static const struct dma_irq_map omap1_dma_irq_map[] = {
 
 /* DMA ports for OMAP1 */
 static int omap_validate_emiff_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_EMIFF_BASE, s->sdram_size, addr);
 }
 
 static int omap_validate_emifs_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_EMIFS_BASE, OMAP_EMIFF_BASE - OMAP_EMIFS_BASE,
                              addr);
 }
 
 static int omap_validate_imif_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_IMIF_BASE, s->sram_size, addr);
 }
 
 static int omap_validate_tipb_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(0xfffb0000, 0xffff0000 - 0xfffb0000, addr);
 }
 
 static int omap_validate_local_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_LOCALBUS_BASE, 0x1000000, addr);
 }
 
 static int omap_validate_tipb_mpui_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(0xe1010000, 0xe1020004 - 0xe1010000, addr);
 }
@@ -3859,7 +3862,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint32(s->ih[0], "size", 0x100);
     qdev_prop_set_ptr(s->ih[0], "clk", omap_findclk(s, "arminth_ck"));
     qdev_init_nofail(s->ih[0]);
-    busdev = sysbus_from_qdev(s->ih[0]);
+    busdev = SYS_BUS_DEVICE(s->ih[0]);
     sysbus_connect_irq(busdev, 0, cpu_irq[ARM_PIC_CPU_IRQ]);
     sysbus_connect_irq(busdev, 1, cpu_irq[ARM_PIC_CPU_FIQ]);
     sysbus_mmio_map(busdev, 0, 0xfffecb00);
@@ -3867,7 +3870,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint32(s->ih[1], "size", 0x800);
     qdev_prop_set_ptr(s->ih[1], "clk", omap_findclk(s, "arminth_ck"));
     qdev_init_nofail(s->ih[1]);
-    busdev = sysbus_from_qdev(s->ih[1]);
+    busdev = SYS_BUS_DEVICE(s->ih[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_15XX_IH2_IRQ));
     /* The second interrupt controller's FIQ output is not wired up */
@@ -3980,9 +3983,9 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_int32(s->gpio, "mpu_model", s->mpu_model);
     qdev_prop_set_ptr(s->gpio, "clk", omap_findclk(s, "arm_gpio_ck"));
     qdev_init_nofail(s->gpio);
-    sysbus_connect_irq(sysbus_from_qdev(s->gpio), 0,
+    sysbus_connect_irq(SYS_BUS_DEVICE(s->gpio), 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_GPIO_BANK1));
-    sysbus_mmio_map(sysbus_from_qdev(s->gpio), 0, 0xfffce000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(s->gpio), 0, 0xfffce000);
 
     s->microwire = omap_uwire_init(system_memory, 0xfffb3000,
                                    qdev_get_gpio_in(s->ih[1], OMAP_INT_uWireTX),
@@ -3998,7 +4001,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint8(s->i2c[0], "revision", 0x11);
     qdev_prop_set_ptr(s->i2c[0], "fclk", omap_findclk(s, "mpuper_ck"));
     qdev_init_nofail(s->i2c[0]);
-    busdev = sysbus_from_qdev(s->i2c[0]);
+    busdev = SYS_BUS_DEVICE(s->i2c[0]);
     sysbus_connect_irq(busdev, 0, qdev_get_gpio_in(s->ih[1], OMAP_INT_I2C));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP_DMA_I2C_TX]);
     sysbus_connect_irq(busdev, 2, s->drq[OMAP_DMA_I2C_RX]);
diff --git a/hw/omap2.c b/hw/omap2.c
index 4278dd19c40..038a82a5172 100644
--- a/hw/omap2.c
+++ b/hw/omap2.c
@@ -18,13 +18,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "flash.h"
 #include "soc_dma.h"
 #include "sysbus.h"
@@ -324,7 +324,7 @@ static void omap_eac_reset(struct omap_eac_s *s)
     omap_eac_interrupt_update(s);
 }
 
-static uint64_t omap_eac_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_eac_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_eac_s *s = (struct omap_eac_s *) opaque;
@@ -440,7 +440,7 @@ static uint64_t omap_eac_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_eac_write(void *opaque, target_phys_addr_t addr,
+static void omap_eac_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_eac_s *s = (struct omap_eac_s *) opaque;
@@ -644,7 +644,7 @@ static void omap_sti_reset(struct omap_sti_s *s)
     omap_sti_interrupt_update(s);
 }
 
-static uint64_t omap_sti_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sti_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -685,7 +685,7 @@ static uint64_t omap_sti_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_sti_write(void *opaque, target_phys_addr_t addr,
+static void omap_sti_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -741,14 +741,14 @@ static const MemoryRegionOps omap_sti_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_sti_fifo_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sti_fifo_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     OMAP_BAD_REG(addr);
     return 0;
 }
 
-static void omap_sti_fifo_write(void *opaque, target_phys_addr_t addr,
+static void omap_sti_fifo_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -780,7 +780,7 @@ static const MemoryRegionOps omap_sti_fifo_ops = {
 
 static struct omap_sti_s *omap_sti_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t channel_base, qemu_irq irq, omap_clk clk,
+                hwaddr channel_base, qemu_irq irq, omap_clk clk,
                 CharDriverState *chr)
 {
     struct omap_sti_s *s = (struct omap_sti_s *)
@@ -1040,7 +1040,7 @@ static void omap_prcm_int_update(struct omap_prcm_s *s, int dom)
     /* XXX or is the mask applied before PRCM_IRQSTATUS_* ? */
 }
 
-static uint64_t omap_prcm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_prcm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_prcm_s *s = (struct omap_prcm_s *) opaque;
@@ -1352,7 +1352,7 @@ static void omap_prcm_dpll_update(struct omap_prcm_s *s)
     }
 }
 
-static void omap_prcm_write(void *opaque, target_phys_addr_t addr,
+static void omap_prcm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_prcm_s *s = (struct omap_prcm_s *) opaque;
@@ -1832,7 +1832,7 @@ struct omap_sysctl_s {
     uint32_t msuspendmux[5];
 };
 
-static uint32_t omap_sysctl_read8(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_sysctl_read8(void *opaque, hwaddr addr)
 {
 
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -1857,7 +1857,7 @@ static uint32_t omap_sysctl_read8(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_sysctl_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_sysctl_read(void *opaque, hwaddr addr)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
 
@@ -1957,7 +1957,7 @@ static uint32_t omap_sysctl_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap_sysctl_write8(void *opaque, target_phys_addr_t addr,
+static void omap_sysctl_write8(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -1981,7 +1981,7 @@ static void omap_sysctl_write8(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_sysctl_write(void *opaque, target_phys_addr_t addr,
+static void omap_sysctl_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -2226,7 +2226,7 @@ static void omap2_mpu_reset(void *opaque)
 }
 
 static int omap2_validate_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return 1;
 }
@@ -2283,7 +2283,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->ih[0], "fclk", omap_findclk(s, "mpu_intc_fclk"));
     qdev_prop_set_ptr(s->ih[0], "iclk", omap_findclk(s, "mpu_intc_iclk"));
     qdev_init_nofail(s->ih[0]);
-    busdev = sysbus_from_qdev(s->ih[0]);
+    busdev = SYS_BUS_DEVICE(s->ih[0]);
     sysbus_connect_irq(busdev, 0, cpu_irq[ARM_PIC_CPU_IRQ]);
     sysbus_connect_irq(busdev, 1, cpu_irq[ARM_PIC_CPU_FIQ]);
     sysbus_mmio_map(busdev, 0, 0x480fe000);
@@ -2398,7 +2398,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[0], "iclk", omap_findclk(s, "i2c1.iclk"));
     qdev_prop_set_ptr(s->i2c[0], "fclk", omap_findclk(s, "i2c1.fclk"));
     qdev_init_nofail(s->i2c[0]);
-    busdev = sysbus_from_qdev(s->i2c[0]);
+    busdev = SYS_BUS_DEVICE(s->i2c[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_I2C1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_I2C1_TX]);
@@ -2410,7 +2410,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[1], "iclk", omap_findclk(s, "i2c2.iclk"));
     qdev_prop_set_ptr(s->i2c[1], "fclk", omap_findclk(s, "i2c2.fclk"));
     qdev_init_nofail(s->i2c[1]);
-    busdev = sysbus_from_qdev(s->i2c[1]);
+    busdev = SYS_BUS_DEVICE(s->i2c[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_I2C2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_I2C2_TX]);
@@ -2428,7 +2428,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
         qdev_prop_set_ptr(s->gpio, "fclk4", omap_findclk(s, "gpio5_dbclk"));
     }
     qdev_init_nofail(s->gpio);
-    busdev = sysbus_from_qdev(s->gpio);
+    busdev = SYS_BUS_DEVICE(s->gpio);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_GPIO_BANK1));
     sysbus_connect_irq(busdev, 3,
diff --git a/hw/omap_dma.c b/hw/omap_dma.c
index 389cb78deef..0c878b6ef25 100644
--- a/hw/omap_dma.c
+++ b/hw/omap_dma.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 #include "irq.h"
 #include "soc_dma.h"
@@ -31,7 +31,7 @@ struct omap_dma_channel_s {
     int endian_lock[2];
     int translate[2];
     enum omap_dma_port port[2];
-    target_phys_addr_t addr[2];
+    hwaddr addr[2];
     omap_dma_addressing_t mode[2];
     uint32_t elements;
     uint16_t frames;
@@ -78,7 +78,7 @@ struct omap_dma_channel_s {
     struct omap_dma_channel_s *sibling;
 
     struct omap_dma_reg_set_s {
-        target_phys_addr_t src, dest;
+        hwaddr src, dest;
         int frame;
         int element;
         int pck_element;
@@ -914,7 +914,7 @@ static int omap_dma_ch_reg_write(struct omap_dma_s *s,
         break;
 
     case 0x06:	/* SYS_DMA_CSR_CH0 */
-        OMAP_RO_REG((target_phys_addr_t) reg);
+        OMAP_RO_REG((hwaddr) reg);
         break;
 
     case 0x08:	/* SYS_DMA_CSSA_L_CH0 */
@@ -954,7 +954,7 @@ static int omap_dma_ch_reg_write(struct omap_dma_s *s,
         break;
 
     case 0x18:	/* SYS_DMA_CPC_CH0 or DMA_CSAC */
-        OMAP_RO_REG((target_phys_addr_t) reg);
+        OMAP_RO_REG((hwaddr) reg);
         break;
 
     case 0x1c:	/* DMA_CDEI */
@@ -1446,7 +1446,7 @@ static int omap_dma_sys_read(struct omap_dma_s *s, int offset,
     return 0;
 }
 
-static uint64_t omap_dma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dma_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1494,7 +1494,7 @@ static uint64_t omap_dma_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_dma_write(void *opaque, target_phys_addr_t addr,
+static void omap_dma_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1618,7 +1618,7 @@ static void omap_dma_setcaps(struct omap_dma_s *s)
     }
 }
 
-struct soc_dma_s *omap_dma_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 qemu_irq lcd_irq, struct omap_mpu_state_s *mpu, omap_clk clk,
                 enum omap_dma_model model)
@@ -1692,7 +1692,7 @@ static void omap_dma_interrupts_4_update(struct omap_dma_s *s)
         qemu_irq_raise(s->irq[3]);
 }
 
-static uint64_t omap_dma4_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dma4_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1709,19 +1709,25 @@ static uint64_t omap_dma4_read(void *opaque, target_phys_addr_t addr,
 
     case 0x14:	/* DMA4_IRQSTATUS_L3 */
         irqn ++;
+        /* fall through */
     case 0x10:	/* DMA4_IRQSTATUS_L2 */
         irqn ++;
+        /* fall through */
     case 0x0c:	/* DMA4_IRQSTATUS_L1 */
         irqn ++;
+        /* fall through */
     case 0x08:	/* DMA4_IRQSTATUS_L0 */
         return s->irqstat[irqn];
 
     case 0x24:	/* DMA4_IRQENABLE_L3 */
         irqn ++;
+        /* fall through */
     case 0x20:	/* DMA4_IRQENABLE_L2 */
         irqn ++;
+        /* fall through */
     case 0x1c:	/* DMA4_IRQENABLE_L1 */
         irqn ++;
+        /* fall through */
     case 0x18:	/* DMA4_IRQENABLE_L0 */
         return s->irqen[irqn];
 
@@ -1842,7 +1848,7 @@ static uint64_t omap_dma4_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
+static void omap_dma4_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1856,10 +1862,13 @@ static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
     switch (addr) {
     case 0x14:	/* DMA4_IRQSTATUS_L3 */
         irqn ++;
+        /* fall through */
     case 0x10:	/* DMA4_IRQSTATUS_L2 */
         irqn ++;
+        /* fall through */
     case 0x0c:	/* DMA4_IRQSTATUS_L1 */
         irqn ++;
+        /* fall through */
     case 0x08:	/* DMA4_IRQSTATUS_L0 */
         s->irqstat[irqn] &= ~value;
         if (!s->irqstat[irqn])
@@ -1868,10 +1877,13 @@ static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
 
     case 0x24:	/* DMA4_IRQENABLE_L3 */
         irqn ++;
+        /* fall through */
     case 0x20:	/* DMA4_IRQENABLE_L2 */
         irqn ++;
+        /* fall through */
     case 0x1c:	/* DMA4_IRQENABLE_L1 */
         irqn ++;
+        /* fall through */
     case 0x18:	/* DMA4_IRQENABLE_L0 */
         s->irqen[irqn] = value;
         return;
@@ -1988,12 +2000,12 @@ static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
         break;
 
     case 0x1c:	/* DMA4_CSSA */
-        ch->addr[0] = (target_phys_addr_t) (uint32_t) value;
+        ch->addr[0] = (hwaddr) (uint32_t) value;
         ch->set_update = 1;
         break;
 
     case 0x20:	/* DMA4_CDSA */
-        ch->addr[1] = (target_phys_addr_t) (uint32_t) value;
+        ch->addr[1] = (hwaddr) (uint32_t) value;
         ch->set_update = 1;
         break;
 
@@ -2040,7 +2052,7 @@ static const MemoryRegionOps omap_dma4_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-struct soc_dma_s *omap_dma4_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 struct omap_mpu_state_s *mpu, int fifo,
                 int chans, omap_clk iclk, omap_clk fclk)
diff --git a/hw/omap_dss.c b/hw/omap_dss.c
index 86ed6ea5d9a..ae51bdfe411 100644
--- a/hw/omap_dss.c
+++ b/hw/omap_dss.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "omap.h"
 
 struct omap_dss_s {
@@ -60,7 +60,7 @@ struct omap_dss_s {
             int nx;
             int ny;
 
-            target_phys_addr_t addr[3];
+            hwaddr addr[3];
 
             uint32_t attr;
             uint32_t tresh;
@@ -168,7 +168,7 @@ void omap_dss_reset(struct omap_dss_s *s)
     omap_dispc_interrupt_update(s);
 }
 
-static uint64_t omap_diss_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_diss_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -206,7 +206,7 @@ static uint64_t omap_diss_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_diss_write(void *opaque, target_phys_addr_t addr,
+static void omap_diss_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -246,7 +246,7 @@ static const MemoryRegionOps omap_diss_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_disc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_disc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -371,7 +371,7 @@ static uint64_t omap_disc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_disc_write(void *opaque, target_phys_addr_t addr,
+static void omap_disc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -502,11 +502,11 @@ static void omap_disc_write(void *opaque, target_phys_addr_t addr,
         s->dispc.invalidate = 1;
         break;
     case 0x080:	/* DISPC_GFX_BA0 */
-        s->dispc.l[0].addr[0] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[0] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
     case 0x084:	/* DISPC_GFX_BA1 */
-        s->dispc.l[0].addr[1] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[1] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
     case 0x088:	/* DISPC_GFX_POSITION */
@@ -543,7 +543,7 @@ static void omap_disc_write(void *opaque, target_phys_addr_t addr,
         s->dispc.l[0].wininc = value;
         break;
     case 0x0b8:	/* DISPC_GFX_TABLE_BA */
-        s->dispc.l[0].addr[2] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[2] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
 
@@ -602,11 +602,11 @@ static void omap_rfbi_transfer_stop(struct omap_dss_s *s)
 static void omap_rfbi_transfer_start(struct omap_dss_s *s)
 {
     void *data;
-    target_phys_addr_t len;
-    target_phys_addr_t data_addr;
+    hwaddr len;
+    hwaddr data_addr;
     int pitch;
     static void *bounce_buffer;
-    static target_phys_addr_t bounce_len;
+    static hwaddr bounce_len;
 
     if (!s->rfbi.enable || s->rfbi.busy)
         return;
@@ -663,7 +663,7 @@ static void omap_rfbi_transfer_start(struct omap_dss_s *s)
     omap_dispc_interrupt_update(s);
 }
 
-static uint64_t omap_rfbi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_rfbi_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -730,7 +730,7 @@ static uint64_t omap_rfbi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_rfbi_write(void *opaque, target_phys_addr_t addr,
+static void omap_rfbi_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -864,7 +864,7 @@ static const MemoryRegionOps omap_rfbi_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_venc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_venc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     if (size != 4) {
@@ -924,7 +924,7 @@ static uint64_t omap_venc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_venc_write(void *opaque, target_phys_addr_t addr,
+static void omap_venc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -986,7 +986,7 @@ static const MemoryRegionOps omap_venc_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_im3_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_im3_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     if (size != 4) {
@@ -1012,7 +1012,7 @@ static uint64_t omap_im3_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_im3_write(void *opaque, target_phys_addr_t addr,
+static void omap_im3_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -1041,7 +1041,7 @@ static const MemoryRegionOps omap_im3_ops = {
 
 struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t l3_base,
+                hwaddr l3_base,
                 qemu_irq irq, qemu_irq drq,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2)
diff --git a/hw/omap_gpio.c b/hw/omap_gpio.c
index 201ff77c362..aadf1cc59f3 100644
--- a/hw/omap_gpio.c
+++ b/hw/omap_gpio.c
@@ -61,7 +61,7 @@ static void omap_gpio_set(void *opaque, int line, int level)
     }
 }
 
-static uint64_t omap_gpio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpio_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpio_s *s = (struct omap_gpio_s *) opaque;
@@ -99,7 +99,7 @@ static uint64_t omap_gpio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_gpio_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpio_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpio_s *s = (struct omap_gpio_s *) opaque;
@@ -300,7 +300,7 @@ static void omap2_gpio_module_reset(struct omap2_gpio_s *s)
     s->delay = 0;
 }
 
-static uint32_t omap2_gpio_module_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap2_gpio_module_read(void *opaque, hwaddr addr)
 {
     struct omap2_gpio_s *s = (struct omap2_gpio_s *) opaque;
 
@@ -372,7 +372,7 @@ static uint32_t omap2_gpio_module_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap2_gpio_module_write(void *opaque, target_phys_addr_t addr,
+static void omap2_gpio_module_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap2_gpio_s *s = (struct omap2_gpio_s *) opaque;
@@ -514,12 +514,12 @@ static void omap2_gpio_module_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint32_t omap2_gpio_module_readp(void *opaque, target_phys_addr_t addr)
+static uint32_t omap2_gpio_module_readp(void *opaque, hwaddr addr)
 {
     return omap2_gpio_module_read(opaque, addr & ~3) >> ((addr & 3) << 3);
 }
 
-static void omap2_gpio_module_writep(void *opaque, target_phys_addr_t addr,
+static void omap2_gpio_module_writep(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint32_t cur = 0;
@@ -588,7 +588,7 @@ static const MemoryRegionOps omap2_gpio_module_ops = {
 static void omap_gpif_reset(DeviceState *dev)
 {
     struct omap_gpif_s *s = FROM_SYSBUS(struct omap_gpif_s,
-                    sysbus_from_qdev(dev));
+                    SYS_BUS_DEVICE(dev));
     omap_gpio_reset(&s->omap1);
 }
 
@@ -596,7 +596,7 @@ static void omap2_gpif_reset(DeviceState *dev)
 {
     int i;
     struct omap2_gpif_s *s = FROM_SYSBUS(struct omap2_gpif_s,
-                    sysbus_from_qdev(dev));
+                    SYS_BUS_DEVICE(dev));
     for (i = 0; i < s->modulecount; i++) {
         omap2_gpio_module_reset(&s->modules[i]);
     }
@@ -604,7 +604,7 @@ static void omap2_gpif_reset(DeviceState *dev)
     s->gpo = 0;
 }
 
-static uint64_t omap2_gpif_top_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap2_gpif_top_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     struct omap2_gpif_s *s = (struct omap2_gpif_s *) opaque;
@@ -633,7 +633,7 @@ static uint64_t omap2_gpif_top_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap2_gpif_top_write(void *opaque, target_phys_addr_t addr,
+static void omap2_gpif_top_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     struct omap2_gpif_s *s = (struct omap2_gpif_s *) opaque;
@@ -747,7 +747,7 @@ static void omap_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = omap_gpio_properties;
 }
 
-static TypeInfo omap_gpio_info = {
+static const TypeInfo omap_gpio_info = {
     .name          = "omap-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_gpif_s),
@@ -776,7 +776,7 @@ static void omap2_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = omap2_gpio_properties;
 }
 
-static TypeInfo omap2_gpio_info = {
+static const TypeInfo omap2_gpio_info = {
     .name          = "omap2-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap2_gpif_s),
diff --git a/hw/omap_gpmc.c b/hw/omap_gpmc.c
index 2fc4137203b..02ab0ab5686 100644
--- a/hw/omap_gpmc.c
+++ b/hw/omap_gpmc.c
@@ -21,8 +21,8 @@
 #include "hw.h"
 #include "flash.h"
 #include "omap.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 /* General-Purpose Memory Controller */
 struct omap_gpmc_s {
@@ -121,7 +121,7 @@ static void omap_gpmc_dma_update(struct omap_gpmc_s *s, int value)
  * all addresses in the region behave like accesses to the relevant
  * GPMC_NAND_DATA_i register (which is actually implemented to call these)
  */
-static uint64_t omap_nand_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_nand_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpmc_cs_file_s *f = (struct omap_gpmc_cs_file_s *)opaque;
@@ -200,7 +200,7 @@ static void omap_nand_setio(DeviceState *dev, uint64_t value,
     }
 }
 
-static void omap_nand_write(void *opaque, target_phys_addr_t addr,
+static void omap_nand_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpmc_cs_file_s *f = (struct omap_gpmc_cs_file_s *)opaque;
@@ -281,7 +281,7 @@ static void fill_prefetch_fifo(struct omap_gpmc_s *s)
  * engine is enabled -- all addresses in the region behave alike:
  * data is read or written to the FIFO.
  */
-static uint64_t omap_gpmc_prefetch_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpmc_prefetch_read(void *opaque, hwaddr addr,
                                         unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -311,7 +311,7 @@ static uint64_t omap_gpmc_prefetch_read(void *opaque, target_phys_addr_t addr,
     return data;
 }
 
-static void omap_gpmc_prefetch_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpmc_prefetch_write(void *opaque, hwaddr addr,
                                      uint64_t value, unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -484,7 +484,7 @@ void omap_gpmc_reset(struct omap_gpmc_s *s)
         ecc_reset(&s->ecc[i]);
 }
 
-static int gpmc_wordaccess_only(target_phys_addr_t addr)
+static int gpmc_wordaccess_only(hwaddr addr)
 {
     /* Return true if the register offset is to a register that
      * only permits word width accesses.
@@ -502,7 +502,7 @@ static int gpmc_wordaccess_only(target_phys_addr_t addr)
     return 1;
 }
 
-static uint64_t omap_gpmc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpmc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -614,7 +614,7 @@ static uint64_t omap_gpmc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_gpmc_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpmc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -819,7 +819,7 @@ static const MemoryRegionOps omap_gpmc_ops = {
 };
 
 struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
-                                   target_phys_addr_t base,
+                                   hwaddr base,
                                    qemu_irq irq, qemu_irq drq)
 {
     int cs;
diff --git a/hw/omap_gptimer.c b/hw/omap_gptimer.c
index 7a145198a42..a5db710dcb9 100644
--- a/hw/omap_gptimer.c
+++ b/hw/omap_gptimer.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 
 /* GP timers */
@@ -258,7 +258,7 @@ void omap_gp_timer_reset(struct omap_gp_timer_s *s)
     omap_gp_timer_update(s);
 }
 
-static uint32_t omap_gp_timer_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_gp_timer_readw(void *opaque, hwaddr addr)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
 
@@ -324,7 +324,7 @@ static uint32_t omap_gp_timer_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_gp_timer_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_gp_timer_readh(void *opaque, hwaddr addr)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
     uint32_t ret;
@@ -338,7 +338,7 @@ static uint32_t omap_gp_timer_readh(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static void omap_gp_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_gp_timer_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
@@ -438,7 +438,7 @@ static void omap_gp_timer_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_gp_timer_writeh(void *opaque, target_phys_addr_t addr,
+static void omap_gp_timer_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
diff --git a/hw/omap_i2c.c b/hw/omap_i2c.c
index 20bc82e3b81..143b198f1d8 100644
--- a/hw/omap_i2c.c
+++ b/hw/omap_i2c.c
@@ -131,7 +131,7 @@ static void omap_i2c_fifo_run(OMAPI2CState *s)
 static void omap_i2c_reset(DeviceState *dev)
 {
     OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState,
-                                  sysbus_from_qdev(dev));
+                                  SYS_BUS_DEVICE(dev));
     s->mask = 0;
     s->stat = 0;
     s->dma = 0;
@@ -149,7 +149,7 @@ static void omap_i2c_reset(DeviceState *dev)
     s->test = 0;
 }
 
-static uint32_t omap_i2c_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_i2c_read(void *opaque, hwaddr addr)
 {
     OMAPI2CState *s = opaque;
     int offset = addr & OMAP_MPUI_REG_MASK;
@@ -248,7 +248,7 @@ static uint32_t omap_i2c_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap_i2c_write(void *opaque, target_phys_addr_t addr,
+static void omap_i2c_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAPI2CState *s = opaque;
@@ -390,7 +390,7 @@ static void omap_i2c_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_i2c_writeb(void *opaque, target_phys_addr_t addr,
+static void omap_i2c_writeb(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAPI2CState *s = opaque;
@@ -471,7 +471,7 @@ static void omap_i2c_class_init(ObjectClass *klass, void *data)
     dc->reset = omap_i2c_reset;
 }
 
-static TypeInfo omap_i2c_info = {
+static const TypeInfo omap_i2c_info = {
     .name = "omap_i2c",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OMAPI2CState),
@@ -485,7 +485,7 @@ static void omap_i2c_register_types(void)
 
 i2c_bus *omap_i2c_bus(DeviceState *omap_i2c)
 {
-    OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState, sysbus_from_qdev(omap_i2c));
+    OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState, SYS_BUS_DEVICE(omap_i2c));
     return s->bus;
 }
 
diff --git a/hw/omap_intc.c b/hw/omap_intc.c
index 5076e07ed50..4b0acd0f332 100644
--- a/hw/omap_intc.c
+++ b/hw/omap_intc.c
@@ -145,7 +145,7 @@ static void omap_set_intr_noedge(void *opaque, int irq, int req)
         bank->irqs = (bank->inputs &= ~(1 << n)) | bank->swi;
 }
 
-static uint64_t omap_inth_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_inth_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -223,7 +223,7 @@ static uint64_t omap_inth_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_inth_write(void *opaque, target_phys_addr_t addr,
+static void omap_inth_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -329,7 +329,7 @@ static const MemoryRegionOps omap_inth_mem_ops = {
 static void omap_inth_reset(DeviceState *dev)
 {
     struct omap_intr_handler_s *s = FROM_SYSBUS(struct omap_intr_handler_s,
-                                                sysbus_from_qdev(dev));
+                                                SYS_BUS_DEVICE(dev));
     int i;
 
     for (i = 0; i < s->nbanks; ++i){
@@ -389,14 +389,14 @@ static void omap_intc_class_init(ObjectClass *klass, void *data)
     dc->props = omap_intc_properties;
 }
 
-static TypeInfo omap_intc_info = {
+static const TypeInfo omap_intc_info = {
     .name          = "omap-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_intr_handler_s),
     .class_init    = omap_intc_class_init,
 };
 
-static uint64_t omap2_inth_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap2_inth_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -477,7 +477,7 @@ static uint64_t omap2_inth_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap2_inth_write(void *opaque, target_phys_addr_t addr,
+static void omap2_inth_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -633,7 +633,7 @@ static void omap2_intc_class_init(ObjectClass *klass, void *data)
     dc->props = omap2_intc_properties;
 }
 
-static TypeInfo omap2_intc_info = {
+static const TypeInfo omap2_intc_info = {
     .name          = "omap2-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_intr_handler_s),
diff --git a/hw/omap_l4.c b/hw/omap_l4.c
index dbad7f67a8f..09e983f319d 100644
--- a/hw/omap_l4.c
+++ b/hw/omap_l4.c
@@ -22,13 +22,13 @@
 
 struct omap_l4_s {
     MemoryRegion *address_space;
-    target_phys_addr_t base;
+    hwaddr base;
     int ta_num;
     struct omap_target_agent_s ta[0];
 };
 
 struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
-                               target_phys_addr_t base, int ta_num)
+                               hwaddr base, int ta_num)
 {
     struct omap_l4_s *bus = g_malloc0(
                     sizeof(*bus) + ta_num * sizeof(*bus->ta));
@@ -40,19 +40,19 @@ struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
     return bus;
 }
 
-target_phys_addr_t omap_l4_region_base(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_base(struct omap_target_agent_s *ta,
                                        int region)
 {
     return ta->bus->base + ta->start[region].offset;
 }
 
-target_phys_addr_t omap_l4_region_size(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_size(struct omap_target_agent_s *ta,
                                        int region)
 {
     return ta->start[region].size;
 }
 
-static uint64_t omap_l4ta_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_l4ta_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_target_agent_s *s = (struct omap_target_agent_s *) opaque;
@@ -76,7 +76,7 @@ static uint64_t omap_l4ta_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_l4ta_write(void *opaque, target_phys_addr_t addr,
+static void omap_l4ta_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_target_agent_s *s = (struct omap_target_agent_s *) opaque;
@@ -143,10 +143,10 @@ struct omap_target_agent_s *omap_l4ta_get(struct omap_l4_s *bus,
     return ta;
 }
 
-target_phys_addr_t omap_l4_attach(struct omap_target_agent_s *ta,
+hwaddr omap_l4_attach(struct omap_target_agent_s *ta,
                                          int region, MemoryRegion *mr)
 {
-    target_phys_addr_t base;
+    hwaddr base;
 
     if (region < 0 || region >= ta->regions) {
         fprintf(stderr, "%s: bad io region (%i)\n", __FUNCTION__, region);
diff --git a/hw/omap_lcdc.c b/hw/omap_lcdc.c
index 4a08e9d002f..936850a6216 100644
--- a/hw/omap_lcdc.c
+++ b/hw/omap_lcdc.c
@@ -17,9 +17,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "framebuffer.h"
+#include "ui/pixel_ops.h"
 
 struct omap_lcd_panel_s {
     MemoryRegion *sysmem;
@@ -66,8 +67,6 @@ static void omap_lcd_interrupts(struct omap_lcd_panel_s *s)
     qemu_irq_lower(s->irq);
 }
 
-#include "pixel_ops.h"
-
 #define draw_line_func drawfn
 
 #define DEPTH 8
@@ -117,7 +116,7 @@ static void omap_update_display(void *opaque)
     draw_line_func draw_line;
     int size, height, first, last;
     int width, linesize, step, bpp, frame_offset;
-    target_phys_addr_t frame_base;
+    hwaddr frame_base;
 
     if (!omap_lcd || omap_lcd->plm == 1 ||
                     !omap_lcd->enable || !ds_get_bits_per_pixel(omap_lcd->state))
@@ -219,23 +218,29 @@ static void omap_update_display(void *opaque)
                                draw_line, omap_lcd->palette,
                                &first, &last);
     if (first >= 0) {
-        dpy_update(omap_lcd->state, 0, first, width, last - first + 1);
+        dpy_gfx_update(omap_lcd->state, 0, first, width, last - first + 1);
     }
     omap_lcd->invalidate = 0;
 }
 
-static int ppm_save(const char *filename, uint8_t *data,
-                int w, int h, int linesize)
+static void omap_ppm_save(const char *filename, uint8_t *data,
+                    int w, int h, int linesize, Error **errp)
 {
     FILE *f;
     uint8_t *d, *d1;
     unsigned int v;
-    int y, x, bpp;
+    int ret, y, x, bpp;
 
     f = fopen(filename, "wb");
-    if (!f)
-        return -1;
-    fprintf(f, "P6\n%d %d\n%d\n", w, h, 255);
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
+        return;
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", w, h, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = data;
     bpp = linesize / w;
     for (y = 0; y < h; y ++) {
@@ -244,35 +249,61 @@ static int ppm_save(const char *filename, uint8_t *data,
             v = *(uint32_t *) d;
             switch (bpp) {
             case 2:
-                fputc((v >> 8) & 0xf8, f);
-                fputc((v >> 3) & 0xfc, f);
-                fputc((v << 3) & 0xf8, f);
+                ret = fputc((v >> 8) & 0xf8, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v >> 3) & 0xfc, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v << 3) & 0xf8, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
                 break;
             case 3:
             case 4:
             default:
-                fputc((v >> 16) & 0xff, f);
-                fputc((v >> 8) & 0xff, f);
-                fputc((v) & 0xff, f);
+                ret = fputc((v >> 16) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v >> 8) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
                 break;
             }
             d += bpp;
         }
         d1 += linesize;
     }
+out:
     fclose(f);
-    return 0;
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
-static void omap_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void omap_screen_dump(void *opaque, const char *filename, bool cswitch,
+                             Error **errp)
 {
     struct omap_lcd_panel_s *omap_lcd = opaque;
 
     omap_update_display(opaque);
     if (omap_lcd && ds_get_data(omap_lcd->state))
-        ppm_save(filename, ds_get_data(omap_lcd->state),
-                omap_lcd->width, omap_lcd->height,
-                ds_get_linesize(omap_lcd->state));
+        omap_ppm_save(filename, ds_get_data(omap_lcd->state),
+                    omap_lcd->width, omap_lcd->height,
+                    ds_get_linesize(omap_lcd->state), errp);
 }
 
 static void omap_invalidate_display(void *opaque) {
@@ -327,7 +358,7 @@ static void omap_lcd_update(struct omap_lcd_panel_s *s) {
     }
 }
 
-static uint64_t omap_lcdc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_lcdc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *) opaque;
@@ -360,7 +391,7 @@ static uint64_t omap_lcdc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_lcdc_write(void *opaque, target_phys_addr_t addr,
+static void omap_lcdc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *) opaque;
@@ -433,7 +464,7 @@ void omap_lcdc_reset(struct omap_lcd_panel_s *s)
 }
 
 struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq irq,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk)
diff --git a/hw/omap_mmc.c b/hw/omap_mmc.c
index aec0285675d..7ecd9bd4caa 100644
--- a/hw/omap_mmc.c
+++ b/hw/omap_mmc.c
@@ -306,7 +306,7 @@ void omap_mmc_reset(struct omap_mmc_s *host)
     host->clkdiv = 0;
 }
 
-static uint64_t omap_mmc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t omap_mmc_read(void *opaque, hwaddr offset,
                               unsigned size)
 {
     uint16_t i;
@@ -399,7 +399,7 @@ static uint64_t omap_mmc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void omap_mmc_write(void *opaque, target_phys_addr_t offset,
+static void omap_mmc_write(void *opaque, hwaddr offset,
                            uint64_t value, unsigned size)
 {
     int i;
@@ -572,7 +572,7 @@ static void omap_mmc_cover_cb(void *opaque, int line, int level)
     }
 }
 
-struct omap_mmc_s *omap_mmc_init(target_phys_addr_t base,
+struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
                 BlockDriverState *bd,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
diff --git a/hw/omap_sdrc.c b/hw/omap_sdrc.c
index 784e3261057..b0f3b8e6756 100644
--- a/hw/omap_sdrc.c
+++ b/hw/omap_sdrc.c
@@ -31,7 +31,7 @@ void omap_sdrc_reset(struct omap_sdrc_s *s)
     s->config = 0x10;
 }
 
-static uint64_t omap_sdrc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sdrc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *) opaque;
@@ -86,7 +86,7 @@ static uint64_t omap_sdrc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_sdrc_write(void *opaque, target_phys_addr_t addr,
+static void omap_sdrc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *) opaque;
@@ -154,7 +154,7 @@ static const MemoryRegionOps omap_sdrc_ops = {
 };
 
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
-                                   target_phys_addr_t base)
+                                   hwaddr base)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *)
             g_malloc0(sizeof(struct omap_sdrc_s));
diff --git a/hw/omap_spi.c b/hw/omap_spi.c
index 8f2b697d2d1..8ff01ed99d0 100644
--- a/hw/omap_spi.c
+++ b/hw/omap_spi.c
@@ -130,7 +130,7 @@ void omap_mcspi_reset(struct omap_mcspi_s *s)
     omap_mcspi_interrupt_update(s);
 }
 
-static uint64_t omap_mcspi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mcspi_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mcspi_s *s = (struct omap_mcspi_s *) opaque;
@@ -167,32 +167,47 @@ static uint64_t omap_mcspi_read(void *opaque, target_phys_addr_t addr,
         return s->control;
 
     case 0x68: ch ++;
+        /* fall through */
     case 0x54: ch ++;
+        /* fall through */
     case 0x40: ch ++;
+        /* fall through */
     case 0x2c:	/* MCSPI_CHCONF */
         return s->ch[ch].config;
 
     case 0x6c: ch ++;
+        /* fall through */
     case 0x58: ch ++;
+        /* fall through */
     case 0x44: ch ++;
+        /* fall through */
     case 0x30:	/* MCSPI_CHSTAT */
         return s->ch[ch].status;
 
     case 0x70: ch ++;
+        /* fall through */
     case 0x5c: ch ++;
+        /* fall through */
     case 0x48: ch ++;
+        /* fall through */
     case 0x34:	/* MCSPI_CHCTRL */
         return s->ch[ch].control;
 
     case 0x74: ch ++;
+        /* fall through */
     case 0x60: ch ++;
+        /* fall through */
     case 0x4c: ch ++;
+        /* fall through */
     case 0x38:	/* MCSPI_TX */
         return s->ch[ch].tx;
 
     case 0x78: ch ++;
+        /* fall through */
     case 0x64: ch ++;
+        /* fall through */
     case 0x50: ch ++;
+        /* fall through */
     case 0x3c:	/* MCSPI_RX */
         s->ch[ch].status &= ~(1 << 0);			/* RXS */
         ret = s->ch[ch].rx;
@@ -204,7 +219,7 @@ static uint64_t omap_mcspi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mcspi_write(void *opaque, target_phys_addr_t addr,
+static void omap_mcspi_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_mcspi_s *s = (struct omap_mcspi_s *) opaque;
@@ -269,8 +284,11 @@ static void omap_mcspi_write(void *opaque, target_phys_addr_t addr,
         break;
 
     case 0x68: ch ++;
+        /* fall through */
     case 0x54: ch ++;
+        /* fall through */
     case 0x40: ch ++;
+        /* fall through */
     case 0x2c:	/* MCSPI_CHCONF */
         if ((value ^ s->ch[ch].config) & (3 << 14))	/* DMAR | DMAW */
             omap_mcspi_dmarequest_update(s->ch + ch);
@@ -283,8 +301,11 @@ static void omap_mcspi_write(void *opaque, target_phys_addr_t addr,
         break;
 
     case 0x70: ch ++;
+        /* fall through */
     case 0x5c: ch ++;
+        /* fall through */
     case 0x48: ch ++;
+        /* fall through */
     case 0x34:	/* MCSPI_CHCTRL */
         if (value & ~s->ch[ch].control & 1) {		/* EN */
             s->ch[ch].control |= 1;
@@ -294,8 +315,11 @@ static void omap_mcspi_write(void *opaque, target_phys_addr_t addr,
         break;
 
     case 0x74: ch ++;
+        /* fall through */
     case 0x60: ch ++;
+        /* fall through */
     case 0x4c: ch ++;
+        /* fall through */
     case 0x38:	/* MCSPI_TX */
         s->ch[ch].tx = value;
         s->ch[ch].status &= ~(1 << 1);			/* TXS */
diff --git a/hw/omap_sx1.c b/hw/omap_sx1.c
index abca3419261..30998c5ff36 100644
--- a/hw/omap_sx1.c
+++ b/hw/omap_sx1.c
@@ -26,13 +26,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "boards.h"
 #include "arm-misc.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 /*****************************************************************************/
 /* Siemens SX1 Cellphone V1 */
@@ -59,7 +59,7 @@
  * - 1 RTC
  */
 
-static uint64_t static_read(void *opaque, target_phys_addr_t offset,
+static uint64_t static_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     uint32_t *val = (uint32_t *) opaque;
@@ -68,7 +68,7 @@ static uint64_t static_read(void *opaque, target_phys_addr_t offset,
     return *val >> ((offset & mask) << 3);
 }
 
-static void static_write(void *opaque, target_phys_addr_t offset,
+static void static_write(void *opaque, hwaddr offset,
                          uint64_t value, unsigned size)
 {
 #ifdef SPY
@@ -97,11 +97,7 @@ static struct arm_boot_info sx1_binfo = {
     .board_id = 0x265,
 };
 
-static void sx1_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model,
-                const int version)
+static void sx1_init(QEMUMachineInitArgs *args, const int version)
 {
     struct omap_mpu_state_s *mpu;
     MemoryRegion *address_space = get_system_memory();
@@ -121,7 +117,7 @@ static void sx1_init(ram_addr_t ram_size,
         flash_size = flash2_size;
     }
 
-    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size, cpu_model);
+    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size, args->cpu_model);
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, "omap_sx1.flash0-0", flash_size);
@@ -192,16 +188,16 @@ static void sx1_init(ram_addr_t ram_size,
                                 OMAP_CS1_BASE, &cs[1]);
     }
 
-    if (!kernel_filename && !fl_idx) {
+    if (!args->kernel_filename && !fl_idx) {
         fprintf(stderr, "Kernel or Flash image must be specified\n");
         exit(1);
     }
 
     /* Load the kernel.  */
-    if (kernel_filename) {
-        sx1_binfo.kernel_filename = kernel_filename;
-        sx1_binfo.kernel_cmdline = kernel_cmdline;
-        sx1_binfo.initrd_filename = initrd_filename;
+    if (args->kernel_filename) {
+        sx1_binfo.kernel_filename = args->kernel_filename;
+        sx1_binfo.kernel_cmdline = args->kernel_cmdline;
+        sx1_binfo.initrd_filename = args->initrd_filename;
         arm_load_kernel(mpu->cpu, &sx1_binfo);
     }
 
@@ -209,34 +205,28 @@ static void sx1_init(ram_addr_t ram_size,
     //~ qemu_console_resize(ds, 640, 480);
 }
 
-static void sx1_init_v1(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void sx1_init_v1(QEMUMachineInitArgs *args)
 {
-    sx1_init(ram_size, boot_device, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, 1);
+    sx1_init(args, 1);
 }
 
-static void sx1_init_v2(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void sx1_init_v2(QEMUMachineInitArgs *args)
 {
-    sx1_init(ram_size, boot_device, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, 2);
+    sx1_init(args, 2);
 }
 
 static QEMUMachine sx1_machine_v2 = {
     .name = "sx1",
     .desc = "Siemens SX1 (OMAP310) V2",
     .init = sx1_init_v2,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sx1_machine_v1 = {
     .name = "sx1-v1",
     .desc = "Siemens SX1 (OMAP310) V1",
     .init = sx1_init_v1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sx1_machine_init(void)
diff --git a/hw/omap_synctimer.c b/hw/omap_synctimer.c
index 367f26e3a1b..945711eff56 100644
--- a/hw/omap_synctimer.c
+++ b/hw/omap_synctimer.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 struct omap_synctimer_s {
     MemoryRegion iomem;
@@ -36,7 +36,7 @@ void omap_synctimer_reset(struct omap_synctimer_s *s)
     s->val = omap_synctimer_read(s);
 }
 
-static uint32_t omap_synctimer_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_synctimer_readw(void *opaque, hwaddr addr)
 {
     struct omap_synctimer_s *s = (struct omap_synctimer_s *) opaque;
 
@@ -52,7 +52,7 @@ static uint32_t omap_synctimer_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_synctimer_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_synctimer_readh(void *opaque, hwaddr addr)
 {
     struct omap_synctimer_s *s = (struct omap_synctimer_s *) opaque;
     uint32_t ret;
@@ -66,7 +66,7 @@ static uint32_t omap_synctimer_readh(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static void omap_synctimer_write(void *opaque, target_phys_addr_t addr,
+static void omap_synctimer_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAP_BAD_REG(addr);
diff --git a/hw/omap_tap.c b/hw/omap_tap.c
index 0277c736529..e273e971ed4 100644
--- a/hw/omap_tap.c
+++ b/hw/omap_tap.c
@@ -22,7 +22,7 @@
 #include "omap.h"
 
 /* TEST-Chip-level TAP */
-static uint64_t omap_tap_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tap_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -91,7 +91,7 @@ static uint64_t omap_tap_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tap_write(void *opaque, target_phys_addr_t addr,
+static void omap_tap_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
diff --git a/hw/omap_uart.c b/hw/omap_uart.c
index 167d5c4e500..0ebfbf8caec 100644
--- a/hw/omap_uart.c
+++ b/hw/omap_uart.c
@@ -17,17 +17,16 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu-char.h"
+#include "char/char.h"
 #include "hw.h"
 #include "omap.h"
-/* We use pc-style serial ports.  */
-#include "pc.h"
-#include "exec-memory.h"
+#include "serial.h"
+#include "exec/address-spaces.h"
 
 /* UARTs */
 struct omap_uart_s {
     MemoryRegion iomem;
-    target_phys_addr_t base;
+    hwaddr base;
     SerialState *serial; /* TODO */
     struct omap_target_agent_s *ta;
     omap_clk fclk;
@@ -51,7 +50,7 @@ void omap_uart_reset(struct omap_uart_s *s)
     s->clksel = 0;
 }
 
-struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
+struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
@@ -69,7 +68,7 @@ struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
     return s;
 }
 
-static uint64_t omap_uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_uart_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_uart_s *s = (struct omap_uart_s *) opaque;
@@ -107,7 +106,7 @@ static uint64_t omap_uart_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_uart_write(void *opaque, target_phys_addr_t addr,
+static void omap_uart_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_uart_s *s = (struct omap_uart_s *) opaque;
@@ -165,7 +164,7 @@ struct omap_uart_s *omap2_uart_init(MemoryRegion *sysmem,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
 {
-    target_phys_addr_t base = omap_l4_attach(ta, 0, NULL);
+    hwaddr base = omap_l4_attach(ta, 0, NULL);
     struct omap_uart_s *s = omap_uart_init(base, irq,
                     fclk, iclk, txdma, rxdma, label, chr);
 
diff --git a/hw/onenand.c b/hw/onenand.c
index db6af682c42..00a8738caf5 100644
--- a/hw/onenand.c
+++ b/hw/onenand.c
@@ -22,11 +22,11 @@
 #include "hw.h"
 #include "flash.h"
 #include "irq.h"
-#include "blockdev.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 /* 11 for 2kB-page OneNAND ("2nd generation") and 10 for 1kB-page chips */
 #define PAGE_SHIFT	11
@@ -42,7 +42,7 @@ typedef struct {
         uint16_t ver;
     } id;
     int shift;
-    target_phys_addr_t base;
+    hwaddr base;
     qemu_irq intr;
     qemu_irq rdy;
     BlockDriverState *bdrv;
@@ -224,7 +224,7 @@ static void onenand_reset(OneNANDState *s, int cold)
 
 static void onenand_system_reset(DeviceState *dev)
 {
-    onenand_reset(FROM_SYSBUS(OneNANDState, sysbus_from_qdev(dev)), 1);
+    onenand_reset(FROM_SYSBUS(OneNANDState, SYS_BUS_DEVICE(dev)), 1);
 }
 
 static inline int onenand_load_main(OneNANDState *s, int sec, int secn,
@@ -351,7 +351,7 @@ static inline int onenand_erase(OneNANDState *s, int sec, int num)
     for (; num > 0; num--, sec++) {
         if (s->bdrv_cur) {
             int erasesec = s->secs_cur + (sec >> 5);
-            if (bdrv_write(s->bdrv_cur, sec, blankbuf, 1)) {
+            if (bdrv_write(s->bdrv_cur, sec, blankbuf, 1) < 0) {
                 goto fail;
             }
             if (bdrv_read(s->bdrv_cur, erasesec, tmpbuf, 1) < 0) {
@@ -588,7 +588,7 @@ static void onenand_command(OneNANDState *s)
     onenand_intr_update(s);
 }
 
-static uint64_t onenand_read(void *opaque, target_phys_addr_t addr,
+static uint64_t onenand_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     OneNANDState *s = (OneNANDState *) opaque;
@@ -653,7 +653,7 @@ static uint64_t onenand_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void onenand_write(void *opaque, target_phys_addr_t addr,
+static void onenand_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     OneNANDState *s = (OneNANDState *) opaque;
@@ -760,7 +760,7 @@ static int onenand_initfn(SysBusDevice *dev)
     OneNANDState *s = (OneNANDState *)dev;
     uint32_t size = 1 << (24 + ((s->id.dev >> 4) & 7));
     void *ram;
-    s->base = (target_phys_addr_t)-1;
+    s->base = (hwaddr)-1;
     s->rdy = NULL;
     s->blocks = size >> BLOCK_SHIFT;
     s->secs = size >> 9;
@@ -821,7 +821,7 @@ static void onenand_class_init(ObjectClass *klass, void *data)
     dc->props = onenand_properties;
 }
 
-static TypeInfo onenand_info = {
+static const TypeInfo onenand_info = {
     .name          = "onenand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OneNANDState),
@@ -835,7 +835,7 @@ static void onenand_register_types(void)
 
 void *onenand_raw_otp(DeviceState *onenand_device)
 {
-    return FROM_SYSBUS(OneNANDState, sysbus_from_qdev(onenand_device))->otp;
+    return FROM_SYSBUS(OneNANDState, SYS_BUS_DEVICE(onenand_device))->otp;
 }
 
 type_init(onenand_register_types)
diff --git a/hw/opencores_eth.c b/hw/opencores_eth.c
index 8c15969e2bf..746a959f6bf 100644
--- a/hw/opencores_eth.c
+++ b/hw/opencores_eth.c
@@ -33,8 +33,8 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 /* RECSMALL is not used because it breaks tap networking in linux:
@@ -528,7 +528,7 @@ static void open_eth_check_start_xmit(OpenEthState *s)
 }
 
 static uint64_t open_eth_reg_read(void *opaque,
-        target_phys_addr_t addr, unsigned int size)
+        hwaddr addr, unsigned int size)
 {
     static uint32_t (*reg_read[REG_MAX])(OpenEthState *s) = {
     };
@@ -620,7 +620,7 @@ static void open_eth_mii_tx_host_write(OpenEthState *s, uint32_t val)
 }
 
 static void open_eth_reg_write(void *opaque,
-        target_phys_addr_t addr, uint64_t val, unsigned int size)
+        hwaddr addr, uint64_t val, unsigned int size)
 {
     static void (*reg_write[REG_MAX])(OpenEthState *s, uint32_t val) = {
         [MODER] = open_eth_moder_host_write,
@@ -644,7 +644,7 @@ static void open_eth_reg_write(void *opaque,
 }
 
 static uint64_t open_eth_desc_read(void *opaque,
-        target_phys_addr_t addr, unsigned int size)
+        hwaddr addr, unsigned int size)
 {
     OpenEthState *s = opaque;
     uint64_t v = 0;
@@ -656,7 +656,7 @@ static uint64_t open_eth_desc_read(void *opaque,
 }
 
 static void open_eth_desc_write(void *opaque,
-        target_phys_addr_t addr, uint64_t val, unsigned int size)
+        hwaddr addr, uint64_t val, unsigned int size)
 {
     OpenEthState *s = opaque;
 
@@ -718,7 +718,7 @@ static void open_eth_class_init(ObjectClass *klass, void *data)
     dc->props = open_eth_properties;
 }
 
-static TypeInfo open_eth_info = {
+static const TypeInfo open_eth_info = {
     .name          = "open_eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OpenEthState),
diff --git a/hw/openpic.c b/hw/openpic.c
index b9d856830a4..20a479c794f 100644
--- a/hw/openpic.c
+++ b/hw/openpic.c
@@ -34,381 +34,463 @@
  *
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
 #include "openpic.h"
+#include "sysbus.h"
+#include "pci/msi.h"
+#include "qemu/bitops.h"
+#include "ppc.h"
 
 //#define DEBUG_OPENPIC
 
 #ifdef DEBUG_OPENPIC
-#define DPRINTF(fmt, ...) do { printf(fmt , ## __VA_ARGS__); } while (0)
+static const int debug_openpic = 1;
 #else
-#define DPRINTF(fmt, ...) do { } while (0)
+static const int debug_openpic = 0;
 #endif
 
-#define USE_MPCxxx /* Intel model is broken, for now */
+#define DPRINTF(fmt, ...) do { \
+        if (debug_openpic) { \
+            printf(fmt , ## __VA_ARGS__); \
+        } \
+    } while (0)
 
-#if defined (USE_INTEL_GW80314)
-/* Intel GW80314 I/O Companion chip */
-
-#define MAX_CPU     4
-#define MAX_IRQ    32
-#define MAX_DBL     4
-#define MAX_MBX     4
+#define MAX_CPU     32
+#define MAX_SRC     256
 #define MAX_TMR     4
-#define VECTOR_BITS 8
 #define MAX_IPI     4
+#define MAX_MSI     8
+#define MAX_IRQ     (MAX_SRC + MAX_IPI + MAX_TMR)
+#define VID         0x03 /* MPIC version ID */
 
-#define VID (0x00000000)
-
-#elif defined(USE_MPCxxx)
+/* OpenPIC capability flags */
+#define OPENPIC_FLAG_IDR_CRIT     (1 << 0)
+#define OPENPIC_FLAG_ILR          (2 << 0)
+
+/* OpenPIC address map */
+#define OPENPIC_GLB_REG_START        0x0
+#define OPENPIC_GLB_REG_SIZE         0x10F0
+#define OPENPIC_TMR_REG_START        0x10F0
+#define OPENPIC_TMR_REG_SIZE         0x220
+#define OPENPIC_MSI_REG_START        0x1600
+#define OPENPIC_MSI_REG_SIZE         0x200
+#define OPENPIC_SUMMARY_REG_START   0x3800
+#define OPENPIC_SUMMARY_REG_SIZE    0x800
+#define OPENPIC_SRC_REG_START        0x10000
+#define OPENPIC_SRC_REG_SIZE         (MAX_SRC * 0x20)
+#define OPENPIC_CPU_REG_START        0x20000
+#define OPENPIC_CPU_REG_SIZE         0x100 + ((MAX_CPU - 1) * 0x1000)
+
+/* Raven */
+#define RAVEN_MAX_CPU      2
+#define RAVEN_MAX_EXT     48
+#define RAVEN_MAX_IRQ     64
+#define RAVEN_MAX_TMR      MAX_TMR
+#define RAVEN_MAX_IPI      MAX_IPI
 
-#define MAX_CPU    15
-#define MAX_IRQ   128
-#define MAX_DBL     0
-#define MAX_MBX     0
-#define MAX_TMR     4
-#define VECTOR_BITS 8
-#define MAX_IPI     4
-#define VID         0x03 /* MPIC version ID */
-#define VENI        0x00000000 /* Vendor ID */
+/* Interrupt definitions */
+#define RAVEN_FE_IRQ     (RAVEN_MAX_EXT)     /* Internal functional IRQ */
+#define RAVEN_ERR_IRQ    (RAVEN_MAX_EXT + 1) /* Error IRQ */
+#define RAVEN_TMR_IRQ    (RAVEN_MAX_EXT + 2) /* First timer IRQ */
+#define RAVEN_IPI_IRQ    (RAVEN_TMR_IRQ + RAVEN_MAX_TMR) /* First IPI IRQ */
+/* First doorbell IRQ */
+#define RAVEN_DBL_IRQ    (RAVEN_IPI_IRQ + (RAVEN_MAX_CPU * RAVEN_MAX_IPI))
+
+typedef struct FslMpicInfo {
+    int max_ext;
+} FslMpicInfo;
+
+static FslMpicInfo fsl_mpic_20 = {
+    .max_ext = 12,
+};
 
-enum {
-    IRQ_IPVP = 0,
-    IRQ_IDE,
+static FslMpicInfo fsl_mpic_42 = {
+    .max_ext = 12,
 };
 
-/* OpenPIC */
-#define OPENPIC_MAX_CPU      2
-#define OPENPIC_MAX_IRQ     64
-#define OPENPIC_EXT_IRQ     48
-#define OPENPIC_MAX_TMR      MAX_TMR
-#define OPENPIC_MAX_IPI      MAX_IPI
+#define FRR_NIRQ_SHIFT    16
+#define FRR_NCPU_SHIFT     8
+#define FRR_VID_SHIFT      0
 
-/* Interrupt definitions */
-#define OPENPIC_IRQ_FE     (OPENPIC_EXT_IRQ)     /* Internal functional IRQ */
-#define OPENPIC_IRQ_ERR    (OPENPIC_EXT_IRQ + 1) /* Error IRQ */
-#define OPENPIC_IRQ_TIM0   (OPENPIC_EXT_IRQ + 2) /* First timer IRQ */
-#if OPENPIC_MAX_IPI > 0
-#define OPENPIC_IRQ_IPI0   (OPENPIC_IRQ_TIM0 + OPENPIC_MAX_TMR) /* First IPI IRQ */
-#define OPENPIC_IRQ_DBL0   (OPENPIC_IRQ_IPI0 + (OPENPIC_MAX_CPU * OPENPIC_MAX_IPI)) /* First doorbell IRQ */
-#else
-#define OPENPIC_IRQ_DBL0   (OPENPIC_IRQ_TIM0 + OPENPIC_MAX_TMR) /* First doorbell IRQ */
-#define OPENPIC_IRQ_MBX0   (OPENPIC_IRQ_DBL0 + OPENPIC_MAX_DBL) /* First mailbox IRQ */
-#endif
+#define VID_REVISION_1_2   2
+#define VID_REVISION_1_3   3
 
-/* MPIC */
-#define MPIC_MAX_CPU      1
-#define MPIC_MAX_EXT     12
-#define MPIC_MAX_INT     64
-#define MPIC_MAX_MSG      4
-#define MPIC_MAX_MSI      8
-#define MPIC_MAX_TMR      MAX_TMR
-#define MPIC_MAX_IPI      MAX_IPI
-#define MPIC_MAX_IRQ     (MPIC_MAX_EXT + MPIC_MAX_INT + MPIC_MAX_TMR + MPIC_MAX_MSG + MPIC_MAX_MSI + (MPIC_MAX_IPI * MPIC_MAX_CPU))
+#define VIR_GENERIC      0x00000000 /* Generic Vendor ID */
 
-/* Interrupt definitions */
-#define MPIC_EXT_IRQ      0
-#define MPIC_INT_IRQ      (MPIC_EXT_IRQ + MPIC_MAX_EXT)
-#define MPIC_TMR_IRQ      (MPIC_INT_IRQ + MPIC_MAX_INT)
-#define MPIC_MSG_IRQ      (MPIC_TMR_IRQ + MPIC_MAX_TMR)
-#define MPIC_MSI_IRQ      (MPIC_MSG_IRQ + MPIC_MAX_MSG)
-#define MPIC_IPI_IRQ      (MPIC_MSI_IRQ + MPIC_MAX_MSI)
-
-#define MPIC_GLB_REG_START        0x0
-#define MPIC_GLB_REG_SIZE         0x10F0
-#define MPIC_TMR_REG_START        0x10F0
-#define MPIC_TMR_REG_SIZE         0x220
-#define MPIC_EXT_REG_START        0x10000
-#define MPIC_EXT_REG_SIZE         0x180
-#define MPIC_INT_REG_START        0x10200
-#define MPIC_INT_REG_SIZE         0x800
-#define MPIC_MSG_REG_START        0x11600
-#define MPIC_MSG_REG_SIZE         0x100
-#define MPIC_MSI_REG_START        0x11C00
-#define MPIC_MSI_REG_SIZE         0x100
-#define MPIC_CPU_REG_START        0x20000
-#define MPIC_CPU_REG_SIZE         0x100 + ((MAX_CPU - 1) * 0x1000)
+#define GCR_RESET        0x80000000
+#define GCR_MODE_PASS    0x00000000
+#define GCR_MODE_MIXED   0x20000000
+#define GCR_MODE_PROXY   0x60000000
 
-/*
- * Block Revision Register1 (BRR1): QEMU does not fully emulate
- * any version on MPIC. So to start with, set the IP version to 0.
- *
- * NOTE: This is Freescale MPIC specific register. Keep it here till
- * this code is refactored for different variants of OPENPIC and MPIC.
- */
-#define FSL_BRR1_IPID (0x0040 << 16) /* 16 bit IP-block ID */
-#define FSL_BRR1_IPMJ (0x00 << 8) /* 8 bit IP major number */
-#define FSL_BRR1_IPMN 0x00 /* 8 bit IP minor number */
-
-enum mpic_ide_bits {
-    IDR_EP     = 31,
-    IDR_CI0     = 30,
-    IDR_CI1     = 29,
-    IDR_P1     = 1,
-    IDR_P0     = 0,
-};
+#define TBCR_CI           0x80000000 /* count inhibit */
+#define TCCR_TOG          0x80000000 /* toggles when decrement to zero */
 
-#else
-#error "Please select which OpenPic implementation is to be emulated"
-#endif
+#define IDR_EP_SHIFT      31
+#define IDR_EP_MASK       (1 << IDR_EP_SHIFT)
+#define IDR_CI0_SHIFT     30
+#define IDR_CI1_SHIFT     29
+#define IDR_P1_SHIFT      1
+#define IDR_P0_SHIFT      0
 
-#define OPENPIC_PAGE_SIZE 4096
+#define ILR_INTTGT_MASK   0x000000ff
+#define ILR_INTTGT_INT    0x00
+#define ILR_INTTGT_CINT   0x01 /* critical */
+#define ILR_INTTGT_MCP    0x02 /* machine check */
 
-#define BF_WIDTH(_bits_) \
-(((_bits_) + (sizeof(uint32_t) * 8) - 1) / (sizeof(uint32_t) * 8))
+/* The currently supported INTTGT values happen to be the same as QEMU's
+ * openpic output codes, but don't depend on this.  The output codes
+ * could change (unlikely, but...) or support could be added for
+ * more INTTGT values.
+ */
+static const int inttgt_output[][2] = {
+    { ILR_INTTGT_INT, OPENPIC_OUTPUT_INT },
+    { ILR_INTTGT_CINT, OPENPIC_OUTPUT_CINT },
+    { ILR_INTTGT_MCP, OPENPIC_OUTPUT_MCK },
+};
 
-static inline void set_bit (uint32_t *field, int bit)
+static int inttgt_to_output(int inttgt)
 {
-    field[bit >> 5] |= 1 << (bit & 0x1F);
-}
+    int i;
 
-static inline void reset_bit (uint32_t *field, int bit)
-{
-    field[bit >> 5] &= ~(1 << (bit & 0x1F));
+    for (i = 0; i < ARRAY_SIZE(inttgt_output); i++) {
+        if (inttgt_output[i][0] == inttgt) {
+            return inttgt_output[i][1];
+        }
+    }
+
+    fprintf(stderr, "%s: unsupported inttgt %d\n", __func__, inttgt);
+    return OPENPIC_OUTPUT_INT;
 }
 
-static inline int test_bit (uint32_t *field, int bit)
+static int output_to_inttgt(int output)
 {
-    return (field[bit >> 5] & 1 << (bit & 0x1F)) != 0;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(inttgt_output); i++) {
+        if (inttgt_output[i][1] == output) {
+            return inttgt_output[i][0];
+        }
+    }
+
+    abort();
 }
 
+#define MSIIR_OFFSET       0x140
+#define MSIIR_SRS_SHIFT    29
+#define MSIIR_SRS_MASK     (0x7 << MSIIR_SRS_SHIFT)
+#define MSIIR_IBS_SHIFT    24
+#define MSIIR_IBS_MASK     (0x1f << MSIIR_IBS_SHIFT)
+
 static int get_current_cpu(void)
 {
-  return cpu_single_env->cpu_index;
+    CPUState *cpu_single_cpu;
+
+    if (!cpu_single_env) {
+        return -1;
+    }
+
+    cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+    return cpu_single_cpu->cpu_index;
 }
 
-static uint32_t openpic_cpu_read_internal(void *opaque, target_phys_addr_t addr,
+static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
                                           int idx);
-static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
+static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx);
 
-enum {
-    IRQ_EXTERNAL = 0x01,
-    IRQ_INTERNAL = 0x02,
-    IRQ_TIMER    = 0x04,
-    IRQ_SPECIAL  = 0x08,
-};
-
-typedef struct IRQ_queue_t {
-    uint32_t queue[BF_WIDTH(MAX_IRQ)];
+typedef enum IRQType {
+    IRQ_TYPE_NORMAL = 0,
+    IRQ_TYPE_FSLINT,        /* FSL internal interrupt -- level only */
+    IRQ_TYPE_FSLSPECIAL,    /* FSL timer/IPI interrupt, edge, no polarity */
+} IRQType;
+
+typedef struct IRQQueue {
+    /* Round up to the nearest 64 IRQs so that the queue length
+     * won't change when moving between 32 and 64 bit hosts.
+     */
+    unsigned long queue[BITS_TO_LONGS((MAX_IRQ + 63) & ~63)];
     int next;
     int priority;
-} IRQ_queue_t;
+} IRQQueue;
 
-typedef struct IRQ_src_t {
-    uint32_t ipvp;  /* IRQ vector/priority register */
-    uint32_t ide;   /* IRQ destination register */
-    int type;
+typedef struct IRQSource {
+    uint32_t ivpr;  /* IRQ vector/priority register */
+    uint32_t idr;   /* IRQ destination register */
+    uint32_t destmask; /* bitmap of CPU destinations */
     int last_cpu;
+    int output;     /* IRQ level, e.g. OPENPIC_OUTPUT_INT */
     int pending;    /* TRUE if IRQ is pending */
-} IRQ_src_t;
-
-enum IPVP_bits {
-    IPVP_MASK     = 31,
-    IPVP_ACTIVITY = 30,
-    IPVP_MODE     = 29,
-    IPVP_POLARITY = 23,
-    IPVP_SENSE    = 22,
-};
-#define IPVP_PRIORITY_MASK     (0x1F << 16)
-#define IPVP_PRIORITY(_ipvpr_) ((int)(((_ipvpr_) & IPVP_PRIORITY_MASK) >> 16))
-#define IPVP_VECTOR_MASK       ((1 << VECTOR_BITS) - 1)
-#define IPVP_VECTOR(_ipvpr_)   ((_ipvpr_) & IPVP_VECTOR_MASK)
-
-typedef struct IRQ_dst_t {
-    uint32_t tfrr;
-    uint32_t pctp; /* CPU current task priority */
-    uint32_t pcsr; /* CPU sensitivity register */
-    IRQ_queue_t raised;
-    IRQ_queue_t servicing;
+    IRQType type;
+    bool level:1;   /* level-triggered */
+    bool nomask:1;  /* critical interrupts ignore mask on some FSL MPICs */
+} IRQSource;
+
+#define IVPR_MASK_SHIFT       31
+#define IVPR_MASK_MASK        (1 << IVPR_MASK_SHIFT)
+#define IVPR_ACTIVITY_SHIFT   30
+#define IVPR_ACTIVITY_MASK    (1 << IVPR_ACTIVITY_SHIFT)
+#define IVPR_MODE_SHIFT       29
+#define IVPR_MODE_MASK        (1 << IVPR_MODE_SHIFT)
+#define IVPR_POLARITY_SHIFT   23
+#define IVPR_POLARITY_MASK    (1 << IVPR_POLARITY_SHIFT)
+#define IVPR_SENSE_SHIFT      22
+#define IVPR_SENSE_MASK       (1 << IVPR_SENSE_SHIFT)
+
+#define IVPR_PRIORITY_MASK     (0xF << 16)
+#define IVPR_PRIORITY(_ivprr_) ((int)(((_ivprr_) & IVPR_PRIORITY_MASK) >> 16))
+#define IVPR_VECTOR(opp, _ivprr_) ((_ivprr_) & (opp)->vector_mask)
+
+/* IDR[EP/CI] are only for FSL MPIC prior to v4.0 */
+#define IDR_EP      0x80000000  /* external pin */
+#define IDR_CI      0x40000000  /* critical interrupt */
+
+typedef struct IRQDest {
+    int32_t ctpr; /* CPU current task priority */
+    IRQQueue raised;
+    IRQQueue servicing;
     qemu_irq *irqs;
-} IRQ_dst_t;
 
-typedef struct openpic_t {
-    PCIDevice pci_dev;
+    /* Count of IRQ sources asserting on non-INT outputs */
+    uint32_t outputs_active[OPENPIC_OUTPUT_NB];
+} IRQDest;
+
+typedef struct OpenPICState {
+    SysBusDevice busdev;
     MemoryRegion mem;
 
+    /* Behavior control */
+    FslMpicInfo *fsl;
+    uint32_t model;
+    uint32_t flags;
+    uint32_t nb_irqs;
+    uint32_t vid;
+    uint32_t vir; /* Vendor identification register */
+    uint32_t vector_mask;
+    uint32_t tfrr_reset;
+    uint32_t ivpr_reset;
+    uint32_t idr_reset;
+    uint32_t brr1;
+    uint32_t mpic_mode_mask;
+
     /* Sub-regions */
-    MemoryRegion sub_io_mem[7];
+    MemoryRegion sub_io_mem[6];
 
     /* Global registers */
-    uint32_t frep; /* Feature reporting register */
-    uint32_t glbc; /* Global configuration register  */
-    uint32_t micr; /* MPIC interrupt configuration register */
-    uint32_t veni; /* Vendor identification register */
-    uint32_t pint; /* Processor initialization register */
+    uint32_t frr; /* Feature reporting register */
+    uint32_t gcr; /* Global configuration register  */
+    uint32_t pir; /* Processor initialization register */
     uint32_t spve; /* Spurious vector register */
-    uint32_t tifr; /* Timer frequency reporting register */
+    uint32_t tfrr; /* Timer frequency reporting register */
     /* Source registers */
-    IRQ_src_t src[MAX_IRQ];
+    IRQSource src[MAX_IRQ];
     /* Local registers per output pin */
-    IRQ_dst_t dst[MAX_CPU];
-    int nb_cpus;
+    IRQDest dst[MAX_CPU];
+    uint32_t nb_cpus;
     /* Timer registers */
     struct {
-        uint32_t ticc;  /* Global timer current count register */
-        uint32_t tibc;  /* Global timer base count register */
+        uint32_t tccr;  /* Global timer current count register */
+        uint32_t tbcr;  /* Global timer base count register */
     } timers[MAX_TMR];
-#if MAX_DBL > 0
-    /* Doorbell registers */
-    uint32_t dar;        /* Doorbell activate register */
+    /* Shared MSI registers */
     struct {
-        uint32_t dmr;    /* Doorbell messaging register */
-    } doorbells[MAX_DBL];
-#endif
-#if MAX_MBX > 0
-    /* Mailbox registers */
-    struct {
-        uint32_t mbr;    /* Mailbox register */
-    } mailboxes[MAX_MAILBOXES];
-#endif
-    /* IRQ out is used when in bypass mode (not implemented) */
-    qemu_irq irq_out;
-    int max_irq;
-    int irq_ipi0;
-    int irq_tim0;
-    void (*reset) (void *);
-    void (*irq_raise) (struct openpic_t *, int, IRQ_src_t *);
-} openpic_t;
-
-static inline void IRQ_setbit (IRQ_queue_t *q, int n_IRQ)
+        uint32_t msir;   /* Shared Message Signaled Interrupt Register */
+    } msi[MAX_MSI];
+    uint32_t max_irq;
+    uint32_t irq_ipi0;
+    uint32_t irq_tim0;
+    uint32_t irq_msi;
+} OpenPICState;
+
+static inline void IRQ_setbit(IRQQueue *q, int n_IRQ)
 {
-    set_bit(q->queue, n_IRQ);
+    set_bit(n_IRQ, q->queue);
 }
 
-static inline void IRQ_resetbit (IRQ_queue_t *q, int n_IRQ)
+static inline void IRQ_resetbit(IRQQueue *q, int n_IRQ)
 {
-    reset_bit(q->queue, n_IRQ);
+    clear_bit(n_IRQ, q->queue);
 }
 
-static inline int IRQ_testbit (IRQ_queue_t *q, int n_IRQ)
+static inline int IRQ_testbit(IRQQueue *q, int n_IRQ)
 {
-    return test_bit(q->queue, n_IRQ);
+    return test_bit(n_IRQ, q->queue);
 }
 
-static void IRQ_check (openpic_t *opp, IRQ_queue_t *q)
+static void IRQ_check(OpenPICState *opp, IRQQueue *q)
 {
-    int next, i;
-    int priority;
+    int irq = -1;
+    int next = -1;
+    int priority = -1;
+
+    for (;;) {
+        irq = find_next_bit(q->queue, opp->max_irq, irq + 1);
+        if (irq == opp->max_irq) {
+            break;
+        }
 
-    next = -1;
-    priority = -1;
-    for (i = 0; i < opp->max_irq; i++) {
-        if (IRQ_testbit(q, i)) {
-            DPRINTF("IRQ_check: irq %d set ipvp_pr=%d pr=%d\n",
-                    i, IPVP_PRIORITY(opp->src[i].ipvp), priority);
-            if (IPVP_PRIORITY(opp->src[i].ipvp) > priority) {
-                next = i;
-                priority = IPVP_PRIORITY(opp->src[i].ipvp);
-            }
+        DPRINTF("IRQ_check: irq %d set ivpr_pr=%d pr=%d\n",
+                irq, IVPR_PRIORITY(opp->src[irq].ivpr), priority);
+
+        if (IVPR_PRIORITY(opp->src[irq].ivpr) > priority) {
+            next = irq;
+            priority = IVPR_PRIORITY(opp->src[irq].ivpr);
         }
     }
+
     q->next = next;
     q->priority = priority;
 }
 
-static int IRQ_get_next (openpic_t *opp, IRQ_queue_t *q)
+static int IRQ_get_next(OpenPICState *opp, IRQQueue *q)
 {
-    if (q->next == -1) {
-        /* XXX: optimize */
-        IRQ_check(opp, q);
-    }
+    /* XXX: optimize */
+    IRQ_check(opp, q);
 
     return q->next;
 }
 
-static void IRQ_local_pipe (openpic_t *opp, int n_CPU, int n_IRQ)
+static void IRQ_local_pipe(OpenPICState *opp, int n_CPU, int n_IRQ,
+                           bool active, bool was_active)
 {
-    IRQ_dst_t *dst;
-    IRQ_src_t *src;
+    IRQDest *dst;
+    IRQSource *src;
     int priority;
 
     dst = &opp->dst[n_CPU];
     src = &opp->src[n_IRQ];
-    priority = IPVP_PRIORITY(src->ipvp);
-    if (priority <= dst->pctp) {
-        /* Too low priority */
-        DPRINTF("%s: IRQ %d has too low priority on CPU %d\n",
-                __func__, n_IRQ, n_CPU);
+
+    DPRINTF("%s: IRQ %d active %d was %d\n",
+            __func__, n_IRQ, active, was_active);
+
+    if (src->output != OPENPIC_OUTPUT_INT) {
+        DPRINTF("%s: output %d irq %d active %d was %d count %d\n",
+                __func__, src->output, n_IRQ, active, was_active,
+                dst->outputs_active[src->output]);
+
+        /* On Freescale MPIC, critical interrupts ignore priority,
+         * IACK, EOI, etc.  Before MPIC v4.1 they also ignore
+         * masking.
+         */
+        if (active) {
+            if (!was_active && dst->outputs_active[src->output]++ == 0) {
+                DPRINTF("%s: Raise OpenPIC output %d cpu %d irq %d\n",
+                        __func__, src->output, n_CPU, n_IRQ);
+                qemu_irq_raise(dst->irqs[src->output]);
+            }
+        } else {
+            if (was_active && --dst->outputs_active[src->output] == 0) {
+                DPRINTF("%s: Lower OpenPIC output %d cpu %d irq %d\n",
+                        __func__, src->output, n_CPU, n_IRQ);
+                qemu_irq_lower(dst->irqs[src->output]);
+            }
+        }
+
         return;
     }
-    if (IRQ_testbit(&dst->raised, n_IRQ)) {
-        /* Interrupt miss */
-        DPRINTF("%s: IRQ %d was missed on CPU %d\n",
-                __func__, n_IRQ, n_CPU);
-        return;
+
+    priority = IVPR_PRIORITY(src->ivpr);
+
+    /* Even if the interrupt doesn't have enough priority,
+     * it is still raised, in case ctpr is lowered later.
+     */
+    if (active) {
+        IRQ_setbit(&dst->raised, n_IRQ);
+    } else {
+        IRQ_resetbit(&dst->raised, n_IRQ);
     }
-    set_bit(&src->ipvp, IPVP_ACTIVITY);
-    IRQ_setbit(&dst->raised, n_IRQ);
-    if (priority < dst->raised.priority) {
-        /* An higher priority IRQ is already raised */
-        DPRINTF("%s: IRQ %d is hidden by raised IRQ %d on CPU %d\n",
-                __func__, n_IRQ, dst->raised.next, n_CPU);
-        return;
+
+    IRQ_check(opp, &dst->raised);
+
+    if (active && priority <= dst->ctpr) {
+        DPRINTF("%s: IRQ %d priority %d too low for ctpr %d on CPU %d\n",
+                __func__, n_IRQ, priority, dst->ctpr, n_CPU);
+        active = 0;
     }
-    IRQ_get_next(opp, &dst->raised);
-    if (IRQ_get_next(opp, &dst->servicing) != -1 &&
-        priority <= dst->servicing.priority) {
-        DPRINTF("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
-                __func__, n_IRQ, dst->servicing.next, n_CPU);
-        /* Already servicing a higher priority IRQ */
-        return;
+
+    if (active) {
+        if (IRQ_get_next(opp, &dst->servicing) >= 0 &&
+                priority <= dst->servicing.priority) {
+            DPRINTF("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
+                    __func__, n_IRQ, dst->servicing.next, n_CPU);
+        } else {
+            DPRINTF("%s: Raise OpenPIC INT output cpu %d irq %d/%d\n",
+                    __func__, n_CPU, n_IRQ, dst->raised.next);
+            qemu_irq_raise(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
+        }
+    } else {
+        IRQ_get_next(opp, &dst->servicing);
+        if (dst->raised.priority > dst->ctpr &&
+                dst->raised.priority > dst->servicing.priority) {
+            DPRINTF("%s: IRQ %d inactive, IRQ %d prio %d above %d/%d, CPU %d\n",
+                    __func__, n_IRQ, dst->raised.next, dst->raised.priority,
+                    dst->ctpr, dst->servicing.priority, n_CPU);
+            /* IRQ line stays asserted */
+        } else {
+            DPRINTF("%s: IRQ %d inactive, current prio %d/%d, CPU %d\n",
+                    __func__, n_IRQ, dst->ctpr, dst->servicing.priority, n_CPU);
+            qemu_irq_lower(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
+        }
     }
-    DPRINTF("Raise OpenPIC INT output cpu %d irq %d\n", n_CPU, n_IRQ);
-    opp->irq_raise(opp, n_CPU, src);
 }
 
 /* update pic state because registers for n_IRQ have changed value */
-static void openpic_update_irq(openpic_t *opp, int n_IRQ)
+static void openpic_update_irq(OpenPICState *opp, int n_IRQ)
 {
-    IRQ_src_t *src;
+    IRQSource *src;
+    bool active, was_active;
     int i;
 
     src = &opp->src[n_IRQ];
+    active = src->pending;
 
-    if (!src->pending) {
-        /* no irq pending */
-        DPRINTF("%s: IRQ %d is not pending\n", __func__, n_IRQ);
-        return;
-    }
-    if (test_bit(&src->ipvp, IPVP_MASK)) {
+    if ((src->ivpr & IVPR_MASK_MASK) && !src->nomask) {
         /* Interrupt source is disabled */
         DPRINTF("%s: IRQ %d is disabled\n", __func__, n_IRQ);
-        return;
+        active = false;
     }
-    if (IPVP_PRIORITY(src->ipvp) == 0) {
-        /* Priority set to zero */
-        DPRINTF("%s: IRQ %d has 0 priority\n", __func__, n_IRQ);
+
+    was_active = !!(src->ivpr & IVPR_ACTIVITY_MASK);
+
+    /*
+     * We don't have a similar check for already-active because
+     * ctpr may have changed and we need to withdraw the interrupt.
+     */
+    if (!active && !was_active) {
+        DPRINTF("%s: IRQ %d is already inactive\n", __func__, n_IRQ);
         return;
     }
-    if (test_bit(&src->ipvp, IPVP_ACTIVITY)) {
-        /* IRQ already active */
-        DPRINTF("%s: IRQ %d is already active\n", __func__, n_IRQ);
-        return;
+
+    if (active) {
+        src->ivpr |= IVPR_ACTIVITY_MASK;
+    } else {
+        src->ivpr &= ~IVPR_ACTIVITY_MASK;
     }
-    if (src->ide == 0x00000000) {
+
+    if (src->destmask == 0) {
         /* No target */
         DPRINTF("%s: IRQ %d has no target\n", __func__, n_IRQ);
         return;
     }
 
-    if (src->ide == (1 << src->last_cpu)) {
+    if (src->destmask == (1 << src->last_cpu)) {
         /* Only one CPU is allowed to receive this IRQ */
-        IRQ_local_pipe(opp, src->last_cpu, n_IRQ);
-    } else if (!test_bit(&src->ipvp, IPVP_MODE)) {
+        IRQ_local_pipe(opp, src->last_cpu, n_IRQ, active, was_active);
+    } else if (!(src->ivpr & IVPR_MODE_MASK)) {
         /* Directed delivery mode */
         for (i = 0; i < opp->nb_cpus; i++) {
-            if (test_bit(&src->ide, i))
-                IRQ_local_pipe(opp, i, n_IRQ);
+            if (src->destmask & (1 << i)) {
+                IRQ_local_pipe(opp, i, n_IRQ, active, was_active);
+            }
         }
     } else {
         /* Distributed delivery mode */
         for (i = src->last_cpu + 1; i != src->last_cpu; i++) {
-            if (i == opp->nb_cpus)
+            if (i == opp->nb_cpus) {
                 i = 0;
-            if (test_bit(&src->ide, i)) {
-                IRQ_local_pipe(opp, i, n_IRQ);
+            }
+            if (src->destmask & (1 << i)) {
+                IRQ_local_pipe(opp, i, n_IRQ, active, was_active);
                 src->last_cpu = i;
                 break;
             }
@@ -418,193 +500,236 @@ static void openpic_update_irq(openpic_t *opp, int n_IRQ)
 
 static void openpic_set_irq(void *opaque, int n_IRQ, int level)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
+    OpenPICState *opp = opaque;
+    IRQSource *src;
+
+    if (n_IRQ >= MAX_IRQ) {
+        fprintf(stderr, "%s: IRQ %d out of range\n", __func__, n_IRQ);
+        abort();
+    }
 
     src = &opp->src[n_IRQ];
-    DPRINTF("openpic: set irq %d = %d ipvp=%08x\n",
-            n_IRQ, level, src->ipvp);
-    if (test_bit(&src->ipvp, IPVP_SENSE)) {
+    DPRINTF("openpic: set irq %d = %d ivpr=0x%08x\n",
+            n_IRQ, level, src->ivpr);
+    if (src->level) {
         /* level-sensitive irq */
         src->pending = level;
-        if (!level)
-            reset_bit(&src->ipvp, IPVP_ACTIVITY);
+        openpic_update_irq(opp, n_IRQ);
     } else {
         /* edge-sensitive irq */
-        if (level)
+        if (level) {
             src->pending = 1;
+            openpic_update_irq(opp, n_IRQ);
+        }
+
+        if (src->output != OPENPIC_OUTPUT_INT) {
+            /* Edge-triggered interrupts shouldn't be used
+             * with non-INT delivery, but just in case,
+             * try to make it do something sane rather than
+             * cause an interrupt storm.  This is close to
+             * what you'd probably see happen in real hardware.
+             */
+            src->pending = 0;
+            openpic_update_irq(opp, n_IRQ);
+        }
     }
-    openpic_update_irq(opp, n_IRQ);
 }
 
-static void openpic_reset (void *opaque)
+static void openpic_reset(DeviceState *d)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = FROM_SYSBUS(typeof(*opp), SYS_BUS_DEVICE(d));
     int i;
 
-    opp->glbc = 0x80000000;
+    opp->gcr = GCR_RESET;
     /* Initialise controller registers */
-    opp->frep = ((OPENPIC_EXT_IRQ - 1) << 16) | ((MAX_CPU - 1) << 8) | VID;
-    opp->veni = VENI;
-    opp->pint = 0x00000000;
-    opp->spve = 0x000000FF;
-    opp->tifr = 0x003F7A00;
-    /* ? */
-    opp->micr = 0x00000000;
+    opp->frr = ((opp->nb_irqs - 1) << FRR_NIRQ_SHIFT) |
+               ((opp->nb_cpus - 1) << FRR_NCPU_SHIFT) |
+               (opp->vid << FRR_VID_SHIFT);
+
+    opp->pir = 0;
+    opp->spve = -1 & opp->vector_mask;
+    opp->tfrr = opp->tfrr_reset;
     /* Initialise IRQ sources */
     for (i = 0; i < opp->max_irq; i++) {
-        opp->src[i].ipvp = 0xA0000000;
-        opp->src[i].ide  = 0x00000000;
+        opp->src[i].ivpr = opp->ivpr_reset;
+        opp->src[i].idr  = opp->idr_reset;
+
+        switch (opp->src[i].type) {
+        case IRQ_TYPE_NORMAL:
+            opp->src[i].level = !!(opp->ivpr_reset & IVPR_SENSE_MASK);
+            break;
+
+        case IRQ_TYPE_FSLINT:
+            opp->src[i].ivpr |= IVPR_POLARITY_MASK;
+            break;
+
+        case IRQ_TYPE_FSLSPECIAL:
+            break;
+        }
     }
     /* Initialise IRQ destinations */
     for (i = 0; i < MAX_CPU; i++) {
-        opp->dst[i].pctp      = 0x0000000F;
-        opp->dst[i].pcsr      = 0x00000000;
-        memset(&opp->dst[i].raised, 0, sizeof(IRQ_queue_t));
+        opp->dst[i].ctpr      = 15;
+        memset(&opp->dst[i].raised, 0, sizeof(IRQQueue));
         opp->dst[i].raised.next = -1;
-        memset(&opp->dst[i].servicing, 0, sizeof(IRQ_queue_t));
+        memset(&opp->dst[i].servicing, 0, sizeof(IRQQueue));
         opp->dst[i].servicing.next = -1;
     }
     /* Initialise timers */
     for (i = 0; i < MAX_TMR; i++) {
-        opp->timers[i].ticc = 0x00000000;
-        opp->timers[i].tibc = 0x80000000;
+        opp->timers[i].tccr = 0;
+        opp->timers[i].tbcr = TBCR_CI;
     }
-    /* Initialise doorbells */
-#if MAX_DBL > 0
-    opp->dar = 0x00000000;
-    for (i = 0; i < MAX_DBL; i++) {
-        opp->doorbells[i].dmr  = 0x00000000;
-    }
-#endif
-    /* Initialise mailboxes */
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MBX; i++) { /* ? */
-        opp->mailboxes[i].mbr   = 0x00000000;
-    }
-#endif
     /* Go out of RESET state */
-    opp->glbc = 0x00000000;
+    opp->gcr = 0;
 }
 
-static inline uint32_t read_IRQreg_ide(openpic_t *opp, int n_IRQ)
+static inline uint32_t read_IRQreg_idr(OpenPICState *opp, int n_IRQ)
 {
-    return opp->src[n_IRQ].ide;
+    return opp->src[n_IRQ].idr;
 }
 
-static inline uint32_t read_IRQreg_ipvp(openpic_t *opp, int n_IRQ)
+static inline uint32_t read_IRQreg_ilr(OpenPICState *opp, int n_IRQ)
 {
-    return opp->src[n_IRQ].ipvp;
-}
-
-static inline void write_IRQreg_ide(openpic_t *opp, int n_IRQ, uint32_t val)
-{
-    uint32_t tmp;
+    if (opp->flags & OPENPIC_FLAG_ILR) {
+        return output_to_inttgt(opp->src[n_IRQ].output);
+    }
 
-    tmp = val & 0xC0000000;
-    tmp |= val & ((1ULL << MAX_CPU) - 1);
-    opp->src[n_IRQ].ide = tmp;
-    DPRINTF("Set IDE %d to 0x%08x\n", n_IRQ, opp->src[n_IRQ].ide);
+    return 0xffffffff;
 }
 
-static inline void write_IRQreg_ipvp(openpic_t *opp, int n_IRQ, uint32_t val)
+static inline uint32_t read_IRQreg_ivpr(OpenPICState *opp, int n_IRQ)
 {
-    /* NOTE: not fully accurate for special IRQs, but simple and sufficient */
-    /* ACTIVITY bit is read-only */
-    opp->src[n_IRQ].ipvp = (opp->src[n_IRQ].ipvp & 0x40000000)
-                         | (val & 0x800F00FF);
-    openpic_update_irq(opp, n_IRQ);
-    DPRINTF("Set IPVP %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
-            opp->src[n_IRQ].ipvp);
+    return opp->src[n_IRQ].ivpr;
 }
 
-#if 0 // Code provision for Intel model
-#if MAX_DBL > 0
-static uint32_t read_doorbell_register (openpic_t *opp,
-                                        int n_dbl, uint32_t offset)
+static inline void write_IRQreg_idr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    uint32_t retval;
+    IRQSource *src = &opp->src[n_IRQ];
+    uint32_t normal_mask = (1UL << opp->nb_cpus) - 1;
+    uint32_t crit_mask = 0;
+    uint32_t mask = normal_mask;
+    int crit_shift = IDR_EP_SHIFT - opp->nb_cpus;
+    int i;
 
-    switch (offset) {
-    case DBL_IPVP_OFFSET:
-        retval = read_IRQreg_ipvp(opp, IRQ_DBL0 + n_dbl);
-        break;
-    case DBL_IDE_OFFSET:
-        retval = read_IRQreg_ide(opp, IRQ_DBL0 + n_dbl);
-        break;
-    case DBL_DMR_OFFSET:
-        retval = opp->doorbells[n_dbl].dmr;
-        break;
+    if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
+        crit_mask = mask << crit_shift;
+        mask |= crit_mask | IDR_EP;
     }
 
-    return retval;
+    src->idr = val & mask;
+    DPRINTF("Set IDR %d to 0x%08x\n", n_IRQ, src->idr);
+
+    if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
+        if (src->idr & crit_mask) {
+            if (src->idr & normal_mask) {
+                DPRINTF("%s: IRQ configured for multiple output types, using "
+                        "critical\n", __func__);
+            }
+
+            src->output = OPENPIC_OUTPUT_CINT;
+            src->nomask = true;
+            src->destmask = 0;
+
+            for (i = 0; i < opp->nb_cpus; i++) {
+                int n_ci = IDR_CI0_SHIFT - i;
+
+                if (src->idr & (1UL << n_ci)) {
+                    src->destmask |= 1UL << i;
+                }
+            }
+        } else {
+            src->output = OPENPIC_OUTPUT_INT;
+            src->nomask = false;
+            src->destmask = src->idr & normal_mask;
+        }
+    } else {
+        src->destmask = src->idr;
+    }
 }
 
-static void write_doorbell_register (penpic_t *opp, int n_dbl,
-                                     uint32_t offset, uint32_t value)
+static inline void write_IRQreg_ilr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    switch (offset) {
-    case DBL_IVPR_OFFSET:
-        write_IRQreg_ipvp(opp, IRQ_DBL0 + n_dbl, value);
-        break;
-    case DBL_IDE_OFFSET:
-        write_IRQreg_ide(opp, IRQ_DBL0 + n_dbl, value);
-        break;
-    case DBL_DMR_OFFSET:
-        opp->doorbells[n_dbl].dmr = value;
-        break;
+    if (opp->flags & OPENPIC_FLAG_ILR) {
+        IRQSource *src = &opp->src[n_IRQ];
+
+        src->output = inttgt_to_output(val & ILR_INTTGT_MASK);
+        DPRINTF("Set ILR %d to 0x%08x, output %d\n", n_IRQ, src->idr,
+                src->output);
+
+        /* TODO: on MPIC v4.0 only, set nomask for non-INT */
     }
 }
-#endif
 
-#if MAX_MBX > 0
-static uint32_t read_mailbox_register (openpic_t *opp,
-                                       int n_mbx, uint32_t offset)
+static inline void write_IRQreg_ivpr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    uint32_t retval;
+    uint32_t mask;
+
+    /* NOTE when implementing newer FSL MPIC models: starting with v4.0,
+     * the polarity bit is read-only on internal interrupts.
+     */
+    mask = IVPR_MASK_MASK | IVPR_PRIORITY_MASK | IVPR_SENSE_MASK |
+           IVPR_POLARITY_MASK | opp->vector_mask;
 
-    switch (offset) {
-    case MBX_MBR_OFFSET:
-        retval = opp->mailboxes[n_mbx].mbr;
+    /* ACTIVITY bit is read-only */
+    opp->src[n_IRQ].ivpr =
+        (opp->src[n_IRQ].ivpr & IVPR_ACTIVITY_MASK) | (val & mask);
+
+    /* For FSL internal interrupts, The sense bit is reserved and zero,
+     * and the interrupt is always level-triggered.  Timers and IPIs
+     * have no sense or polarity bits, and are edge-triggered.
+     */
+    switch (opp->src[n_IRQ].type) {
+    case IRQ_TYPE_NORMAL:
+        opp->src[n_IRQ].level = !!(opp->src[n_IRQ].ivpr & IVPR_SENSE_MASK);
         break;
-    case MBX_IVPR_OFFSET:
-        retval = read_IRQreg_ipvp(opp, IRQ_MBX0 + n_mbx);
+
+    case IRQ_TYPE_FSLINT:
+        opp->src[n_IRQ].ivpr &= ~IVPR_SENSE_MASK;
         break;
-    case MBX_DMR_OFFSET:
-        retval = read_IRQreg_ide(opp, IRQ_MBX0 + n_mbx);
+
+    case IRQ_TYPE_FSLSPECIAL:
+        opp->src[n_IRQ].ivpr &= ~(IVPR_POLARITY_MASK | IVPR_SENSE_MASK);
         break;
     }
 
-    return retval;
+    openpic_update_irq(opp, n_IRQ);
+    DPRINTF("Set IVPR %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
+            opp->src[n_IRQ].ivpr);
 }
 
-static void write_mailbox_register (openpic_t *opp, int n_mbx,
-                                    uint32_t address, uint32_t value)
+static void openpic_gcr_write(OpenPICState *opp, uint64_t val)
 {
-    switch (offset) {
-    case MBX_MBR_OFFSET:
-        opp->mailboxes[n_mbx].mbr = value;
-        break;
-    case MBX_IVPR_OFFSET:
-        write_IRQreg_ipvp(opp, IRQ_MBX0 + n_mbx, value);
-        break;
-    case MBX_DMR_OFFSET:
-        write_IRQreg_ide(opp, IRQ_MBX0 + n_mbx, value);
-        break;
+    bool mpic_proxy = false;
+
+    if (val & GCR_RESET) {
+        openpic_reset(&opp->busdev.qdev);
+        return;
+    }
+
+    opp->gcr &= ~opp->mpic_mode_mask;
+    opp->gcr |= val & opp->mpic_mode_mask;
+
+    /* Set external proxy mode */
+    if ((val & opp->mpic_mode_mask) == GCR_MODE_PROXY) {
+        mpic_proxy = true;
     }
+
+    ppce500_set_mpic_proxy(mpic_proxy);
 }
-#endif
-#endif /* 0 : Code provision for Intel model */
 
-static void openpic_gbl_write (void *opaque, target_phys_addr_t addr, uint32_t val)
+static void openpic_gbl_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
-    openpic_t *opp = opaque;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQDest *dst;
     int idx;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
         return;
+    }
     switch (addr) {
     case 0x00: /* Block Revision Register1 (BRR1) is Readonly */
         break;
@@ -618,73 +743,71 @@ static void openpic_gbl_write (void *opaque, target_phys_addr_t addr, uint32_t v
     case 0xB0:
         openpic_cpu_write_internal(opp, addr, val, get_current_cpu());
         break;
-    case 0x1000: /* FREP */
+    case 0x1000: /* FRR */
         break;
-    case 0x1020: /* GLBC */
-        if (val & 0x80000000 && opp->reset)
-            opp->reset(opp);
-        opp->glbc = val & ~0x80000000;
+    case 0x1020: /* GCR */
+        openpic_gcr_write(opp, val);
         break;
-    case 0x1080: /* VENI */
+    case 0x1080: /* VIR */
         break;
-    case 0x1090: /* PINT */
+    case 0x1090: /* PIR */
         for (idx = 0; idx < opp->nb_cpus; idx++) {
-            if ((val & (1 << idx)) && !(opp->pint & (1 << idx))) {
+            if ((val & (1 << idx)) && !(opp->pir & (1 << idx))) {
                 DPRINTF("Raise OpenPIC RESET output for CPU %d\n", idx);
                 dst = &opp->dst[idx];
                 qemu_irq_raise(dst->irqs[OPENPIC_OUTPUT_RESET]);
-            } else if (!(val & (1 << idx)) && (opp->pint & (1 << idx))) {
+            } else if (!(val & (1 << idx)) && (opp->pir & (1 << idx))) {
                 DPRINTF("Lower OpenPIC RESET output for CPU %d\n", idx);
                 dst = &opp->dst[idx];
                 qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_RESET]);
             }
         }
-        opp->pint = val;
+        opp->pir = val;
         break;
-    case 0x10A0: /* IPI_IPVP */
+    case 0x10A0: /* IPI_IVPR */
     case 0x10B0:
     case 0x10C0:
     case 0x10D0:
         {
             int idx;
             idx = (addr - 0x10A0) >> 4;
-            write_IRQreg_ipvp(opp, opp->irq_ipi0 + idx, val);
+            write_IRQreg_ivpr(opp, opp->irq_ipi0 + idx, val);
         }
         break;
     case 0x10E0: /* SPVE */
-        opp->spve = val & 0x000000FF;
-        break;
-    case 0x10F0: /* TIFR */
-        opp->tifr = val;
+        opp->spve = val & opp->vector_mask;
         break;
     default:
         break;
     }
 }
 
-static uint32_t openpic_gbl_read (void *opaque, target_phys_addr_t addr)
+static uint64_t openpic_gbl_read(void *opaque, hwaddr addr, unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     uint32_t retval;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
+    if (addr & 0xF) {
         return retval;
+    }
     switch (addr) {
-    case 0x1000: /* FREP */
-        retval = opp->frep;
+    case 0x1000: /* FRR */
+        retval = opp->frr;
         break;
-    case 0x1020: /* GLBC */
-        retval = opp->glbc;
+    case 0x1020: /* GCR */
+        retval = opp->gcr;
         break;
-    case 0x1080: /* VENI */
-        retval = opp->veni;
+    case 0x1080: /* VIR */
+        retval = opp->vir;
         break;
-    case 0x1090: /* PINT */
+    case 0x1090: /* PIR */
         retval = 0x00000000;
         break;
     case 0x00: /* Block Revision Register1 (BRR1) */
+        retval = opp->brr1;
+        break;
     case 0x40:
     case 0x50:
     case 0x60:
@@ -695,179 +818,309 @@ static uint32_t openpic_gbl_read (void *opaque, target_phys_addr_t addr)
     case 0xB0:
         retval = openpic_cpu_read_internal(opp, addr, get_current_cpu());
         break;
-    case 0x10A0: /* IPI_IPVP */
+    case 0x10A0: /* IPI_IVPR */
     case 0x10B0:
     case 0x10C0:
     case 0x10D0:
         {
             int idx;
             idx = (addr - 0x10A0) >> 4;
-            retval = read_IRQreg_ipvp(opp, opp->irq_ipi0 + idx);
+            retval = read_IRQreg_ivpr(opp, opp->irq_ipi0 + idx);
         }
         break;
     case 0x10E0: /* SPVE */
         retval = opp->spve;
         break;
-    case 0x10F0: /* TIFR */
-        retval = opp->tifr;
-        break;
     default:
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static void openpic_timer_write (void *opaque, uint32_t addr, uint32_t val)
+static void openpic_tmr_write(void *opaque, hwaddr addr, uint64_t val,
+                                unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     int idx;
 
-    DPRINTF("%s: addr %08x <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
+    addr += 0x10f0;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
+        return;
+    }
+
+    if (addr == 0x10f0) {
+        /* TFRR */
+        opp->tfrr = val;
         return;
-    addr -= 0x10;
-    addr &= 0xFFFF;
-    idx = (addr & 0xFFF0) >> 6;
+    }
+
+    idx = (addr >> 6) & 0x3;
     addr = addr & 0x30;
-    switch (addr) {
-    case 0x00: /* TICC */
+
+    switch (addr & 0x30) {
+    case 0x00: /* TCCR */
         break;
-    case 0x10: /* TIBC */
-        if ((opp->timers[idx].ticc & 0x80000000) != 0 &&
-            (val & 0x80000000) == 0 &&
-            (opp->timers[idx].tibc & 0x80000000) != 0)
-            opp->timers[idx].ticc &= ~0x80000000;
-        opp->timers[idx].tibc = val;
+    case 0x10: /* TBCR */
+        if ((opp->timers[idx].tccr & TCCR_TOG) != 0 &&
+            (val & TBCR_CI) == 0 &&
+            (opp->timers[idx].tbcr & TBCR_CI) != 0) {
+            opp->timers[idx].tccr &= ~TCCR_TOG;
+        }
+        opp->timers[idx].tbcr = val;
         break;
-    case 0x20: /* TIVP */
-        write_IRQreg_ipvp(opp, opp->irq_tim0 + idx, val);
+    case 0x20: /* TVPR */
+        write_IRQreg_ivpr(opp, opp->irq_tim0 + idx, val);
         break;
-    case 0x30: /* TIDE */
-        write_IRQreg_ide(opp, opp->irq_tim0 + idx, val);
+    case 0x30: /* TDR */
+        write_IRQreg_idr(opp, opp->irq_tim0 + idx, val);
         break;
     }
 }
 
-static uint32_t openpic_timer_read (void *opaque, uint32_t addr)
+static uint64_t openpic_tmr_read(void *opaque, hwaddr addr, unsigned len)
 {
-    openpic_t *opp = opaque;
-    uint32_t retval;
+    OpenPICState *opp = opaque;
+    uint32_t retval = -1;
     int idx;
 
-    DPRINTF("%s: addr %08x\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr -= 0x10;
-    addr &= 0xFFFF;
-    idx = (addr & 0xFFF0) >> 6;
-    addr = addr & 0x30;
-    switch (addr) {
-    case 0x00: /* TICC */
-        retval = opp->timers[idx].ticc;
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+    if (addr & 0xF) {
+        goto out;
+    }
+    idx = (addr >> 6) & 0x3;
+    if (addr == 0x0) {
+        /* TFRR */
+        retval = opp->tfrr;
+        goto out;
+    }
+    switch (addr & 0x30) {
+    case 0x00: /* TCCR */
+        retval = opp->timers[idx].tccr;
         break;
-    case 0x10: /* TIBC */
-        retval = opp->timers[idx].tibc;
+    case 0x10: /* TBCR */
+        retval = opp->timers[idx].tbcr;
         break;
     case 0x20: /* TIPV */
-        retval = read_IRQreg_ipvp(opp, opp->irq_tim0 + idx);
+        retval = read_IRQreg_ivpr(opp, opp->irq_tim0 + idx);
         break;
-    case 0x30: /* TIDE */
-        retval = read_IRQreg_ide(opp, opp->irq_tim0 + idx);
+    case 0x30: /* TIDE (TIDR) */
+        retval = read_IRQreg_idr(opp, opp->irq_tim0 + idx);
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+
+out:
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static void openpic_src_write (void *opaque, uint32_t addr, uint32_t val)
+static void openpic_src_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     int idx;
 
-    DPRINTF("%s: addr %08x <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-    addr = addr & 0xFFF0;
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+
+    addr = addr & 0xffff;
     idx = addr >> 5;
-    if (addr & 0x10) {
-        /* EXDE / IFEDE / IEEDE */
-        write_IRQreg_ide(opp, idx, val);
-    } else {
-        /* EXVP / IFEVP / IEEVP */
-        write_IRQreg_ipvp(opp, idx, val);
+
+    switch (addr & 0x1f) {
+    case 0x00:
+        write_IRQreg_ivpr(opp, idx, val);
+        break;
+    case 0x10:
+        write_IRQreg_idr(opp, idx, val);
+        break;
+    case 0x18:
+        write_IRQreg_ilr(opp, idx, val);
+        break;
     }
 }
 
-static uint32_t openpic_src_read (void *opaque, uint32_t addr)
+static uint64_t openpic_src_read(void *opaque, uint64_t addr, unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     uint32_t retval;
     int idx;
 
-    DPRINTF("%s: addr %08x\n", __func__, addr);
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr = addr & 0xFFF0;
+
+    addr = addr & 0xffff;
     idx = addr >> 5;
-    if (addr & 0x10) {
-        /* EXDE / IFEDE / IEEDE */
-        retval = read_IRQreg_ide(opp, idx);
-    } else {
-        /* EXVP / IFEVP / IEEVP */
-        retval = read_IRQreg_ipvp(opp, idx);
+
+    switch (addr & 0x1f) {
+    case 0x00:
+        retval = read_IRQreg_ivpr(opp, idx);
+        break;
+    case 0x10:
+        retval = read_IRQreg_idr(opp, idx);
+        break;
+    case 0x18:
+        retval = read_IRQreg_ilr(opp, idx);
+        break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
 
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
     return retval;
 }
 
-static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
+static void openpic_msi_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
+{
+    OpenPICState *opp = opaque;
+    int idx = opp->irq_msi;
+    int srs, ibs;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= 0x%08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
+        return;
+    }
+
+    switch (addr) {
+    case MSIIR_OFFSET:
+        srs = val >> MSIIR_SRS_SHIFT;
+        idx += srs;
+        ibs = (val & MSIIR_IBS_MASK) >> MSIIR_IBS_SHIFT;
+        opp->msi[srs].msir |= 1 << ibs;
+        openpic_set_irq(opp, idx, 1);
+        break;
+    default:
+        /* most registers are read-only, thus ignored */
+        break;
+    }
+}
+
+static uint64_t openpic_msi_read(void *opaque, hwaddr addr, unsigned size)
+{
+    OpenPICState *opp = opaque;
+    uint64_t r = 0;
+    int i, srs;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+    if (addr & 0xF) {
+        return -1;
+    }
+
+    srs = addr >> 4;
+
+    switch (addr) {
+    case 0x00:
+    case 0x10:
+    case 0x20:
+    case 0x30:
+    case 0x40:
+    case 0x50:
+    case 0x60:
+    case 0x70: /* MSIRs */
+        r = opp->msi[srs].msir;
+        /* Clear on read */
+        opp->msi[srs].msir = 0;
+        openpic_set_irq(opp, opp->irq_msi + srs, 0);
+        break;
+    case 0x120: /* MSISR */
+        for (i = 0; i < MAX_MSI; i++) {
+            r |= (opp->msi[i].msir ? 1 : 0) << i;
+        }
+        break;
+    }
+
+    return r;
+}
+
+static uint64_t openpic_summary_read(void *opaque, hwaddr addr, unsigned size)
+{
+    uint64_t r = 0;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+
+    /* TODO: EISR/EIMR */
+
+    return r;
+}
+
+static void openpic_summary_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned size)
+{
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= 0x%08" PRIx64 "\n",
+            __func__, addr, val);
+
+    /* TODO: EISR/EIMR */
+}
+
+static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQSource *src;
+    IRQDest *dst;
     int s_IRQ, n_IRQ;
 
-    DPRINTF("%s: cpu %d addr " TARGET_FMT_plx " <= %08x\n", __func__, idx,
+    DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx " <= 0x%08x\n", __func__, idx,
             addr, val);
-    if (addr & 0xF)
+
+    if (idx < 0) {
         return;
+    }
+
+    if (addr & 0xF) {
+        return;
+    }
     dst = &opp->dst[idx];
     addr &= 0xFF0;
     switch (addr) {
-#if MAX_IPI > 0
     case 0x40: /* IPIDR */
     case 0x50:
     case 0x60:
     case 0x70:
         idx = (addr - 0x40) >> 4;
         /* we use IDE as mask which CPUs to deliver the IPI to still. */
-        write_IRQreg_ide(opp, opp->irq_ipi0 + idx,
-                         opp->src[opp->irq_ipi0 + idx].ide | val);
+        opp->src[opp->irq_ipi0 + idx].destmask |= val;
         openpic_set_irq(opp, opp->irq_ipi0 + idx, 1);
         openpic_set_irq(opp, opp->irq_ipi0 + idx, 0);
         break;
-#endif
-    case 0x80: /* PCTP */
-        dst->pctp = val & 0x0000000F;
+    case 0x80: /* CTPR */
+        dst->ctpr = val & 0x0000000F;
+
+        DPRINTF("%s: set CPU %d ctpr to %d, raised %d servicing %d\n",
+                __func__, idx, dst->ctpr, dst->raised.priority,
+                dst->servicing.priority);
+
+        if (dst->raised.priority <= dst->ctpr) {
+            DPRINTF("%s: Lower OpenPIC INT output cpu %d due to ctpr\n",
+                    __func__, idx);
+            qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
+        } else if (dst->raised.priority > dst->servicing.priority) {
+            DPRINTF("%s: Raise OpenPIC INT output cpu %d irq %d\n",
+                    __func__, idx, dst->raised.next);
+            qemu_irq_raise(dst->irqs[OPENPIC_OUTPUT_INT]);
+        }
+
         break;
     case 0x90: /* WHOAMI */
         /* Read-only register */
         break;
-    case 0xA0: /* PIAC */
+    case 0xA0: /* IACK */
         /* Read-only register */
         break;
-    case 0xB0: /* PEOI */
-        DPRINTF("PEOI\n");
+    case 0xB0: /* EOI */
+        DPRINTF("EOI\n");
         s_IRQ = IRQ_get_next(opp, &dst->servicing);
+
+        if (s_IRQ < 0) {
+            DPRINTF("%s: EOI with no interrupt in service\n", __func__);
+            break;
+        }
+
         IRQ_resetbit(&dst->servicing, s_IRQ);
-        dst->servicing.next = -1;
         /* Set up next servicing IRQ */
         s_IRQ = IRQ_get_next(opp, &dst->servicing);
         /* Check queued interrupts. */
@@ -875,10 +1128,10 @@ static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
         src = &opp->src[n_IRQ];
         if (n_IRQ != -1 &&
             (s_IRQ == -1 ||
-             IPVP_PRIORITY(src->ipvp) > dst->servicing.priority)) {
+             IVPR_PRIORITY(src->ivpr) > dst->servicing.priority)) {
             DPRINTF("Raise OpenPIC INT output cpu %d irq %d\n",
                     idx, n_IRQ);
-            opp->irq_raise(opp, idx, src);
+            qemu_irq_raise(opp->dst[idx].irqs[OPENPIC_OUTPUT_INT]);
         }
         break;
     default:
@@ -886,188 +1139,222 @@ static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void openpic_cpu_write(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void openpic_cpu_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
     openpic_cpu_write_internal(opaque, addr, val, (addr & 0x1f000) >> 12);
 }
 
-static uint32_t openpic_cpu_read_internal(void *opaque, target_phys_addr_t addr,
+
+static uint32_t openpic_iack(OpenPICState *opp, IRQDest *dst, int cpu)
+{
+    IRQSource *src;
+    int retval, irq;
+
+    DPRINTF("Lower OpenPIC INT output\n");
+    qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
+
+    irq = IRQ_get_next(opp, &dst->raised);
+    DPRINTF("IACK: irq=%d\n", irq);
+
+    if (irq == -1) {
+        /* No more interrupt pending */
+        return opp->spve;
+    }
+
+    src = &opp->src[irq];
+    if (!(src->ivpr & IVPR_ACTIVITY_MASK) ||
+            !(IVPR_PRIORITY(src->ivpr) > dst->ctpr)) {
+        fprintf(stderr, "%s: bad raised IRQ %d ctpr %d ivpr 0x%08x\n",
+                __func__, irq, dst->ctpr, src->ivpr);
+        openpic_update_irq(opp, irq);
+        retval = opp->spve;
+    } else {
+        /* IRQ enter servicing state */
+        IRQ_setbit(&dst->servicing, irq);
+        retval = IVPR_VECTOR(opp, src->ivpr);
+    }
+
+    if (!src->level) {
+        /* edge-sensitive IRQ */
+        src->ivpr &= ~IVPR_ACTIVITY_MASK;
+        src->pending = 0;
+        IRQ_resetbit(&dst->raised, irq);
+    }
+
+    if ((irq >= opp->irq_ipi0) &&  (irq < (opp->irq_ipi0 + MAX_IPI))) {
+        src->destmask &= ~(1 << cpu);
+        if (src->destmask && !src->level) {
+            /* trigger on CPUs that didn't know about it yet */
+            openpic_set_irq(opp, irq, 1);
+            openpic_set_irq(opp, irq, 0);
+            /* if all CPUs knew about it, set active bit again */
+            src->ivpr |= IVPR_ACTIVITY_MASK;
+        }
+    }
+
+    return retval;
+}
+
+static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
                                           int idx)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQDest *dst;
     uint32_t retval;
-    int n_IRQ;
 
-    DPRINTF("%s: cpu %d addr " TARGET_FMT_plx "\n", __func__, idx, addr);
+    DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx "\n", __func__, idx, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
+
+    if (idx < 0) {
+        return retval;
+    }
+
+    if (addr & 0xF) {
         return retval;
+    }
     dst = &opp->dst[idx];
     addr &= 0xFF0;
     switch (addr) {
-    case 0x00: /* Block Revision Register1 (BRR1) */
-        retval = FSL_BRR1_IPID | FSL_BRR1_IPMJ | FSL_BRR1_IPMN;
-        break;
-    case 0x80: /* PCTP */
-        retval = dst->pctp;
+    case 0x80: /* CTPR */
+        retval = dst->ctpr;
         break;
     case 0x90: /* WHOAMI */
         retval = idx;
         break;
-    case 0xA0: /* PIAC */
-        DPRINTF("Lower OpenPIC INT output\n");
-        qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
-        n_IRQ = IRQ_get_next(opp, &dst->raised);
-        DPRINTF("PIAC: irq=%d\n", n_IRQ);
-        if (n_IRQ == -1) {
-            /* No more interrupt pending */
-            retval = IPVP_VECTOR(opp->spve);
-        } else {
-            src = &opp->src[n_IRQ];
-            if (!test_bit(&src->ipvp, IPVP_ACTIVITY) ||
-                !(IPVP_PRIORITY(src->ipvp) > dst->pctp)) {
-                /* - Spurious level-sensitive IRQ
-                 * - Priorities has been changed
-                 *   and the pending IRQ isn't allowed anymore
-                 */
-                reset_bit(&src->ipvp, IPVP_ACTIVITY);
-                retval = IPVP_VECTOR(opp->spve);
-            } else {
-                /* IRQ enter servicing state */
-                IRQ_setbit(&dst->servicing, n_IRQ);
-                retval = IPVP_VECTOR(src->ipvp);
-            }
-            IRQ_resetbit(&dst->raised, n_IRQ);
-            dst->raised.next = -1;
-            if (!test_bit(&src->ipvp, IPVP_SENSE)) {
-                /* edge-sensitive IRQ */
-                reset_bit(&src->ipvp, IPVP_ACTIVITY);
-                src->pending = 0;
-            }
-
-            if ((n_IRQ >= opp->irq_ipi0) &&  (n_IRQ < (opp->irq_ipi0 + MAX_IPI))) {
-                src->ide &= ~(1 << idx);
-                if (src->ide && !test_bit(&src->ipvp, IPVP_SENSE)) {
-                    /* trigger on CPUs that didn't know about it yet */
-                    openpic_set_irq(opp, n_IRQ, 1);
-                    openpic_set_irq(opp, n_IRQ, 0);
-                    /* if all CPUs knew about it, set active bit again */
-                    set_bit(&src->ipvp, IPVP_ACTIVITY);
-                }
-            }
-        }
+    case 0xA0: /* IACK */
+        retval = openpic_iack(opp, dst, idx);
         break;
-    case 0xB0: /* PEOI */
+    case 0xB0: /* EOI */
         retval = 0;
         break;
     default:
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static uint32_t openpic_cpu_read(void *opaque, target_phys_addr_t addr)
+static uint64_t openpic_cpu_read(void *opaque, hwaddr addr, unsigned len)
 {
     return openpic_cpu_read_internal(opaque, addr, (addr & 0x1f000) >> 12);
 }
 
-static void openpic_buggy_write (void *opaque,
-                                 target_phys_addr_t addr, uint32_t val)
-{
-    printf("Invalid OPENPIC write access !\n");
-}
-
-static uint32_t openpic_buggy_read (void *opaque, target_phys_addr_t addr)
-{
-    printf("Invalid OPENPIC read access !\n");
-
-    return -1;
-}
+static const MemoryRegionOps openpic_glb_ops_le = {
+    .write = openpic_gbl_write,
+    .read  = openpic_gbl_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static void openpic_writel (void *opaque,
-                            target_phys_addr_t addr, uint32_t val)
-{
-    openpic_t *opp = opaque;
-
-    addr &= 0x3FFFF;
-    DPRINTF("%s: offset %08x val: %08x\n", __func__, (int)addr, val);
-    if (addr < 0x1100) {
-        /* Global registers */
-        openpic_gbl_write(opp, addr, val);
-    } else if (addr < 0x10000) {
-        /* Timers registers */
-        openpic_timer_write(opp, addr, val);
-    } else if (addr < 0x20000) {
-        /* Source registers */
-        openpic_src_write(opp, addr, val);
-    } else {
-        /* CPU registers */
-        openpic_cpu_write(opp, addr, val);
-    }
-}
+static const MemoryRegionOps openpic_glb_ops_be = {
+    .write = openpic_gbl_write,
+    .read  = openpic_gbl_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static uint32_t openpic_readl (void *opaque,target_phys_addr_t addr)
-{
-    openpic_t *opp = opaque;
-    uint32_t retval;
+static const MemoryRegionOps openpic_tmr_ops_le = {
+    .write = openpic_tmr_write,
+    .read  = openpic_tmr_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    addr &= 0x3FFFF;
-    DPRINTF("%s: offset %08x\n", __func__, (int)addr);
-    if (addr < 0x1100) {
-        /* Global registers */
-        retval = openpic_gbl_read(opp, addr);
-    } else if (addr < 0x10000) {
-        /* Timers registers */
-        retval = openpic_timer_read(opp, addr);
-    } else if (addr < 0x20000) {
-        /* Source registers */
-        retval = openpic_src_read(opp, addr);
-    } else {
-        /* CPU registers */
-        retval = openpic_cpu_read(opp, addr);
-    }
+static const MemoryRegionOps openpic_tmr_ops_be = {
+    .write = openpic_tmr_write,
+    .read  = openpic_tmr_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    return retval;
-}
+static const MemoryRegionOps openpic_cpu_ops_le = {
+    .write = openpic_cpu_write,
+    .read  = openpic_cpu_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static uint64_t openpic_read(void *opaque, target_phys_addr_t addr,
-                             unsigned size)
-{
-    openpic_t *opp = opaque;
+static const MemoryRegionOps openpic_cpu_ops_be = {
+    .write = openpic_cpu_write,
+    .read  = openpic_cpu_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    switch (size) {
-    case 4: return openpic_readl(opp, addr);
-    default: return openpic_buggy_read(opp, addr);
-    }
-}
+static const MemoryRegionOps openpic_src_ops_le = {
+    .write = openpic_src_write,
+    .read  = openpic_src_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static void openpic_write(void *opaque, target_phys_addr_t addr,
-                          uint64_t data, unsigned size)
-{
-    openpic_t *opp = opaque;
+static const MemoryRegionOps openpic_src_ops_be = {
+    .write = openpic_src_write,
+    .read  = openpic_src_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    switch (size) {
-    case 4: return openpic_writel(opp, addr, data);
-    default: return openpic_buggy_write(opp, addr, data);
-    }
-}
+static const MemoryRegionOps openpic_msi_ops_be = {
+    .read = openpic_msi_read,
+    .write = openpic_msi_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static const MemoryRegionOps openpic_ops = {
-    .read = openpic_read,
-    .write = openpic_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
+static const MemoryRegionOps openpic_summary_ops_be = {
+    .read = openpic_summary_read,
+    .write = openpic_summary_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
 };
 
-static void openpic_save_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
+static void openpic_save_IRQ_queue(QEMUFile* f, IRQQueue *q)
 {
     unsigned int i;
 
-    for (i = 0; i < BF_WIDTH(MAX_IRQ); i++)
-        qemu_put_be32s(f, &q->queue[i]);
+    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
+        /* Always put the lower half of a 64-bit long first, in case we
+         * restore on a 32-bit host.  The least significant bits correspond
+         * to lower IRQ numbers in the bitmap.
+         */
+        qemu_put_be32(f, (uint32_t)q->queue[i]);
+#if LONG_MAX > 0x7FFFFFFF
+        qemu_put_be32(f, (uint32_t)(q->queue[i] >> 32));
+#endif
+    }
 
     qemu_put_sbe32s(f, &q->next);
     qemu_put_sbe32s(f, &q->priority);
@@ -1075,63 +1362,54 @@ static void openpic_save_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
 
 static void openpic_save(QEMUFile* f, void *opaque)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = (OpenPICState *)opaque;
     unsigned int i;
 
-    qemu_put_be32s(f, &opp->frep);
-    qemu_put_be32s(f, &opp->glbc);
-    qemu_put_be32s(f, &opp->micr);
-    qemu_put_be32s(f, &opp->veni);
-    qemu_put_be32s(f, &opp->pint);
+    qemu_put_be32s(f, &opp->gcr);
+    qemu_put_be32s(f, &opp->vir);
+    qemu_put_be32s(f, &opp->pir);
     qemu_put_be32s(f, &opp->spve);
-    qemu_put_be32s(f, &opp->tifr);
-
-    for (i = 0; i < opp->max_irq; i++) {
-        qemu_put_be32s(f, &opp->src[i].ipvp);
-        qemu_put_be32s(f, &opp->src[i].ide);
-        qemu_put_sbe32s(f, &opp->src[i].type);
-        qemu_put_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_put_sbe32s(f, &opp->src[i].pending);
-    }
+    qemu_put_be32s(f, &opp->tfrr);
 
-    qemu_put_sbe32s(f, &opp->nb_cpus);
+    qemu_put_be32s(f, &opp->nb_cpus);
 
     for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_put_be32s(f, &opp->dst[i].tfrr);
-        qemu_put_be32s(f, &opp->dst[i].pctp);
-        qemu_put_be32s(f, &opp->dst[i].pcsr);
+        qemu_put_sbe32s(f, &opp->dst[i].ctpr);
         openpic_save_IRQ_queue(f, &opp->dst[i].raised);
         openpic_save_IRQ_queue(f, &opp->dst[i].servicing);
+        qemu_put_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
+                        sizeof(opp->dst[i].outputs_active));
     }
 
     for (i = 0; i < MAX_TMR; i++) {
-        qemu_put_be32s(f, &opp->timers[i].ticc);
-        qemu_put_be32s(f, &opp->timers[i].tibc);
+        qemu_put_be32s(f, &opp->timers[i].tccr);
+        qemu_put_be32s(f, &opp->timers[i].tbcr);
     }
 
-#if MAX_DBL > 0
-    qemu_put_be32s(f, &opp->dar);
-
-    for (i = 0; i < MAX_DBL; i++) {
-        qemu_put_be32s(f, &opp->doorbells[i].dmr);
-    }
-#endif
-
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MAILBOXES; i++) {
-        qemu_put_be32s(f, &opp->mailboxes[i].mbr);
+    for (i = 0; i < opp->max_irq; i++) {
+        qemu_put_be32s(f, &opp->src[i].ivpr);
+        qemu_put_be32s(f, &opp->src[i].idr);
+        qemu_get_be32s(f, &opp->src[i].destmask);
+        qemu_put_sbe32s(f, &opp->src[i].last_cpu);
+        qemu_put_sbe32s(f, &opp->src[i].pending);
     }
-#endif
-
-    pci_device_save(&opp->pci_dev, f);
 }
 
-static void openpic_load_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
+static void openpic_load_IRQ_queue(QEMUFile* f, IRQQueue *q)
 {
     unsigned int i;
 
-    for (i = 0; i < BF_WIDTH(MAX_IRQ); i++)
-        qemu_get_be32s(f, &q->queue[i]);
+    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
+        unsigned long val;
+
+        val = qemu_get_be32(f);
+#if LONG_MAX > 0x7FFFFFFF
+        val <<= 32;
+        val |= qemu_get_be32(f);
+#endif
+
+        q->queue[i] = val;
+    }
 
     qemu_get_sbe32s(f, &q->next);
     qemu_get_sbe32s(f, &q->priority);
@@ -1139,572 +1417,245 @@ static void openpic_load_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
 
 static int openpic_load(QEMUFile* f, void *opaque, int version_id)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = (OpenPICState *)opaque;
     unsigned int i;
 
-    if (version_id != 1)
+    if (version_id != 1) {
         return -EINVAL;
+    }
 
-    qemu_get_be32s(f, &opp->frep);
-    qemu_get_be32s(f, &opp->glbc);
-    qemu_get_be32s(f, &opp->micr);
-    qemu_get_be32s(f, &opp->veni);
-    qemu_get_be32s(f, &opp->pint);
+    qemu_get_be32s(f, &opp->gcr);
+    qemu_get_be32s(f, &opp->vir);
+    qemu_get_be32s(f, &opp->pir);
     qemu_get_be32s(f, &opp->spve);
-    qemu_get_be32s(f, &opp->tifr);
-
-    for (i = 0; i < opp->max_irq; i++) {
-        qemu_get_be32s(f, &opp->src[i].ipvp);
-        qemu_get_be32s(f, &opp->src[i].ide);
-        qemu_get_sbe32s(f, &opp->src[i].type);
-        qemu_get_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_get_sbe32s(f, &opp->src[i].pending);
-    }
+    qemu_get_be32s(f, &opp->tfrr);
 
-    qemu_get_sbe32s(f, &opp->nb_cpus);
+    qemu_get_be32s(f, &opp->nb_cpus);
 
     for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_get_be32s(f, &opp->dst[i].tfrr);
-        qemu_get_be32s(f, &opp->dst[i].pctp);
-        qemu_get_be32s(f, &opp->dst[i].pcsr);
+        qemu_get_sbe32s(f, &opp->dst[i].ctpr);
         openpic_load_IRQ_queue(f, &opp->dst[i].raised);
         openpic_load_IRQ_queue(f, &opp->dst[i].servicing);
+        qemu_get_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
+                        sizeof(opp->dst[i].outputs_active));
     }
 
     for (i = 0; i < MAX_TMR; i++) {
-        qemu_get_be32s(f, &opp->timers[i].ticc);
-        qemu_get_be32s(f, &opp->timers[i].tibc);
+        qemu_get_be32s(f, &opp->timers[i].tccr);
+        qemu_get_be32s(f, &opp->timers[i].tbcr);
     }
 
-#if MAX_DBL > 0
-    qemu_get_be32s(f, &opp->dar);
+    for (i = 0; i < opp->max_irq; i++) {
+        uint32_t val;
 
-    for (i = 0; i < MAX_DBL; i++) {
-        qemu_get_be32s(f, &opp->doorbells[i].dmr);
-    }
-#endif
+        val = qemu_get_be32(f);
+        write_IRQreg_idr(opp, i, val);
+        val = qemu_get_be32(f);
+        write_IRQreg_ivpr(opp, i, val);
 
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MAILBOXES; i++) {
-        qemu_get_be32s(f, &opp->mailboxes[i].mbr);
+        qemu_get_be32s(f, &opp->src[i].ivpr);
+        qemu_get_be32s(f, &opp->src[i].idr);
+        qemu_get_be32s(f, &opp->src[i].destmask);
+        qemu_get_sbe32s(f, &opp->src[i].last_cpu);
+        qemu_get_sbe32s(f, &opp->src[i].pending);
     }
-#endif
 
-    return pci_device_load(&opp->pci_dev, f);
+    return 0;
 }
 
-static void openpic_irq_raise(openpic_t *opp, int n_CPU, IRQ_src_t *src)
-{
-    qemu_irq_raise(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
-}
+typedef struct MemReg {
+    const char             *name;
+    MemoryRegionOps const  *ops;
+    hwaddr      start_addr;
+    ram_addr_t              size;
+} MemReg;
 
-qemu_irq *openpic_init (MemoryRegion **pmem, int nb_cpus,
-                        qemu_irq **irqs, qemu_irq irq_out)
+static void fsl_common_init(OpenPICState *opp)
 {
-    openpic_t *opp;
-    int i, m;
-
-    /* XXX: for now, only one CPU is supported */
-    if (nb_cpus != 1)
-        return NULL;
-    opp = g_malloc0(sizeof(openpic_t));
-    memory_region_init_io(&opp->mem, &openpic_ops, opp, "openpic", 0x40000);
-
-    //    isu_base &= 0xFFFC0000;
-    opp->nb_cpus = nb_cpus;
-    opp->max_irq = OPENPIC_MAX_IRQ;
-    opp->irq_ipi0 = OPENPIC_IRQ_IPI0;
-    opp->irq_tim0 = OPENPIC_IRQ_TIM0;
-    /* Set IRQ types */
-    for (i = 0; i < OPENPIC_EXT_IRQ; i++) {
-        opp->src[i].type = IRQ_EXTERNAL;
-    }
-    for (; i < OPENPIC_IRQ_TIM0; i++) {
-        opp->src[i].type = IRQ_SPECIAL;
-    }
-#if MAX_IPI > 0
-    m = OPENPIC_IRQ_IPI0;
-#else
-    m = OPENPIC_IRQ_DBL0;
-#endif
-    for (; i < m; i++) {
-        opp->src[i].type = IRQ_TIMER;
-    }
-    for (; i < OPENPIC_MAX_IRQ; i++) {
-        opp->src[i].type = IRQ_INTERNAL;
-    }
-    for (i = 0; i < nb_cpus; i++)
-        opp->dst[i].irqs = irqs[i];
-    opp->irq_out = irq_out;
-
-    register_savevm(&opp->pci_dev.qdev, "openpic", 0, 2,
-                    openpic_save, openpic_load, opp);
-    qemu_register_reset(openpic_reset, opp);
+    int i;
+    int virq = MAX_SRC;
 
-    opp->irq_raise = openpic_irq_raise;
-    opp->reset = openpic_reset;
+    opp->vid = VID_REVISION_1_2;
+    opp->vir = VIR_GENERIC;
+    opp->vector_mask = 0xFFFF;
+    opp->tfrr_reset = 0;
+    opp->ivpr_reset = IVPR_MASK_MASK;
+    opp->idr_reset = 1 << 0;
+    opp->max_irq = MAX_IRQ;
 
-    if (pmem)
-        *pmem = &opp->mem;
+    opp->irq_ipi0 = virq;
+    virq += MAX_IPI;
+    opp->irq_tim0 = virq;
+    virq += MAX_TMR;
 
-    return qemu_allocate_irqs(openpic_set_irq, opp, opp->max_irq);
-}
+    assert(virq <= MAX_IRQ);
 
-static void mpic_irq_raise(openpic_t *mpp, int n_CPU, IRQ_src_t *src)
-{
-    int n_ci = IDR_CI0 - n_CPU;
+    opp->irq_msi = 224;
 
-    if(test_bit(&src->ide, n_ci)) {
-        qemu_irq_raise(mpp->dst[n_CPU].irqs[OPENPIC_OUTPUT_CINT]);
+    msi_supported = true;
+    for (i = 0; i < opp->fsl->max_ext; i++) {
+        opp->src[i].level = false;
     }
-    else {
-        qemu_irq_raise(mpp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
-    }
-}
-
-static void mpic_reset (void *opaque)
-{
-    openpic_t *mpp = (openpic_t *)opaque;
-    int i;
 
-    mpp->glbc = 0x80000000;
-    /* Initialise controller registers */
-    mpp->frep = 0x004f0002 | ((mpp->nb_cpus - 1) << 8);
-    mpp->veni = VENI;
-    mpp->pint = 0x00000000;
-    mpp->spve = 0x0000FFFF;
-    /* Initialise IRQ sources */
-    for (i = 0; i < mpp->max_irq; i++) {
-        mpp->src[i].ipvp = 0x80800000;
-        mpp->src[i].ide  = 0x00000001;
-    }
-    /* Set IDE for IPIs to 0 so we don't get spurious interrupts */
-    for (i = mpp->irq_ipi0; i < (mpp->irq_ipi0 + MAX_IPI); i++) {
-        mpp->src[i].ide = 0;
-    }
-    /* Initialise IRQ destinations */
-    for (i = 0; i < MAX_CPU; i++) {
-        mpp->dst[i].pctp      = 0x0000000F;
-        mpp->dst[i].tfrr      = 0x00000000;
-        memset(&mpp->dst[i].raised, 0, sizeof(IRQ_queue_t));
-        mpp->dst[i].raised.next = -1;
-        memset(&mpp->dst[i].servicing, 0, sizeof(IRQ_queue_t));
-        mpp->dst[i].servicing.next = -1;
-    }
-    /* Initialise timers */
-    for (i = 0; i < MAX_TMR; i++) {
-        mpp->timers[i].ticc = 0x00000000;
-        mpp->timers[i].tibc = 0x80000000;
-    }
-    /* Go out of RESET state */
-    mpp->glbc = 0x00000000;
-}
-
-static void mpic_timer_write (void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx, cpu;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-    addr &= 0xFFFF;
-    cpu = addr >> 12;
-    idx = (addr >> 6) & 0x3;
-    switch (addr & 0x30) {
-    case 0x00: /* gtccr */
-        break;
-    case 0x10: /* gtbcr */
-        if ((mpp->timers[idx].ticc & 0x80000000) != 0 &&
-            (val & 0x80000000) == 0 &&
-            (mpp->timers[idx].tibc & 0x80000000) != 0)
-            mpp->timers[idx].ticc &= ~0x80000000;
-        mpp->timers[idx].tibc = val;
-        break;
-    case 0x20: /* GTIVPR */
-        write_IRQreg_ipvp(mpp, MPIC_TMR_IRQ + idx, val);
-        break;
-    case 0x30: /* GTIDR & TFRR */
-        if ((addr & 0xF0) == 0xF0)
-            mpp->dst[cpu].tfrr = val;
-        else
-            write_IRQreg_ide(mpp, MPIC_TMR_IRQ + idx, val);
-        break;
+    /* Internal interrupts, including message and MSI */
+    for (i = 16; i < MAX_SRC; i++) {
+        opp->src[i].type = IRQ_TYPE_FSLINT;
+        opp->src[i].level = true;
     }
-}
-
-static uint32_t mpic_timer_read (void *opaque, target_phys_addr_t addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx, cpu;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr &= 0xFFFF;
-    cpu = addr >> 12;
-    idx = (addr >> 6) & 0x3;
-    switch (addr & 0x30) {
-    case 0x00: /* gtccr */
-        retval = mpp->timers[idx].ticc;
-        break;
-    case 0x10: /* gtbcr */
-        retval = mpp->timers[idx].tibc;
-        break;
-    case 0x20: /* TIPV */
-        retval = read_IRQreg_ipvp(mpp, MPIC_TMR_IRQ + idx);
-        break;
-    case 0x30: /* TIDR */
-        if ((addr &0xF0) == 0XF0)
-            retval = mpp->dst[cpu].tfrr;
-        else
-            retval = read_IRQreg_ide(mpp, MPIC_TMR_IRQ + idx);
-        break;
+    /* timers and IPIs */
+    for (i = MAX_SRC; i < virq; i++) {
+        opp->src[i].type = IRQ_TYPE_FSLSPECIAL;
+        opp->src[i].level = false;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
-
-    return retval;
 }
 
-static void mpic_src_ext_write (void *opaque, target_phys_addr_t addr,
-                                uint32_t val)
+static void map_list(OpenPICState *opp, const MemReg *list, int *count)
 {
-    openpic_t *mpp = opaque;
-    int idx = MPIC_EXT_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-
-    if (addr < MPIC_EXT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
-        }
-    }
-}
+    while (list->name) {
+        assert(*count < ARRAY_SIZE(opp->sub_io_mem));
 
-static uint32_t mpic_src_ext_read (void *opaque, target_phys_addr_t addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_EXT_IRQ;
+        memory_region_init_io(&opp->sub_io_mem[*count], list->ops, opp,
+                              list->name, list->size);
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+        memory_region_add_subregion(&opp->mem, list->start_addr,
+                                    &opp->sub_io_mem[*count]);
 
-    if (addr < MPIC_EXT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
+        (*count)++;
+        list++;
     }
-
-    return retval;
 }
 
-static void mpic_src_int_write (void *opaque, target_phys_addr_t addr,
-                                uint32_t val)
+static int openpic_init(SysBusDevice *dev)
 {
-    openpic_t *mpp = opaque;
-    int idx = MPIC_INT_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
+    OpenPICState *opp = FROM_SYSBUS(typeof (*opp), dev);
+    int i, j;
+    int list_count = 0;
+    static const MemReg list_le[] = {
+        {"glb", &openpic_glb_ops_le,
+                OPENPIC_GLB_REG_START, OPENPIC_GLB_REG_SIZE},
+        {"tmr", &openpic_tmr_ops_le,
+                OPENPIC_TMR_REG_START, OPENPIC_TMR_REG_SIZE},
+        {"src", &openpic_src_ops_le,
+                OPENPIC_SRC_REG_START, OPENPIC_SRC_REG_SIZE},
+        {"cpu", &openpic_cpu_ops_le,
+                OPENPIC_CPU_REG_START, OPENPIC_CPU_REG_SIZE},
+        {NULL}
+    };
+    static const MemReg list_be[] = {
+        {"glb", &openpic_glb_ops_be,
+                OPENPIC_GLB_REG_START, OPENPIC_GLB_REG_SIZE},
+        {"tmr", &openpic_tmr_ops_be,
+                OPENPIC_TMR_REG_START, OPENPIC_TMR_REG_SIZE},
+        {"src", &openpic_src_ops_be,
+                OPENPIC_SRC_REG_START, OPENPIC_SRC_REG_SIZE},
+        {"cpu", &openpic_cpu_ops_be,
+                OPENPIC_CPU_REG_START, OPENPIC_CPU_REG_SIZE},
+        {NULL}
+    };
+    static const MemReg list_fsl[] = {
+        {"msi", &openpic_msi_ops_be,
+                OPENPIC_MSI_REG_START, OPENPIC_MSI_REG_SIZE},
+        {"summary", &openpic_summary_ops_be,
+                OPENPIC_SUMMARY_REG_START, OPENPIC_SUMMARY_REG_SIZE},
+        {NULL}
+    };
 
-    if (addr < MPIC_INT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
-        }
-    }
-}
+    memory_region_init(&opp->mem, "openpic", 0x40000);
 
-static uint32_t mpic_src_int_read (void *opaque, target_phys_addr_t addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_INT_IRQ;
+    switch (opp->model) {
+    case OPENPIC_MODEL_FSL_MPIC_20:
+    default:
+        opp->fsl = &fsl_mpic_20;
+        opp->brr1 = 0x00400200;
+        opp->flags |= OPENPIC_FLAG_IDR_CRIT;
+        opp->nb_irqs = 80;
+        opp->mpic_mode_mask = GCR_MODE_MIXED;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+        fsl_common_init(opp);
+        map_list(opp, list_be, &list_count);
+        map_list(opp, list_fsl, &list_count);
 
-    if (addr < MPIC_INT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
-    }
+        break;
 
-    return retval;
-}
+    case OPENPIC_MODEL_FSL_MPIC_42:
+        opp->fsl = &fsl_mpic_42;
+        opp->brr1 = 0x00400402;
+        opp->flags |= OPENPIC_FLAG_ILR;
+        opp->nb_irqs = 196;
+        opp->mpic_mode_mask = GCR_MODE_PROXY;
 
-static void mpic_src_msg_write (void *opaque, target_phys_addr_t addr,
-                                uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx = MPIC_MSG_IRQ;
+        fsl_common_init(opp);
+        map_list(opp, list_be, &list_count);
+        map_list(opp, list_fsl, &list_count);
 
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
+        break;
 
-    if (addr < MPIC_MSG_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
+    case OPENPIC_MODEL_RAVEN:
+        opp->nb_irqs = RAVEN_MAX_EXT;
+        opp->vid = VID_REVISION_1_3;
+        opp->vir = VIR_GENERIC;
+        opp->vector_mask = 0xFF;
+        opp->tfrr_reset = 4160000;
+        opp->ivpr_reset = IVPR_MASK_MASK | IVPR_MODE_MASK;
+        opp->idr_reset = 0;
+        opp->max_irq = RAVEN_MAX_IRQ;
+        opp->irq_ipi0 = RAVEN_IPI_IRQ;
+        opp->irq_tim0 = RAVEN_TMR_IRQ;
+        opp->brr1 = -1;
+        opp->mpic_mode_mask = GCR_MODE_MIXED;
+
+        /* Only UP supported today */
+        if (opp->nb_cpus != 1) {
+            return -EINVAL;
         }
-    }
-}
-
-static uint32_t mpic_src_msg_read (void *opaque, target_phys_addr_t addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_MSG_IRQ;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-
-    if (addr < MPIC_MSG_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
+        map_list(opp, list_le, &list_count);
+        break;
     }
 
-    return retval;
-}
-
-static void mpic_src_msi_write (void *opaque, target_phys_addr_t addr,
-                                uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx = MPIC_MSI_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-
-    if (addr < MPIC_MSI_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
+    for (i = 0; i < opp->nb_cpus; i++) {
+        opp->dst[i].irqs = g_new(qemu_irq, OPENPIC_OUTPUT_NB);
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_init_irq(dev, &opp->dst[i].irqs[j]);
         }
     }
-}
-static uint32_t mpic_src_msi_read (void *opaque, target_phys_addr_t addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_MSI_IRQ;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+    register_savevm(&opp->busdev.qdev, "openpic", 0, 2,
+                    openpic_save, openpic_load, opp);
 
-    if (addr < MPIC_MSI_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
-    }
+    sysbus_init_mmio(dev, &opp->mem);
+    qdev_init_gpio_in(&dev->qdev, openpic_set_irq, opp->max_irq);
 
-    return retval;
+    return 0;
 }
 
-static const MemoryRegionOps mpic_glb_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   openpic_gbl_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   openpic_gbl_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-static const MemoryRegionOps mpic_tmr_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_timer_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_timer_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
+static Property openpic_properties[] = {
+    DEFINE_PROP_UINT32("model", OpenPICState, model, OPENPIC_MODEL_FSL_MPIC_20),
+    DEFINE_PROP_UINT32("nb_cpus", OpenPICState, nb_cpus, 1),
+    DEFINE_PROP_END_OF_LIST(),
 };
 
-static const MemoryRegionOps mpic_cpu_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   openpic_cpu_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   openpic_cpu_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-static const MemoryRegionOps mpic_ext_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_ext_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_ext_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
+static void openpic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-static const MemoryRegionOps mpic_int_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_int_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_int_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
+    k->init = openpic_init;
+    dc->props = openpic_properties;
+    dc->reset = openpic_reset;
+}
 
-static const MemoryRegionOps mpic_msg_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_msg_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_msg_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
+static const TypeInfo openpic_info = {
+    .name          = "openpic",
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(OpenPICState),
+    .class_init    = openpic_class_init,
 };
 
-static const MemoryRegionOps mpic_msi_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_msi_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_msi_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-qemu_irq *mpic_init (MemoryRegion *address_space, target_phys_addr_t base,
-                     int nb_cpus, qemu_irq **irqs, qemu_irq irq_out)
+static void openpic_register_types(void)
 {
-    openpic_t    *mpp;
-    int           i;
-    struct {
-        const char             *name;
-        MemoryRegionOps const  *ops;
-        target_phys_addr_t      start_addr;
-        ram_addr_t              size;
-    } const list[] = {
-        {"glb", &mpic_glb_ops, MPIC_GLB_REG_START, MPIC_GLB_REG_SIZE},
-        {"tmr", &mpic_tmr_ops, MPIC_TMR_REG_START, MPIC_TMR_REG_SIZE},
-        {"ext", &mpic_ext_ops, MPIC_EXT_REG_START, MPIC_EXT_REG_SIZE},
-        {"int", &mpic_int_ops, MPIC_INT_REG_START, MPIC_INT_REG_SIZE},
-        {"msg", &mpic_msg_ops, MPIC_MSG_REG_START, MPIC_MSG_REG_SIZE},
-        {"msi", &mpic_msi_ops, MPIC_MSI_REG_START, MPIC_MSI_REG_SIZE},
-        {"cpu", &mpic_cpu_ops, MPIC_CPU_REG_START, MPIC_CPU_REG_SIZE},
-    };
-
-    mpp = g_malloc0(sizeof(openpic_t));
-
-    memory_region_init(&mpp->mem, "mpic", 0x40000);
-    memory_region_add_subregion(address_space, base, &mpp->mem);
-
-    for (i = 0; i < sizeof(list)/sizeof(list[0]); i++) {
-
-        memory_region_init_io(&mpp->sub_io_mem[i], list[i].ops, mpp,
-                              list[i].name, list[i].size);
-
-        memory_region_add_subregion(&mpp->mem, list[i].start_addr,
-                                    &mpp->sub_io_mem[i]);
-    }
-
-    mpp->nb_cpus = nb_cpus;
-    mpp->max_irq = MPIC_MAX_IRQ;
-    mpp->irq_ipi0 = MPIC_IPI_IRQ;
-    mpp->irq_tim0 = MPIC_TMR_IRQ;
-
-    for (i = 0; i < nb_cpus; i++)
-        mpp->dst[i].irqs = irqs[i];
-    mpp->irq_out = irq_out;
-
-    mpp->irq_raise = mpic_irq_raise;
-    mpp->reset = mpic_reset;
-
-    register_savevm(NULL, "mpic", 0, 2, openpic_save, openpic_load, mpp);
-    qemu_register_reset(mpic_reset, mpp);
-
-    return qemu_allocate_irqs(openpic_set_irq, mpp, mpp->max_irq);
+    type_register_static(&openpic_info);
 }
+
+type_init(openpic_register_types)
diff --git a/hw/openpic.h b/hw/openpic.h
index 855603026b6..9dcaf0e7cdf 100644
--- a/hw/openpic.h
+++ b/hw/openpic.h
@@ -11,8 +11,8 @@ enum {
     OPENPIC_OUTPUT_NB,
 };
 
-qemu_irq *openpic_init (MemoryRegion **pmem, int nb_cpus,
-                        qemu_irq **irqs, qemu_irq irq_out);
-qemu_irq *mpic_init (MemoryRegion *address_space, target_phys_addr_t base,
-                     int nb_cpus, qemu_irq **irqs, qemu_irq irq_out);
+#define OPENPIC_MODEL_RAVEN       0
+#define OPENPIC_MODEL_FSL_MPIC_20 1
+#define OPENPIC_MODEL_FSL_MPIC_42 2
+
 #endif /* __OPENPIC_H__ */
diff --git a/hw/openrisc_sim.c b/hw/openrisc_sim.c
index 55e97f0959d..30947dee6e1 100644
--- a/hw/openrisc_sim.c
+++ b/hw/openrisc_sim.c
@@ -21,12 +21,13 @@
 #include "hw.h"
 #include "boards.h"
 #include "elf.h"
-#include "pc.h"
+#include "serial.h"
+#include "net/net.h"
 #include "loader.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "qtest.h"
+#include "sysemu/qtest.h"
 
 #define KERNEL_LOAD_ADDR 0x100
 
@@ -38,8 +39,8 @@ static void main_cpu_reset(void *opaque)
 }
 
 static void openrisc_sim_net_init(MemoryRegion *address_space,
-                                  target_phys_addr_t base,
-                                  target_phys_addr_t descriptors,
+                                  hwaddr base,
+                                  hwaddr descriptors,
                                   qemu_irq irq, NICInfo *nd)
 {
     DeviceState *dev;
@@ -49,7 +50,7 @@ static void openrisc_sim_net_init(MemoryRegion *address_space,
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(address_space, base,
                                 sysbus_mmio_get_region(s, 0));
@@ -63,7 +64,7 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
 {
     long kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
 
     if (kernel_filename && !qtest_enabled()) {
         kernel_size = load_elf(kernel_filename, NULL, NULL,
@@ -90,13 +91,11 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
     cpu->env.pc = entry;
 }
 
-static void openrisc_sim_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void openrisc_sim_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
    OpenRISCCPU *cpu = NULL;
     MemoryRegion *ram;
     int n;
@@ -108,7 +107,7 @@ static void openrisc_sim_init(ram_addr_t ram_size,
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_openrisc_init(cpu_model);
         if (cpu == NULL) {
-            qemu_log("Unable to find CPU defineition!\n");
+            qemu_log("Unable to find CPU definition!\n");
             exit(1);
         }
         qemu_register_reset(main_cpu_reset, cpu);
@@ -140,6 +139,7 @@ static QEMUMachine openrisc_sim_machine = {
     .init = openrisc_sim_init,
     .max_cpus = 1,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void openrisc_sim_machine_init(void)
diff --git a/hw/openrisc_timer.c b/hw/openrisc_timer.c
index 7916e61d24b..d965be77dee 100644
--- a/hw/openrisc_timer.c
+++ b/hw/openrisc_timer.c
@@ -20,7 +20,7 @@
 
 #include "cpu.h"
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define TIMER_FREQ    (20 * 1000 * 1000)    /* 20MHz */
 
diff --git a/hw/palm.c b/hw/palm.c
index bacdc90d4a7..a633dfc4b17 100644
--- a/hw/palm.c
+++ b/hw/palm.c
@@ -18,34 +18,34 @@
  */
 #include "hw.h"
 #include "audio/audio.h"
-#include "sysemu.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "boards.h"
 #include "arm-misc.h"
 #include "devices.h"
 #include "loader.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
-static uint32_t static_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readb(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 3) << 3);
 }
 
-static uint32_t static_readh(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readh(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 1) << 3);
 }
 
-static uint32_t static_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readw(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 0) << 3);
 }
 
-static void static_write(void *opaque, target_phys_addr_t offset,
+static void static_write(void *opaque, hwaddr offset,
                 uint32_t value)
 {
 #ifdef SPY
@@ -190,11 +190,12 @@ static struct arm_boot_info palmte_binfo = {
     .board_id = 0x331,
 };
 
-static void palmte_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void palmte_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     struct omap_mpu_state_s *mpu;
     int flash_size = 0x00800000;
@@ -272,13 +273,14 @@ static void palmte_init(ram_addr_t ram_size,
        will set the size once configured, so this just sets an initial
        size until the guest activates the display.  */
     ds->surface = qemu_resize_displaysurface(ds, 320, 320);
-    dpy_resize(ds);
+    dpy_gfx_resize(ds);
 }
 
 static QEMUMachine palmte_machine = {
     .name = "cheetah",
     .desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)",
     .init = palmte_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void palmte_machine_init(void)
diff --git a/hw/pam.c b/hw/pam.c
new file mode 100644
index 00000000000..1d72e88e626
--- /dev/null
+++ b/hw/pam.c
@@ -0,0 +1,87 @@
+/*
+ * QEMU i440FX/PIIX3 PCI Bridge Emulation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2011 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * Split out from piix_pci.c
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "sysemu/sysemu.h"
+#include "pam.h"
+
+void smram_update(MemoryRegion *smram_region, uint8_t smram,
+                  uint8_t smm_enabled)
+{
+    bool smram_enabled;
+
+    smram_enabled = ((smm_enabled && (smram & SMRAM_G_SMRAME)) ||
+                        (smram & SMRAM_D_OPEN));
+    memory_region_set_enabled(smram_region, !smram_enabled);
+}
+
+void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
+                   MemoryRegion *smram_region)
+{
+    uint8_t smm_enabled = (smm != 0);
+    if (*host_smm_enabled != smm_enabled) {
+        *host_smm_enabled = smm_enabled;
+        smram_update(smram_region, smram, *host_smm_enabled);
+    }
+}
+
+void init_pam(MemoryRegion *ram_memory, MemoryRegion *system_memory,
+              MemoryRegion *pci_address_space, PAMMemoryRegion *mem,
+              uint32_t start, uint32_t size)
+{
+    int i;
+
+    /* RAM */
+    memory_region_init_alias(&mem->alias[3], "pam-ram", ram_memory,
+                             start, size);
+    /* ROM (XXX: not quite correct) */
+    memory_region_init_alias(&mem->alias[1], "pam-rom", ram_memory,
+                             start, size);
+    memory_region_set_readonly(&mem->alias[1], true);
+
+    /* XXX: should distinguish read/write cases */
+    memory_region_init_alias(&mem->alias[0], "pam-pci", pci_address_space,
+                             start, size);
+    memory_region_init_alias(&mem->alias[2], "pam-pci", pci_address_space,
+                             start, size);
+
+    for (i = 0; i < 4; ++i) {
+        memory_region_set_enabled(&mem->alias[i], false);
+        memory_region_add_subregion_overlap(system_memory, start,
+                                            &mem->alias[i], 1);
+    }
+    mem->current = 0;
+}
+
+void pam_update(PAMMemoryRegion *pam, int idx, uint8_t val)
+{
+    assert(0 <= idx && idx <= 12);
+
+    memory_region_set_enabled(&pam->alias[pam->current], false);
+    pam->current = (val >> ((!(idx & 1)) * 4)) & PAM_ATTR_MASK;
+    memory_region_set_enabled(&pam->alias[pam->current], true);
+}
diff --git a/hw/pam.h b/hw/pam.h
new file mode 100644
index 00000000000..8e9e349b1d0
--- /dev/null
+++ b/hw/pam.h
@@ -0,0 +1,97 @@
+#ifndef QEMU_PAM_H
+#define QEMU_PAM_H
+
+/*
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2011 Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * Split out from piix_pci.c
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * SMRAM memory area and PAM memory area in Legacy address range for PC.
+ * PAM: Programmable Attribute Map registers
+ *
+ * 0xa0000 - 0xbffff compatible SMRAM
+ *
+ * 0xc0000 - 0xc3fff Expansion area memory segments
+ * 0xc4000 - 0xc7fff
+ * 0xc8000 - 0xcbfff
+ * 0xcc000 - 0xcffff
+ * 0xd0000 - 0xd3fff
+ * 0xd4000 - 0xd7fff
+ * 0xd8000 - 0xdbfff
+ * 0xdc000 - 0xdffff
+ * 0xe0000 - 0xe3fff Extended System BIOS Area Memory Segments
+ * 0xe4000 - 0xe7fff
+ * 0xe8000 - 0xebfff
+ * 0xec000 - 0xeffff
+ *
+ * 0xf0000 - 0xfffff System BIOS Area Memory Segments
+ */
+
+#include "qemu-common.h"
+#include "exec/memory.h"
+
+#define SMRAM_C_BASE    0xa0000
+#define SMRAM_C_END     0xc0000
+#define SMRAM_C_SIZE    0x20000
+
+#define PAM_EXPAN_BASE  0xc0000
+#define PAM_EXPAN_SIZE  0x04000
+
+#define PAM_EXBIOS_BASE 0xe0000
+#define PAM_EXBIOS_SIZE 0x04000
+
+#define PAM_BIOS_BASE   0xf0000
+#define PAM_BIOS_END    0xfffff
+/* 64KB: Intel 3 series express chipset family p. 58*/
+#define PAM_BIOS_SIZE   0x10000
+
+/* PAM registers: log nibble and high nibble*/
+#define PAM_ATTR_WE     ((uint8_t)2)
+#define PAM_ATTR_RE     ((uint8_t)1)
+#define PAM_ATTR_MASK   ((uint8_t)3)
+
+/* SMRAM register */
+#define SMRAM_D_OPEN           ((uint8_t)(1 << 6))
+#define SMRAM_D_CLS            ((uint8_t)(1 << 5))
+#define SMRAM_D_LCK            ((uint8_t)(1 << 4))
+#define SMRAM_G_SMRAME         ((uint8_t)(1 << 3))
+#define SMRAM_C_BASE_SEG_MASK  ((uint8_t)0x7)
+#define SMRAM_C_BASE_SEG       ((uint8_t)0x2)  /* hardwired to b010 */
+
+typedef struct PAMMemoryRegion {
+    MemoryRegion alias[4];  /* index = PAM value */
+    unsigned current;
+} PAMMemoryRegion;
+
+void smram_update(MemoryRegion *smram_region, uint8_t smram,
+                  uint8_t smm_enabled);
+void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
+                   MemoryRegion *smram_region);
+void init_pam(MemoryRegion *ram, MemoryRegion *system, MemoryRegion *pci,
+              PAMMemoryRegion *mem, uint32_t start, uint32_t size);
+void pam_update(PAMMemoryRegion *mem, int idx, uint8_t val);
+
+#endif /* QEMU_PAM_H */
diff --git a/hw/parallel.c b/hw/parallel.c
index 219f38436ff..3a4e06bab0c 100644
--- a/hw/parallel.c
+++ b/hw/parallel.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "isa.h"
 #include "pc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG_PARALLEL
 
@@ -511,7 +511,7 @@ static int parallel_isa_initfn(ISADevice *dev)
 }
 
 /* Memory mapped interface */
-static uint32_t parallel_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readb (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -519,14 +519,14 @@ static uint32_t parallel_mm_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writeb (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
     parallel_ioport_write_sw(s, addr >> s->it_shift, value & 0xFF);
 }
 
-static uint32_t parallel_mm_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readw (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -534,14 +534,14 @@ static uint32_t parallel_mm_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writew (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
     parallel_ioport_write_sw(s, addr >> s->it_shift, value & 0xFFFF);
 }
 
-static uint32_t parallel_mm_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readl (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -549,7 +549,7 @@ static uint32_t parallel_mm_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writel (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
@@ -566,7 +566,7 @@ static const MemoryRegionOps parallel_mm_ops = {
 
 /* If fd is zero, it means that the parallel device uses the console */
 bool parallel_mm_init(MemoryRegion *address_space,
-                      target_phys_addr_t base, int it_shift, qemu_irq irq,
+                      hwaddr base, int it_shift, qemu_irq irq,
                       CharDriverState *chr)
 {
     ParallelState *s;
@@ -599,7 +599,7 @@ static void parallel_isa_class_initfn(ObjectClass *klass, void *data)
     dc->props = parallel_isa_properties;
 }
 
-static TypeInfo parallel_isa_info = {
+static const TypeInfo parallel_isa_info = {
     .name          = "isa-parallel",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAParallelState),
diff --git a/hw/pc-testdev.c b/hw/pc-testdev.c
new file mode 100644
index 00000000000..cf64a1f2035
--- /dev/null
+++ b/hw/pc-testdev.c
@@ -0,0 +1,187 @@
+/*
+ * QEMU x86 ISA testdev
+ *
+ * Copyright (c) 2012 Avi Kivity, Gerd Hoffmann, Marcelo Tosatti
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * This device is used to test KVM features specific to the x86 port, such
+ * as emulation, power management, interrupt routing, among others. It's meant
+ * to be used like:
+ *
+ * qemu-system-x86_64 -device pc-testdev -serial stdio \
+ * -device isa-debug-exit,iobase=0xf4,iosize=0x4 \
+ * -kernel /home/lmr/Code/virt-test.git/kvm/unittests/msr.flat
+ *
+ * Where msr.flat is one of the KVM unittests, present on a separate repo,
+ * git://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git
+*/
+
+#include "config-host.h"
+#if defined(CONFIG_POSIX)
+#include <sys/mman.h>
+#endif
+#include "hw.h"
+#include "qdev.h"
+#include "isa.h"
+
+#define IOMEM_LEN    0x10000
+
+typedef struct PCTestdev {
+    ISADevice parent_obj;
+
+    MemoryRegion ioport;
+    MemoryRegion flush;
+    MemoryRegion irq;
+    MemoryRegion iomem;
+    uint32_t ioport_data;
+    char iomem_buf[IOMEM_LEN];
+} PCTestdev;
+
+#define TYPE_TESTDEV "pc-testdev"
+#define TESTDEV(obj) \
+     OBJECT_CHECK(PCTestdev, (obj), TYPE_TESTDEV)
+
+static void test_irq_line(void *opaque, hwaddr addr, uint64_t data,
+                          unsigned len)
+{
+    PCTestdev *dev = opaque;
+    ISADevice *isa = ISA_DEVICE(dev);
+
+    qemu_set_irq(isa_get_irq(isa, addr), !!data);
+}
+
+static const MemoryRegionOps test_irq_ops = {
+    .write = test_irq_line,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void test_ioport_write(void *opaque, hwaddr addr, uint64_t data,
+                              unsigned len)
+{
+    PCTestdev *dev = opaque;
+    dev->ioport_data = data;
+}
+
+static uint64_t test_ioport_read(void *opaque, hwaddr addr, unsigned len)
+{
+    PCTestdev *dev = opaque;
+    return dev->ioport_data;
+}
+
+static const MemoryRegionOps test_ioport_ops = {
+    .read = test_ioport_read,
+    .write = test_ioport_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void test_flush_page(void *opaque, hwaddr addr, uint64_t data,
+                            unsigned len)
+{
+    hwaddr page = 4096;
+    void *a = cpu_physical_memory_map(data & ~0xffful, &page, 0);
+
+    /* We might not be able to get the full page, only mprotect what we actually
+       have mapped */
+#if defined(CONFIG_POSIX)
+    mprotect(a, page, PROT_NONE);
+    mprotect(a, page, PROT_READ|PROT_WRITE);
+#endif
+    cpu_physical_memory_unmap(a, page, 0, 0);
+}
+
+static const MemoryRegionOps test_flush_ops = {
+    .write = test_flush_page,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t test_iomem_read(void *opaque, hwaddr addr, unsigned len)
+{
+    PCTestdev *dev = opaque;
+    uint64_t ret = 0;
+    memcpy(&ret, &dev->iomem_buf[addr], len);
+    ret = le64_to_cpu(ret);
+
+    return ret;
+}
+
+static void test_iomem_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned len)
+{
+    PCTestdev *dev = opaque;
+    val = cpu_to_le64(val);
+    memcpy(&dev->iomem_buf[addr], &val, len);
+    dev->iomem_buf[addr] = val;
+}
+
+static const MemoryRegionOps test_iomem_ops = {
+    .read = test_iomem_read,
+    .write = test_iomem_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int init_test_device(ISADevice *isa)
+{
+    PCTestdev *dev = TESTDEV(isa);
+    MemoryRegion *mem = isa_address_space(isa);
+    MemoryRegion *io = isa_address_space_io(isa);
+
+    memory_region_init_io(&dev->ioport, &test_ioport_ops, dev,
+                          "pc-testdev-ioport", 4);
+    memory_region_init_io(&dev->flush, &test_flush_ops, dev,
+                          "pc-testdev-flush-page", 4);
+    memory_region_init_io(&dev->irq, &test_irq_ops, dev,
+                          "pc-testdev-irq-line", 24);
+    memory_region_init_io(&dev->iomem, &test_iomem_ops, dev,
+                          "pc-testdev-iomem", IOMEM_LEN);
+
+    memory_region_add_subregion(io,  0xe0,       &dev->ioport);
+    memory_region_add_subregion(io,  0xe4,       &dev->flush);
+    memory_region_add_subregion(io,  0x2000,     &dev->irq);
+    memory_region_add_subregion(mem, 0xff000000, &dev->iomem);
+
+    return 0;
+}
+
+static void testdev_class_init(ObjectClass *klass, void *data)
+{
+    ISADeviceClass *k = ISA_DEVICE_CLASS(klass);
+
+    k->init = init_test_device;
+}
+
+static const TypeInfo testdev_info = {
+    .name           = TYPE_TESTDEV,
+    .parent         = TYPE_ISA_DEVICE,
+    .instance_size  = sizeof(PCTestdev),
+    .class_init     = testdev_class_init,
+};
+
+static void testdev_register_types(void)
+{
+    type_register_static(&testdev_info);
+}
+
+type_init(testdev_register_types)
diff --git a/hw/pc.c b/hw/pc.c
index 112739a2789..34b6dff6866 100644
--- a/hw/pc.c
+++ b/hw/pc.c
@@ -23,12 +23,12 @@
  */
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "apic.h"
 #include "fdc.h"
 #include "ide.h"
-#include "pci.h"
-#include "vmware_vga.h"
-#include "monitor.h"
+#include "pci/pci.h"
+#include "monitor/monitor.h"
 #include "fw_cfg.h"
 #include "hpet_emul.h"
 #include "smbios.h"
@@ -38,23 +38,19 @@
 #include "mc146818rtc.h"
 #include "i8254.h"
 #include "pcspk.h"
-#include "msi.h"
+#include "pci/msi.h"
 #include "sysbus.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #include "xen.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
 #include "ui/qemu-spice.h"
-#include "memory.h"
-#include "exec-memory.h"
-#include "arch_init.h"
-#include "bitmap.h"
-#include "vga-pci.h"
-
-/* output Bochs bios info messages */
-//#define DEBUG_BIOS
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "sysemu/arch_init.h"
+#include "qemu/bitmap.h"
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -75,8 +71,6 @@
 #define FW_CFG_E820_TABLE (FW_CFG_ARCH_LOCAL + 3)
 #define FW_CFG_HPET (FW_CFG_ARCH_LOCAL + 4)
 
-#define MSI_ADDR_BASE 0xfee00000
-
 #define E820_NR_ENTRIES		16
 
 struct e820_entry {
@@ -104,10 +98,16 @@ void gsi_handler(void *opaque, int n, int level)
     qemu_set_irq(s->ioapic_irq[n], level);
 }
 
-static void ioport80_write(void *opaque, uint32_t addr, uint32_t data)
+static void ioport80_write(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned size)
 {
 }
 
+static uint64_t ioport80_read(void *opaque, hwaddr addr, unsigned size)
+{
+    return 0xffffffffffffffffULL;
+}
+
 /* MSDOS compatibility mode FPU exception support */
 static qemu_irq ferr_irq;
 
@@ -122,11 +122,17 @@ void cpu_set_ferr(CPUX86State *s)
     qemu_irq_raise(ferr_irq);
 }
 
-static void ioportF0_write(void *opaque, uint32_t addr, uint32_t data)
+static void ioportF0_write(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned size)
 {
     qemu_irq_lower(ferr_irq);
 }
 
+static uint64_t ioportF0_read(void *opaque, hwaddr addr, unsigned size)
+{
+    return 0xffffffffffffffffULL;
+}
+
 /* TSC handling */
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
@@ -425,7 +431,8 @@ typedef struct Port92State {
     qemu_irq *a20_out;
 } Port92State;
 
-static void port92_write(void *opaque, uint32_t addr, uint32_t val)
+static void port92_write(void *opaque, hwaddr addr, uint64_t val,
+                         unsigned size)
 {
     Port92State *s = opaque;
 
@@ -437,7 +444,8 @@ static void port92_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t port92_read(void *opaque, uint32_t addr)
+static uint64_t port92_read(void *opaque, hwaddr addr,
+                            unsigned size)
 {
     Port92State *s = opaque;
     uint32_t ret;
@@ -472,13 +480,14 @@ static void port92_reset(DeviceState *d)
     s->outport &= ~1;
 }
 
-static const MemoryRegionPortio port92_portio[] = {
-    { 0, 1, 1, .read = port92_read, .write = port92_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps port92_ops = {
-    .old_portio = port92_portio
+    .read = port92_read,
+    .write = port92_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int port92_initfn(ISADevice *dev)
@@ -502,7 +511,7 @@ static void port92_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_port92_isa;
 }
 
-static TypeInfo port92_info = {
+static const TypeInfo port92_info = {
     .name          = "port92",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(Port92State),
@@ -525,52 +534,6 @@ static void handle_a20_line_change(void *opaque, int irq, int level)
     cpu_x86_set_a20(cpu, level);
 }
 
-/***********************************************************/
-/* Bochs BIOS debug ports */
-
-static void bochs_bios_write(void *opaque, uint32_t addr, uint32_t val)
-{
-    static const char shutdown_str[8] = "Shutdown";
-    static int shutdown_index = 0;
-
-    switch(addr) {
-        /* Bochs BIOS messages */
-    case 0x400:
-    case 0x401:
-        /* used to be panic, now unused */
-        break;
-    case 0x402:
-    case 0x403:
-#ifdef DEBUG_BIOS
-        fprintf(stderr, "%c", val);
-#endif
-        break;
-    case 0x8900:
-        /* same as Bochs power off */
-        if (val == shutdown_str[shutdown_index]) {
-            shutdown_index++;
-            if (shutdown_index == 8) {
-                shutdown_index = 0;
-                qemu_system_shutdown_request();
-            }
-        } else {
-            shutdown_index = 0;
-        }
-        break;
-
-        /* LGPL'ed VGA BIOS messages */
-    case 0x501:
-    case 0x502:
-        exit((val << 1) | 1);
-    case 0x500:
-    case 0x503:
-#ifdef DEBUG_BIOS
-        fprintf(stderr, "%c", val);
-#endif
-        break;
-    }
-}
-
 int e820_add_entry(uint64_t address, uint64_t length, uint32_t type)
 {
     int index = le32_to_cpu(e820_table.count);
@@ -588,6 +551,18 @@ int e820_add_entry(uint64_t address, uint64_t length, uint32_t type)
     return index;
 }
 
+/* Calculates the limit to CPU APIC ID values
+ *
+ * This function returns the limit for the APIC ID value, so that all
+ * CPU APIC IDs are < pc_apic_id_limit().
+ *
+ * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
+ */
+static unsigned int pc_apic_id_limit(unsigned int max_cpus)
+{
+    return x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+}
+
 static void *bochs_bios_init(void)
 {
     void *fw_cfg;
@@ -595,55 +570,60 @@ static void *bochs_bios_init(void)
     size_t smbios_len;
     uint64_t *numa_fw_cfg;
     int i, j;
-
-    register_ioport_write(0x400, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x401, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x402, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x403, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x8900, 1, 1, bochs_bios_write, NULL);
-
-    register_ioport_write(0x501, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x501, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x502, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x500, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x503, 1, 1, bochs_bios_write, NULL);
+    unsigned int apic_id_limit = pc_apic_id_limit(max_cpus);
 
     fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
-
+    /* FW_CFG_MAX_CPUS is a bit confusing/problematic on x86:
+     *
+     * SeaBIOS needs FW_CFG_MAX_CPUS for CPU hotplug, but the CPU hotplug
+     * QEMU<->SeaBIOS interface is not based on the "CPU index", but on the APIC
+     * ID of hotplugged CPUs[1]. This means that FW_CFG_MAX_CPUS is not the
+     * "maximum number of CPUs", but the "limit to the APIC ID values SeaBIOS
+     * may see".
+     *
+     * So, this means we must not use max_cpus, here, but the maximum possible
+     * APIC ID value, plus one.
+     *
+     * [1] The only kind of "CPU identifier" used between SeaBIOS and QEMU is
+     *     the APIC ID, not the "CPU index"
+     */
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)apic_id_limit);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES, (uint8_t *)acpi_tables,
-                     acpi_tables_len);
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES,
+                     acpi_tables, acpi_tables_len);
     fw_cfg_add_i32(fw_cfg, FW_CFG_IRQ0_OVERRIDE, kvm_allows_irq0_override());
 
     smbios_table = smbios_get_table(&smbios_len);
     if (smbios_table)
         fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
                          smbios_table, smbios_len);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE, (uint8_t *)&e820_table,
-                     sizeof(struct e820_table));
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE,
+                     &e820_table, sizeof(e820_table));
 
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_HPET, (uint8_t *)&hpet_cfg,
-                     sizeof(struct hpet_fw_config));
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_HPET, &hpet_cfg, sizeof(hpet_cfg));
     /* allocate memory for the NUMA channel: one (64bit) word for the number
      * of nodes, one word for each VCPU->node and one word for each node to
      * hold the amount of memory.
      */
-    numa_fw_cfg = g_malloc0((1 + max_cpus + nb_numa_nodes) * 8);
+    numa_fw_cfg = g_new0(uint64_t, 1 + apic_id_limit + nb_numa_nodes);
     numa_fw_cfg[0] = cpu_to_le64(nb_numa_nodes);
     for (i = 0; i < max_cpus; i++) {
+        unsigned int apic_id = x86_cpu_apic_id_from_index(i);
+        assert(apic_id < apic_id_limit);
         for (j = 0; j < nb_numa_nodes; j++) {
             if (test_bit(i, node_cpumask[j])) {
-                numa_fw_cfg[i + 1] = cpu_to_le64(j);
+                numa_fw_cfg[apic_id + 1] = cpu_to_le64(j);
                 break;
             }
         }
     }
     for (i = 0; i < nb_numa_nodes; i++) {
-        numa_fw_cfg[max_cpus + 1 + i] = cpu_to_le64(node_mem[i]);
+        numa_fw_cfg[apic_id_limit + 1 + i] = cpu_to_le64(node_mem[i]);
     }
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, (uint8_t *)numa_fw_cfg,
-                     (1 + max_cpus + nb_numa_nodes) * 8);
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, numa_fw_cfg,
+                     (1 + apic_id_limit + nb_numa_nodes) *
+                     sizeof(*numa_fw_cfg));
 
     return fw_cfg;
 }
@@ -666,13 +646,13 @@ static void load_linux(void *fw_cfg,
                        const char *kernel_filename,
 		       const char *initrd_filename,
 		       const char *kernel_cmdline,
-                       target_phys_addr_t max_ram_size)
+                       hwaddr max_ram_size)
 {
     uint16_t protocol;
     int setup_size, kernel_size, initrd_size = 0, cmdline_size;
     uint32_t initrd_max;
     uint8_t header[8192], *setup, *kernel, *initrd_data;
-    target_phys_addr_t real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
+    hwaddr real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
     FILE *f;
     char *vmode;
 
@@ -742,9 +722,7 @@ static void load_linux(void *fw_cfg,
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, strlen(kernel_cmdline)+1);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                     (uint8_t*)strdup(kernel_cmdline),
-                     strlen(kernel_cmdline)+1);
+    fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
 
     if (protocol >= 0x202) {
 	stl_p(header+0x228, cmdline_addr);
@@ -874,35 +852,6 @@ DeviceState *cpu_get_current_apic(void)
     }
 }
 
-static DeviceState *apic_init(void *env, uint8_t apic_id)
-{
-    DeviceState *dev;
-    static int apic_mapped;
-
-    if (kvm_irqchip_in_kernel()) {
-        dev = qdev_create(NULL, "kvm-apic");
-    } else if (xen_enabled()) {
-        dev = qdev_create(NULL, "xen-apic");
-    } else {
-        dev = qdev_create(NULL, "apic");
-    }
-
-    qdev_prop_set_uint8(dev, "id", apic_id);
-    qdev_prop_set_ptr(dev, "cpu_env", env);
-    qdev_init_nofail(dev);
-
-    /* XXX: mapping more APICs at the same memory location */
-    if (apic_mapped == 0) {
-        /* NOTE: the APIC is directly connected to the CPU - it is not
-           on the global memory bus. */
-        /* XXX: what if the base changes? */
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, MSI_ADDR_BASE);
-        apic_mapped = 1;
-    }
-
-    return dev;
-}
-
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
 {
     CPUX86State *s = opaque;
@@ -912,24 +861,6 @@ void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
     }
 }
 
-static X86CPU *pc_new_cpu(const char *cpu_model)
-{
-    X86CPU *cpu;
-    CPUX86State *env;
-
-    cpu = cpu_x86_init(cpu_model);
-    if (cpu == NULL) {
-        fprintf(stderr, "Unable to find x86 CPU definition\n");
-        exit(1);
-    }
-    env = &cpu->env;
-    if ((env->cpuid_features & CPUID_APIC) || smp_cpus > 1) {
-        env->apic_state = apic_init(env, env->cpuid_apic_id);
-    }
-    cpu_reset(CPU(cpu));
-    return cpu;
-}
-
 void pc_cpus_init(const char *cpu_model)
 {
     int i;
@@ -943,11 +874,37 @@ void pc_cpus_init(const char *cpu_model)
 #endif
     }
 
-    for(i = 0; i < smp_cpus; i++) {
-        pc_new_cpu(cpu_model);
+    for (i = 0; i < smp_cpus; i++) {
+        if (!cpu_x86_init(cpu_model)) {
+            fprintf(stderr, "Unable to find x86 CPU definition\n");
+            exit(1);
+        }
     }
 }
 
+void pc_acpi_init(const char *default_dsdt)
+{
+    char *filename = NULL, *arg = NULL;
+
+    if (acpi_tables != NULL) {
+        /* manually set via -acpitable, leave it alone */
+        return;
+    }
+
+    filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, default_dsdt);
+    if (filename == NULL) {
+        fprintf(stderr, "WARNING: failed to find %s\n", default_dsdt);
+        return;
+    }
+
+    arg = g_strdup_printf("file=%s", filename);
+    if (acpi_table_add(arg) != 0) {
+        fprintf(stderr, "WARNING: failed to load %s\n", filename);
+    }
+    g_free(arg);
+    g_free(filename);
+}
+
 void *pc_memory_init(MemoryRegion *system_memory,
                     const char *kernel_filename,
                     const char *kernel_cmdline,
@@ -1019,34 +976,13 @@ DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     DeviceState *dev = NULL;
 
-    if (cirrus_vga_enabled) {
-        if (pci_bus) {
-            dev = pci_cirrus_vga_init(pci_bus);
-        } else {
-            dev = &isa_create_simple(isa_bus, "isa-cirrus-vga")->qdev;
-        }
-    } else if (vmsvga_enabled) {
-        if (pci_bus) {
-            dev = pci_vmsvga_init(pci_bus);
-        } else {
-            fprintf(stderr, "%s: vmware_vga: no PCI bus\n", __FUNCTION__);
-        }
-#ifdef CONFIG_SPICE
-    } else if (qxl_enabled) {
-        if (pci_bus) {
-            dev = &pci_create_simple(pci_bus, -1, "qxl-vga")->qdev;
-        } else {
-            fprintf(stderr, "%s: qxl: no PCI bus\n", __FUNCTION__);
-        }
-#endif
-    } else if (std_vga_enabled) {
-        if (pci_bus) {
-            dev = pci_vga_init(pci_bus);
-        } else {
-            dev = isa_vga_init(isa_bus);
-        }
+    if (pci_bus) {
+        PCIDevice *pcidev = pci_vga_init(pci_bus);
+        dev = pcidev ? &pcidev->qdev : NULL;
+    } else if (isa_bus) {
+        ISADevice *isadev = isa_vga_init(isa_bus);
+        dev = isadev ? &isadev->qdev : NULL;
     }
-
     return dev;
 }
 
@@ -1059,6 +995,26 @@ static void cpu_request_exit(void *opaque, int irq, int level)
     }
 }
 
+static const MemoryRegionOps ioport80_io_ops = {
+    .write = ioport80_write,
+    .read = ioport80_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static const MemoryRegionOps ioportF0_io_ops = {
+    .write = ioportF0_write,
+    .read = ioportF0_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
                           ISADevice **floppy,
@@ -1073,10 +1029,14 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     qemu_irq *a20_line;
     ISADevice *i8042, *port92, *vmmouse, *pit = NULL;
     qemu_irq *cpu_exit_irq;
+    MemoryRegion *ioport80_io = g_new(MemoryRegion, 1);
+    MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1);
 
-    register_ioport_write(0x80, 1, 1, ioport80_write, NULL);
+    memory_region_init_io(ioport80_io, &ioport80_io_ops, NULL, "ioport80", 1);
+    memory_region_add_subregion(isa_bus->address_space_io, 0x80, ioport80_io);
 
-    register_ioport_write(0xf0, 1, 1, ioportF0_write, NULL);
+    memory_region_init_io(ioportF0_io, &ioportF0_io_ops, NULL, "ioportF0", 1);
+    memory_region_add_subregion(isa_bus->address_space_io, 0xf0, ioportF0_io);
 
     /*
      * Check if an HPET shall be created.
@@ -1089,7 +1049,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
 
         if (hpet) {
             for (i = 0; i < GSI_NUM_PINS; i++) {
-                sysbus_connect_irq(sysbus_from_qdev(hpet), i, gsi[i]);
+                sysbus_connect_irq(SYS_BUS_DEVICE(hpet), i, gsi[i]);
             }
             pit_isa_irq = -1;
             pit_alt_irq = qdev_get_gpio_in(hpet, HPET_LEGACY_PIT_INT);
@@ -1150,6 +1110,21 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     *floppy = fdctrl_init_isa(isa_bus, fd);
 }
 
+void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus)
+{
+    int i;
+
+    for (i = 0; i < nb_nics; i++) {
+        NICInfo *nd = &nd_table[i];
+
+        if (!pci_bus || (nd->model && strcmp(nd->model, "ne2k_isa") == 0)) {
+            pc_init_ne2k_isa(isa_bus, nd);
+        } else {
+            pci_nic_init_nofail(nd, "e1000", NULL);
+        }
+    }
+}
+
 void pc_pci_device_init(PCIBus *pci_bus)
 {
     int max_bus;
@@ -1160,3 +1135,27 @@ void pc_pci_device_init(PCIBus *pci_bus)
         pci_create_simple(pci_bus, -1, "lsi53c895a");
     }
 }
+
+void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name)
+{
+    DeviceState *dev;
+    SysBusDevice *d;
+    unsigned int i;
+
+    if (kvm_irqchip_in_kernel()) {
+        dev = qdev_create(NULL, "kvm-ioapic");
+    } else {
+        dev = qdev_create(NULL, "ioapic");
+    }
+    if (parent_name) {
+        object_property_add_child(object_resolve_path(parent_name, NULL),
+                                  "ioapic", OBJECT(dev), NULL);
+    }
+    qdev_init_nofail(dev);
+    d = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(d, 0, 0xfec00000);
+
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        gsi_state->ioapic_irq[i] = qdev_get_gpio_in(dev, i);
+    }
+}
diff --git a/hw/pc.h b/hw/pc.h
index e4db0715b81..fbcf43d7176 100644
--- a/hw/pc.h
+++ b/hw/pc.h
@@ -2,43 +2,16 @@
 #define HW_PC_H
 
 #include "qemu-common.h"
-#include "memory.h"
-#include "ioport.h"
+#include "exec/memory.h"
+#include "exec/ioport.h"
 #include "isa.h"
 #include "fdc.h"
-#include "net.h"
-#include "memory.h"
+#include "net/net.h"
+#include "exec/memory.h"
 #include "ioapic.h"
 
 /* PC-style peripherals (also used by other machines).  */
 
-/* serial.c */
-
-SerialState *serial_init(int base, qemu_irq irq, int baudbase,
-                         CharDriverState *chr);
-SerialState *serial_mm_init(MemoryRegion *address_space,
-                            target_phys_addr_t base, int it_shift,
-                            qemu_irq irq, int baudbase,
-                            CharDriverState *chr, enum device_endian);
-static inline bool serial_isa_init(ISABus *bus, int index,
-                                   CharDriverState *chr)
-{
-    ISADevice *dev;
-
-    dev = isa_try_create(bus, "isa-serial");
-    if (!dev) {
-        return false;
-    }
-    qdev_prop_set_uint32(&dev->qdev, "index", index);
-    qdev_prop_set_chr(&dev->qdev, "chardev", chr);
-    if (qdev_init(&dev->qdev) < 0) {
-        return false;
-    }
-    return true;
-}
-
-void serial_set_frequency(SerialState *s, uint32_t frequency);
-
 /* parallel.c */
 static inline bool parallel_init(ISABus *bus, int index, CharDriverState *chr)
 {
@@ -57,7 +30,7 @@ static inline bool parallel_init(ISABus *bus, int index, CharDriverState *chr)
 }
 
 bool parallel_mm_init(MemoryRegion *address_space,
-                      target_phys_addr_t base, int it_shift, qemu_irq irq,
+                      hwaddr base, int it_shift, qemu_irq irq,
                       CharDriverState *chr);
 
 /* i8259.c */
@@ -67,8 +40,8 @@ qemu_irq *i8259_init(ISABus *bus, qemu_irq parent_irq);
 qemu_irq *kvm_i8259_init(ISABus *bus);
 int pic_read_irq(DeviceState *d);
 int pic_get_output(DeviceState *d);
-void pic_info(Monitor *mon);
-void irq_info(Monitor *mon);
+void pic_info(Monitor *mon, const QDict *qdict);
+void irq_info(Monitor *mon, const QDict *qdict);
 
 /* Global System Interrupts */
 
@@ -95,7 +68,7 @@ void vmmouse_set_data(const uint32_t *data);
 void i8042_init(qemu_irq kbd_irq, qemu_irq mouse_irq, uint32_t io_base);
 void i8042_mm_init(qemu_irq kbd_irq, qemu_irq mouse_irq,
                    MemoryRegion *region, ram_addr_t size,
-                   target_phys_addr_t mask);
+                   hwaddr mask);
 void i8042_isa_mouse_fake_event(void *opaque);
 void i8042_setup_a20_line(ISADevice *dev, qemu_irq *a20_out);
 
@@ -106,6 +79,7 @@ void pc_register_ferr_irq(qemu_irq irq);
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level);
 
 void pc_cpus_init(const char *cpu_model);
+void pc_acpi_init(const char *default_dsdt);
 void *pc_memory_init(MemoryRegion *system_memory,
                     const char *kernel_filename,
                     const char *kernel_cmdline,
@@ -125,11 +99,14 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
                   const char *boot_device,
                   ISADevice *floppy, BusState *ide0, BusState *ide1,
                   ISADevice *s);
+void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
 void pc_pci_device_init(PCIBus *pci_bus);
 
 typedef void (*cpu_set_smm_t)(int smm, void *arg);
 void cpu_smm_register(cpu_set_smm_t callback, void *arg);
 
+void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name);
+
 /* acpi.c */
 extern int acpi_enabled;
 extern char *acpi_tables;
@@ -157,10 +134,10 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    target_phys_addr_t pci_hole_start,
-                    target_phys_addr_t pci_hole_size,
-                    target_phys_addr_t pci_hole64_start,
-                    target_phys_addr_t pci_hole64_size,
+                    hwaddr pci_hole_start,
+                    hwaddr pci_hole_size,
+                    hwaddr pci_hole64_start,
+                    hwaddr pci_hole64_size,
                     MemoryRegion *pci_memory,
                     MemoryRegion *ram_memory);
 
@@ -176,21 +153,8 @@ enum vga_retrace_method {
 
 extern enum vga_retrace_method vga_retrace_method;
 
-static inline DeviceState *isa_vga_init(ISABus *bus)
-{
-    ISADevice *dev;
-
-    dev = isa_try_create(bus, "isa-vga");
-    if (!dev) {
-        fprintf(stderr, "Warning: isa-vga not available\n");
-        return NULL;
-    }
-    qdev_init_nofail(&dev->qdev);
-    return &dev->qdev;
-}
-
-int isa_vga_mm_init(target_phys_addr_t vram_base,
-                    target_phys_addr_t ctrl_base, int it_shift,
+int isa_vga_mm_init(hwaddr vram_base,
+                    hwaddr ctrl_base, int it_shift,
                     MemoryRegion *address_space);
 
 /* ne2000.c */
diff --git a/hw/pc87312.c b/hw/pc87312.c
new file mode 100644
index 00000000000..38af4c1d107
--- /dev/null
+++ b/hw/pc87312.c
@@ -0,0 +1,401 @@
+/*
+ * QEMU National Semiconductor PC87312 (Super I/O)
+ *
+ * Copyright (c) 2010-2012 Herve Poussineau
+ * Copyright (c) 2011-2012 Andreas Färber
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "pc87312.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
+#include "trace.h"
+
+
+#define REG_FER 0
+#define REG_FAR 1
+#define REG_PTR 2
+
+#define FER_PARALLEL_EN   0x01
+#define FER_UART1_EN      0x02
+#define FER_UART2_EN      0x04
+#define FER_FDC_EN        0x08
+#define FER_FDC_4         0x10
+#define FER_FDC_ADDR      0x20
+#define FER_IDE_EN        0x40
+#define FER_IDE_ADDR      0x80
+
+#define FAR_PARALLEL_ADDR 0x03
+#define FAR_UART1_ADDR    0x0C
+#define FAR_UART2_ADDR    0x30
+#define FAR_UART_3_4      0xC0
+
+#define PTR_POWER_DOWN    0x01
+#define PTR_CLOCK_DOWN    0x02
+#define PTR_PWDN          0x04
+#define PTR_IRQ_5_7       0x08
+#define PTR_UART1_TEST    0x10
+#define PTR_UART2_TEST    0x20
+#define PTR_LOCK_CONF     0x40
+#define PTR_EPP_MODE      0x80
+
+
+/* Parallel port */
+
+static inline bool is_parallel_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_PARALLEL_EN;
+}
+
+static const uint32_t parallel_base[] = { 0x378, 0x3bc, 0x278, 0x00 };
+
+static inline uint32_t get_parallel_iobase(PC87312State *s)
+{
+    return parallel_base[s->regs[REG_FAR] & FAR_PARALLEL_ADDR];
+}
+
+static const uint32_t parallel_irq[] = { 5, 7, 5, 0 };
+
+static inline uint32_t get_parallel_irq(PC87312State *s)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] & FAR_PARALLEL_ADDR);
+    if (idx == 0) {
+        return (s->regs[REG_PTR] & PTR_IRQ_5_7) ? 7 : 5;
+    } else {
+        return parallel_irq[idx];
+    }
+}
+
+static inline bool is_parallel_epp(PC87312State *s)
+{
+    return s->regs[REG_PTR] & PTR_EPP_MODE;
+}
+
+
+/* UARTs */
+
+static const uint32_t uart_base[2][4] = {
+    { 0x3e8, 0x338, 0x2e8, 0x220 },
+    { 0x2e8, 0x238, 0x2e0, 0x228 }
+};
+
+static inline uint32_t get_uart_iobase(PC87312State *s, int i)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] >> (2 * i + 2)) & 0x3;
+    if (idx == 0) {
+        return 0x3f8;
+    } else if (idx == 1) {
+        return 0x2f8;
+    } else {
+        return uart_base[idx & 1][(s->regs[REG_FAR] & FAR_UART_3_4) >> 6];
+    }
+}
+
+static inline uint32_t get_uart_irq(PC87312State *s, int i)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] >> (2 * i + 2)) & 0x3;
+    return (idx & 1) ? 3 : 4;
+}
+
+static inline bool is_uart_enabled(PC87312State *s, int i)
+{
+    return s->regs[REG_FER] & (FER_UART1_EN << i);
+}
+
+
+/* Floppy controller */
+
+static inline bool is_fdc_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_FDC_EN;
+}
+
+static inline uint32_t get_fdc_iobase(PC87312State *s)
+{
+    return (s->regs[REG_FER] & FER_FDC_ADDR) ? 0x370 : 0x3f0;
+}
+
+
+/* IDE controller */
+
+static inline bool is_ide_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_IDE_EN;
+}
+
+static inline uint32_t get_ide_iobase(PC87312State *s)
+{
+    return (s->regs[REG_FER] & FER_IDE_ADDR) ? 0x170 : 0x1f0;
+}
+
+
+static void reconfigure_devices(PC87312State *s)
+{
+    error_report("pc87312: unsupported device reconfiguration (%02x %02x %02x)",
+                 s->regs[REG_FER], s->regs[REG_FAR], s->regs[REG_PTR]);
+}
+
+static void pc87312_soft_reset(PC87312State *s)
+{
+    static const uint8_t fer_init[] = {
+        0x4f, 0x4f, 0x4f, 0x4f, 0x4f, 0x4f, 0x4b, 0x4b,
+        0x4b, 0x4b, 0x4b, 0x4b, 0x0f, 0x0f, 0x0f, 0x0f,
+        0x49, 0x49, 0x49, 0x49, 0x07, 0x07, 0x07, 0x07,
+        0x47, 0x47, 0x47, 0x47, 0x47, 0x47, 0x08, 0x00,
+    };
+    static const uint8_t far_init[] = {
+        0x10, 0x11, 0x11, 0x39, 0x24, 0x38, 0x00, 0x01,
+        0x01, 0x09, 0x08, 0x08, 0x10, 0x11, 0x39, 0x24,
+        0x00, 0x01, 0x01, 0x00, 0x10, 0x11, 0x39, 0x24,
+        0x10, 0x11, 0x11, 0x39, 0x24, 0x38, 0x10, 0x10,
+    };
+    static const uint8_t ptr_init[] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+    };
+
+    s->read_id_step = 0;
+    s->selected_index = REG_FER;
+
+    s->regs[REG_FER] = fer_init[s->config & 0x1f];
+    s->regs[REG_FAR] = far_init[s->config & 0x1f];
+    s->regs[REG_PTR] = ptr_init[s->config & 0x1f];
+}
+
+static void pc87312_hard_reset(PC87312State *s)
+{
+    pc87312_soft_reset(s);
+}
+
+static void pc87312_io_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned int size)
+{
+    PC87312State *s = opaque;
+
+    trace_pc87312_io_write(addr, val);
+
+    if ((addr & 1) == 0) {
+        /* Index register */
+        s->read_id_step = 2;
+        s->selected_index = val;
+    } else {
+        /* Data register */
+        if (s->selected_index < 3) {
+            s->regs[s->selected_index] = val;
+            reconfigure_devices(s);
+        }
+    }
+}
+
+static uint64_t pc87312_io_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    PC87312State *s = opaque;
+    uint32_t val;
+
+    if ((addr & 1) == 0) {
+        /* Index register */
+        if (s->read_id_step++ == 0) {
+            val = 0x88;
+        } else if (s->read_id_step++ == 1) {
+            val = 0;
+        } else {
+            val = s->selected_index;
+        }
+    } else {
+        /* Data register */
+        if (s->selected_index < 3) {
+            val = s->regs[s->selected_index];
+        } else {
+            /* Invalid selected index */
+            val = 0;
+        }
+    }
+
+    trace_pc87312_io_read(addr, val);
+    return val;
+}
+
+static const MemoryRegionOps pc87312_io_ops = {
+    .read  = pc87312_io_read,
+    .write = pc87312_io_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static int pc87312_post_load(void *opaque, int version_id)
+{
+    PC87312State *s = opaque;
+
+    reconfigure_devices(s);
+    return 0;
+}
+
+static void pc87312_reset(DeviceState *d)
+{
+    PC87312State *s = PC87312(d);
+
+    pc87312_soft_reset(s);
+}
+
+static int pc87312_init(ISADevice *dev)
+{
+    PC87312State *s;
+    DeviceState *d;
+    ISADevice *isa;
+    ISABus *bus;
+    CharDriverState *chr;
+    DriveInfo *drive;
+    char name[5];
+    int i;
+
+    s = PC87312(dev);
+    bus = isa_bus_from_device(dev);
+    pc87312_hard_reset(s);
+    isa_register_ioport(dev, &s->io, s->iobase);
+
+    if (is_parallel_enabled(s)) {
+        chr = parallel_hds[0];
+        if (chr == NULL) {
+            chr = qemu_chr_new("par0", "null", NULL);
+        }
+        isa = isa_create(bus, "isa-parallel");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "index", 0);
+        qdev_prop_set_uint32(d, "iobase", get_parallel_iobase(s));
+        qdev_prop_set_uint32(d, "irq", get_parallel_irq(s));
+        qdev_prop_set_chr(d, "chardev", chr);
+        qdev_init_nofail(d);
+        s->parallel.dev = isa;
+        trace_pc87312_info_parallel(get_parallel_iobase(s),
+                                    get_parallel_irq(s));
+    }
+
+    for (i = 0; i < 2; i++) {
+        if (is_uart_enabled(s, i)) {
+            chr = serial_hds[i];
+            if (chr == NULL) {
+                snprintf(name, sizeof(name), "ser%d", i);
+                chr = qemu_chr_new(name, "null", NULL);
+            }
+            isa = isa_create(bus, "isa-serial");
+            d = DEVICE(isa);
+            qdev_prop_set_uint32(d, "index", i);
+            qdev_prop_set_uint32(d, "iobase", get_uart_iobase(s, i));
+            qdev_prop_set_uint32(d, "irq", get_uart_irq(s, i));
+            qdev_prop_set_chr(d, "chardev", chr);
+            qdev_init_nofail(d);
+            s->uart[i].dev = isa;
+            trace_pc87312_info_serial(i, get_uart_iobase(s, i),
+                                      get_uart_irq(s, i));
+        }
+    }
+
+    if (is_fdc_enabled(s)) {
+        isa = isa_create(bus, "isa-fdc");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "iobase", get_fdc_iobase(s));
+        qdev_prop_set_uint32(d, "irq", 6);
+        drive = drive_get(IF_FLOPPY, 0, 0);
+        if (drive != NULL) {
+            qdev_prop_set_drive_nofail(d, "driveA", drive->bdrv);
+        }
+        drive = drive_get(IF_FLOPPY, 0, 1);
+        if (drive != NULL) {
+            qdev_prop_set_drive_nofail(d, "driveB", drive->bdrv);
+        }
+        qdev_init_nofail(d);
+        s->fdc.dev = isa;
+        trace_pc87312_info_floppy(get_fdc_iobase(s));
+    }
+
+    if (is_ide_enabled(s)) {
+        isa = isa_create(bus, "isa-ide");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "iobase", get_ide_iobase(s));
+        qdev_prop_set_uint32(d, "iobase2", get_ide_iobase(s) + 0x206);
+        qdev_prop_set_uint32(d, "irq", 14);
+        qdev_init_nofail(d);
+        s->ide.dev = isa;
+        trace_pc87312_info_ide(get_ide_iobase(s));
+    }
+
+    return 0;
+}
+
+static void pc87312_initfn(Object *obj)
+{
+    PC87312State *s = PC87312(obj);
+
+    memory_region_init_io(&s->io, &pc87312_io_ops, s, "pc87312", 2);
+}
+
+static const VMStateDescription vmstate_pc87312 = {
+    .name = "pc87312",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = pc87312_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(read_id_step, PC87312State),
+        VMSTATE_UINT8(selected_index, PC87312State),
+        VMSTATE_UINT8_ARRAY(regs, PC87312State, 3),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property pc87312_properties[] = {
+    DEFINE_PROP_HEX32("iobase", PC87312State, iobase, 0x398),
+    DEFINE_PROP_UINT8("config", PC87312State, config, 1),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void pc87312_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+
+    ic->init = pc87312_init;
+    dc->reset = pc87312_reset;
+    dc->vmsd = &vmstate_pc87312;
+    dc->props = pc87312_properties;
+}
+
+static const TypeInfo pc87312_type_info = {
+    .name          = TYPE_PC87312,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(PC87312State),
+    .instance_init = pc87312_initfn,
+    .class_init    = pc87312_class_init,
+};
+
+static void pc87312_register_types(void)
+{
+    type_register_static(&pc87312_type_info);
+}
+
+type_init(pc87312_register_types)
diff --git a/hw/pc87312.h b/hw/pc87312.h
new file mode 100644
index 00000000000..7b9e6f61320
--- /dev/null
+++ b/hw/pc87312.h
@@ -0,0 +1,68 @@
+/*
+ * QEMU National Semiconductor PC87312 (Super I/O)
+ *
+ * Copyright (c) 2010-2012 Herve Poussineau
+ * Copyright (c) 2011-2012 Andreas Färber
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef QEMU_PC87312_H
+#define QEMU_PC87312_H
+
+#include "isa.h"
+
+
+#define TYPE_PC87312 "pc87312"
+#define PC87312(obj) OBJECT_CHECK(PC87312State, (obj), TYPE_PC87312)
+
+typedef struct PC87312State {
+    ISADevice dev;
+
+    uint32_t iobase;
+    uint8_t config; /* initial configuration */
+
+    struct {
+        ISADevice *dev;
+    } parallel;
+
+    struct {
+        ISADevice *dev;
+    } uart[2];
+
+    struct {
+        ISADevice *dev;
+        BlockDriverState *drive[2];
+        uint32_t base;
+    } fdc;
+
+    struct {
+        ISADevice *dev;
+        uint32_t base;
+    } ide;
+
+    MemoryRegion io;
+
+    uint8_t read_id_step;
+    uint8_t selected_index;
+
+    uint8_t regs[3];
+} PC87312State;
+
+
+#endif
diff --git a/hw/pc_piix.c b/hw/pc_piix.c
index 88ff0411cae..b9a9b2efe1d 100644
--- a/hw/pc_piix.c
+++ b/hw/pc_piix.c
@@ -27,22 +27,23 @@
 #include "hw.h"
 #include "pc.h"
 #include "apic.h"
-#include "pci.h"
-#include "pci_ids.h"
+#include "pci/pci.h"
+#include "pci/pci_ids.h"
 #include "usb.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "ide.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm/clock.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "arch_init.h"
-#include "blockdev.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/blockdev.h"
 #include "smbus.h"
 #include "xen.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "cpu.h"
 #ifdef CONFIG_XEN
 #  include <xen/hvm/hvm_info_table.h>
 #endif
@@ -53,70 +54,6 @@ static const int ide_iobase[MAX_IDE_BUS] = { 0x1f0, 0x170 };
 static const int ide_iobase2[MAX_IDE_BUS] = { 0x3f6, 0x376 };
 static const int ide_irq[MAX_IDE_BUS] = { 14, 15 };
 
-static void kvm_piix3_setup_irq_routing(bool pci_enabled)
-{
-#ifdef CONFIG_KVM
-    KVMState *s = kvm_state;
-    int i;
-
-    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
-        for (i = 0; i < 8; ++i) {
-            if (i == 2) {
-                continue;
-            }
-            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_MASTER, i);
-        }
-        for (i = 8; i < 16; ++i) {
-            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_SLAVE, i - 8);
-        }
-        if (pci_enabled) {
-            for (i = 0; i < 24; ++i) {
-                if (i == 0) {
-                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, 2);
-                } else if (i != 2) {
-                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, i);
-                }
-            }
-        }
-    }
-#endif /* CONFIG_KVM */
-}
-
-static void kvm_piix3_gsi_handler(void *opaque, int n, int level)
-{
-    GSIState *s = opaque;
-
-    if (n < ISA_NUM_IRQS) {
-        /* Kernel will forward to both PIC and IOAPIC */
-        qemu_set_irq(s->i8259_irq[n], level);
-    } else {
-        qemu_set_irq(s->ioapic_irq[n], level);
-    }
-}
-
-static void ioapic_init(GSIState *gsi_state)
-{
-    DeviceState *dev;
-    SysBusDevice *d;
-    unsigned int i;
-
-    if (kvm_irqchip_in_kernel()) {
-        dev = qdev_create(NULL, "kvm-ioapic");
-    } else {
-        dev = qdev_create(NULL, "ioapic");
-    }
-    /* FIXME: this should be under the piix3.  */
-    object_property_add_child(object_resolve_path("i440fx", NULL),
-                              "ioapic", OBJECT(dev), NULL);
-    qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
-    sysbus_mmio_map(d, 0, 0xfec00000);
-
-    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
-        gsi_state->ioapic_irq[i] = qdev_get_gpio_in(dev, i);
-    }
-}
-
 /* PC hardware initialisation */
 static void pc_init1(MemoryRegion *system_memory,
                      MemoryRegion *system_io,
@@ -150,6 +87,7 @@ static void pc_init1(MemoryRegion *system_memory,
     void *fw_cfg = NULL;
 
     pc_cpus_init(cpu_model);
+    pc_acpi_init("acpi-dsdt.aml");
 
     if (kvmclock_enabled) {
         kvmclock_create();
@@ -177,13 +115,13 @@ static void pc_init1(MemoryRegion *system_memory,
         fw_cfg = pc_memory_init(system_memory,
                        kernel_filename, kernel_cmdline, initrd_filename,
                        below_4g_mem_size, above_4g_mem_size,
-                       pci_enabled ? rom_memory : system_memory, &ram_memory);
+                       rom_memory, &ram_memory);
     }
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
     if (kvm_irqchip_in_kernel()) {
-        kvm_piix3_setup_irq_routing(pci_enabled);
-        gsi = qemu_allocate_irqs(kvm_piix3_gsi_handler, gsi_state,
+        kvm_pc_setup_irq_routing(pci_enabled);
+        gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
         gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
@@ -195,7 +133,7 @@ static void pc_init1(MemoryRegion *system_memory,
                               below_4g_mem_size,
                               0x100000000ULL - below_4g_mem_size,
                               0x100000000ULL + above_4g_mem_size,
-                              (sizeof(target_phys_addr_t) == 4
+                              (sizeof(hwaddr) == 4
                                ? 0
                                : ((uint64_t)1 << 62)),
                               pci_memory, ram_memory);
@@ -220,7 +158,7 @@ static void pc_init1(MemoryRegion *system_memory,
         gsi_state->i8259_irq[i] = i8259[i];
     }
     if (pci_enabled) {
-        ioapic_init(gsi_state);
+        ioapic_init_gsi(gsi_state, "i440fx");
     }
 
     pc_register_ferr_irq(gsi[13]);
@@ -233,14 +171,7 @@ static void pc_init1(MemoryRegion *system_memory,
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, xen_enabled());
 
-    for(i = 0; i < nb_nics; i++) {
-        NICInfo *nd = &nd_table[i];
-
-        if (!pci_enabled || (nd->model && strcmp(nd->model, "ne2k_isa") == 0))
-            pc_init_ne2k_isa(isa_bus, nd);
-        else
-            pci_nic_init_nofail(nd, "e1000", NULL);
-    }
+    pc_nic_init(isa_bus, pci_bus);
 
     ide_drive_get(hd, MAX_IDE_BUS);
     if (pci_enabled) {
@@ -267,7 +198,7 @@ static void pc_init1(MemoryRegion *system_memory,
     pc_cmos_init(below_4g_mem_size, above_4g_mem_size, boot_device,
                  floppy, idebus[0], idebus[1], rtc_state);
 
-    if (pci_enabled && usb_enabled) {
+    if (pci_enabled && usb_enabled(false)) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
     }
 
@@ -287,13 +218,14 @@ static void pc_init1(MemoryRegion *system_memory,
     }
 }
 
-static void pc_init_pci(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void pc_init_pci(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -301,13 +233,30 @@ static void pc_init_pci(ram_addr_t ram_size,
              initrd_filename, cpu_model, 1, 1);
 }
 
-static void pc_init_pci_no_kvmclock(ram_addr_t ram_size,
-                                    const char *boot_device,
-                                    const char *kernel_filename,
-                                    const char *kernel_cmdline,
-                                    const char *initrd_filename,
-                                    const char *cpu_model)
+static void pc_init_pci_1_3(QEMUMachineInitArgs *args)
+{
+    enable_compat_apic_id_mode();
+    pc_init_pci(args);
+}
+
+/* PC machine init function for pc-0.14 to pc-1.2 */
+static void pc_init_pci_1_2(QEMUMachineInitArgs *args)
 {
+    disable_kvm_pv_eoi();
+    pc_init_pci_1_3(args);
+}
+
+/* PC init function for pc-0.10 to pc-0.13, and reused by xenfv */
+static void pc_init_pci_no_kvmclock(QEMUMachineInitArgs *args)
+{
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
+    disable_kvm_pv_eoi();
+    enable_compat_apic_id_mode();
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -315,15 +264,18 @@ static void pc_init_pci_no_kvmclock(ram_addr_t ram_size,
              initrd_filename, cpu_model, 1, 0);
 }
 
-static void pc_init_isa(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void pc_init_isa(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     if (cpu_model == NULL)
         cpu_model = "486";
+    disable_kvm_pv_eoi();
+    enable_compat_apic_id_mode();
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -332,33 +284,87 @@ static void pc_init_isa(ram_addr_t ram_size,
 }
 
 #ifdef CONFIG_XEN
-static void pc_xen_hvm_init(ram_addr_t ram_size,
-                            const char *boot_device,
-                            const char *kernel_filename,
-                            const char *kernel_cmdline,
-                            const char *initrd_filename,
-                            const char *cpu_model)
+static void pc_xen_hvm_init(QEMUMachineInitArgs *args)
 {
     if (xen_hvm_init() != 0) {
         hw_error("xen hardware virtual machine initialisation failed");
     }
-    pc_init_pci_no_kvmclock(ram_size, boot_device,
-                            kernel_filename, kernel_cmdline,
-                            initrd_filename, cpu_model);
+    pc_init_pci_no_kvmclock(args);
     xen_vcpu_init();
 }
 #endif
 
-static QEMUMachine pc_machine_v1_2 = {
-    .name = "pc-1.2",
+static QEMUMachine pc_i440fx_machine_v1_4 = {
+    .name = "pc-i440fx-1.4",
     .alias = "pc",
-    .desc = "Standard PC",
+    .desc = "Standard PC (i440FX + PIIX, 1996)",
     .init = pc_init_pci,
     .max_cpus = 255,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+#define PC_COMPAT_1_3 \
+        {\
+            .driver   = "usb-tablet",\
+            .property = "usb_version",\
+            .value    = stringify(1),\
+        }
+
+static QEMUMachine pc_machine_v1_3 = {
+    .name = "pc-1.3",
+    .desc = "Standard PC",
+    .init = pc_init_pci_1_3,
+    .max_cpus = 255,
+    .compat_props = (GlobalProperty[]) {
+        PC_COMPAT_1_3,
+        { /* end of list */ }
+    },
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+#define PC_COMPAT_1_2 \
+        PC_COMPAT_1_3,\
+        {\
+            .driver   = "nec-usb-xhci",\
+            .property = "msi",\
+            .value    = "off",\
+        },{\
+            .driver   = "nec-usb-xhci",\
+            .property = "msix",\
+            .value    = "off",\
+        },{\
+            .driver   = "ivshmem",\
+            .property = "use64",\
+            .value    = "0",\
+        },{\
+            .driver   = "qxl",\
+            .property = "revision",\
+            .value    = stringify(3),\
+        },{\
+            .driver   = "qxl-vga",\
+            .property = "revision",\
+            .value    = stringify(3),\
+        },{\
+            .driver   = "VGA",\
+            .property = "mmio",\
+            .value    = "off",\
+        }
+
+static QEMUMachine pc_machine_v1_2 = {
+    .name = "pc-1.2",
+    .desc = "Standard PC",
+    .init = pc_init_pci_1_2,
+    .max_cpus = 255,
+    .compat_props = (GlobalProperty[]) {
+        PC_COMPAT_1_2,
+        { /* end of list */ }
+    },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_1_1 \
+        PC_COMPAT_1_2,\
         {\
             .driver   = "virtio-scsi-pci",\
             .property = "hotplug",\
@@ -392,12 +398,13 @@ static QEMUMachine pc_machine_v1_2 = {
 static QEMUMachine pc_machine_v1_1 = {
     .name = "pc-1.1",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_1_1,
         { /* end of list */ }
     },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_1_0 \
@@ -427,13 +434,14 @@ static QEMUMachine pc_machine_v1_1 = {
 static QEMUMachine pc_machine_v1_0 = {
     .name = "pc-1.0",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_1_0,
         { /* end of list */ }
     },
     .hw_version = "1.0",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_15 \
@@ -442,13 +450,14 @@ static QEMUMachine pc_machine_v1_0 = {
 static QEMUMachine pc_machine_v0_15 = {
     .name = "pc-0.15",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_0_15,
         { /* end of list */ }
     },
     .hw_version = "0.15",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_14 \
@@ -474,7 +483,7 @@ static QEMUMachine pc_machine_v0_15 = {
 static QEMUMachine pc_machine_v0_14 = {
     .name = "pc-0.14",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_0_14, 
@@ -490,6 +499,7 @@ static QEMUMachine pc_machine_v0_14 = {
         { /* end of list */ }
     },
     .hw_version = "0.14",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_13 \
@@ -527,6 +537,7 @@ static QEMUMachine pc_machine_v0_13 = {
         { /* end of list */ }
     },
     .hw_version = "0.13",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_12 \
@@ -560,6 +571,7 @@ static QEMUMachine pc_machine_v0_12 = {
         { /* end of list */ }
     },
     .hw_version = "0.12",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_11 \
@@ -593,6 +605,7 @@ static QEMUMachine pc_machine_v0_11 = {
         { /* end of list */ }
     },
     .hw_version = "0.11",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine pc_machine_v0_10 = {
@@ -626,6 +639,7 @@ static QEMUMachine pc_machine_v0_10 = {
         { /* end of list */ }
     },
     .hw_version = "0.10",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine isapc_machine = {
@@ -641,6 +655,7 @@ static QEMUMachine isapc_machine = {
         },
         { /* end of list */ }
     },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #ifdef CONFIG_XEN
@@ -650,11 +665,14 @@ static QEMUMachine xenfv_machine = {
     .init = pc_xen_hvm_init,
     .max_cpus = HVM_MAX_VCPUS,
     .default_machine_opts = "accel=xen",
+    DEFAULT_MACHINE_OPTIONS,
 };
 #endif
 
 static void pc_machine_init(void)
 {
+    qemu_register_machine(&pc_i440fx_machine_v1_4);
+    qemu_register_machine(&pc_machine_v1_3);
     qemu_register_machine(&pc_machine_v1_2);
     qemu_register_machine(&pc_machine_v1_1);
     qemu_register_machine(&pc_machine_v1_0);
diff --git a/hw/pc_q35.c b/hw/pc_q35.c
new file mode 100644
index 00000000000..d82353e84f0
--- /dev/null
+++ b/hw/pc_q35.c
@@ -0,0 +1,225 @@
+/*
+ * Q35 chipset based pc system emulator
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2009, 2010
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on pc.c, but heavily modified.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "hw.h"
+#include "sysemu/arch_init.h"
+#include "smbus.h"
+#include "boards.h"
+#include "mc146818rtc.h"
+#include "xen.h"
+#include "sysemu/kvm.h"
+#include "kvm/clock.h"
+#include "q35.h"
+#include "exec/address-spaces.h"
+#include "ich9.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
+#include "hw/usb.h"
+
+/* ICH9 AHCI has 6 ports */
+#define MAX_SATA_PORTS     6
+
+/* set CMOS shutdown status register (index 0xF) as S3_resume(0xFE)
+ *    BIOS will read it and start S3 resume at POST Entry */
+static void pc_cmos_set_s3_resume(void *opaque, int irq, int level)
+{
+    ISADevice *s = opaque;
+
+    if (level) {
+        rtc_set_memory(s, 0xF, 0xFE);
+    }
+}
+
+/* PC hardware initialisation */
+static void pc_q35_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
+    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    Q35PCIHost *q35_host;
+    PCIBus *host_bus;
+    PCIDevice *lpc;
+    BusState *idebus[MAX_SATA_PORTS];
+    ISADevice *rtc_state;
+    ISADevice *floppy;
+    MemoryRegion *pci_memory;
+    MemoryRegion *rom_memory;
+    MemoryRegion *ram_memory;
+    GSIState *gsi_state;
+    ISABus *isa_bus;
+    int pci_enabled = 1;
+    qemu_irq *cpu_irq;
+    qemu_irq *gsi;
+    qemu_irq *i8259;
+    int i;
+    ICH9LPCState *ich9_lpc;
+    PCIDevice *ahci;
+    qemu_irq *cmos_s3;
+
+    pc_cpus_init(cpu_model);
+    pc_acpi_init("q35-acpi-dsdt.aml");
+
+    kvmclock_create();
+
+    if (ram_size >= 0xb0000000) {
+        above_4g_mem_size = ram_size - 0xb0000000;
+        below_4g_mem_size = 0xb0000000;
+    } else {
+        above_4g_mem_size = 0;
+        below_4g_mem_size = ram_size;
+    }
+
+    /* pci enabled */
+    if (pci_enabled) {
+        pci_memory = g_new(MemoryRegion, 1);
+        memory_region_init(pci_memory, "pci", INT64_MAX);
+        rom_memory = pci_memory;
+    } else {
+        pci_memory = NULL;
+        rom_memory = get_system_memory();
+    }
+
+    /* allocate ram and load rom/bios */
+    if (!xen_enabled()) {
+        pc_memory_init(get_system_memory(), kernel_filename, kernel_cmdline,
+                       initrd_filename, below_4g_mem_size, above_4g_mem_size,
+                       rom_memory, &ram_memory);
+    }
+
+    /* irq lines */
+    gsi_state = g_malloc0(sizeof(*gsi_state));
+    if (kvm_irqchip_in_kernel()) {
+        kvm_pc_setup_irq_routing(pci_enabled);
+        gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
+                                 GSI_NUM_PINS);
+    } else {
+        gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
+    }
+
+    /* create pci host bus */
+    q35_host = Q35_HOST_DEVICE(qdev_create(NULL, TYPE_Q35_HOST_DEVICE));
+
+    q35_host->mch.ram_memory = ram_memory;
+    q35_host->mch.pci_address_space = pci_memory;
+    q35_host->mch.system_memory = get_system_memory();
+    q35_host->mch.address_space_io = get_system_io();;
+    q35_host->mch.below_4g_mem_size = below_4g_mem_size;
+    q35_host->mch.above_4g_mem_size = above_4g_mem_size;
+    /* pci */
+    qdev_init_nofail(DEVICE(q35_host));
+    host_bus = q35_host->host.pci.bus;
+    /* create ISA bus */
+    lpc = pci_create_simple_multifunction(host_bus, PCI_DEVFN(ICH9_LPC_DEV,
+                                          ICH9_LPC_FUNC), true,
+                                          TYPE_ICH9_LPC_DEVICE);
+    ich9_lpc = ICH9_LPC_DEVICE(lpc);
+    ich9_lpc->pic = gsi;
+    ich9_lpc->ioapic = gsi_state->ioapic_irq;
+    pci_bus_irqs(host_bus, ich9_lpc_set_irq, ich9_lpc_map_irq, ich9_lpc,
+                 ICH9_LPC_NB_PIRQS);
+    isa_bus = ich9_lpc->isa_bus;
+
+    /*end early*/
+    isa_bus_irqs(isa_bus, gsi);
+
+    if (kvm_irqchip_in_kernel()) {
+        i8259 = kvm_i8259_init(isa_bus);
+    } else if (xen_enabled()) {
+        i8259 = xen_interrupt_controller_init();
+    } else {
+        cpu_irq = pc_allocate_cpu_irq();
+        i8259 = i8259_init(isa_bus, cpu_irq[0]);
+    }
+
+    for (i = 0; i < ISA_NUM_IRQS; i++) {
+        gsi_state->i8259_irq[i] = i8259[i];
+    }
+    if (pci_enabled) {
+        ioapic_init_gsi(gsi_state, NULL);
+    }
+
+    pc_register_ferr_irq(gsi[13]);
+
+    /* init basic PC hardware */
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, false);
+
+    /* connect pm stuff to lpc */
+    cmos_s3 = qemu_allocate_irqs(pc_cmos_set_s3_resume, rtc_state, 1);
+    ich9_lpc_pm_init(lpc, *cmos_s3);
+
+    /* ahci and SATA device, for q35 1 ahci controller is built-in */
+    ahci = pci_create_simple_multifunction(host_bus,
+                                           PCI_DEVFN(ICH9_SATA1_DEV,
+                                                     ICH9_SATA1_FUNC),
+                                           true, "ich9-ahci");
+    idebus[0] = qdev_get_child_bus(&ahci->qdev, "ide.0");
+    idebus[1] = qdev_get_child_bus(&ahci->qdev, "ide.1");
+
+    if (usb_enabled(false)) {
+        /* Should we create 6 UHCI according to ich9 spec? */
+        ehci_create_ich9_with_companions(host_bus, 0x1d);
+    }
+
+    /* TODO: Populate SPD eeprom data.  */
+    smbus_eeprom_init(ich9_smb_init(host_bus,
+                                    PCI_DEVFN(ICH9_SMB_DEV, ICH9_SMB_FUNC),
+                                    0xb100),
+                      8, NULL, 0);
+
+    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, boot_device,
+                 floppy, idebus[0], idebus[1], rtc_state);
+
+    /* the rest devices to which pci devfn is automatically assigned */
+    pc_vga_init(isa_bus, host_bus);
+    audio_init(isa_bus, host_bus);
+    pc_nic_init(isa_bus, host_bus);
+    if (pci_enabled) {
+        pc_pci_device_init(host_bus);
+    }
+}
+
+static QEMUMachine pc_q35_machine = {
+    .name = "pc-q35-1.4",
+    .alias = "q35",
+    .desc = "Standard PC (Q35 + ICH9, 2009)",
+    .init = pc_q35_init,
+    .max_cpus = 255,
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+static void pc_q35_machine_init(void)
+{
+    qemu_register_machine(&pc_q35_machine);
+}
+
+machine_init(pc_q35_machine_init);
diff --git a/hw/pc_sysfw.c b/hw/pc_sysfw.c
index b45f0acc7dd..7f6c12c8a86 100644
--- a/hw/pc_sysfw.c
+++ b/hw/pc_sysfw.c
@@ -23,15 +23,15 @@
  * THE SOFTWARE.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
 #include "hw.h"
 #include "pc.h"
 #include "hw/boards.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "flash.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define BIOS_FILENAME "bios.bin"
 
@@ -84,6 +84,10 @@ static void pc_fw_add_pflash_drv(void)
         bios_name = BIOS_FILENAME;
     }
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+    if (!filename) {
+        error_report("Can't open BIOS image %s", bios_name);
+        exit(1);
+    }
 
     opts = drive_add(IF_PFLASH, -1, filename, "readonly=on");
 
@@ -98,7 +102,9 @@ static void pc_fw_add_pflash_drv(void)
       return;
     }
 
-    drive_init(opts, machine->use_scsi);
+    if (!drive_init(opts, machine->block_default_type)) {
+        qemu_opts_del(opts);
+    }
 }
 
 static void pc_system_flash_init(MemoryRegion *rom_memory,
@@ -106,7 +112,7 @@ static void pc_system_flash_init(MemoryRegion *rom_memory,
 {
     BlockDriverState *bdrv;
     int64_t size;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int sector_bits, sector_size;
     pflash_t *system_flash;
     MemoryRegion *flash_mem;
@@ -250,7 +256,7 @@ static void pcsysfw_class_init (ObjectClass *klass, void *data)
     dc->props = pcsysfw_properties;
 }
 
-static TypeInfo pcsysfw_info = {
+static const TypeInfo pcsysfw_info = {
     .name          = "pc-sysfw",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (PcSysFwDevice),
diff --git a/hw/pci/Makefile.objs b/hw/pci/Makefile.objs
new file mode 100644
index 00000000000..1cd6cde2ee0
--- /dev/null
+++ b/hw/pci/Makefile.objs
@@ -0,0 +1,9 @@
+common-obj-$(CONFIG_PCI) += pci.o pci_bridge.o
+common-obj-$(CONFIG_PCI) += msix.o msi.o
+common-obj-$(CONFIG_PCI) += shpc.o
+common-obj-$(CONFIG_PCI) += slotid_cap.o
+common-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
+common-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
+common-obj-$(CONFIG_NO_PCI) += pci-stub.o
+
+common-obj-$(CONFIG_ALL) += pci-stub.o
diff --git a/hw/msi.c b/hw/pci/msi.c
index e2273a09ae1..2a04d188846 100644
--- a/hw/msi.c
+++ b/hw/pci/msi.c
@@ -18,8 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "msi.h"
-#include "range.h"
+#include "hw/pci/msi.h"
+#include "qemu/range.h"
 
 /* Eventually those constants should go to Linux pci_regs.h */
 #define PCI_MSI_PENDING_32      0x10
@@ -122,6 +122,31 @@ void msi_set_message(PCIDevice *dev, MSIMessage msg)
     pci_set_word(dev->config + msi_data_off(dev, msi64bit), msg.data);
 }
 
+MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector)
+{
+    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
+    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
+    unsigned int nr_vectors = msi_nr_vectors(flags);
+    MSIMessage msg;
+
+    assert(vector < nr_vectors);
+
+    if (msi64bit) {
+        msg.address = pci_get_quad(dev->config + msi_address_lo_off(dev));
+    } else {
+        msg.address = pci_get_long(dev->config + msi_address_lo_off(dev));
+    }
+
+    /* upper bit 31:16 is zero */
+    msg.data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
+    if (nr_vectors > 1) {
+        msg.data &= ~(nr_vectors - 1);
+        msg.data |= vector;
+    }
+
+    return msg;
+}
+
 bool msi_enabled(const PCIDevice *dev)
 {
     return msi_present(dev) &&
@@ -249,8 +274,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
     unsigned int nr_vectors = msi_nr_vectors(flags);
-    uint64_t address;
-    uint32_t data;
+    MSIMessage msg;
 
     assert(vector < nr_vectors);
     if (msi_is_masked(dev, vector)) {
@@ -261,24 +285,13 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
         return;
     }
 
-    if (msi64bit) {
-        address = pci_get_quad(dev->config + msi_address_lo_off(dev));
-    } else {
-        address = pci_get_long(dev->config + msi_address_lo_off(dev));
-    }
-
-    /* upper bit 31:16 is zero */
-    data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
-    if (nr_vectors > 1) {
-        data &= ~(nr_vectors - 1);
-        data |= vector;
-    }
+    msg = msi_get_message(dev, vector);
 
     MSI_DEV_PRINTF(dev,
                    "notify vector 0x%x"
                    " address: 0x%"PRIx64" data: 0x%"PRIx32"\n",
-                   vector, address, data);
-    stl_le_phys(address, data);
+                   vector, msg.address, msg.data);
+    stl_le_phys(msg.address, msg.data);
 }
 
 /* Normally called by pci_default_write_config(). */
diff --git a/hw/msi.h b/hw/pci/msi.h
index 6ec1f99f809..81a3848a313 100644
--- a/hw/msi.h
+++ b/hw/pci/msi.h
@@ -22,7 +22,7 @@
 #define QEMU_MSI_H
 
 #include "qemu-common.h"
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 struct MSIMessage {
     uint64_t address;
@@ -32,6 +32,7 @@ struct MSIMessage {
 extern bool msi_supported;
 
 void msi_set_message(PCIDevice *dev, MSIMessage msg);
+MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector);
 bool msi_enabled(const PCIDevice *dev);
 int msi_init(struct PCIDevice *dev, uint8_t offset,
              unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask);
diff --git a/hw/msix.c b/hw/pci/msix.c
index d8120941392..e231a0dc4bb 100644
--- a/hw/msix.c
+++ b/hw/pci/msix.c
@@ -14,11 +14,11 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw.h"
-#include "msi.h"
-#include "msix.h"
-#include "pci.h"
-#include "range.h"
+#include "hw/hw.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/pci.h"
+#include "qemu/range.h"
 
 #define MSIX_CAP_LENGTH 12
 
@@ -27,7 +27,7 @@
 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
 
-static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
+MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
 {
     uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
     MSIMessage msg;
@@ -65,7 +65,7 @@ static int msix_is_pending(PCIDevice *dev, int vector)
     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
 }
 
-static void msix_set_pending(PCIDevice *dev, int vector)
+void msix_set_pending(PCIDevice *dev, unsigned int vector)
 {
     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 }
@@ -75,13 +75,13 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 }
 
-static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
+static bool msix_vector_masked(PCIDevice *dev, unsigned int vector, bool fmask)
 {
     unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
     return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
-static bool msix_is_masked(PCIDevice *dev, int vector)
+bool msix_is_masked(PCIDevice *dev, unsigned int vector)
 {
     return msix_vector_masked(dev, vector, dev->msix_function_masked);
 }
@@ -157,7 +157,7 @@ void msix_write_config(PCIDevice *dev, uint32_t addr,
     }
 }
 
-static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     PCIDevice *dev = opaque;
@@ -165,7 +165,7 @@ static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
     return pci_get_long(dev->msix_table + addr);
 }
 
-static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
+static void msix_table_mmio_write(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     PCIDevice *dev = opaque;
@@ -180,26 +180,29 @@ static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
 static const MemoryRegionOps msix_table_mmio_ops = {
     .read = msix_table_mmio_read,
     .write = msix_table_mmio_write,
-    /* TODO: MSIX should be LITTLE_ENDIAN. */
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
         .max_access_size = 4,
     },
 };
 
-static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     PCIDevice *dev = opaque;
+    if (dev->msix_vector_poll_notifier) {
+        unsigned vector_start = addr * 8;
+        unsigned vector_end = MIN(addr + size * 8, dev->msix_entries_nr);
+        dev->msix_vector_poll_notifier(dev, vector_start, vector_end);
+    }
 
     return pci_get_long(dev->msix_pba + addr);
 }
 
 static const MemoryRegionOps msix_pba_mmio_ops = {
     .read = msix_pba_mmio_read,
-    /* TODO: MSIX should be LITTLE_ENDIAN. */
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
         .max_access_size = 4,
@@ -366,7 +369,6 @@ void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
     g_free(dev->msix_entry_used);
     dev->msix_entry_used = NULL;
     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
-    return;
 }
 
 void msix_uninit_exclusive_bar(PCIDevice *dev)
@@ -516,7 +518,8 @@ static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
 
 int msix_set_vector_notifiers(PCIDevice *dev,
                               MSIVectorUseNotifier use_notifier,
-                              MSIVectorReleaseNotifier release_notifier)
+                              MSIVectorReleaseNotifier release_notifier,
+                              MSIVectorPollNotifier poll_notifier)
 {
     int vector, ret;
 
@@ -524,6 +527,7 @@ int msix_set_vector_notifiers(PCIDevice *dev,
 
     dev->msix_vector_use_notifier = use_notifier;
     dev->msix_vector_release_notifier = release_notifier;
+    dev->msix_vector_poll_notifier = poll_notifier;
 
     if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
         (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
@@ -534,6 +538,9 @@ int msix_set_vector_notifiers(PCIDevice *dev,
             }
         }
     }
+    if (dev->msix_vector_poll_notifier) {
+        dev->msix_vector_poll_notifier(dev, 0, dev->msix_entries_nr);
+    }
     return 0;
 
 undo:
@@ -560,4 +567,5 @@ void msix_unset_vector_notifiers(PCIDevice *dev)
     }
     dev->msix_vector_use_notifier = NULL;
     dev->msix_vector_release_notifier = NULL;
+    dev->msix_vector_poll_notifier = NULL;
 }
diff --git a/hw/msix.h b/hw/pci/msix.h
index 15211cb5927..e6484105351 100644
--- a/hw/msix.h
+++ b/hw/pci/msix.h
@@ -2,9 +2,10 @@
 #define QEMU_MSIX_H
 
 #include "qemu-common.h"
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 void msix_set_message(PCIDevice *dev, int vector, MSIMessage msg);
+MSIMessage msix_get_message(PCIDevice *dev, unsigned int vector);
 int msix_init(PCIDevice *dev, unsigned short nentries,
               MemoryRegion *table_bar, uint8_t table_bar_nr,
               unsigned table_offset, MemoryRegion *pba_bar,
@@ -26,6 +27,9 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
 int msix_enabled(PCIDevice *dev);
 int msix_present(PCIDevice *dev);
 
+bool msix_is_masked(PCIDevice *dev, unsigned vector);
+void msix_set_pending(PCIDevice *dev, unsigned vector);
+
 int msix_vector_use(PCIDevice *dev, unsigned vector);
 void msix_vector_unuse(PCIDevice *dev, unsigned vector);
 void msix_unuse_all_vectors(PCIDevice *dev);
@@ -36,6 +40,7 @@ void msix_reset(PCIDevice *dev);
 
 int msix_set_vector_notifiers(PCIDevice *dev,
                               MSIVectorUseNotifier use_notifier,
-                              MSIVectorReleaseNotifier release_notifier);
+                              MSIVectorReleaseNotifier release_notifier,
+                              MSIVectorPollNotifier poll_notifier);
 void msix_unset_vector_notifiers(PCIDevice *dev);
 #endif
diff --git a/hw/pci-hotplug.c b/hw/pci/pci-hotplug.c
index e7fb780a088..f38df30540f 100644
--- a/hw/pci-hotplug.c
+++ b/hw/pci/pci-hotplug.c
@@ -22,17 +22,17 @@
  * THE SOFTWARE.
  */
 
-#include "hw.h"
-#include "boards.h"
-#include "pci.h"
-#include "net.h"
-#include "pc.h"
-#include "monitor.h"
-#include "scsi.h"
-#include "virtio-blk.h"
-#include "qemu-config.h"
-#include "blockdev.h"
-#include "error.h"
+#include "hw/hw.h"
+#include "hw/boards.h"
+#include "hw/pci/pci.h"
+#include "net/net.h"
+#include "hw/pc.h"
+#include "monitor/monitor.h"
+#include "hw/scsi.h"
+#include "hw/virtio-blk.h"
+#include "qemu/config-file.h"
+#include "sysemu/blockdev.h"
+#include "qapi/error.h"
 
 #if defined(TARGET_I386)
 static PCIDevice *qemu_pci_hot_add_nic(Monitor *mon,
@@ -80,7 +80,13 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
     SCSIBus *scsibus;
     SCSIDevice *scsidev;
 
-    scsibus = SCSI_BUS(QLIST_FIRST(&adapter->child_bus));
+    scsibus = (SCSIBus *)
+        object_dynamic_cast(OBJECT(QLIST_FIRST(&adapter->child_bus)),
+                            TYPE_SCSI_BUS);
+    if (!scsibus) {
+	error_report("Device is not a SCSI adapter");
+	return -1;
+    }
 
     /*
      * drive_init() tries to find a default for dinfo->unit.  Doesn't
@@ -105,15 +111,14 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
     return 0;
 }
 
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type)
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
 {
     int dom, pci_bus;
     unsigned slot;
     PCIDevice *dev;
     const char *pci_addr = qdict_get_str(qdict, "pci_addr");
 
-    switch (type) {
+    switch (dinfo->type) {
     case IF_SCSI:
         if (pci_read_devaddr(mon, pci_addr, &dom, &pci_bus, &slot)) {
             goto err;
@@ -129,7 +134,7 @@ int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
         }
         break;
     default:
-        monitor_printf(mon, "Can't hot-add drive to type %d\n", type);
+        monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
         goto err;
     }
 
diff --git a/hw/pci-stub.c b/hw/pci/pci-stub.c
index 134c4484b68..1dda89b5938 100644
--- a/hw/pci-stub.c
+++ b/hw/pci/pci-stub.c
@@ -18,9 +18,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sysemu.h"
-#include "monitor.h"
-#include "pci.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "hw/pci/pci.h"
 #include "qmp-commands.h"
 
 PciInfoList *qmp_query_pci(Error **errp)
diff --git a/hw/pci.c b/hw/pci/pci.c
index 4d959848079..5fd1bcf08e6 100644
--- a/hw/pci.c
+++ b/hw/pci/pci.c
@@ -21,18 +21,19 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "pci.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
-#include "monitor.h"
-#include "net.h"
-#include "sysemu.h"
-#include "loader.h"
-#include "range.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
+#include "monitor/monitor.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/loader.h"
+#include "qemu/range.h"
 #include "qmp-commands.h"
-#include "msi.h"
-#include "msix.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -300,9 +301,9 @@ PCIBus *pci_bus_new(DeviceState *parent, const char *name,
     PCIBus *bus;
 
     bus = g_malloc0(sizeof(*bus));
-    bus->qbus.glib_allocated = true;
     pci_bus_new_inplace(bus, parent, name, address_space_mem,
                         address_space_io, devfn_min);
+    OBJECT(bus)->free = g_free;
     return bus;
 }
 
@@ -366,6 +367,10 @@ static int get_pci_config_device(QEMUFile *f, void *pv, size_t size)
 
     pci_update_mappings(s);
 
+    memory_region_set_enabled(&s->bus_master_enable_region,
+                              pci_get_word(s->config + PCI_COMMAND)
+                              & PCI_COMMAND_MASTER);
+
     g_free(config);
     return 0;
 }
@@ -439,7 +444,7 @@ const VMStateDescription vmstate_pci_device = {
 };
 
 const VMStateDescription vmstate_pcie_device = {
-    .name = "PCIDevice",
+    .name = "PCIEDevice",
     .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
@@ -777,6 +782,17 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
     pci_dev->bus = bus;
     if (bus->dma_context_fn) {
         pci_dev->dma = bus->dma_context_fn(bus, bus->dma_context_opaque, devfn);
+    } else {
+        /* FIXME: Make dma_context_fn use MemoryRegions instead, so this path is
+         * taken unconditionally */
+        /* FIXME: inherit memory region from bus creator */
+        memory_region_init_alias(&pci_dev->bus_master_enable_region, "bus master",
+                                 get_system_memory(), 0,
+                                 memory_region_size(get_system_memory()));
+        memory_region_set_enabled(&pci_dev->bus_master_enable_region, false);
+        address_space_init(&pci_dev->bus_master_as, &pci_dev->bus_master_enable_region);
+        pci_dev->dma = g_new(DMAContext, 1);
+        dma_context_init(pci_dev->dma, &pci_dev->bus_master_as, NULL, NULL, NULL);
     }
     pci_dev->devfn = devfn;
     pstrcpy(pci_dev->name, sizeof(pci_dev->name), name);
@@ -830,6 +846,13 @@ static void do_pci_unregister_device(PCIDevice *pci_dev)
     qemu_free_irqs(pci_dev->irq);
     pci_dev->bus->devices[pci_dev->devfn] = NULL;
     pci_config_free(pci_dev);
+
+    if (!pci_dev->bus->dma_context_fn) {
+        address_space_destroy(&pci_dev->bus_master_as);
+        memory_region_destroy(&pci_dev->bus_master_enable_region);
+        g_free(pci_dev->dma);
+        pci_dev->dma = NULL;
+    }
 }
 
 static void pci_unregister_io_regions(PCIDevice *pci_dev)
@@ -968,7 +991,7 @@ static pcibus_t pci_bar_address(PCIDevice *d,
      * to >4G. Check it. TODO: we might need to support
      * it in the future for e.g. PAE.
      */
-    if (last_addr >= TARGET_PHYS_ADDR_MAX) {
+    if (last_addr >= HWADDR_MAX) {
         return PCI_BAR_UNMAPPED;
     }
 
@@ -1051,8 +1074,12 @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int l)
         range_covers_byte(addr, l, PCI_COMMAND))
         pci_update_mappings(d);
 
-    if (range_covers_byte(addr, l, PCI_COMMAND))
+    if (range_covers_byte(addr, l, PCI_COMMAND)) {
         pci_update_irq_disabled(d, was_irq_disabled);
+        memory_region_set_enabled(&d->bus_master_enable_region,
+                                  pci_get_word(d->config + PCI_COMMAND)
+                                    & PCI_COMMAND_MASTER);
+    }
 
     msi_write_config(d, addr, val, l);
     msix_write_config(d, addr, val, l);
@@ -1094,10 +1121,21 @@ PCIINTxRoute pci_device_route_intx_to_irq(PCIDevice *dev, int pin)
          pin = bus->map_irq(dev, pin);
          dev = bus->parent_dev;
     } while (dev);
-    assert(bus->route_intx_to_irq);
+
+    if (!bus->route_intx_to_irq) {
+        error_report("PCI: Bug - unimplemented PCI INTx routing (%s)\n",
+                     object_get_typename(OBJECT(bus->qbus.parent)));
+        return (PCIINTxRoute) { PCI_INTX_DISABLED, -1 };
+    }
+
     return bus->route_intx_to_irq(bus->irq_opaque, pin);
 }
 
+bool pci_intx_route_changed(PCIINTxRoute *old, PCIINTxRoute *new)
+{
+    return old->mode != new->mode || old->irq != new->irq;
+}
+
 void pci_bus_fire_intx_routing_notifier(PCIBus *bus)
 {
     PCIDevice *dev;
@@ -1121,6 +1159,24 @@ void pci_device_set_intx_routing_notifier(PCIDevice *dev,
     dev->intx_routing_notifier = notifier;
 }
 
+/*
+ * PCI-to-PCI bridge specification
+ * 9.1: Interrupt routing. Table 9-1
+ *
+ * the PCI Express Base Specification, Revision 2.1
+ * 2.2.8.1: INTx interrutp signaling - Rules
+ *          the Implementation Note
+ *          Table 2-20
+ */
+/*
+ * 0 <= pin <= 3 0 = INTA, 1 = INTB, 2 = INTC, 3 = INTD
+ * 0-origin unlike PCI interrupt pin register.
+ */
+int pci_swizzle_map_irq_fn(PCIDevice *pci_dev, int pin)
+{
+    return (pin + PCI_SLOT(pci_dev->devfn)) % PCI_NUM_PINS;
+}
+
 /***********************************************************/
 /* monitor info on PCI */
 
@@ -1185,6 +1241,7 @@ static const pci_class_desc pci_class_descriptions[] =
     { 0x0c02, "SSA controller", "ssa"},
     { 0x0c03, "USB controller", "usb"},
     { 0x0c04, "Fibre channel controller", "fibre-channel"},
+    { 0x0c05, "SMBus"},
     { 0, NULL}
 };
 
@@ -1474,6 +1531,24 @@ PCIDevice *pci_nic_init_nofail(NICInfo *nd, const char *default_model,
     return res;
 }
 
+PCIDevice *pci_vga_init(PCIBus *bus)
+{
+    switch (vga_interface_type) {
+    case VGA_CIRRUS:
+        return pci_create_simple(bus, -1, "cirrus-vga");
+    case VGA_QXL:
+        return pci_create_simple(bus, -1, "qxl-vga");
+    case VGA_STD:
+        return pci_create_simple(bus, -1, "VGA");
+    case VGA_VMWARE:
+        return pci_create_simple(bus, -1, "vmware-svga");
+    case VGA_NONE:
+    default: /* Other non-PCI types. Checking for unsupported types is already
+                done in vl.c. */
+        return NULL;
+    }
+}
+
 /* Whether a given bus number is in range of the secondary
  * bus of the given bridge device. */
 static bool pci_secondary_bus_in_range(PCIDevice *dev, int bus_num)
@@ -1626,16 +1701,16 @@ PCIDevice *pci_create_simple(PCIBus *bus, int devfn, const char *name)
     return pci_create_simple_multifunction(bus, devfn, false, name);
 }
 
-static int pci_find_space(PCIDevice *pdev, uint8_t size)
+static uint8_t pci_find_space(PCIDevice *pdev, uint8_t size)
 {
-    int config_size = pci_config_size(pdev);
     int offset = PCI_CONFIG_HEADER_SIZE;
     int i;
-    for (i = PCI_CONFIG_HEADER_SIZE; i < config_size; ++i)
+    for (i = PCI_CONFIG_HEADER_SIZE; i < PCI_CONFIG_SPACE_SIZE; ++i) {
         if (pdev->used[i])
             offset = i + 1;
         else if (i - offset + 1 == size)
             return offset;
+    }
     return 0;
 }
 
@@ -1854,7 +1929,7 @@ int pci_add_capability(PCIDevice *pdev, uint8_t cap_id,
     config[PCI_CAP_LIST_NEXT] = pdev->config[PCI_CAPABILITY_LIST];
     pdev->config[PCI_CAPABILITY_LIST] = offset;
     pdev->config[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
-    memset(pdev->used + offset, 0xFF, size);
+    memset(pdev->used + offset, 0xFF, QEMU_ALIGN_UP(size, 4));
     /* Make capability read-only by default */
     memset(pdev->wmask + offset, 0, size);
     /* Check capability by default */
@@ -1874,7 +1949,7 @@ void pci_del_capability(PCIDevice *pdev, uint8_t cap_id, uint8_t size)
     memset(pdev->w1cmask + offset, 0, size);
     /* Clear cmask as device-specific registers can't be checked */
     memset(pdev->cmask + offset, 0, size);
-    memset(pdev->used + offset, 0, size);
+    memset(pdev->used + offset, 0, QEMU_ALIGN_UP(size, 4));
 
     if (!pdev->config[PCI_CAPABILITY_LIST])
         pdev->config[PCI_STATUS] &= ~PCI_STATUS_CAP_LIST;
@@ -1962,7 +2037,7 @@ static char *pcibus_get_fw_dev_path(DeviceState *dev)
                    PCI_SLOT(d->devfn));
     if (PCI_FUNC(d->devfn))
         snprintf(path + off, sizeof(path) + off, ",%x", PCI_FUNC(d->devfn));
-    return strdup(path);
+    return g_strdup(path);
 }
 
 static char *pcibus_get_dev_path(DeviceState *dev)
@@ -2075,7 +2150,7 @@ void pci_setup_iommu(PCIBus *bus, PCIDMAContextFunc fn, void *opaque)
     bus->dma_context_opaque = opaque;
 }
 
-static TypeInfo pci_device_type_info = {
+static const TypeInfo pci_device_type_info = {
     .name = TYPE_PCI_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(PCIDevice),
diff --git a/hw/pci.h b/hw/pci/pci.h
index 4b6ab3d1908..f340fe57c9f 100644
--- a/hw/pci.h
+++ b/hw/pci/pci.h
@@ -3,14 +3,14 @@
 
 #include "qemu-common.h"
 
-#include "qdev.h"
-#include "memory.h"
-#include "dma.h"
+#include "hw/qdev.h"
+#include "exec/memory.h"
+#include "sysemu/dma.h"
 
 /* PCI includes legacy ISA access.  */
-#include "isa.h"
+#include "hw/isa.h"
 
-#include "pcie.h"
+#include "hw/pci/pcie.h"
 
 /* PCI bus */
 
@@ -21,7 +21,7 @@
 #define PCI_FUNC_MAX            8
 
 /* Class, Vendor and Device IDs from Linux's pci_ids.h */
-#include "pci_ids.h"
+#include "hw/pci/pci_ids.h"
 
 /* QEMU-specific Vendor and Device ID definitions */
 
@@ -76,6 +76,15 @@
 #define PCI_DEVICE_ID_VIRTIO_BALLOON     0x1002
 #define PCI_DEVICE_ID_VIRTIO_CONSOLE     0x1003
 #define PCI_DEVICE_ID_VIRTIO_SCSI        0x1004
+#define PCI_DEVICE_ID_VIRTIO_RNG         0x1005
+#define PCI_DEVICE_ID_VIRTIO_9P          0x1009
+
+#define PCI_VENDOR_ID_REDHAT             0x1b36
+#define PCI_DEVICE_ID_REDHAT_BRIDGE      0x0001
+#define PCI_DEVICE_ID_REDHAT_SERIAL      0x0002
+#define PCI_DEVICE_ID_REDHAT_SERIAL2     0x0003
+#define PCI_DEVICE_ID_REDHAT_SERIAL4     0x0004
+#define PCI_DEVICE_ID_REDHAT_QXL         0x0100
 
 #define FMT_PCIBUS                      PRIx64
 
@@ -99,7 +108,7 @@ typedef struct PCIIORegion {
 #define PCI_ROM_SLOT 6
 #define PCI_NUM_REGIONS 7
 
-#include "pci_regs.h"
+#include "hw/pci/pci_regs.h"
 
 /* PCI HEADER_TYPE */
 #define  PCI_HEADER_TYPE_MULTI_FUNCTION 0x80
@@ -186,6 +195,9 @@ typedef void (*PCIINTxRoutingNotifier)(PCIDevice *dev);
 typedef int (*MSIVectorUseNotifier)(PCIDevice *dev, unsigned int vector,
                                       MSIMessage msg);
 typedef void (*MSIVectorReleaseNotifier)(PCIDevice *dev, unsigned int vector);
+typedef void (*MSIVectorPollNotifier)(PCIDevice *dev,
+                                      unsigned int vector_start,
+                                      unsigned int vector_end);
 
 struct PCIDevice {
     DeviceState qdev;
@@ -211,6 +223,8 @@ struct PCIDevice {
     int32_t devfn;
     char name[64];
     PCIIORegion io_regions[PCI_NUM_REGIONS];
+    AddressSpace bus_master_as;
+    MemoryRegion bus_master_enable_region;
     DMAContext *dma;
 
     /* do not access the following fields */
@@ -268,6 +282,7 @@ struct PCIDevice {
     /* MSI-X notifiers */
     MSIVectorUseNotifier msix_vector_use_notifier;
     MSIVectorReleaseNotifier msix_vector_release_notifier;
+    MSIVectorPollNotifier msix_vector_poll_notifier;
 };
 
 void pci_register_bar(PCIDevice *pci_dev, int region_num,
@@ -316,6 +331,8 @@ void pci_bus_irqs(PCIBus *bus, pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                   void *irq_opaque, int nirq);
 int pci_bus_get_irq_level(PCIBus *bus, int irq_num);
 void pci_bus_hotplug(PCIBus *bus, pci_hotplug_fn hotplug, DeviceState *dev);
+/* 0 <= pin <= 3 0 = INTA, 1 = INTB, 2 = INTC, 3 = INTD */
+int pci_swizzle_map_irq_fn(PCIDevice *pci_dev, int pin);
 PCIBus *pci_register_bus(DeviceState *parent, const char *name,
                          pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                          void *irq_opaque,
@@ -324,6 +341,7 @@ PCIBus *pci_register_bus(DeviceState *parent, const char *name,
                          uint8_t devfn_min, int nirq);
 void pci_bus_set_route_irq_fn(PCIBus *, pci_route_irq_fn);
 PCIINTxRoute pci_device_route_intx_to_irq(PCIDevice *dev, int pin);
+bool pci_intx_route_changed(PCIINTxRoute *old, PCIINTxRoute *new);
 void pci_bus_fire_intx_routing_notifier(PCIBus *bus);
 void pci_device_set_intx_routing_notifier(PCIDevice *dev,
                                           PCIINTxRoutingNotifier notifier);
@@ -334,6 +352,9 @@ PCIDevice *pci_nic_init(NICInfo *nd, const char *default_model,
                         const char *default_devaddr);
 PCIDevice *pci_nic_init_nofail(NICInfo *nd, const char *default_model,
                                const char *default_devaddr);
+
+PCIDevice *pci_vga_init(PCIBus *bus);
+
 int pci_bus_num(PCIBus *s);
 void pci_for_each_device(PCIBus *bus, int bus_num,
                          void (*fn)(PCIBus *bus, PCIDevice *d, void *opaque),
diff --git a/hw/pci_bridge.c b/hw/pci/pci_bridge.c
index 5c6455f6fac..995842a72d0 100644
--- a/hw/pci_bridge.c
+++ b/hw/pci/pci_bridge.c
@@ -29,9 +29,9 @@
  *                    VA Linux Systems Japan K.K.
  */
 
-#include "pci_bridge.h"
-#include "pci_internals.h"
-#include "range.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
+#include "qemu/range.h"
 
 /* PCI bridge subsystem vendor ID helper functions */
 #define PCI_SSVID_SIZEOF        8
@@ -151,58 +151,63 @@ static void pci_bridge_init_alias(PCIBridge *bridge, MemoryRegion *alias,
     memory_region_add_subregion_overlap(parent_space, base, alias, 1);
 }
 
-static void pci_bridge_cleanup_alias(MemoryRegion *alias,
-                                     MemoryRegion *parent_space)
-{
-    memory_region_del_subregion(parent_space, alias);
-    memory_region_destroy(alias);
-}
-
-static void pci_bridge_region_init(PCIBridge *br)
+static PCIBridgeWindows *pci_bridge_region_init(PCIBridge *br)
 {
     PCIBus *parent = br->dev.bus;
+    PCIBridgeWindows *w = g_new(PCIBridgeWindows, 1);
     uint16_t cmd = pci_get_word(br->dev.config + PCI_COMMAND);
 
-    pci_bridge_init_alias(br, &br->alias_pref_mem,
+    pci_bridge_init_alias(br, &w->alias_pref_mem,
                           PCI_BASE_ADDRESS_MEM_PREFETCH,
                           "pci_bridge_pref_mem",
                           &br->address_space_mem,
                           parent->address_space_mem,
                           cmd & PCI_COMMAND_MEMORY);
-    pci_bridge_init_alias(br, &br->alias_mem,
+    pci_bridge_init_alias(br, &w->alias_mem,
                           PCI_BASE_ADDRESS_SPACE_MEMORY,
                           "pci_bridge_mem",
                           &br->address_space_mem,
                           parent->address_space_mem,
                           cmd & PCI_COMMAND_MEMORY);
-    pci_bridge_init_alias(br, &br->alias_io,
+    pci_bridge_init_alias(br, &w->alias_io,
                           PCI_BASE_ADDRESS_SPACE_IO,
                           "pci_bridge_io",
                           &br->address_space_io,
                           parent->address_space_io,
                           cmd & PCI_COMMAND_IO);
    /* TODO: optinal VGA and VGA palette snooping support. */
+
+    return w;
 }
 
-static void pci_bridge_region_cleanup(PCIBridge *br)
+static void pci_bridge_region_del(PCIBridge *br, PCIBridgeWindows *w)
 {
     PCIBus *parent = br->dev.bus;
-    pci_bridge_cleanup_alias(&br->alias_io,
-                             parent->address_space_io);
-    pci_bridge_cleanup_alias(&br->alias_mem,
-                             parent->address_space_mem);
-    pci_bridge_cleanup_alias(&br->alias_pref_mem,
-                             parent->address_space_mem);
+
+    memory_region_del_subregion(parent->address_space_io, &w->alias_io);
+    memory_region_del_subregion(parent->address_space_mem, &w->alias_mem);
+    memory_region_del_subregion(parent->address_space_mem, &w->alias_pref_mem);
+}
+
+static void pci_bridge_region_cleanup(PCIBridge *br, PCIBridgeWindows *w)
+{
+    memory_region_destroy(&w->alias_io);
+    memory_region_destroy(&w->alias_mem);
+    memory_region_destroy(&w->alias_pref_mem);
+    g_free(w);
 }
 
 static void pci_bridge_update_mappings(PCIBridge *br)
 {
+    PCIBridgeWindows *w = br->windows;
+
     /* Make updates atomic to: handle the case of one VCPU updating the bridge
      * while another accesses an unaffected region. */
     memory_region_transaction_begin();
-    pci_bridge_region_cleanup(br);
-    pci_bridge_region_init(br);
+    pci_bridge_region_del(br, br->windows);
+    br->windows = pci_bridge_region_init(br);
     memory_region_transaction_commit();
+    pci_bridge_region_cleanup(br, w);
 }
 
 /* default write_config function for PCI-to-PCI bridge */
@@ -326,7 +331,7 @@ int pci_bridge_initfn(PCIDevice *dev)
     memory_region_init(&br->address_space_mem, "pci_bridge_pci", INT64_MAX);
     sec_bus->address_space_io = &br->address_space_io;
     memory_region_init(&br->address_space_io, "pci_bridge_io", 65536);
-    pci_bridge_region_init(br);
+    br->windows = pci_bridge_region_init(br);
     QLIST_INIT(&sec_bus->child);
     QLIST_INSERT_HEAD(&parent->child, sec_bus, sibling);
     return 0;
@@ -338,7 +343,8 @@ void pci_bridge_exitfn(PCIDevice *pci_dev)
     PCIBridge *s = DO_UPCAST(PCIBridge, dev, pci_dev);
     assert(QLIST_EMPTY(&s->sec_bus.child));
     QLIST_REMOVE(&s->sec_bus, sibling);
-    pci_bridge_region_cleanup(s);
+    pci_bridge_region_del(s, s->windows);
+    pci_bridge_region_cleanup(s, s->windows);
     memory_region_destroy(&s->address_space_mem);
     memory_region_destroy(&s->address_space_io);
     /* qbus_free() is called automatically by qdev_free() */
diff --git a/hw/pci_bridge.h b/hw/pci/pci_bridge.h
index a00accc172f..455cb6677a7 100644
--- a/hw/pci_bridge.h
+++ b/hw/pci/pci_bridge.h
@@ -26,7 +26,7 @@
 #ifndef QEMU_PCI_BRIDGE_H
 #define QEMU_PCI_BRIDGE_H
 
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 int pci_bridge_ssvid_init(PCIDevice *dev, uint8_t offset,
                           uint16_t svid, uint16_t ssid);
diff --git a/hw/pci_internals.h b/hw/pci/pci_bus.h
index c931b64b46e..f905b9e11e1 100644
--- a/hw/pci_internals.h
+++ b/hw/pci/pci_bus.h
@@ -1,15 +1,11 @@
-#ifndef QEMU_PCI_INTERNALS_H
-#define QEMU_PCI_INTERNALS_H
+#ifndef QEMU_PCI_BUS_H
+#define QEMU_PCI_BUS_H
 
 /*
- * This header files is private to pci.c and pci_bridge.c
- * So following structures are opaque to others and shouldn't be
- * accessed.
+ * PCI Bus and Bridge datastructures.
  *
- * For pci-to-pci bridge needs to include this header file to embed
- * PCIBridge in its structure or to get sizeof(PCIBridge),
- * However, they shouldn't access those following members directly.
- * Use accessor function in pci.h, pci_bridge.h
+ * Do not access the following members directly;
+ * use accessor functions in pci.h, pci_bridge.h
  */
 
 #define TYPE_PCI_BUS "PCI"
@@ -40,6 +36,19 @@ struct PCIBus {
     int *irq_count;
 };
 
+typedef struct PCIBridgeWindows PCIBridgeWindows;
+
+/*
+ * Aliases for each of the address space windows that the bridge
+ * can forward. Mapped into the bridge's parent's address space,
+ * as subregions.
+ */
+struct PCIBridgeWindows {
+    MemoryRegion alias_pref_mem;
+    MemoryRegion alias_mem;
+    MemoryRegion alias_io;
+};
+
 struct PCIBridge {
     PCIDevice dev;
 
@@ -55,16 +64,11 @@ struct PCIBridge {
      */
     MemoryRegion address_space_mem;
     MemoryRegion address_space_io;
-    /*
-     * Aliases for each of the address space windows that the bridge
-     * can forward. Mapped into the bridge's parent's address space,
-     * as subregions.
-     */
-    MemoryRegion alias_pref_mem;
-    MemoryRegion alias_mem;
-    MemoryRegion alias_io;
+
+    PCIBridgeWindows *windows;
+
     pci_map_irq_fn map_irq;
     const char *bus_name;
 };
 
-#endif /* QEMU_PCI_INTERNALS_H */
+#endif /* QEMU_PCI_BUS_H */
diff --git a/hw/pci_host.c b/hw/pci/pci_host.c
index 3950e943c08..daca1c1ea0c 100644
--- a/hw/pci_host.c
+++ b/hw/pci/pci_host.c
@@ -18,8 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci.h"
-#include "pci_host.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
 
 /* debug PCI */
 //#define DEBUG_PCI
@@ -94,7 +94,7 @@ uint32_t pci_data_read(PCIBus *s, uint32_t addr, int len)
     return val;
 }
 
-static void pci_host_config_write(void *opaque, target_phys_addr_t addr,
+static void pci_host_config_write(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned len)
 {
     PCIHostState *s = opaque;
@@ -107,7 +107,7 @@ static void pci_host_config_write(void *opaque, target_phys_addr_t addr,
     s->config_reg = val;
 }
 
-static uint64_t pci_host_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_host_config_read(void *opaque, hwaddr addr,
                                      unsigned len)
 {
     PCIHostState *s = opaque;
@@ -118,7 +118,7 @@ static uint64_t pci_host_config_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void pci_host_data_write(void *opaque, target_phys_addr_t addr,
+static void pci_host_data_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned len)
 {
     PCIHostState *s = opaque;
@@ -129,7 +129,7 @@ static void pci_host_data_write(void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t pci_host_data_read(void *opaque,
-                                   target_phys_addr_t addr, unsigned len)
+                                   hwaddr addr, unsigned len)
 {
     PCIHostState *s = opaque;
     uint32_t val;
diff --git a/hw/pci_host.h b/hw/pci/pci_host.h
index 4b9c300fcf9..1845d4dfd58 100644
--- a/hw/pci_host.h
+++ b/hw/pci/pci_host.h
@@ -28,7 +28,7 @@
 #ifndef PCI_HOST_H
 #define PCI_HOST_H
 
-#include "sysbus.h"
+#include "hw/sysbus.h"
 
 #define TYPE_PCI_HOST_BRIDGE "pci-host-bridge"
 #define PCI_HOST_BRIDGE(obj) \
diff --git a/hw/pci_ids.h b/hw/pci/pci_ids.h
index 301bf1cd86b..d8dc2f1bf7e 100644
--- a/hw/pci_ids.h
+++ b/hw/pci/pci_ids.h
@@ -7,6 +7,8 @@
  *
  *      QEMU-specific definitions belong in pci.h
  */
+#ifndef HW_PCI_IDS_H
+#define HW_PCI_IDS_H 1
 
 /* Device classes and subclasses */
 
@@ -31,12 +33,15 @@
 #define PCI_CLASS_SYSTEM_OTHER           0x0880
 
 #define PCI_CLASS_SERIAL_USB             0x0c03
+#define PCI_CLASS_SERIAL_SMBUS           0x0c05
 
 #define PCI_CLASS_BRIDGE_HOST            0x0600
 #define PCI_CLASS_BRIDGE_ISA             0x0601
 #define PCI_CLASS_BRIDGE_PCI             0x0604
+#define PCI_CLASS_BRDIGE_PCI_INF_SUB     0x01
 #define PCI_CLASS_BRIDGE_OTHER           0x0680
 
+#define PCI_CLASS_COMMUNICATION_SERIAL   0x0700
 #define PCI_CLASS_COMMUNICATION_OTHER    0x0780
 
 #define PCI_CLASS_PROCESSOR_CO           0x0b40
@@ -104,6 +109,7 @@
 #define PCI_DEVICE_ID_INTEL_82378        0x0484
 #define PCI_DEVICE_ID_INTEL_82441        0x1237
 #define PCI_DEVICE_ID_INTEL_82801AA_5    0x2415
+#define PCI_DEVICE_ID_INTEL_82801BA_11   0x244e
 #define PCI_DEVICE_ID_INTEL_82801D       0x24CD
 #define PCI_DEVICE_ID_INTEL_ESB_9        0x25ab
 #define PCI_DEVICE_ID_INTEL_82371SB_0    0x7000
@@ -113,6 +119,17 @@
 #define PCI_DEVICE_ID_INTEL_82371AB      0x7111
 #define PCI_DEVICE_ID_INTEL_82371AB_2    0x7112
 #define PCI_DEVICE_ID_INTEL_82371AB_3    0x7113
+
+#define PCI_DEVICE_ID_INTEL_ICH9_0       0x2910
+#define PCI_DEVICE_ID_INTEL_ICH9_1       0x2917
+#define PCI_DEVICE_ID_INTEL_ICH9_2       0x2912
+#define PCI_DEVICE_ID_INTEL_ICH9_3       0x2913
+#define PCI_DEVICE_ID_INTEL_ICH9_4       0x2914
+#define PCI_DEVICE_ID_INTEL_ICH9_5       0x2919
+#define PCI_DEVICE_ID_INTEL_ICH9_6       0x2930
+#define PCI_DEVICE_ID_INTEL_ICH9_7       0x2916
+#define PCI_DEVICE_ID_INTEL_ICH9_8       0x2918
+
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI1 0x2934
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI2 0x2935
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI3 0x2936
@@ -123,8 +140,15 @@
 #define PCI_DEVICE_ID_INTEL_82801I_EHCI2 0x293c
 #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
 
+#define PCI_DEVICE_ID_INTEL_Q35_MCH      0x29c0
+
 #define PCI_VENDOR_ID_XEN               0x5853
 #define PCI_DEVICE_ID_XEN_PLATFORM      0x0001
 
 #define PCI_VENDOR_ID_NEC                0x1033
 #define PCI_DEVICE_ID_NEC_UPD720200      0x0194
+
+#define PCI_VENDOR_ID_TEWS               0x1498
+#define PCI_DEVICE_ID_TEWS_TPCI200       0x30C8
+
+#endif
diff --git a/hw/pci_regs.h b/hw/pci/pci_regs.h
index 56a404be6e1..56a404be6e1 100644
--- a/hw/pci_regs.h
+++ b/hw/pci/pci_regs.h
diff --git a/hw/pcie.c b/hw/pci/pcie.c
index 7c92f193e7d..485c94c1b25 100644
--- a/hw/pcie.c
+++ b/hw/pci/pcie.c
@@ -19,13 +19,13 @@
  */
 
 #include "qemu-common.h"
-#include "pci_bridge.h"
-#include "pcie.h"
-#include "msix.h"
-#include "msi.h"
-#include "pci_internals.h"
-#include "pcie_regs.h"
-#include "range.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pcie.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pcie_regs.h"
+#include "qemu/range.h"
 
 //#define DEBUG_PCIE
 #ifdef DEBUG_PCIE
@@ -494,7 +494,7 @@ uint16_t pcie_find_capability(PCIDevice *dev, uint16_t cap_id)
 
 static void pcie_ext_cap_set_next(PCIDevice *dev, uint16_t pos, uint16_t next)
 {
-    uint16_t header = pci_get_long(dev->config + pos);
+    uint32_t header = pci_get_long(dev->config + pos);
     assert(!(next & (PCI_EXT_CAP_ALIGN - 1)));
     header = (header & ~PCI_EXT_CAP_NEXT_MASK) |
         ((next << PCI_EXT_CAP_NEXT_SHIFT) & PCI_EXT_CAP_NEXT_MASK);
diff --git a/hw/pcie.h b/hw/pci/pcie.h
index b8ab0c7b4f1..31604e27423 100644
--- a/hw/pcie.h
+++ b/hw/pci/pcie.h
@@ -21,10 +21,10 @@
 #ifndef QEMU_PCIE_H
 #define QEMU_PCIE_H
 
-#include "hw.h"
-#include "pci_regs.h"
-#include "pcie_regs.h"
-#include "pcie_aer.h"
+#include "hw/hw.h"
+#include "hw/pci/pci_regs.h"
+#include "hw/pci/pcie_regs.h"
+#include "hw/pci/pcie_aer.h"
 
 typedef enum {
     /* for attention and power indicator */
@@ -133,7 +133,6 @@ extern const VMStateDescription vmstate_pcie_device;
 
 #define VMSTATE_PCIE_DEVICE(_field, _state) {                        \
     .name       = (stringify(_field)),                               \
-    .version_id = 2,                                                 \
     .size       = sizeof(PCIDevice),                                 \
     .vmsd       = &vmstate_pcie_device,                              \
     .flags      = VMS_STRUCT,                                        \
diff --git a/hw/pcie_aer.c b/hw/pci/pcie_aer.c
index 3b6981c7b76..1ce72ce944d 100644
--- a/hw/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -18,15 +18,15 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sysemu.h"
-#include "qemu-objects.h"
-#include "monitor.h"
-#include "pci_bridge.h"
-#include "pcie.h"
-#include "msix.h"
-#include "msi.h"
-#include "pci_internals.h"
-#include "pcie_regs.h"
+#include "sysemu/sysemu.h"
+#include "qapi/qmp/types.h"
+#include "monitor/monitor.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pcie.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pcie_regs.h"
 
 //#define DEBUG_PCIE
 #ifdef DEBUG_PCIE
@@ -738,6 +738,11 @@ void pcie_aer_root_init(PCIDevice *dev)
                  PCI_ERR_ROOT_CMD_EN_MASK);
     pci_set_long(dev->w1cmask + pos + PCI_ERR_ROOT_STATUS,
                  PCI_ERR_ROOT_STATUS_REPORT_MASK);
+    /* PCI_ERR_ROOT_IRQ is RO but devices change it using a
+     * device-specific method.
+     */
+    pci_set_long(dev->cmask + pos + PCI_ERR_ROOT_STATUS,
+                 ~PCI_ERR_ROOT_IRQ);
 }
 
 void pcie_aer_root_reset(PCIDevice *dev)
diff --git a/hw/pcie_aer.h b/hw/pci/pcie_aer.h
index 7539500cd8b..bcac80a7b07 100644
--- a/hw/pcie_aer.h
+++ b/hw/pci/pcie_aer.h
@@ -21,7 +21,7 @@
 #ifndef QEMU_PCIE_AER_H
 #define QEMU_PCIE_AER_H
 
-#include "hw.h"
+#include "hw/hw.h"
 
 /* definitions which PCIExpressDevice uses */
 
diff --git a/hw/pcie_host.c b/hw/pci/pcie_host.c
index 28bbe72b370..b2d942bce1b 100644
--- a/hw/pcie_host.c
+++ b/hw/pci/pcie_host.c
@@ -19,10 +19,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw.h"
-#include "pci.h"
-#include "pcie_host.h"
-#include "exec-memory.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pcie_host.h"
+#include "exec/address-spaces.h"
 
 /*
  * PCI express mmcfig address
@@ -53,7 +53,7 @@ static inline PCIDevice *pcie_dev_find_by_mmcfg_addr(PCIBus *s,
                            PCIE_MMCFG_DEVFN(mmcfg_addr));
 }
 
-static void pcie_mmcfg_data_write(void *opaque, target_phys_addr_t mmcfg_addr,
+static void pcie_mmcfg_data_write(void *opaque, hwaddr mmcfg_addr,
                                   uint64_t val, unsigned len)
 {
     PCIExpressHost *e = opaque;
@@ -76,7 +76,7 @@ static void pcie_mmcfg_data_write(void *opaque, target_phys_addr_t mmcfg_addr,
 }
 
 static uint64_t pcie_mmcfg_data_read(void *opaque,
-                                     target_phys_addr_t mmcfg_addr,
+                                     hwaddr mmcfg_addr,
                                      unsigned len)
 {
     PCIExpressHost *e = opaque;
@@ -105,16 +105,11 @@ static const MemoryRegionOps pcie_mmcfg_ops = {
 };
 
 /* pcie_host::base_addr == PCIE_BASE_ADDR_UNMAPPED when it isn't mapped. */
-#define PCIE_BASE_ADDR_UNMAPPED  ((target_phys_addr_t)-1ULL)
+#define PCIE_BASE_ADDR_UNMAPPED  ((hwaddr)-1ULL)
 
-int pcie_host_init(PCIExpressHost *e, uint32_t size)
+int pcie_host_init(PCIExpressHost *e)
 {
-    assert(!(size & (size - 1)));       /* power of 2 */
-    assert(size >= PCIE_MMCFG_SIZE_MIN);
-    assert(size <= PCIE_MMCFG_SIZE_MAX);
     e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
-    e->size = size;
-    memory_region_init_io(&e->mmio, &pcie_mmcfg_ops, e, "pcie-mmcfg", e->size);
 
     return 0;
 }
@@ -123,22 +118,44 @@ void pcie_host_mmcfg_unmap(PCIExpressHost *e)
 {
     if (e->base_addr != PCIE_BASE_ADDR_UNMAPPED) {
         memory_region_del_subregion(get_system_memory(), &e->mmio);
+        memory_region_destroy(&e->mmio);
         e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
     }
 }
 
-void pcie_host_mmcfg_map(PCIExpressHost *e, target_phys_addr_t addr)
+void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr,
+                         uint32_t size)
 {
+    assert(!(size & (size - 1)));       /* power of 2 */
+    assert(size >= PCIE_MMCFG_SIZE_MIN);
+    assert(size <= PCIE_MMCFG_SIZE_MAX);
+    e->size = size;
+    memory_region_init_io(&e->mmio, &pcie_mmcfg_ops, e, "pcie-mmcfg", e->size);
     e->base_addr = addr;
     memory_region_add_subregion(get_system_memory(), e->base_addr, &e->mmio);
 }
 
 void pcie_host_mmcfg_update(PCIExpressHost *e,
                             int enable,
-                            target_phys_addr_t addr)
+                            hwaddr addr,
+                            uint32_t size)
 {
     pcie_host_mmcfg_unmap(e);
     if (enable) {
-        pcie_host_mmcfg_map(e, addr);
+        pcie_host_mmcfg_map(e, addr, size);
     }
 }
+
+static const TypeInfo pcie_host_type_info = {
+    .name = TYPE_PCIE_HOST_BRIDGE,
+    .parent = TYPE_PCI_HOST_BRIDGE,
+    .abstract = true,
+    .instance_size = sizeof(PCIExpressHost),
+};
+
+static void pcie_host_register_types(void)
+{
+    type_register_static(&pcie_host_type_info);
+}
+
+type_init(pcie_host_register_types)
diff --git a/hw/pcie_host.h b/hw/pci/pcie_host.h
index 0074508b434..1228e36cb26 100644
--- a/hw/pcie_host.h
+++ b/hw/pci/pcie_host.h
@@ -21,8 +21,12 @@
 #ifndef PCIE_HOST_H
 #define PCIE_HOST_H
 
-#include "pci_host.h"
-#include "memory.h"
+#include "hw/pci/pci_host.h"
+#include "exec/memory.h"
+
+#define TYPE_PCIE_HOST_BRIDGE "pcie-host-bridge"
+#define PCIE_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(PCIExpressHost, (obj), TYPE_PCIE_HOST_BRIDGE)
 
 struct PCIExpressHost {
     PCIHostState pci;
@@ -30,20 +34,21 @@ struct PCIExpressHost {
     /* express part */
 
     /* base address where MMCONFIG area is mapped. */
-    target_phys_addr_t  base_addr;
+    hwaddr  base_addr;
 
     /* the size of MMCONFIG area. It's host bridge dependent */
-    target_phys_addr_t  size;
+    hwaddr  size;
 
     /* MMCONFIG mmio area */
     MemoryRegion mmio;
 };
 
-int pcie_host_init(PCIExpressHost *e, uint32_t size);
+int pcie_host_init(PCIExpressHost *e);
 void pcie_host_mmcfg_unmap(PCIExpressHost *e);
-void pcie_host_mmcfg_map(PCIExpressHost *e, target_phys_addr_t addr);
+void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr, uint32_t size);
 void pcie_host_mmcfg_update(PCIExpressHost *e,
                             int enable,
-                            target_phys_addr_t addr);
+                            hwaddr addr,
+                            uint32_t size);
 
 #endif /* PCIE_HOST_H */
diff --git a/hw/pcie_port.c b/hw/pci/pcie_port.c
index d6350e5e73f..33a6b0a08ac 100644
--- a/hw/pcie_port.c
+++ b/hw/pci/pcie_port.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pcie_port.h"
+#include "hw/pci/pcie_port.h"
 
 void pcie_port_init_reg(PCIDevice *d)
 {
diff --git a/hw/pcie_port.h b/hw/pci/pcie_port.h
index 3709583cc0f..d89aa615c5e 100644
--- a/hw/pcie_port.h
+++ b/hw/pci/pcie_port.h
@@ -21,8 +21,8 @@
 #ifndef QEMU_PCIE_PORT_H
 #define QEMU_PCIE_PORT_H
 
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
 
 struct PCIEPort {
     PCIBridge   br;
diff --git a/hw/pcie_regs.h b/hw/pci/pcie_regs.h
index 4d123d9fccc..4d123d9fccc 100644
--- a/hw/pcie_regs.h
+++ b/hw/pci/pcie_regs.h
diff --git a/hw/shpc.c b/hw/pci/shpc.c
index a5baf246f17..f07266da66e 100644
--- a/hw/shpc.c
+++ b/hw/pci/shpc.c
@@ -1,11 +1,11 @@
 #include <strings.h>
 #include <stdint.h>
-#include "range.h"
-#include "range.h"
-#include "shpc.h"
-#include "pci.h"
-#include "pci_internals.h"
-#include "msi.h"
+#include "qemu/range.h"
+#include "qemu/range.h"
+#include "hw/pci/shpc.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/msi.h"
 
 /* TODO: model power only and disabled slot states. */
 /* TODO: handle SERR and wakeups */
@@ -466,13 +466,13 @@ static int shpc_cap_add_config(PCIDevice *d)
     return 0;
 }
 
-static uint64_t shpc_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t shpc_mmio_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     return shpc_read(opaque, addr, size);
 }
 
-static void shpc_mmio_write(void *opaque, target_phys_addr_t addr,
+static void shpc_mmio_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     shpc_write(opaque, addr, val, size);
diff --git a/hw/shpc.h b/hw/pci/shpc.h
index 130b71df301..467911a558d 100644
--- a/hw/shpc.h
+++ b/hw/pci/shpc.h
@@ -2,8 +2,8 @@
 #define SHPC_H
 
 #include "qemu-common.h"
-#include "memory.h"
-#include "vmstate.h"
+#include "exec/memory.h"
+#include "migration/vmstate.h"
 
 struct SHPCDevice {
     /* Capability offset in device's config space */
diff --git a/hw/slotid_cap.c b/hw/pci/slotid_cap.c
index 01064521a94..99a30f429da 100644
--- a/hw/slotid_cap.c
+++ b/hw/pci/slotid_cap.c
@@ -1,5 +1,5 @@
-#include "slotid_cap.h"
-#include "pci.h"
+#include "hw/pci/slotid_cap.h"
+#include "hw/pci/pci.h"
 
 #define SLOTID_CAP_LENGTH 4
 #define SLOTID_NSLOTS_SHIFT (ffs(PCI_SID_ESR_NSLOTS) - 1)
diff --git a/hw/slotid_cap.h b/hw/pci/slotid_cap.h
index 70db0470b04..70db0470b04 100644
--- a/hw/slotid_cap.h
+++ b/hw/pci/slotid_cap.h
diff --git a/hw/pci_bridge_dev.c b/hw/pci_bridge_dev.c
index f7063961a00..1124c53b8c0 100644
--- a/hw/pci_bridge_dev.c
+++ b/hw/pci_bridge_dev.c
@@ -19,17 +19,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_bridge.h"
-#include "pci_ids.h"
-#include "msi.h"
-#include "shpc.h"
-#include "slotid_cap.h"
-#include "memory.h"
-#include "pci_internals.h"
-
-#define REDHAT_PCI_VENDOR_ID 0x1b36
-#define PCI_BRIDGE_DEV_VENDOR_ID REDHAT_PCI_VENDOR_ID
-#define PCI_BRIDGE_DEV_DEVICE_ID 0x1
+#include "pci/pci_bridge.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/shpc.h"
+#include "pci/slotid_cap.h"
+#include "exec/memory.h"
+#include "pci/pci_bus.h"
 
 struct PCIBridgeDev {
     PCIBridge bridge;
@@ -146,8 +142,8 @@ static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
     k->init = pci_bridge_dev_initfn;
     k->exit = pci_bridge_dev_exitfn;
     k->config_write = pci_bridge_dev_write_config;
-    k->vendor_id = PCI_BRIDGE_DEV_VENDOR_ID;
-    k->device_id = PCI_BRIDGE_DEV_DEVICE_ID;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_BRIDGE;
     k->class_id = PCI_CLASS_BRIDGE_PCI;
     k->is_bridge = 1,
     dc->desc = "Standard PCI Bridge";
@@ -156,7 +152,7 @@ static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &pci_bridge_dev_vmstate;
 }
 
-static TypeInfo pci_bridge_dev_info = {
+static const TypeInfo pci_bridge_dev_info = {
     .name = "pci-bridge",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBridgeDev),
diff --git a/hw/pckbd.c b/hw/pckbd.c
index 69857bade97..3bad09baf28 100644
--- a/hw/pckbd.c
+++ b/hw/pckbd.c
@@ -25,7 +25,7 @@
 #include "isa.h"
 #include "pc.h"
 #include "ps2.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* debug PC keyboard */
 //#define DEBUG_KBD
@@ -139,7 +139,7 @@ typedef struct KBDState {
     qemu_irq irq_kbd;
     qemu_irq irq_mouse;
     qemu_irq *a20_out;
-    target_phys_addr_t mask;
+    hwaddr mask;
 } KBDState;
 
 /* update irq and KBD_STAT_[MOUSE_]OBF */
@@ -194,7 +194,8 @@ static void kbd_update_aux_irq(void *opaque, int level)
     kbd_update_irq(s);
 }
 
-static uint32_t kbd_read_status(void *opaque, uint32_t addr)
+static uint64_t kbd_read_status(void *opaque, hwaddr addr,
+                                unsigned size)
 {
     KBDState *s = opaque;
     int val;
@@ -223,7 +224,8 @@ static void outport_write(KBDState *s, uint32_t val)
     }
 }
 
-static void kbd_write_command(void *opaque, uint32_t addr, uint32_t val)
+static void kbd_write_command(void *opaque, hwaddr addr,
+                              uint64_t val, unsigned size)
 {
     KBDState *s = opaque;
 
@@ -303,12 +305,13 @@ static void kbd_write_command(void *opaque, uint32_t addr, uint32_t val)
         /* ignore that */
         break;
     default:
-        fprintf(stderr, "qemu: unsupported keyboard cmd=0x%02x\n", val);
+        fprintf(stderr, "qemu: unsupported keyboard cmd=0x%02x\n", (int)val);
         break;
     }
 }
 
-static uint32_t kbd_read_data(void *opaque, uint32_t addr)
+static uint64_t kbd_read_data(void *opaque, hwaddr addr,
+                              unsigned size)
 {
     KBDState *s = opaque;
     uint32_t val;
@@ -322,7 +325,8 @@ static uint32_t kbd_read_data(void *opaque, uint32_t addr)
     return val;
 }
 
-static void kbd_write_data(void *opaque, uint32_t addr, uint32_t val)
+static void kbd_write_data(void *opaque, hwaddr addr,
+                           uint64_t val, unsigned size)
 {
     KBDState *s = opaque;
 
@@ -380,24 +384,24 @@ static const VMStateDescription vmstate_kbd = {
 };
 
 /* Memory mapped interface */
-static uint32_t kbd_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t kbd_mm_readb (void *opaque, hwaddr addr)
 {
     KBDState *s = opaque;
 
     if (addr & s->mask)
-        return kbd_read_status(s, 0) & 0xff;
+        return kbd_read_status(s, 0, 1) & 0xff;
     else
-        return kbd_read_data(s, 0) & 0xff;
+        return kbd_read_data(s, 0, 1) & 0xff;
 }
 
-static void kbd_mm_writeb (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void kbd_mm_writeb (void *opaque, hwaddr addr, uint32_t value)
 {
     KBDState *s = opaque;
 
     if (addr & s->mask)
-        kbd_write_command(s, 0, value & 0xff);
+        kbd_write_command(s, 0, value & 0xff, 1);
     else
-        kbd_write_data(s, 0, value & 0xff);
+        kbd_write_data(s, 0, value & 0xff, 1);
 }
 
 static const MemoryRegionOps i8042_mmio_ops = {
@@ -410,7 +414,7 @@ static const MemoryRegionOps i8042_mmio_ops = {
 
 void i8042_mm_init(qemu_irq kbd_irq, qemu_irq mouse_irq,
                    MemoryRegion *region, ram_addr_t size,
-                   target_phys_addr_t mask)
+                   hwaddr mask)
 {
     KBDState *s = g_malloc0(sizeof(KBDState));
 
@@ -459,22 +463,24 @@ static const VMStateDescription vmstate_kbd_isa = {
     }
 };
 
-static const MemoryRegionPortio i8042_data_portio[] = {
-    { 0, 1, 1, .read = kbd_read_data, .write = kbd_write_data },
-    PORTIO_END_OF_LIST()
-};
-
-static const MemoryRegionPortio i8042_cmd_portio[] = {
-    { 0, 1, 1, .read = kbd_read_status, .write = kbd_write_command },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps i8042_data_ops = {
-    .old_portio = i8042_data_portio
+    .read = kbd_read_data,
+    .write = kbd_write_data,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static const MemoryRegionOps i8042_cmd_ops = {
-    .old_portio = i8042_cmd_portio
+    .read = kbd_read_status,
+    .write = kbd_write_command,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int i8042_initfn(ISADevice *dev)
@@ -506,7 +512,7 @@ static void i8042_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_kbd_isa;
 }
 
-static TypeInfo i8042_info = {
+static const TypeInfo i8042_info = {
     .name          = "i8042",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAKBDState),
diff --git a/hw/pcmcia.h b/hw/pcmcia.h
index 50648c973ff..f91669305e1 100644
--- a/hw/pcmcia.h
+++ b/hw/pcmcia.h
@@ -1,3 +1,6 @@
+#ifndef HW_PCMCIA_H
+#define HW_PCMCIA_H 1
+
 /* PCMCIA/Cardbus */
 
 #include "qemu-common.h"
@@ -11,7 +14,7 @@ typedef struct {
 
 void pcmcia_socket_register(PCMCIASocket *socket);
 void pcmcia_socket_unregister(PCMCIASocket *socket);
-void pcmcia_info(Monitor *mon);
+void pcmcia_info(Monitor *mon, const QDict *qdict);
 
 struct PCMCIACardState {
     void *state;
@@ -49,3 +52,5 @@ struct PCMCIACardState {
 
 /* dscm1xxxx.c */
 PCMCIACardState *dscm1xxxx_init(DriveInfo *bdrv);
+
+#endif
diff --git a/hw/pcnet-pci.c b/hw/pcnet-pci.c
index 48fd447996d..a94f642136c 100644
--- a/hw/pcnet-pci.c
+++ b/hw/pcnet-pci.c
@@ -27,11 +27,11 @@
  * AMD Publication# 19436  Rev:E  Amendment/0  Issue Date: June 2000
  */
 
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "loader.h"
-#include "qemu-timer.h"
-#include "dma.h"
+#include "qemu/timer.h"
+#include "sysemu/dma.h"
 
 #include "pcnet.h"
 
@@ -71,7 +71,7 @@ static uint32_t pcnet_aprom_readb(void *opaque, uint32_t addr)
     return val;
 }
 
-static uint64_t pcnet_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pcnet_ioport_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     PCNetState *d = opaque;
@@ -98,7 +98,7 @@ static uint64_t pcnet_ioport_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void pcnet_ioport_write(void *opaque, target_phys_addr_t addr,
+static void pcnet_ioport_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     PCNetState *d = opaque;
@@ -130,7 +130,7 @@ static const MemoryRegionOps pcnet_io_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pcnet_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -141,7 +141,7 @@ static void pcnet_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t va
         pcnet_aprom_writeb(d, addr & 0x0f, val);
 }
 
-static uint32_t pcnet_mmio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readb(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
@@ -154,7 +154,7 @@ static uint32_t pcnet_mmio_readb(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void pcnet_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -170,7 +170,7 @@ static void pcnet_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t va
     }
 }
 
-static uint32_t pcnet_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readw(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
@@ -189,7 +189,7 @@ static uint32_t pcnet_mmio_readw(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void pcnet_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -207,7 +207,7 @@ static void pcnet_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t va
     }
 }
 
-static uint32_t pcnet_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readl(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val;
@@ -252,13 +252,13 @@ static const MemoryRegionOps pcnet_mmio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pci_physical_memory_write(void *dma_opaque, target_phys_addr_t addr,
+static void pci_physical_memory_write(void *dma_opaque, hwaddr addr,
                                       uint8_t *buf, int len, int do_bswap)
 {
     pci_dma_write(dma_opaque, addr, buf, len);
 }
 
-static void pci_physical_memory_read(void *dma_opaque, target_phys_addr_t addr,
+static void pci_physical_memory_read(void *dma_opaque, hwaddr addr,
                                      uint8_t *buf, int len, int do_bswap)
 {
     pci_dma_read(dma_opaque, addr, buf, len);
@@ -361,7 +361,7 @@ static void pcnet_class_init(ObjectClass *klass, void *data)
     dc->props = pcnet_properties;
 }
 
-static TypeInfo pcnet_info = {
+static const TypeInfo pcnet_info = {
     .name          = "pcnet",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIPCNetState),
diff --git a/hw/pcnet.c b/hw/pcnet.c
index 40820b36321..30f100007a6 100644
--- a/hw/pcnet.c
+++ b/hw/pcnet.c
@@ -36,10 +36,10 @@
  */
 
 #include "qdev.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "qemu_socket.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "qemu/sockets.h"
+#include "sysemu/sysemu.h"
 
 #include "pcnet.h"
 
@@ -293,7 +293,7 @@ struct pcnet_RMD {
         GET_FIELD((R)->msg_length, RMDM, ZEROS))
 
 static inline void pcnet_tmd_load(PCNetState *s, struct pcnet_TMD *tmd,
-                                  target_phys_addr_t addr)
+                                  hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -323,7 +323,7 @@ static inline void pcnet_tmd_load(PCNetState *s, struct pcnet_TMD *tmd,
 }
 
 static inline void pcnet_tmd_store(PCNetState *s, const struct pcnet_TMD *tmd,
-                                   target_phys_addr_t addr)
+                                   hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -359,7 +359,7 @@ static inline void pcnet_tmd_store(PCNetState *s, const struct pcnet_TMD *tmd,
 }
 
 static inline void pcnet_rmd_load(PCNetState *s, struct pcnet_RMD *rmd,
-                                  target_phys_addr_t addr)
+                                  hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -389,7 +389,7 @@ static inline void pcnet_rmd_load(PCNetState *s, struct pcnet_RMD *rmd,
 }
 
 static inline void pcnet_rmd_store(PCNetState *s, struct pcnet_RMD *rmd,
-                                   target_phys_addr_t addr)
+                                   hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -660,7 +660,7 @@ static inline int ladr_match(PCNetState *s, const uint8_t *buf, int size)
     return 0;
 }
 
-static inline target_phys_addr_t pcnet_rdra_addr(PCNetState *s, int idx)
+static inline hwaddr pcnet_rdra_addr(PCNetState *s, int idx)
 {
     while (idx < 1) idx += CSR_RCVRL(s);
     return s->rdra + ((CSR_RCVRL(s) - idx) * (BCR_SWSTYLE(s) ? 16 : 8));
@@ -898,19 +898,19 @@ static void pcnet_rdte_poll(PCNetState *s)
     if (s->rdra) {
         int bad = 0;
 #if 1
-        target_phys_addr_t crda = pcnet_rdra_addr(s, CSR_RCVRC(s));
-        target_phys_addr_t nrda = pcnet_rdra_addr(s, -1 + CSR_RCVRC(s));
-        target_phys_addr_t nnrd = pcnet_rdra_addr(s, -2 + CSR_RCVRC(s));
+        hwaddr crda = pcnet_rdra_addr(s, CSR_RCVRC(s));
+        hwaddr nrda = pcnet_rdra_addr(s, -1 + CSR_RCVRC(s));
+        hwaddr nnrd = pcnet_rdra_addr(s, -2 + CSR_RCVRC(s));
 #else
-        target_phys_addr_t crda = s->rdra +
+        hwaddr crda = s->rdra +
             (CSR_RCVRL(s) - CSR_RCVRC(s)) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
         int nrdc = CSR_RCVRC(s)<=1 ? CSR_RCVRL(s) : CSR_RCVRC(s)-1;
-        target_phys_addr_t nrda = s->rdra +
+        hwaddr nrda = s->rdra +
             (CSR_RCVRL(s) - nrdc) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
         int nnrc = nrdc<=1 ? CSR_RCVRL(s) : nrdc-1;
-        target_phys_addr_t nnrd = s->rdra +
+        hwaddr nnrd = s->rdra +
             (CSR_RCVRL(s) - nnrc) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
 #endif
@@ -970,7 +970,7 @@ static int pcnet_tdte_poll(PCNetState *s)
 {
     s->csr[34] = s->csr[35] = 0;
     if (s->tdra) {
-        target_phys_addr_t cxda = s->tdra +
+        hwaddr cxda = s->tdra +
             (CSR_XMTRL(s) - CSR_XMTRC(s)) *
             (BCR_SWSTYLE(s) ? 16 : 8);
         int bad = 0;
@@ -1050,7 +1050,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
         if (!(CSR_CRST(s) & 0x8000) && s->rdra) {
             struct pcnet_RMD rmd;
             int rcvrc = CSR_RCVRC(s)-1,i;
-            target_phys_addr_t nrda;
+            hwaddr nrda;
             for (i = CSR_RCVRL(s)-1; i > 0; i--, rcvrc--) {
                 if (rcvrc <= 1)
                     rcvrc = CSR_RCVRL(s);
@@ -1078,7 +1078,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             CSR_MISSC(s)++;
         } else {
             uint8_t *src = s->buffer;
-            target_phys_addr_t crda = CSR_CRDA(s);
+            hwaddr crda = CSR_CRDA(s);
             struct pcnet_RMD rmd;
             int pktcount = 0;
 
@@ -1118,7 +1118,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 
 #define PCNET_RECV_STORE() do {                                 \
     int count = MIN(4096 - GET_FIELD(rmd.buf_length, RMDL, BCNT),remaining); \
-    target_phys_addr_t rbadr = PHYSADDR(s, rmd.rbadr);          \
+    hwaddr rbadr = PHYSADDR(s, rmd.rbadr);          \
     s->phys_mem_write(s->dma_opaque, rbadr, src, count, CSR_BSWP(s)); \
     src += count; remaining -= count;                           \
     SET_FIELD(&rmd.status, RMDS, OWN, 0);                       \
@@ -1129,7 +1129,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             remaining = size;
             PCNET_RECV_STORE();
             if ((remaining > 0) && CSR_NRDA(s)) {
-                target_phys_addr_t nrda = CSR_NRDA(s);
+                hwaddr nrda = CSR_NRDA(s);
 #ifdef PCNET_DEBUG_RMD
                 PRINT_RMD(&rmd);
 #endif
@@ -1206,7 +1206,7 @@ void pcnet_set_link_status(NetClientState *nc)
 
 static void pcnet_transmit(PCNetState *s)
 {
-    target_phys_addr_t xmit_cxda = 0;
+    hwaddr xmit_cxda = 0;
     int count = CSR_XMTRL(s)-1;
     int add_crc = 0;
 
diff --git a/hw/pcnet.h b/hw/pcnet.h
index d0af54a46aa..9dee6f3e2c5 100644
--- a/hw/pcnet.h
+++ b/hw/pcnet.h
@@ -1,10 +1,13 @@
+#ifndef HW_PCNET_H
+#define HW_PCNET_H 1
+
 #define PCNET_IOPORT_SIZE       0x20
 #define PCNET_PNPMMIO_SIZE      0x20
 
 #define PCNET_LOOPTEST_CRC	1
 #define PCNET_LOOPTEST_NOCRC	2
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* BUS CONFIGURATION REGISTERS */
 #define BCR_MSRDA    0
@@ -42,9 +45,9 @@ struct PCNetState_st {
     MemoryRegion mmio;
     uint8_t buffer[4096];
     qemu_irq irq;
-    void (*phys_mem_read)(void *dma_opaque, target_phys_addr_t addr,
+    void (*phys_mem_read)(void *dma_opaque, hwaddr addr,
                          uint8_t *buf, int len, int do_bswap);
-    void (*phys_mem_write)(void *dma_opaque, target_phys_addr_t addr,
+    void (*phys_mem_write)(void *dma_opaque, hwaddr addr,
                           uint8_t *buf, int len, int do_bswap);
     void *dma_opaque;
     int tx_busy;
@@ -63,3 +66,5 @@ void pcnet_set_link_status(NetClientState *nc);
 void pcnet_common_cleanup(PCNetState *d);
 int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info);
 extern const VMStateDescription vmstate_pcnet;
+
+#endif
diff --git a/hw/pcspk.c b/hw/pcspk.c
index e4303247d4a..dfab9559ae9 100644
--- a/hw/pcspk.c
+++ b/hw/pcspk.c
@@ -26,7 +26,7 @@
 #include "pc.h"
 #include "isa.h"
 #include "audio/audio.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "pcspk.h"
 
@@ -121,7 +121,7 @@ int pcspk_audio_init(ISABus *bus)
     return 0;
 }
 
-static uint64_t pcspk_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pcspk_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PCSpkState *s = opaque;
@@ -135,7 +135,7 @@ static uint64_t pcspk_io_read(void *opaque, target_phys_addr_t addr,
        (ch.out << 5);
 }
 
-static void pcspk_io_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void pcspk_io_write(void *opaque, hwaddr addr, uint64_t val,
                            unsigned size)
 {
     PCSpkState *s = opaque;
@@ -187,7 +187,7 @@ static void pcspk_class_initfn(ObjectClass *klass, void *data)
     dc->props = pcspk_properties;
 }
 
-static TypeInfo pcspk_info = {
+static const TypeInfo pcspk_info = {
     .name           = "isa-pcspk",
     .parent         = TYPE_ISA_DEVICE,
     .instance_size  = sizeof(PCSpkState),
diff --git a/hw/petalogix_ml605_mmu.c b/hw/petalogix_ml605_mmu.c
index dced648f458..bdfc6ce3654 100644
--- a/hw/petalogix_ml605_mmu.c
+++ b/hw/petalogix_ml605_mmu.c
@@ -27,15 +27,16 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "xilinx.h"
-#include "blockdev.h"
-#include "pc.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "serial.h"
+#include "exec/address-spaces.h"
+#include "ssi.h"
 
 #include "microblaze_boot.h"
 #include "microblaze_pic_cpu.h"
@@ -47,6 +48,8 @@
 
 #define BINARY_DEVICE_TREE_FILE "petalogix-ml605.dtb"
 
+#define NUM_SPI_FLASHES 4
+
 #define MEMORY_BASEADDR 0x50000000
 #define FLASH_BASEADDR 0x86000000
 #define INTC_BASEADDR 0x81800000
@@ -70,19 +73,18 @@ static void machine_cpu_reset(MicroBlazeCPU *cpu)
 }
 
 static void
-petalogix_ml605_init(ram_addr_t ram_size,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+petalogix_ml605_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     MemoryRegion *address_space_mem = get_system_memory();
     DeviceState *dev, *dma, *eth0;
     MicroBlazeCPU *cpu;
+    SysBusDevice *busdev;
     CPUMBState *env;
     DriveInfo *dinfo;
     int i;
-    target_phys_addr_t ddr_base = MEMORY_BASEADDR;
+    hwaddr ddr_base = MEMORY_BASEADDR;
     MemoryRegion *phys_lmb_bram = g_new(MemoryRegion, 1);
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
@@ -139,6 +141,29 @@ petalogix_ml605_init(ram_addr_t ram_size,
     xilinx_axiethernetdma_init(dma, STREAM_SLAVE(eth0),
                                0x84600000, irq[1], irq[0], 100 * 1000000);
 
+    {
+        SSIBus *spi;
+
+        dev = qdev_create(NULL, "xlnx.xps-spi");
+        qdev_prop_set_uint8(dev, "num-ss-bits", NUM_SPI_FLASHES);
+        qdev_init_nofail(dev);
+        busdev = SYS_BUS_DEVICE(dev);
+        sysbus_mmio_map(busdev, 0, 0x40a00000);
+        sysbus_connect_irq(busdev, 0, irq[4]);
+
+        spi = (SSIBus *)qdev_get_child_bus(dev, "spi");
+
+        for (i = 0; i < NUM_SPI_FLASHES; i++) {
+            qemu_irq cs_line;
+
+            dev = ssi_create_slave_no_init(spi, "m25p80");
+            qdev_prop_set_string(dev, "partname", "n25q128");
+            qdev_init_nofail(dev);
+            cs_line = qdev_get_gpio_in(dev, 0);
+            sysbus_connect_irq(busdev, i+1, cs_line);
+        }
+    }
+
     microblaze_load_kernel(cpu, ddr_base, ram_size, BINARY_DEVICE_TREE_FILE,
                                                             machine_cpu_reset);
 
@@ -148,7 +173,8 @@ static QEMUMachine petalogix_ml605_machine = {
     .name = "petalogix-ml605",
     .desc = "PetaLogix linux refdesign for xilinx ml605 little endian",
     .init = petalogix_ml605_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void petalogix_ml605_machine_init(void)
diff --git a/hw/petalogix_s3adsp1800_mmu.c b/hw/petalogix_s3adsp1800_mmu.c
index 2cf68828ed9..8605fb8c003 100644
--- a/hw/petalogix_s3adsp1800_mmu.c
+++ b/hw/petalogix_s3adsp1800_mmu.c
@@ -25,14 +25,14 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "xilinx.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #include "microblaze_boot.h"
 #include "microblaze_pic_cpu.h"
@@ -57,18 +57,16 @@ static void machine_cpu_reset(MicroBlazeCPU *cpu)
 }
 
 static void
-petalogix_s3adsp1800_init(ram_addr_t ram_size,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+petalogix_s3adsp1800_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     DeviceState *dev;
     MicroBlazeCPU *cpu;
     CPUMBState *env;
     DriveInfo *dinfo;
     int i;
-    target_phys_addr_t ddr_base = MEMORY_BASEADDR;
+    hwaddr ddr_base = MEMORY_BASEADDR;
     MemoryRegion *phys_lmb_bram = g_new(MemoryRegion, 1);
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
@@ -117,7 +115,8 @@ static QEMUMachine petalogix_s3adsp1800_machine = {
     .name = "petalogix-s3adsp1800",
     .desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800",
     .init = petalogix_s3adsp1800_init,
-    .is_default = 1
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void petalogix_s3adsp1800_machine_init(void)
diff --git a/hw/pflash_cfi01.c b/hw/pflash_cfi01.c
index d1c742379ba..9e6ff523365 100644
--- a/hw/pflash_cfi01.c
+++ b/hw/pflash_cfi01.c
@@ -38,44 +38,51 @@
 
 #include "hw.h"
 #include "flash.h"
-#include "block.h"
-#include "qemu-timer.h"
-#include "exec-memory.h"
+#include "block/block.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
+#include "sysbus.h"
 
 #define PFLASH_BUG(fmt, ...) \
 do { \
-    printf("PFLASH: Possible BUG - " fmt, ## __VA_ARGS__); \
+    fprintf(stderr, "PFLASH: Possible BUG - " fmt, ## __VA_ARGS__); \
     exit(1); \
 } while(0)
 
 /* #define PFLASH_DEBUG */
 #ifdef PFLASH_DEBUG
-#define DPRINTF(fmt, ...)                          \
-do {                                               \
-    printf("PFLASH: " fmt , ## __VA_ARGS__);       \
+#define DPRINTF(fmt, ...)                                   \
+do {                                                        \
+    fprintf(stderr, "PFLASH: " fmt , ## __VA_ARGS__);       \
 } while (0)
 #else
 #define DPRINTF(fmt, ...) do { } while (0)
 #endif
 
 struct pflash_t {
+    SysBusDevice busdev;
     BlockDriverState *bs;
-    target_phys_addr_t base;
-    target_phys_addr_t sector_len;
-    target_phys_addr_t total_len;
-    int width;
+    uint32_t nb_blocs;
+    uint64_t sector_len;
+    uint8_t width;
+    uint8_t be;
     int wcycle; /* if 0, the flash is read normally */
     int bypass;
     int ro;
     uint8_t cmd;
     uint8_t status;
-    uint16_t ident[4];
+    uint16_t ident0;
+    uint16_t ident1;
+    uint16_t ident2;
+    uint16_t ident3;
     uint8_t cfi_len;
     uint8_t cfi_table[0x52];
-    target_phys_addr_t counter;
+    hwaddr counter;
     unsigned int writeblock_size;
     QEMUTimer *timer;
     MemoryRegion mem;
+    char *name;
     void *storage;
 };
 
@@ -95,10 +102,10 @@ static void pflash_timer (void *opaque)
     pfl->cmd = 0;
 }
 
-static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
+static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                              int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint32_t ret;
     uint8_t *p;
 
@@ -167,15 +174,16 @@ static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
     case 0x90:
         switch (boff) {
         case 0:
-            ret = pfl->ident[0] << 8 | pfl->ident[1];
+            ret = pfl->ident0 << 8 | pfl->ident1;
             DPRINTF("%s: Manufacturer Code %04x\n", __func__, ret);
             break;
         case 1:
-            ret = pfl->ident[2] << 8 | pfl->ident[3];
+            ret = pfl->ident2 << 8 | pfl->ident3;
             DPRINTF("%s: Device ID Code %04x\n", __func__, ret);
             break;
         default:
-            DPRINTF("%s: Read Device Information boff=%x\n", __func__, boff);
+            DPRINTF("%s: Read Device Information boff=%x\n", __func__,
+                    (unsigned)boff);
             ret = 0;
             break;
         }
@@ -210,7 +218,7 @@ static void pflash_update(pflash_t *pfl, int offset,
     }
 }
 
-static inline void pflash_data_write(pflash_t *pfl, target_phys_addr_t offset,
+static inline void pflash_data_write(pflash_t *pfl, hwaddr offset,
                                      uint32_t value, int width, int be)
 {
     uint8_t *p = pfl->storage;
@@ -248,7 +256,7 @@ static inline void pflash_data_write(pflash_t *pfl, target_phys_addr_t offset,
 
 }
 
-static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
+static void pflash_write(pflash_t *pfl, hwaddr offset,
                          uint32_t value, int width, int be)
 {
     uint8_t *p;
@@ -278,9 +286,8 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
             p = pfl->storage;
             offset &= ~(pfl->sector_len - 1);
 
-            DPRINTF("%s: block erase at " TARGET_FMT_plx " bytes "
-                    TARGET_FMT_plx "\n",
-                    __func__, offset, pfl->sector_len);
+            DPRINTF("%s: block erase at " TARGET_FMT_plx " bytes %x\n",
+                    __func__, offset, (unsigned)pfl->sector_len);
 
             if (!pfl->ro) {
                 memset(p + offset, 0xff, pfl->sector_len);
@@ -312,6 +319,9 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
             DPRINTF("%s: Write to buffer\n", __func__);
             pfl->status |= 0x80; /* Ready! */
             break;
+        case 0xf0: /* Probe for AMD flash */
+            DPRINTF("%s: Probe for AMD flash\n", __func__);
+            goto reset_flash;
         case 0xff: /* Read array mode */
             DPRINTF("%s: Read array mode\n", __func__);
             goto reset_flash;
@@ -320,7 +330,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         }
         pfl->wcycle++;
         pfl->cmd = cmd;
-        return;
+        break;
     case 1:
         switch (pfl->cmd) {
         case 0x10: /* Single Byte Program */
@@ -375,7 +385,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     case 2:
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
@@ -388,7 +398,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
             pfl->status |= 0x80;
 
             if (!pfl->counter) {
-                target_phys_addr_t mask = pfl->writeblock_size - 1;
+                hwaddr mask = pfl->writeblock_size - 1;
                 mask = ~mask;
 
                 DPRINTF("%s: block write finished\n", __func__);
@@ -406,7 +416,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     case 3: /* Confirm mode */
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
@@ -422,7 +432,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     default:
         /* Should never happen */
         DPRINTF("%s: invalid write state\n",  __func__);
@@ -431,9 +441,9 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
     return;
 
  error_flash:
-    printf("%s: Unimplemented flash cmd sequence "
-           "(offset " TARGET_FMT_plx ", wcycle 0x%x cmd 0x%x value 0x%x)\n",
-           __func__, offset, pfl->wcycle, pfl->cmd, value);
+    qemu_log_mask(LOG_UNIMP, "%s: Unimplemented flash cmd sequence "
+                  "(offset " TARGET_FMT_plx ", wcycle 0x%x cmd 0x%x value 0x%x)"
+                  "\n", __func__, offset, pfl->wcycle, pfl->cmd, value);
 
  reset_flash:
     memory_region_rom_device_set_readable(&pfl->mem, true);
@@ -441,61 +451,60 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
     pfl->bypass = 0;
     pfl->wcycle = 0;
     pfl->cmd = 0;
-    return;
 }
 
 
-static uint32_t pflash_readb_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_be(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 1);
 }
 
-static uint32_t pflash_readb_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_le(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 0);
 }
 
-static uint32_t pflash_readw_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 1);
 }
 
-static uint32_t pflash_readw_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 0);
 }
 
-static uint32_t pflash_readl_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 1);
 }
 
-static uint32_t pflash_readl_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 0);
 }
 
-static void pflash_writeb_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 1);
 }
 
-static void pflash_writeb_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 0);
 }
 
-static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -503,7 +512,7 @@ static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 1);
 }
 
-static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -511,7 +520,7 @@ static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 0);
 }
 
-static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -519,7 +528,7 @@ static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 4, 1);
 }
 
-static void pflash_writel_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -543,55 +552,13 @@ static const MemoryRegionOps pflash_cfi01_ops_le = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-/* Count trailing zeroes of a 32 bits quantity */
-static int ctz32 (uint32_t n)
-{
-    int ret;
-
-    ret = 0;
-    if (!(n & 0xFFFF)) {
-        ret += 16;
-        n = n >> 16;
-    }
-    if (!(n & 0xFF)) {
-        ret += 8;
-        n = n >> 8;
-    }
-    if (!(n & 0xF)) {
-        ret += 4;
-        n = n >> 4;
-    }
-    if (!(n & 0x3)) {
-        ret += 2;
-        n = n >> 2;
-    }
-    if (!(n & 0x1)) {
-        ret++;
-#if 0 /* This is not necessary as n is never 0 */
-        n = n >> 1;
-#endif
-    }
-#if 0 /* This is not necessary as n is never 0 */
-    if (!n)
-        ret++;
-#endif
-
-    return ret;
-}
-
-pflash_t *pflash_cfi01_register(target_phys_addr_t base,
-                                DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
-                                BlockDriverState *bs, uint32_t sector_len,
-                                int nb_blocs, int width,
-                                uint16_t id0, uint16_t id1,
-                                uint16_t id2, uint16_t id3, int be)
+static int pflash_cfi01_init(SysBusDevice *dev)
 {
-    pflash_t *pfl;
-    target_phys_addr_t total_len;
+    pflash_t *pfl = FROM_SYSBUS(typeof(*pfl), dev);
+    uint64_t total_len;
     int ret;
 
-    total_len = sector_len * nb_blocs;
+    total_len = pfl->sector_len * pfl->nb_blocs;
 
     /* XXX: to be fixed */
 #if 0
@@ -600,27 +567,22 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
         return NULL;
 #endif
 
-    pfl = g_malloc0(sizeof(pflash_t));
-
     memory_region_init_rom_device(
-        &pfl->mem, be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
-        name, size);
-    vmstate_register_ram(&pfl->mem, qdev);
+        &pfl->mem, pfl->be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
+        pfl->name, total_len);
+    vmstate_register_ram(&pfl->mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->mem);
-    memory_region_add_subregion(get_system_memory(), base, &pfl->mem);
+    sysbus_init_mmio(dev, &pfl->mem);
 
-    pfl->bs = bs;
     if (pfl->bs) {
         /* read the initial flash content */
         ret = bdrv_read(pfl->bs, 0, pfl->storage, total_len >> 9);
+
         if (ret < 0) {
-            memory_region_del_subregion(get_system_memory(), &pfl->mem);
-            vmstate_unregister_ram(&pfl->mem, qdev);
+            vmstate_unregister_ram(&pfl->mem, DEVICE(pfl));
             memory_region_destroy(&pfl->mem);
-            g_free(pfl);
-            return NULL;
+            return 1;
         }
-        bdrv_attach_dev_nofail(pfl->bs, pfl);
     }
 
     if (pfl->bs) {
@@ -630,17 +592,9 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     }
 
     pfl->timer = qemu_new_timer_ns(vm_clock, pflash_timer, pfl);
-    pfl->base = base;
-    pfl->sector_len = sector_len;
-    pfl->total_len = total_len;
-    pfl->width = width;
     pfl->wcycle = 0;
     pfl->cmd = 0;
     pfl->status = 0;
-    pfl->ident[0] = id0;
-    pfl->ident[1] = id1;
-    pfl->ident[2] = id2;
-    pfl->ident[3] = id3;
     /* Hardcoded CFI table */
     pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
@@ -689,7 +643,7 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x28] = 0x02;
     pfl->cfi_table[0x29] = 0x00;
     /* Max number of bytes in multi-bytes write */
-    if (width == 1) {
+    if (pfl->width == 1) {
         pfl->cfi_table[0x2A] = 0x08;
     } else {
         pfl->cfi_table[0x2A] = 0x0B;
@@ -700,10 +654,10 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     /* Number of erase block regions (uniform) */
     pfl->cfi_table[0x2C] = 0x01;
     /* Erase block region 1 */
-    pfl->cfi_table[0x2D] = nb_blocs - 1;
-    pfl->cfi_table[0x2E] = (nb_blocs - 1) >> 8;
-    pfl->cfi_table[0x2F] = sector_len >> 8;
-    pfl->cfi_table[0x30] = sector_len >> 16;
+    pfl->cfi_table[0x2D] = pfl->nb_blocs - 1;
+    pfl->cfi_table[0x2E] = (pfl->nb_blocs - 1) >> 8;
+    pfl->cfi_table[0x2F] = pfl->sector_len >> 8;
+    pfl->cfi_table[0x30] = pfl->sector_len >> 16;
 
     /* Extended */
     pfl->cfi_table[0x31] = 'P';
@@ -711,7 +665,7 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x33] = 'I';
 
     pfl->cfi_table[0x34] = '1';
-    pfl->cfi_table[0x35] = '1';
+    pfl->cfi_table[0x35] = '0';
 
     pfl->cfi_table[0x36] = 0x00;
     pfl->cfi_table[0x37] = 0x00;
@@ -723,6 +677,77 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x3b] = 0x00;
     pfl->cfi_table[0x3c] = 0x00;
 
+    pfl->cfi_table[0x3f] = 0x01; /* Number of protection fields */
+
+    return 0;
+}
+
+static Property pflash_cfi01_properties[] = {
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
+    DEFINE_PROP_UINT64("sector-length", struct pflash_t, sector_len, 0),
+    DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
+    DEFINE_PROP_UINT8("big-endian", struct pflash_t, be, 0),
+    DEFINE_PROP_UINT16("id0", struct pflash_t, ident0, 0),
+    DEFINE_PROP_UINT16("id1", struct pflash_t, ident1, 0),
+    DEFINE_PROP_UINT16("id2", struct pflash_t, ident2, 0),
+    DEFINE_PROP_UINT16("id3", struct pflash_t, ident3, 0),
+    DEFINE_PROP_STRING("name", struct pflash_t, name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pflash_cfi01_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = pflash_cfi01_init;
+    dc->props = pflash_cfi01_properties;
+}
+
+
+static const TypeInfo pflash_cfi01_info = {
+    .name           = "cfi.pflash01",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(struct pflash_t),
+    .class_init     = pflash_cfi01_class_init,
+};
+
+static void pflash_cfi01_register_types(void)
+{
+    type_register_static(&pflash_cfi01_info);
+}
+
+type_init(pflash_cfi01_register_types)
+
+pflash_t *pflash_cfi01_register(hwaddr base,
+                                DeviceState *qdev, const char *name,
+                                hwaddr size,
+                                BlockDriverState *bs,
+                                uint32_t sector_len, int nb_blocs, int width,
+                                uint16_t id0, uint16_t id1,
+                                uint16_t id2, uint16_t id3, int be)
+{
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+    SysBusDevice *busdev = SYS_BUS_DEVICE(dev);
+    pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
+                                                    "cfi.pflash01");
+
+    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+        abort();
+    }
+    qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
+    qdev_prop_set_uint64(dev, "sector-length", sector_len);
+    qdev_prop_set_uint8(dev, "width", width);
+    qdev_prop_set_uint8(dev, "big-endian", !!be);
+    qdev_prop_set_uint16(dev, "id0", id0);
+    qdev_prop_set_uint16(dev, "id1", id1);
+    qdev_prop_set_uint16(dev, "id2", id2);
+    qdev_prop_set_uint16(dev, "id3", id3);
+    qdev_prop_set_string(dev, "name", name);
+    qdev_init_nofail(dev);
+
+    sysbus_mmio_map(busdev, 0, base);
     return pfl;
 }
 
diff --git a/hw/pflash_cfi02.c b/hw/pflash_cfi02.c
index 3e2002e4b36..44bd4654f0f 100644
--- a/hw/pflash_cfi02.c
+++ b/hw/pflash_cfi02.c
@@ -37,15 +37,17 @@
 
 #include "hw.h"
 #include "flash.h"
-#include "qemu-timer.h"
-#include "block.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "block/block.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
+#include "sysbus.h"
 
 //#define PFLASH_DEBUG
 #ifdef PFLASH_DEBUG
-#define DPRINTF(fmt, ...)                          \
-do {                                               \
-    printf("PFLASH: " fmt , ## __VA_ARGS__);       \
+#define DPRINTF(fmt, ...)                                  \
+do {                                                       \
+    fprintf(stderr "PFLASH: " fmt , ## __VA_ARGS__);       \
 } while (0)
 #else
 #define DPRINTF(fmt, ...) do { } while (0)
@@ -54,19 +56,26 @@ do {                                               \
 #define PFLASH_LAZY_ROMD_THRESHOLD 42
 
 struct pflash_t {
+    SysBusDevice busdev;
     BlockDriverState *bs;
-    target_phys_addr_t base;
     uint32_t sector_len;
+    uint32_t nb_blocs;
     uint32_t chip_len;
-    int mappings;
-    int width;
+    uint8_t mappings;
+    uint8_t width;
+    uint8_t be;
     int wcycle; /* if 0, the flash is read normally */
     int bypass;
     int ro;
     uint8_t cmd;
     uint8_t status;
-    uint16_t ident[4];
-    uint16_t unlock_addr[2];
+    /* FIXME: implement array device properties */
+    uint16_t ident0;
+    uint16_t ident1;
+    uint16_t ident2;
+    uint16_t ident3;
+    uint16_t unlock_addr0;
+    uint16_t unlock_addr1;
     uint8_t cfi_len;
     uint8_t cfi_table[0x52];
     QEMUTimer *timer;
@@ -79,6 +88,7 @@ struct pflash_t {
     MemoryRegion orig_mem;
     int rom_mode;
     int read_counter; /* used for lazy switch-back to rom mode */
+    char *name;
     void *storage;
 };
 
@@ -88,7 +98,7 @@ struct pflash_t {
 static void pflash_setup_mappings(pflash_t *pfl)
 {
     unsigned i;
-    target_phys_addr_t size = memory_region_size(&pfl->orig_mem);
+    hwaddr size = memory_region_size(&pfl->orig_mem);
 
     memory_region_init(&pfl->mem, "pflash", pfl->mappings * size);
     pfl->mem_mappings = g_new(MemoryRegion, pfl->mappings);
@@ -121,10 +131,10 @@ static void pflash_timer (void *opaque)
     pfl->cmd = 0;
 }
 
-static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
+static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                              int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint32_t ret;
     uint8_t *p;
 
@@ -147,6 +157,7 @@ static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
         DPRINTF("%s: unknown command state: %x\n", __func__, pfl->cmd);
         pfl->wcycle = 0;
         pfl->cmd = 0;
+        /* fall through to the read code */
     case 0x80:
         /* We accept reads during second unlock sequence... */
     case 0x00:
@@ -189,16 +200,17 @@ static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
         switch (boff) {
         case 0x00:
         case 0x01:
-            ret = pfl->ident[boff & 0x01];
+            ret = boff & 0x01 ? pfl->ident1 : pfl->ident0;
             break;
         case 0x02:
             ret = 0x00; /* Pretend all sectors are unprotected */
             break;
         case 0x0E:
         case 0x0F:
-            if (pfl->ident[2 + (boff & 0x01)] == (uint8_t)-1)
+            ret = boff & 0x01 ? pfl->ident3 : pfl->ident2;
+            if (ret == (uint8_t)-1) {
                 goto flash_read;
-            ret = pfl->ident[2 + (boff & 0x01)];
+            }
             break;
         default:
             goto flash_read;
@@ -241,10 +253,10 @@ static void pflash_update(pflash_t *pfl, int offset,
     }
 }
 
-static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
+static void pflash_write (pflash_t *pfl, hwaddr offset,
                           uint32_t value, int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint8_t *p;
     uint8_t cmd;
 
@@ -282,9 +294,9 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
             pfl->cmd = 0x98;
             return;
         }
-        if (boff != pfl->unlock_addr[0] || cmd != 0xAA) {
+        if (boff != pfl->unlock_addr0 || cmd != 0xAA) {
             DPRINTF("%s: unlock0 failed " TARGET_FMT_plx " %02x %04x\n",
-                    __func__, boff, cmd, pfl->unlock_addr[0]);
+                    __func__, boff, cmd, pfl->unlock_addr0);
             goto reset_flash;
         }
         DPRINTF("%s: unlock sequence started\n", __func__);
@@ -292,7 +304,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
     case 1:
         /* We started an unlock sequence */
     check_unlock1:
-        if (boff != pfl->unlock_addr[1] || cmd != 0x55) {
+        if (boff != pfl->unlock_addr1 || cmd != 0x55) {
             DPRINTF("%s: unlock1 failed " TARGET_FMT_plx " %02x\n", __func__,
                     boff, cmd);
             goto reset_flash;
@@ -301,7 +313,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
         break;
     case 2:
         /* We finished an unlock sequence */
-        if (!pfl->bypass && boff != pfl->unlock_addr[0]) {
+        if (!pfl->bypass && boff != pfl->unlock_addr0) {
             DPRINTF("%s: command failed " TARGET_FMT_plx " %02x\n", __func__,
                     boff, cmd);
             goto reset_flash;
@@ -399,7 +411,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
     case 5:
         switch (cmd) {
         case 0x10:
-            if (boff != pfl->unlock_addr[0]) {
+            if (boff != pfl->unlock_addr0) {
                 DPRINTF("%s: chip erase: invalid address " TARGET_FMT_plx "\n",
                         __func__, offset);
                 goto reset_flash;
@@ -473,61 +485,60 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
  do_bypass:
     pfl->wcycle = 2;
     pfl->cmd = 0;
-    return;
 }
 
 
-static uint32_t pflash_readb_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_be(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 1);
 }
 
-static uint32_t pflash_readb_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_le(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 0);
 }
 
-static uint32_t pflash_readw_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 1);
 }
 
-static uint32_t pflash_readw_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 0);
 }
 
-static uint32_t pflash_readl_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 1);
 }
 
-static uint32_t pflash_readl_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 0);
 }
 
-static void pflash_writeb_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 1);
 }
 
-static void pflash_writeb_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 0);
 }
 
-static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -535,7 +546,7 @@ static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 1);
 }
 
-static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -543,7 +554,7 @@ static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 0);
 }
 
-static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -551,7 +562,7 @@ static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 4, 1);
 }
 
-static void pflash_writel_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -575,86 +586,38 @@ static const MemoryRegionOps pflash_cfi02_ops_le = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-/* Count trailing zeroes of a 32 bits quantity */
-static int ctz32 (uint32_t n)
+static int pflash_cfi02_init(SysBusDevice *dev)
 {
+    pflash_t *pfl = FROM_SYSBUS(typeof(*pfl), dev);
+    uint32_t chip_len;
     int ret;
 
-    ret = 0;
-    if (!(n & 0xFFFF)) {
-        ret += 16;
-        n = n >> 16;
-    }
-    if (!(n & 0xFF)) {
-        ret += 8;
-        n = n >> 8;
-    }
-    if (!(n & 0xF)) {
-        ret += 4;
-        n = n >> 4;
-    }
-    if (!(n & 0x3)) {
-        ret += 2;
-        n = n >> 2;
-    }
-    if (!(n & 0x1)) {
-        ret++;
-#if 0 /* This is not necessary as n is never 0 */
-        n = n >> 1;
-#endif
-    }
-#if 0 /* This is not necessary as n is never 0 */
-    if (!n)
-        ret++;
-#endif
-
-    return ret;
-}
-
-pflash_t *pflash_cfi02_register(target_phys_addr_t base,
-                                DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
-                                BlockDriverState *bs, uint32_t sector_len,
-                                int nb_blocs, int nb_mappings, int width,
-                                uint16_t id0, uint16_t id1,
-                                uint16_t id2, uint16_t id3,
-                                uint16_t unlock_addr0, uint16_t unlock_addr1,
-                                int be)
-{
-    pflash_t *pfl;
-    int32_t chip_len;
-    int ret;
-
-    chip_len = sector_len * nb_blocs;
+    chip_len = pfl->sector_len * pfl->nb_blocs;
     /* XXX: to be fixed */
 #if 0
     if (total_len != (8 * 1024 * 1024) && total_len != (16 * 1024 * 1024) &&
         total_len != (32 * 1024 * 1024) && total_len != (64 * 1024 * 1024))
         return NULL;
 #endif
-    pfl = g_malloc0(sizeof(pflash_t));
-    memory_region_init_rom_device(
-        &pfl->orig_mem, be ? &pflash_cfi02_ops_be : &pflash_cfi02_ops_le, pfl,
-        name, size);
-    vmstate_register_ram(&pfl->orig_mem, qdev);
+
+    memory_region_init_rom_device(&pfl->orig_mem, pfl->be ?
+                                  &pflash_cfi02_ops_be : &pflash_cfi02_ops_le,
+                                  pfl, pfl->name, chip_len);
+    vmstate_register_ram(&pfl->orig_mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->orig_mem);
-    pfl->base = base;
     pfl->chip_len = chip_len;
-    pfl->mappings = nb_mappings;
-    pfl->bs = bs;
     if (pfl->bs) {
         /* read the initial flash content */
         ret = bdrv_read(pfl->bs, 0, pfl->storage, chip_len >> 9);
         if (ret < 0) {
             g_free(pfl);
-            return NULL;
+            return 1;
         }
-        bdrv_attach_dev_nofail(pfl->bs, pfl);
     }
 
     pflash_setup_mappings(pfl);
     pfl->rom_mode = 1;
-    memory_region_add_subregion(get_system_memory(), pfl->base, &pfl->mem);
+    sysbus_init_mmio(dev, &pfl->mem);
 
     if (pfl->bs) {
         pfl->ro = bdrv_is_read_only(pfl->bs);
@@ -663,17 +626,9 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     }
 
     pfl->timer = qemu_new_timer_ns(vm_clock, pflash_timer, pfl);
-    pfl->sector_len = sector_len;
-    pfl->width = width;
     pfl->wcycle = 0;
     pfl->cmd = 0;
     pfl->status = 0;
-    pfl->ident[0] = id0;
-    pfl->ident[1] = id1;
-    pfl->ident[2] = id2;
-    pfl->ident[3] = id3;
-    pfl->unlock_addr[0] = unlock_addr0;
-    pfl->unlock_addr[1] = unlock_addr1;
     /* Hardcoded CFI table (mostly from SG29 Spansion flash) */
     pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
@@ -729,10 +684,10 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     /* Number of erase block regions (uniform) */
     pfl->cfi_table[0x2C] = 0x01;
     /* Erase block region 1 */
-    pfl->cfi_table[0x2D] = nb_blocs - 1;
-    pfl->cfi_table[0x2E] = (nb_blocs - 1) >> 8;
-    pfl->cfi_table[0x2F] = sector_len >> 8;
-    pfl->cfi_table[0x30] = sector_len >> 16;
+    pfl->cfi_table[0x2D] = pfl->nb_blocs - 1;
+    pfl->cfi_table[0x2E] = (pfl->nb_blocs - 1) >> 8;
+    pfl->cfi_table[0x2F] = pfl->sector_len >> 8;
+    pfl->cfi_table[0x30] = pfl->sector_len >> 16;
 
     /* Extended */
     pfl->cfi_table[0x31] = 'P';
@@ -752,5 +707,81 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     pfl->cfi_table[0x3b] = 0x00;
     pfl->cfi_table[0x3c] = 0x00;
 
+    return 0;
+}
+
+static Property pflash_cfi02_properties[] = {
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
+    DEFINE_PROP_UINT32("sector-length", struct pflash_t, sector_len, 0),
+    DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
+    DEFINE_PROP_UINT8("mappings", struct pflash_t, mappings, 0),
+    DEFINE_PROP_UINT8("big-endian", struct pflash_t, be, 0),
+    DEFINE_PROP_UINT16("id0", struct pflash_t, ident0, 0),
+    DEFINE_PROP_UINT16("id1", struct pflash_t, ident1, 0),
+    DEFINE_PROP_UINT16("id2", struct pflash_t, ident2, 0),
+    DEFINE_PROP_UINT16("id3", struct pflash_t, ident3, 0),
+    DEFINE_PROP_UINT16("unlock-addr0", struct pflash_t, unlock_addr0, 0),
+    DEFINE_PROP_UINT16("unlock-addr1", struct pflash_t, unlock_addr1, 0),
+    DEFINE_PROP_STRING("name", struct pflash_t, name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pflash_cfi02_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = pflash_cfi02_init;
+    dc->props = pflash_cfi02_properties;
+}
+
+static const TypeInfo pflash_cfi02_info = {
+    .name           = "cfi.pflash02",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(struct pflash_t),
+    .class_init     = pflash_cfi02_class_init,
+};
+
+static void pflash_cfi02_register_types(void)
+{
+    type_register_static(&pflash_cfi02_info);
+}
+
+type_init(pflash_cfi02_register_types)
+
+pflash_t *pflash_cfi02_register(hwaddr base,
+                                DeviceState *qdev, const char *name,
+                                hwaddr size,
+                                BlockDriverState *bs, uint32_t sector_len,
+                                int nb_blocs, int nb_mappings, int width,
+                                uint16_t id0, uint16_t id1,
+                                uint16_t id2, uint16_t id3,
+                                uint16_t unlock_addr0, uint16_t unlock_addr1,
+                                int be)
+{
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash02");
+    SysBusDevice *busdev = SYS_BUS_DEVICE(dev);
+    pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
+                                                    "cfi.pflash02");
+
+    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+        abort();
+    }
+    qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
+    qdev_prop_set_uint32(dev, "sector-length", sector_len);
+    qdev_prop_set_uint8(dev, "width", width);
+    qdev_prop_set_uint8(dev, "mappings", nb_mappings);
+    qdev_prop_set_uint8(dev, "big-endian", !!be);
+    qdev_prop_set_uint16(dev, "id0", id0);
+    qdev_prop_set_uint16(dev, "id1", id1);
+    qdev_prop_set_uint16(dev, "id2", id2);
+    qdev_prop_set_uint16(dev, "id3", id3);
+    qdev_prop_set_uint16(dev, "unlock-addr0", unlock_addr0);
+    qdev_prop_set_uint16(dev, "unlock-addr1", unlock_addr1);
+    qdev_prop_set_string(dev, "name", name);
+    qdev_init_nofail(dev);
+
+    sysbus_mmio_map(busdev, 0, base);
     return pfl;
 }
diff --git a/hw/piix4.c b/hw/piix4.c
index ce4eb0d1ae3..c1cb94d39f5 100644
--- a/hw/piix4.c
+++ b/hw/piix4.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "isa.h"
 #include "sysbus.h"
 
@@ -117,7 +117,7 @@ static void piix4_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_piix4;
 }
 
-static TypeInfo piix4_info = {
+static const TypeInfo piix4_info = {
     .name          = "PIIX4",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PIIX4State),
diff --git a/hw/piix_pci.c b/hw/piix_pci.c
index 537fc1973cc..3d79c73fdaa 100644
--- a/hw/piix_pci.c
+++ b/hw/piix_pci.c
@@ -24,12 +24,13 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "isa.h"
 #include "sysbus.h"
-#include "range.h"
+#include "qemu/range.h"
 #include "xen.h"
+#include "pam.h"
 
 /*
  * I440FX chipset data sheet.
@@ -68,11 +69,6 @@ typedef struct PIIX3State {
     int32_t pci_irq_levels_vmstate[PIIX_NUM_PIRQS];
 } PIIX3State;
 
-typedef struct PAMMemoryRegion {
-    MemoryRegion mem;
-    bool initialized;
-} PAMMemoryRegion;
-
 struct PCII440FXState {
     PCIDevice dev;
     MemoryRegion *system_memory;
@@ -105,56 +101,16 @@ static int pci_slot_get_pirq(PCIDevice *pci_dev, int pci_intx)
     return (pci_intx + slot_addend) & 3;
 }
 
-static void update_pam(PCII440FXState *d, uint32_t start, uint32_t end, int r,
-                       PAMMemoryRegion *mem)
-{
-    if (mem->initialized) {
-        memory_region_del_subregion(d->system_memory, &mem->mem);
-        memory_region_destroy(&mem->mem);
-    }
-
-    //    printf("ISA mapping %08x-0x%08x: %d\n", start, end, r);
-    switch(r) {
-    case 3:
-        /* RAM */
-        memory_region_init_alias(&mem->mem, "pam-ram", d->ram_memory,
-                                 start, end - start);
-        break;
-    case 1:
-        /* ROM (XXX: not quite correct) */
-        memory_region_init_alias(&mem->mem, "pam-rom", d->ram_memory,
-                                 start, end - start);
-        memory_region_set_readonly(&mem->mem, true);
-        break;
-    case 2:
-    case 0:
-        /* XXX: should distinguish read/write cases */
-        memory_region_init_alias(&mem->mem, "pam-pci", d->pci_address_space,
-                                 start, end - start);
-        break;
-    }
-    memory_region_add_subregion_overlap(d->system_memory,
-                                        start, &mem->mem, 1);
-    mem->initialized = true;
-}
-
 static void i440fx_update_memory_mappings(PCII440FXState *d)
 {
-    int i, r;
-    uint32_t smram;
-    bool smram_enabled;
+    int i;
 
     memory_region_transaction_begin();
-    update_pam(d, 0xf0000, 0x100000, (d->dev.config[I440FX_PAM] >> 4) & 3,
-               &d->pam_regions[0]);
-    for(i = 0; i < 12; i++) {
-        r = (d->dev.config[(i >> 1) + (I440FX_PAM + 1)] >> ((i & 1) * 4)) & 3;
-        update_pam(d, 0xc0000 + 0x4000 * i, 0xc0000 + 0x4000 * (i + 1), r,
-                   &d->pam_regions[i+1]);
+    for (i = 0; i < 13; i++) {
+        pam_update(&d->pam_regions[i], i,
+                   d->dev.config[I440FX_PAM + ((i + 1) / 2)]);
     }
-    smram = d->dev.config[I440FX_SMRAM];
-    smram_enabled = (d->smm_enabled && (smram & 0x08)) || (smram & 0x40);
-    memory_region_set_enabled(&d->smram_region, !smram_enabled);
+    smram_update(&d->smram_region, d->dev.config[I440FX_SMRAM], d->smm_enabled);
     memory_region_transaction_commit();
 }
 
@@ -162,11 +118,10 @@ static void i440fx_set_smm(int val, void *arg)
 {
     PCII440FXState *d = arg;
 
-    val = (val != 0);
-    if (d->smm_enabled != val) {
-        d->smm_enabled = val;
-        i440fx_update_memory_mappings(d);
-    }
+    memory_region_transaction_begin();
+    smram_set_smm(&d->smm_enabled, val, d->dev.config[I440FX_SMRAM],
+                  &d->smram_region);
+    memory_region_transaction_commit();
 }
 
 
@@ -259,10 +214,10 @@ static PCIBus *i440fx_common_init(const char *device_name,
                                   MemoryRegion *address_space_mem,
                                   MemoryRegion *address_space_io,
                                   ram_addr_t ram_size,
-                                  target_phys_addr_t pci_hole_start,
-                                  target_phys_addr_t pci_hole_size,
-                                  target_phys_addr_t pci_hole64_start,
-                                  target_phys_addr_t pci_hole64_size,
+                                  hwaddr pci_hole_start,
+                                  hwaddr pci_hole_size,
+                                  hwaddr pci_hole64_start,
+                                  hwaddr pci_hole64_size,
                                   MemoryRegion *pci_address_space,
                                   MemoryRegion *ram_memory)
 {
@@ -272,6 +227,7 @@ static PCIBus *i440fx_common_init(const char *device_name,
     PCIHostState *s;
     PIIX3State *piix3;
     PCII440FXState *f;
+    unsigned i;
 
     dev = qdev_create(NULL, "i440FX-pcihost");
     s = PCI_HOST_BRIDGE(dev);
@@ -303,6 +259,13 @@ static PCIBus *i440fx_common_init(const char *device_name,
     memory_region_add_subregion_overlap(f->system_memory, 0xa0000,
                                         &f->smram_region, 1);
     memory_region_set_enabled(&f->smram_region, false);
+    init_pam(f->ram_memory, f->system_memory, f->pci_address_space,
+             &f->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
+    for (i = 0; i < 12; ++i) {
+        init_pam(f->ram_memory, f->system_memory, f->pci_address_space,
+                 &f->pam_regions[i+1], PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE,
+                 PAM_EXPAN_SIZE);
+    }
 
     /* Xen supports additional interrupt routes from the PCI devices to
      * the IOAPIC: the four pins of each PCI device on the bus are also
@@ -341,10 +304,10 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix3_devfn,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    target_phys_addr_t pci_hole_start,
-                    target_phys_addr_t pci_hole_size,
-                    target_phys_addr_t pci_hole64_start,
-                    target_phys_addr_t pci_hole64_size,
+                    hwaddr pci_hole_start,
+                    hwaddr pci_hole_size,
+                    hwaddr pci_hole64_start,
+                    hwaddr pci_hole64_size,
                     MemoryRegion *pci_memory, MemoryRegion *ram_memory)
 
 {
diff --git a/hw/pl011.c b/hw/pl011.c
index 3245702df0b..002a50e16a8 100644
--- a/hw/pl011.c
+++ b/hw/pl011.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -54,7 +54,7 @@ static void pl011_update(pl011_state *s)
     qemu_set_irq(s->irq, flags != 0);
 }
 
-static uint64_t pl011_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl011_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl011_state *s = (pl011_state *)opaque;
@@ -107,7 +107,8 @@ static uint64_t pl011_read(void *opaque, target_phys_addr_t offset,
     case 18: /* UARTDMACR */
         return s->dmacr;
     default:
-        hw_error("pl011_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl011_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
@@ -126,7 +127,7 @@ static void pl011_set_read_trigger(pl011_state *s)
         s->read_trigger = 1;
 }
 
-static void pl011_write(void *opaque, target_phys_addr_t offset,
+static void pl011_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl011_state *s = (pl011_state *)opaque;
@@ -178,11 +179,13 @@ static void pl011_write(void *opaque, target_phys_addr_t offset,
         break;
     case 18: /* UARTDMACR */
         s->dmacr = value;
-        if (value & 3)
-            hw_error("PL011: DMA not implemented\n");
+        if (value & 3) {
+            qemu_log_mask(LOG_UNIMP, "pl011: DMA not implemented\n");
+        }
         break;
     default:
-        hw_error("pl011_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl011_write: Bad offset %x\n", (int)offset);
     }
 }
 
@@ -297,7 +300,7 @@ static void pl011_arm_class_init(ObjectClass *klass, void *data)
     sdc->init = pl011_arm_init;
 }
 
-static TypeInfo pl011_arm_info = {
+static const TypeInfo pl011_arm_info = {
     .name          = "pl011",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl011_state),
@@ -311,7 +314,7 @@ static void pl011_luminary_class_init(ObjectClass *klass, void *data)
     sdc->init = pl011_luminary_init;
 }
 
-static TypeInfo pl011_luminary_info = {
+static const TypeInfo pl011_luminary_info = {
     .name          = "pl011_luminary",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl011_state),
diff --git a/hw/pl022.c b/hw/pl022.c
index 60e35daeb58..c160e9061c5 100644
--- a/hw/pl022.c
+++ b/hw/pl022.c
@@ -130,7 +130,7 @@ static void pl022_xfer(pl022_state *s)
     pl022_update(s);
 }
 
-static uint64_t pl022_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl022_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl022_state *s = (pl022_state *)opaque;
@@ -168,12 +168,13 @@ static uint64_t pl022_read(void *opaque, target_phys_addr_t offset,
         /* Not implemented.  */
         return 0;
     default:
-        hw_error("pl022_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl022_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl022_write(void *opaque, target_phys_addr_t offset,
+static void pl022_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl022_state *s = (pl022_state *)opaque;
@@ -211,11 +212,12 @@ static void pl022_write(void *opaque, target_phys_addr_t offset,
         break;
     case 0x20: /* DMACR */
         if (value) {
-            hw_error("pl022: DMA not implemented\n");
+            qemu_log_mask(LOG_UNIMP, "pl022: DMA not implemented\n");
         }
         break;
     default:
-        hw_error("pl022_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl022_write: Bad offset %x\n", (int)offset);
     }
 }
 
@@ -291,7 +293,7 @@ static void pl022_class_init(ObjectClass *klass, void *data)
     sdc->init = pl022_init;
 }
 
-static TypeInfo pl022_info = {
+static const TypeInfo pl022_info = {
     .name          = "pl022",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl022_state),
diff --git a/hw/pl031.c b/hw/pl031.c
index 9602664da6a..757867ff799 100644
--- a/hw/pl031.c
+++ b/hw/pl031.c
@@ -12,8 +12,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG_PL031
 
@@ -95,7 +95,7 @@ static void pl031_set_alarm(pl031_state *s)
     }
 }
 
-static uint64_t pl031_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl031_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl031_state *s = (pl031_state *)opaque;
@@ -120,18 +120,20 @@ static uint64_t pl031_read(void *opaque, target_phys_addr_t offset,
     case RTC_MIS:
         return s->is & s->im;
     case RTC_ICR:
-        fprintf(stderr, "qemu: pl031_read: Unexpected offset 0x%x\n",
-                (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031: read of write-only register at offset 0x%x\n",
+                      (int)offset);
         break;
     default:
-        hw_error("pl031_read: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031_read: Bad offset 0x%x\n", (int)offset);
         break;
     }
 
     return 0;
 }
 
-static void pl031_write(void * opaque, target_phys_addr_t offset,
+static void pl031_write(void * opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl031_state *s = (pl031_state *)opaque;
@@ -167,12 +169,14 @@ static void pl031_write(void * opaque, target_phys_addr_t offset,
     case RTC_DR:
     case RTC_MIS:
     case RTC_RIS:
-        fprintf(stderr, "qemu: pl031_write: Unexpected offset 0x%x\n",
-                (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031: write to read-only register at offset 0x%x\n",
+                      (int)offset);
         break;
 
     default:
-        hw_error("pl031_write: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031_write: Bad offset 0x%x\n", (int)offset);
         break;
     }
 }
@@ -246,7 +250,7 @@ static void pl031_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl031;
 }
 
-static TypeInfo pl031_info = {
+static const TypeInfo pl031_info = {
     .name          = "pl031",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl031_state),
diff --git a/hw/pl041.c b/hw/pl041.c
index b6723be0a95..0b71c457483 100644
--- a/hw/pl041.c
+++ b/hw/pl041.c
@@ -97,7 +97,7 @@ static const char *pl041_regs_name[] = {
 
 
 #if defined(PL041_DEBUG_LEVEL)
-static const char *get_reg_name(target_phys_addr_t offset)
+static const char *get_reg_name(hwaddr offset)
 {
     if (offset <= PL041_dr1_7) {
         return pl041_regs_name[offset >> 2];
@@ -327,7 +327,7 @@ static void pl041_request_data(void *opaque)
     pl041_isr1_update(s);
 }
 
-static uint64_t pl041_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl041_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     pl041_state *s = (pl041_state *)opaque;
@@ -361,7 +361,7 @@ static uint64_t pl041_read(void *opaque, target_phys_addr_t offset,
     return value;
 }
 
-static void pl041_write(void *opaque, target_phys_addr_t offset,
+static void pl041_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     pl041_state *s = (pl041_state *)opaque;
@@ -536,8 +536,9 @@ static int pl041_init(SysBusDevice *dev)
     default:
         /* NC FIFO depth of 16 is not allowed because its id bits in
            AACIPERIPHID3 overlap with the id for the default NC FIFO depth */
-        fprintf(stderr, "pl041: unsupported non-compact fifo depth [%i]\n",
-                s->fifo_depth);
+        qemu_log_mask(LOG_UNIMP,
+                      "pl041: unsupported non-compact fifo depth [%i]\n",
+                      s->fifo_depth);
         return -1;
     }
 
@@ -631,7 +632,7 @@ static void pl041_device_class_init(ObjectClass *klass, void *data)
     dc->props = pl041_device_properties;
 }
 
-static TypeInfo pl041_device_info = {
+static const TypeInfo pl041_device_info = {
     .name          = "pl041",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl041_state),
diff --git a/hw/pl050.c b/hw/pl050.c
index b13924a160a..5d06bc9a3fa 100644
--- a/hw/pl050.c
+++ b/hw/pl050.c
@@ -58,7 +58,7 @@ static void pl050_update(void *opaque, int level)
     qemu_set_irq(s->irq, raise);
 }
 
-static uint64_t pl050_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl050_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl050_state *s = (pl050_state *)opaque;
@@ -95,12 +95,13 @@ static uint64_t pl050_read(void *opaque, target_phys_addr_t offset,
     case 4: /* KMIIR */
         return s->pending | 2;
     default:
-        hw_error("pl050_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl050_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl050_write(void *opaque, target_phys_addr_t offset,
+static void pl050_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl050_state *s = (pl050_state *)opaque;
@@ -123,7 +124,8 @@ static void pl050_write(void *opaque, target_phys_addr_t offset,
         s->clk = value;
         return;
     default:
-        hw_error("pl050_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl050_write: Bad offset %x\n", (int)offset);
     }
 }
 static const MemoryRegionOps pl050_ops = {
@@ -166,7 +168,7 @@ static void pl050_kbd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl050;
 }
 
-static TypeInfo pl050_kbd_info = {
+static const TypeInfo pl050_kbd_info = {
     .name          = "pl050_keyboard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl050_state),
@@ -182,7 +184,7 @@ static void pl050_mouse_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl050;
 }
 
-static TypeInfo pl050_mouse_info = {
+static const TypeInfo pl050_mouse_info = {
     .name          = "pl050_mouse",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl050_state),
diff --git a/hw/pl061.c b/hw/pl061.c
index 2aac7e8e9ee..a78e819d967 100644
--- a/hw/pl061.c
+++ b/hw/pl061.c
@@ -113,7 +113,7 @@ static void pl061_update(pl061_state *s)
     /* FIXME: Implement input interrupts.  */
 }
 
-static uint64_t pl061_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl061_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl061_state *s = (pl061_state *)opaque;
@@ -164,12 +164,13 @@ static uint64_t pl061_read(void *opaque, target_phys_addr_t offset,
     case 0x528: /* Analog mode select */
         return s->amsel;
     default:
-        hw_error("pl061_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl061_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl061_write(void *opaque, target_phys_addr_t offset,
+static void pl061_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl061_state *s = (pl061_state *)opaque;
@@ -239,7 +240,8 @@ static void pl061_write(void *opaque, target_phys_addr_t offset,
         s->amsel = value & 0xff;
         break;
     default:
-        hw_error("pl061_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl061_write: Bad offset %x\n", (int)offset);
     }
     pl061_update(s);
 }
@@ -302,7 +304,7 @@ static void pl061_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl061;
 }
 
-static TypeInfo pl061_info = {
+static const TypeInfo pl061_info = {
     .name          = "pl061",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl061_state),
@@ -318,7 +320,7 @@ static void pl061_luminary_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl061;
 }
 
-static TypeInfo pl061_luminary_info = {
+static const TypeInfo pl061_luminary_info = {
     .name          = "pl061_luminary",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl061_state),
diff --git a/hw/pl080.c b/hw/pl080.c
index b3cf651ab7c..f6bbf98a7ee 100644
--- a/hw/pl080.c
+++ b/hw/pl080.c
@@ -218,7 +218,7 @@ again:
     }
 }
 
-static uint64_t pl080_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl080_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl080_state *s = (pl080_state *)opaque;
@@ -281,12 +281,13 @@ static uint64_t pl080_read(void *opaque, target_phys_addr_t offset,
         return s->sync;
     default:
     bad_offset:
-        hw_error("pl080_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl080_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl080_write(void *opaque, target_phys_addr_t offset,
+static void pl080_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl080_state *s = (pl080_state *)opaque;
@@ -327,12 +328,13 @@ static void pl080_write(void *opaque, target_phys_addr_t offset,
     case 10: /* SoftLBReq */
     case 11: /* SoftLSReq */
         /* ??? Implement these.  */
-        hw_error("pl080_write: Soft DMA not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "pl080_write: Soft DMA not implemented\n");
         break;
     case 12: /* Configuration */
         s->conf = value;
         if (s->conf & (PL080_CONF_M1 | PL080_CONF_M1)) {
-            hw_error("pl080_write: Big-endian DMA not implemented\n");
+            qemu_log_mask(LOG_UNIMP,
+                          "pl080_write: Big-endian DMA not implemented\n");
         }
         pl080_run(s);
         break;
@@ -341,7 +343,8 @@ static void pl080_write(void *opaque, target_phys_addr_t offset,
         break;
     default:
     bad_offset:
-        hw_error("pl080_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl080_write: Bad offset %x\n", (int)offset);
     }
     pl080_update(s);
 }
@@ -383,7 +386,7 @@ static void pl080_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl080;
 }
 
-static TypeInfo pl080_info = {
+static const TypeInfo pl080_info = {
     .name          = "pl080",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl080_state),
@@ -400,7 +403,7 @@ static void pl081_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl080;
 }
 
-static TypeInfo pl081_info = {
+static const TypeInfo pl081_info = {
     .name          = "pl081",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl080_state),
diff --git a/hw/pl110.c b/hw/pl110.c
index a5826407aba..3d0ac00ade8 100644
--- a/hw/pl110.c
+++ b/hw/pl110.c
@@ -8,8 +8,9 @@
  */
 
 #include "sysbus.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
+#include "ui/pixel_ops.h"
 
 #define PL110_CR_EN   0x001
 #define PL110_CR_BGR  0x100
@@ -109,8 +110,6 @@ static const unsigned char *idregs[] = {
     pl111_id
 };
 
-#include "pixel_ops.h"
-
 #define BITS 8
 #include "pl110_template.h"
 #define BITS 15
@@ -239,7 +238,7 @@ static void pl110_update_display(void *opaque)
                                fn, s->palette,
                                &first, &last);
     if (first >= 0) {
-        dpy_update(s->ds, 0, first, s->cols, last - first + 1);
+        dpy_gfx_update(s->ds, 0, first, s->cols, last - first + 1);
     }
     s->invalidate = 0;
 }
@@ -305,7 +304,7 @@ static void pl110_update(pl110_state *s)
   /* TODO: Implement interrupts.  */
 }
 
-static uint64_t pl110_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl110_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl110_state *s = (pl110_state *)opaque;
@@ -349,12 +348,13 @@ static uint64_t pl110_read(void *opaque, target_phys_addr_t offset,
     case 12: /* LCDLPCURR */
         return s->lpbase;
     default:
-        hw_error("pl110_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl110_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl110_write(void *opaque, target_phys_addr_t offset,
+static void pl110_write(void *opaque, hwaddr offset,
                         uint64_t val, unsigned size)
 {
     pl110_state *s = (pl110_state *)opaque;
@@ -417,7 +417,8 @@ static void pl110_write(void *opaque, target_phys_addr_t offset,
         pl110_update(s);
         break;
     default:
-        hw_error("pl110_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl110_write: Bad offset %x\n", (int)offset);
     }
 }
 
@@ -479,7 +480,7 @@ static void pl110_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl110_info = {
+static const TypeInfo pl110_info = {
     .name          = "pl110",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
@@ -496,7 +497,7 @@ static void pl110_versatile_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl110_versatile_info = {
+static const TypeInfo pl110_versatile_info = {
     .name          = "pl110_versatile",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
@@ -513,7 +514,7 @@ static void pl111_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl111_info = {
+static const TypeInfo pl111_info = {
     .name          = "pl111",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
diff --git a/hw/pl181.c b/hw/pl181.c
index 7d91fbba1d0..98529f78211 100644
--- a/hw/pl181.c
+++ b/hw/pl181.c
@@ -7,7 +7,7 @@
  * This code is licensed under the GPL.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
 #include "sd.h"
 
@@ -285,7 +285,7 @@ static void pl181_fifo_run(pl181_state *s)
     }
 }
 
-static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl181_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl181_state *s = (pl181_state *)opaque;
@@ -352,7 +352,7 @@ static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
     case 0xa0: case 0xa4: case 0xa8: case 0xac:
     case 0xb0: case 0xb4: case 0xb8: case 0xbc:
         if (s->fifo_len == 0) {
-            fprintf(stderr, "pl181: Unexpected FIFO read\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "pl181: Unexpected FIFO read\n");
             return 0;
         } else {
             uint32_t value;
@@ -363,12 +363,13 @@ static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
             return value;
         }
     default:
-        hw_error("pl181_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl181_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl181_write(void *opaque, target_phys_addr_t offset,
+static void pl181_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl181_state *s = (pl181_state *)opaque;
@@ -387,11 +388,11 @@ static void pl181_write(void *opaque, target_phys_addr_t offset,
         s->cmd = value;
         if (s->cmd & PL181_CMD_ENABLE) {
             if (s->cmd & PL181_CMD_INTERRUPT) {
-                fprintf(stderr, "pl181: Interrupt mode not implemented\n");
-                abort();
+                qemu_log_mask(LOG_UNIMP,
+                              "pl181: Interrupt mode not implemented\n");
             } if (s->cmd & PL181_CMD_PENDING) {
-                fprintf(stderr, "pl181: Pending commands not implemented\n");
-                abort();
+                qemu_log_mask(LOG_UNIMP,
+                              "pl181: Pending commands not implemented\n");
             } else {
                 pl181_send_command(s);
                 pl181_fifo_run(s);
@@ -427,14 +428,15 @@ static void pl181_write(void *opaque, target_phys_addr_t offset,
     case 0xa0: case 0xa4: case 0xa8: case 0xac:
     case 0xb0: case 0xb4: case 0xb8: case 0xbc:
         if (s->datacnt == 0) {
-            fprintf(stderr, "pl181: Unexpected FIFO write\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "pl181: Unexpected FIFO write\n");
         } else {
             pl181_fifo_push(s, value);
             pl181_fifo_run(s);
         }
         break;
     default:
-        hw_error("pl181_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl181_write: Bad offset %x\n", (int)offset);
     }
     pl181_update(s);
 }
@@ -498,7 +500,7 @@ static void pl181_class_init(ObjectClass *klass, void *data)
     k->no_user = 1;
 }
 
-static TypeInfo pl181_info = {
+static const TypeInfo pl181_info = {
     .name          = "pl181",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl181_state),
diff --git a/hw/pl190.c b/hw/pl190.c
index cb50afb9f4c..76ac159374c 100644
--- a/hw/pl190.c
+++ b/hw/pl190.c
@@ -85,7 +85,7 @@ static void pl190_update_vectors(pl190_state *s)
     pl190_update(s);
 }
 
-static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl190_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl190_state *s = (pl190_state *)opaque;
@@ -117,12 +117,18 @@ static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
         return s->protected;
     case 12: /* VECTADDR */
         /* Read vector address at the start of an ISR.  Increases the
-           current priority level to that of the current interrupt.  */
-        for (i = 0; i < s->priority; i++)
-          {
-            if ((s->level | s->soft_level) & s->prio_mask[i])
-              break;
-          }
+         * current priority level to that of the current interrupt.
+         *
+         * Since an enabled interrupt X at priority P causes prio_mask[Y]
+         * to have bit X set for all Y > P, this loop will stop with
+         * i == the priority of the highest priority set interrupt.
+         */
+        for (i = 0; i < s->priority; i++) {
+            if ((s->level | s->soft_level) & s->prio_mask[i + 1]) {
+                break;
+            }
+        }
+
         /* Reading this value with no pending interrupts is undefined.
            We return the default address.  */
         if (i == PL190_NUM_PRIO)
@@ -137,12 +143,13 @@ static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
     case 13: /* DEFVECTADDR */
         return s->vect_addr[16];
     default:
-        hw_error("pl190_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl190_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl190_write(void *opaque, target_phys_addr_t offset,
+static void pl190_write(void *opaque, hwaddr offset,
                         uint64_t val, unsigned size)
 {
     pl190_state *s = (pl190_state *)opaque;
@@ -192,11 +199,12 @@ static void pl190_write(void *opaque, target_phys_addr_t offset,
         break;
     case 0xc0: /* ITCR */
         if (val) {
-            hw_error("pl190: Test mode not implemented\n");
+            qemu_log_mask(LOG_UNIMP, "pl190: Test mode not implemented\n");
         }
         break;
     default:
-        hw_error("pl190_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                     "pl190_write: Bad offset %x\n", (int)offset);
         return;
     }
     pl190_update(s);
@@ -266,7 +274,7 @@ static void pl190_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl190;
 }
 
-static TypeInfo pl190_info = {
+static const TypeInfo pl190_info = {
     .name          = "pl190",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl190_state),
diff --git a/hw/pm_smbus.c b/hw/pm_smbus.c
index 5d6046de5a8..ea1380ca680 100644
--- a/hw/pm_smbus.c
+++ b/hw/pm_smbus.c
@@ -94,10 +94,11 @@ static void smb_transaction(PMSMBus *s)
     s->smb_stat |= 0x04;
 }
 
-void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void smb_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
 {
     PMSMBus *s = opaque;
-    addr &= 0x3f;
+
     SMBUS_DPRINTF("SMB writeb port=0x%04x val=0x%02x\n", addr, val);
     switch(addr) {
     case SMBHSTSTS:
@@ -131,12 +132,11 @@ void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-uint32_t smb_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t smb_ioport_readb(void *opaque, hwaddr addr, unsigned width)
 {
     PMSMBus *s = opaque;
     uint32_t val;
 
-    addr &= 0x3f;
     switch(addr) {
     case SMBHSTSTS:
         val = s->smb_stat;
@@ -170,7 +170,16 @@ uint32_t smb_ioport_readb(void *opaque, uint32_t addr)
     return val;
 }
 
+static const MemoryRegionOps pm_smbus_ops = {
+    .read = smb_ioport_readb,
+    .write = smb_ioport_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 void pm_smbus_init(DeviceState *parent, PMSMBus *smb)
 {
     smb->smbus = i2c_init_bus(parent, "i2c");
+    memory_region_init_io(&smb->io, &pm_smbus_ops, smb, "pm-smbus", 64);
 }
diff --git a/hw/pm_smbus.h b/hw/pm_smbus.h
index 4750a409f9e..e3069bf7d4e 100644
--- a/hw/pm_smbus.h
+++ b/hw/pm_smbus.h
@@ -3,6 +3,7 @@
 
 typedef struct PMSMBus {
     i2c_bus *smbus;
+    MemoryRegion io;
 
     uint8_t smb_stat;
     uint8_t smb_ctl;
@@ -15,7 +16,5 @@ typedef struct PMSMBus {
 } PMSMBus;
 
 void pm_smbus_init(DeviceState *parent, PMSMBus *smb);
-void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val);
-uint32_t smb_ioport_readb(void *opaque, uint32_t addr);
 
 #endif /* !PM_SMBUS_H */
diff --git a/hw/ppc.c b/hw/ppc.c
index 98546de9915..c52e22f708e 100644
--- a/hw/ppc.c
+++ b/hw/ppc.c
@@ -23,12 +23,12 @@
  */
 #include "hw.h"
 #include "ppc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "nvram.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 //#define PPC_DEBUG_IRQ
@@ -50,8 +50,9 @@
 static void cpu_ppc_tb_stop (CPUPPCState *env);
 static void cpu_ppc_tb_start (CPUPPCState *env);
 
-void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
+void ppc_set_irq(PowerPCCPU *cpu, int n_IRQ, int level)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned int old_pending = env->pending_interrupts;
 
     if (level) {
@@ -65,7 +66,7 @@ void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
 
     if (old_pending != env->pending_interrupts) {
 #ifdef CONFIG_KVM
-        kvmppc_set_interrupt(env, n_IRQ, level);
+        kvmppc_set_interrupt(cpu, n_IRQ, level);
 #endif
     }
 
@@ -75,9 +76,10 @@ void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
 }
 
 /* PowerPC 6xx / 7xx internal IRQ controller */
-static void ppc6xx_set_irq (void *opaque, int pin, int level)
+static void ppc6xx_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -99,13 +101,13 @@ static void ppc6xx_set_irq (void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC6xx_INPUT_SMI:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the SMI IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_SMI, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_SMI, level);
             break;
         case PPC6xx_INPUT_MCP:
             /* Negative edge sensitive */
@@ -115,7 +117,7 @@ static void ppc6xx_set_irq (void *opaque, int pin, int level)
             if (cur_level == 1 && level == 0) {
                 LOG_IRQ("%s: raise machine check state\n",
                             __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, 1);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, 1);
             }
             break;
         case PPC6xx_INPUT_CKSTP_IN:
@@ -137,7 +139,7 @@ static void ppc6xx_set_irq (void *opaque, int pin, int level)
         case PPC6xx_INPUT_SRESET:
             LOG_IRQ("%s: set the RESET IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_RESET, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_RESET, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -151,17 +153,20 @@ static void ppc6xx_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc6xx_irq_init (CPUPPCState *env)
+void ppc6xx_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc6xx_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc6xx_set_irq, cpu,
                                                   PPC6xx_INPUT_NB);
 }
 
 #if defined(TARGET_PPC64)
 /* PowerPC 970 internal IRQ controller */
-static void ppc970_set_irq (void *opaque, int pin, int level)
+static void ppc970_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -174,13 +179,13 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC970_INPUT_THINT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the SMI IRQ state to %d\n", __func__,
                         level);
-            ppc_set_irq(env, PPC_INTERRUPT_THERM, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_THERM, level);
             break;
         case PPC970_INPUT_MCP:
             /* Negative edge sensitive */
@@ -190,7 +195,7 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
             if (cur_level == 1 && level == 0) {
                 LOG_IRQ("%s: raise machine check state\n",
                             __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, 1);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, 1);
             }
             break;
         case PPC970_INPUT_CKSTP:
@@ -202,7 +207,7 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
             } else {
                 LOG_IRQ("%s: restart the CPU\n", __func__);
                 env->halted = 0;
-                qemu_cpu_kick(env);
+                qemu_cpu_kick(CPU(cpu));
             }
             break;
         case PPC970_INPUT_HRESET:
@@ -214,7 +219,7 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
         case PPC970_INPUT_SRESET:
             LOG_IRQ("%s: set the RESET IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_RESET, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_RESET, level);
             break;
         case PPC970_INPUT_TBEN:
             LOG_IRQ("%s: set the TBEN state to %d\n", __func__,
@@ -233,16 +238,19 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc970_irq_init (CPUPPCState *env)
+void ppc970_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc970_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc970_set_irq, cpu,
                                                   PPC970_INPUT_NB);
 }
 
 /* POWER7 internal IRQ controller */
-static void power7_set_irq (void *opaque, int pin, int level)
+static void power7_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
                 env, pin, level);
@@ -252,7 +260,7 @@ static void power7_set_irq (void *opaque, int pin, int level)
         /* Level sensitive - active high */
         LOG_IRQ("%s: set the external IRQ state to %d\n",
                 __func__, level);
-        ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+        ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
         break;
     default:
         /* Unknown pin - do nothing */
@@ -266,17 +274,20 @@ static void power7_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppcPOWER7_irq_init (CPUPPCState *env)
+void ppcPOWER7_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&power7_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&power7_set_irq, cpu,
                                                   POWER7_INPUT_NB);
 }
 #endif /* defined(TARGET_PPC64) */
 
 /* PowerPC 40x internal IRQ controller */
-static void ppc40x_set_irq (void *opaque, int pin, int level)
+static void ppc40x_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -309,13 +320,13 @@ static void ppc40x_set_irq (void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the critical IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_CEXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_CEXT, level);
             break;
         case PPC40x_INPUT_INT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC40x_INPUT_HALT:
             /* Level sensitive - active low */
@@ -325,14 +336,14 @@ static void ppc40x_set_irq (void *opaque, int pin, int level)
             } else {
                 LOG_IRQ("%s: restart the CPU\n", __func__);
                 env->halted = 0;
-                qemu_cpu_kick(env);
+                qemu_cpu_kick(CPU(cpu));
             }
             break;
         case PPC40x_INPUT_DEBUG:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the debug pin state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_DEBUG, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_DEBUG, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -346,16 +357,19 @@ static void ppc40x_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc40x_irq_init (CPUPPCState *env)
+void ppc40x_irq_init(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppc40x_set_irq,
-                                                  env, PPC40x_INPUT_NB);
+                                                  cpu, PPC40x_INPUT_NB);
 }
 
 /* PowerPC E500 internal IRQ controller */
-static void ppce500_set_irq (void *opaque, int pin, int level)
+static void ppce500_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -374,26 +388,26 @@ static void ppce500_set_irq (void *opaque, int pin, int level)
         case PPCE500_INPUT_RESET_CORE:
             if (level) {
                 LOG_IRQ("%s: reset the PowerPC core\n", __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, level);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, level);
             }
             break;
         case PPCE500_INPUT_CINT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the critical IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_CEXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_CEXT, level);
             break;
         case PPCE500_INPUT_INT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the core IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPCE500_INPUT_DEBUG:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the debug pin state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_DEBUG, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_DEBUG, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -407,11 +421,30 @@ static void ppce500_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppce500_irq_init (CPUPPCState *env)
+void ppce500_irq_init(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppce500_set_irq,
-                                        env, PPCE500_INPUT_NB);
+                                                  cpu, PPCE500_INPUT_NB);
 }
+
+/* Enable or Disable the E500 EPR capability */
+void ppce500_set_mpic_proxy(bool enabled)
+{
+    CPUPPCState *env;
+
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        PowerPCCPU *cpu = ppc_env_get_cpu(env);
+        CPUState *cs = CPU(cpu);
+
+        env->mpic_proxy = enabled;
+        if (kvm_enabled()) {
+            kvmppc_set_mpic_proxy(POWERPC_CPU(cs), enabled);
+        }
+    }
+}
+
 /*****************************************************************************/
 /* PowerPC time base and decrementer emulation */
 
@@ -628,26 +661,27 @@ uint64_t cpu_ppc_load_purr (CPUPPCState *env)
 /* When decrementer expires,
  * all we need to do is generate or queue a CPU exception
  */
-static inline void cpu_ppc_decr_excp(CPUPPCState *env)
+static inline void cpu_ppc_decr_excp(PowerPCCPU *cpu)
 {
     /* Raise it */
     LOG_TB("raise decrementer exception\n");
-    ppc_set_irq(env, PPC_INTERRUPT_DECR, 1);
+    ppc_set_irq(cpu, PPC_INTERRUPT_DECR, 1);
 }
 
-static inline void cpu_ppc_hdecr_excp(CPUPPCState *env)
+static inline void cpu_ppc_hdecr_excp(PowerPCCPU *cpu)
 {
     /* Raise it */
     LOG_TB("raise decrementer exception\n");
-    ppc_set_irq(env, PPC_INTERRUPT_HDECR, 1);
+    ppc_set_irq(cpu, PPC_INTERRUPT_HDECR, 1);
 }
 
-static void __cpu_ppc_store_decr (CPUPPCState *env, uint64_t *nextp,
-                                  struct QEMUTimer *timer,
-                                  void (*raise_excp)(CPUPPCState *),
-                                  uint32_t decr, uint32_t value,
-                                  int is_excp)
+static void __cpu_ppc_store_decr(PowerPCCPU *cpu, uint64_t *nextp,
+                                 struct QEMUTimer *timer,
+                                 void (*raise_excp)(PowerPCCPU *),
+                                 uint32_t decr, uint32_t value,
+                                 int is_excp)
 {
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env = env->tb_env;
     uint64_t now, next;
 
@@ -677,53 +711,61 @@ static void __cpu_ppc_store_decr (CPUPPCState *env, uint64_t *nextp,
     if ((tb_env->flags & PPC_DECR_UNDERFLOW_TRIGGERED)
         && (value & 0x80000000)
         && !(decr & 0x80000000)) {
-        (*raise_excp)(env);
+        (*raise_excp)(cpu);
     }
 }
 
-static inline void _cpu_ppc_store_decr(CPUPPCState *env, uint32_t decr,
+static inline void _cpu_ppc_store_decr(PowerPCCPU *cpu, uint32_t decr,
                                        uint32_t value, int is_excp)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
-    __cpu_ppc_store_decr(env, &tb_env->decr_next, tb_env->decr_timer,
+    __cpu_ppc_store_decr(cpu, &tb_env->decr_next, tb_env->decr_timer,
                          &cpu_ppc_decr_excp, decr, value, is_excp);
 }
 
 void cpu_ppc_store_decr (CPUPPCState *env, uint32_t value)
 {
-    _cpu_ppc_store_decr(env, cpu_ppc_load_decr(env), value, 0);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    _cpu_ppc_store_decr(cpu, cpu_ppc_load_decr(env), value, 0);
 }
 
-static void cpu_ppc_decr_cb (void *opaque)
+static void cpu_ppc_decr_cb(void *opaque)
 {
-    _cpu_ppc_store_decr(opaque, 0x00000000, 0xFFFFFFFF, 1);
+    PowerPCCPU *cpu = opaque;
+
+    _cpu_ppc_store_decr(cpu, 0x00000000, 0xFFFFFFFF, 1);
 }
 
-static inline void _cpu_ppc_store_hdecr(CPUPPCState *env, uint32_t hdecr,
+static inline void _cpu_ppc_store_hdecr(PowerPCCPU *cpu, uint32_t hdecr,
                                         uint32_t value, int is_excp)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
     if (tb_env->hdecr_timer != NULL) {
-        __cpu_ppc_store_decr(env, &tb_env->hdecr_next, tb_env->hdecr_timer,
+        __cpu_ppc_store_decr(cpu, &tb_env->hdecr_next, tb_env->hdecr_timer,
                              &cpu_ppc_hdecr_excp, hdecr, value, is_excp);
     }
 }
 
 void cpu_ppc_store_hdecr (CPUPPCState *env, uint32_t value)
 {
-    _cpu_ppc_store_hdecr(env, cpu_ppc_load_hdecr(env), value, 0);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    _cpu_ppc_store_hdecr(cpu, cpu_ppc_load_hdecr(env), value, 0);
 }
 
-static void cpu_ppc_hdecr_cb (void *opaque)
+static void cpu_ppc_hdecr_cb(void *opaque)
 {
-    _cpu_ppc_store_hdecr(opaque, 0x00000000, 0xFFFFFFFF, 1);
+    PowerPCCPU *cpu = opaque;
+
+    _cpu_ppc_store_hdecr(cpu, 0x00000000, 0xFFFFFFFF, 1);
 }
 
-void cpu_ppc_store_purr (CPUPPCState *env, uint64_t value)
+static void cpu_ppc_store_purr(PowerPCCPU *cpu, uint64_t value)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
     tb_env->purr_load = value;
     tb_env->purr_start = qemu_get_clock_ns(vm_clock);
@@ -732,6 +774,7 @@ void cpu_ppc_store_purr (CPUPPCState *env, uint64_t value)
 static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
 {
     CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env = env->tb_env;
 
     tb_env->tb_freq = freq;
@@ -740,25 +783,27 @@ static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
      * if a decrementer exception is pending when it enables msr_ee at startup,
      * it's not ready to handle it...
      */
-    _cpu_ppc_store_decr(env, 0xFFFFFFFF, 0xFFFFFFFF, 0);
-    _cpu_ppc_store_hdecr(env, 0xFFFFFFFF, 0xFFFFFFFF, 0);
-    cpu_ppc_store_purr(env, 0x0000000000000000ULL);
+    _cpu_ppc_store_decr(cpu, 0xFFFFFFFF, 0xFFFFFFFF, 0);
+    _cpu_ppc_store_hdecr(cpu, 0xFFFFFFFF, 0xFFFFFFFF, 0);
+    cpu_ppc_store_purr(cpu, 0x0000000000000000ULL);
 }
 
 /* Set up (once) timebase frequency (in Hz) */
 clk_setup_cb cpu_ppc_tb_init (CPUPPCState *env, uint32_t freq)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env;
 
     tb_env = g_malloc0(sizeof(ppc_tb_t));
     env->tb_env = tb_env;
     tb_env->flags = PPC_DECR_UNDERFLOW_TRIGGERED;
     /* Create new timer */
-    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_decr_cb, env);
+    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_decr_cb, cpu);
     if (0) {
         /* XXX: find a suitable condition to enable the hypervisor decrementer
          */
-        tb_env->hdecr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_hdecr_cb, env);
+        tb_env->hdecr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_hdecr_cb,
+                                                cpu);
     } else {
         tb_env->hdecr_timer = NULL;
     }
@@ -814,12 +859,14 @@ struct ppc40x_timer_t {
 /* Fixed interval timer */
 static void cpu_4xx_fit_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
     uint64_t now, next;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     now = qemu_get_clock_ns(vm_clock);
@@ -845,8 +892,9 @@ static void cpu_4xx_fit_cb (void *opaque)
         next++;
     qemu_mod_timer(ppc40x_timer->fit_timer, next);
     env->spr[SPR_40x_TSR] |= 1 << 26;
-    if ((env->spr[SPR_40x_TCR] >> 23) & 0x1)
-        ppc_set_irq(env, PPC_INTERRUPT_FIT, 1);
+    if ((env->spr[SPR_40x_TCR] >> 23) & 0x1) {
+        ppc_set_irq(cpu, PPC_INTERRUPT_FIT, 1);
+    }
     LOG_TB("%s: ir %d TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx "\n", __func__,
            (int)((env->spr[SPR_40x_TCR] >> 23) & 0x1),
            env->spr[SPR_40x_TCR], env->spr[SPR_40x_TSR]);
@@ -882,16 +930,19 @@ static void start_stop_pit (CPUPPCState *env, ppc_tb_t *tb_env, int is_excp)
 
 static void cpu_4xx_pit_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     env->spr[SPR_40x_TSR] |= 1 << 27;
-    if ((env->spr[SPR_40x_TCR] >> 26) & 0x1)
-        ppc_set_irq(env, ppc40x_timer->decr_excp, 1);
+    if ((env->spr[SPR_40x_TCR] >> 26) & 0x1) {
+        ppc_set_irq(cpu, ppc40x_timer->decr_excp, 1);
+    }
     start_stop_pit(env, tb_env, 1);
     LOG_TB("%s: ar %d ir %d TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx " "
            "%016" PRIx64 "\n", __func__,
@@ -904,12 +955,14 @@ static void cpu_4xx_pit_cb (void *opaque)
 /* Watchdog timer */
 static void cpu_4xx_wdt_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
     uint64_t now, next;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     now = qemu_get_clock_ns(vm_clock);
@@ -946,8 +999,9 @@ static void cpu_4xx_wdt_cb (void *opaque)
         qemu_mod_timer(ppc40x_timer->wdt_timer, next);
         ppc40x_timer->wdt_next = next;
         env->spr[SPR_40x_TSR] |= 1 << 30;
-        if ((env->spr[SPR_40x_TCR] >> 27) & 0x1)
-            ppc_set_irq(env, PPC_INTERRUPT_WDT, 1);
+        if ((env->spr[SPR_40x_TCR] >> 27) & 0x1) {
+            ppc_set_irq(cpu, PPC_INTERRUPT_WDT, 1);
+        }
         break;
     case 0x3:
         env->spr[SPR_40x_TSR] &= ~0x30000000;
@@ -1152,23 +1206,23 @@ static inline void nvram_write (nvram_t *nvram, uint32_t addr, uint32_t val)
     (*nvram->write_fn)(nvram->opaque, addr, val);
 }
 
-void NVRAM_set_byte (nvram_t *nvram, uint32_t addr, uint8_t value)
+static void NVRAM_set_byte(nvram_t *nvram, uint32_t addr, uint8_t value)
 {
     nvram_write(nvram, addr, value);
 }
 
-uint8_t NVRAM_get_byte (nvram_t *nvram, uint32_t addr)
+static uint8_t NVRAM_get_byte(nvram_t *nvram, uint32_t addr)
 {
     return nvram_read(nvram, addr);
 }
 
-void NVRAM_set_word (nvram_t *nvram, uint32_t addr, uint16_t value)
+static void NVRAM_set_word(nvram_t *nvram, uint32_t addr, uint16_t value)
 {
     nvram_write(nvram, addr, value >> 8);
     nvram_write(nvram, addr + 1, value & 0xFF);
 }
 
-uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr)
+static uint16_t NVRAM_get_word(nvram_t *nvram, uint32_t addr)
 {
     uint16_t tmp;
 
@@ -1178,7 +1232,7 @@ uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr)
     return tmp;
 }
 
-void NVRAM_set_lword (nvram_t *nvram, uint32_t addr, uint32_t value)
+static void NVRAM_set_lword(nvram_t *nvram, uint32_t addr, uint32_t value)
 {
     nvram_write(nvram, addr, value >> 24);
     nvram_write(nvram, addr + 1, (value >> 16) & 0xFF);
@@ -1198,8 +1252,8 @@ uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr)
     return tmp;
 }
 
-void NVRAM_set_string (nvram_t *nvram, uint32_t addr,
-                       const char *str, uint32_t max)
+static void NVRAM_set_string(nvram_t *nvram, uint32_t addr, const char *str,
+                             uint32_t max)
 {
     int i;
 
diff --git a/hw/ppc.h b/hw/ppc.h
index 2f3ea277bc1..ee0cd16ee53 100644
--- a/hw/ppc.h
+++ b/hw/ppc.h
@@ -1,4 +1,7 @@
-void ppc_set_irq (CPUPPCState *env, int n_IRQ, int level);
+#ifndef HW_PPC_H
+#define HW_PPC_H 1
+
+void ppc_set_irq(PowerPCCPU *cpu, int n_IRQ, int level);
 
 /* PowerPC hardware exceptions management helpers */
 typedef void (*clk_setup_cb)(void *opaque, uint32_t freq);
@@ -70,6 +73,8 @@ void ppc6xx_irq_init (CPUPPCState *env);
 void ppc970_irq_init (CPUPPCState *env);
 void ppcPOWER7_irq_init (CPUPPCState *env);
 
+void ppce500_set_mpic_proxy(bool enabled);
+
 /* PPC machines for OpenBIOS */
 enum {
     ARCH_PREP = 0,
@@ -89,4 +94,6 @@ enum {
 #define PPC_SERIAL_MM_BAUDBASE 399193
 
 /* ppc_booke.c */
-void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags);
+void ppc_booke_timers_init(PowerPCCPU *cpu, uint32_t freq, uint32_t flags);
+
+#endif
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index 951e407f143..462146b0b0a 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -3,14 +3,11 @@ obj-y = ppc.o ppc_booke.o
 # PREP target
 obj-y += mc146818rtc.o
 obj-y += ppc_prep.o
-# OldWorld PowerMac
-obj-y += ppc_oldworld.o
-# NewWorld PowerMac
-obj-y += ppc_newworld.o
 # IBM pSeries (sPAPR)
 obj-$(CONFIG_PSERIES) += spapr.o spapr_hcall.o spapr_rtas.o spapr_vio.o
 obj-$(CONFIG_PSERIES) += xics.o spapr_vty.o spapr_llan.o spapr_vscsi.o
-obj-$(CONFIG_PSERIES) += spapr_pci.o pci-hotplug.o spapr_iommu.o
+obj-$(CONFIG_PSERIES) += spapr_pci.o pci/pci-hotplug.o spapr_iommu.o
+obj-$(CONFIG_PSERIES) += spapr_events.o spapr_nvram.o
 # PowerPC 4xx boards
 obj-y += ppc4xx_devs.o ppc4xx_pci.o ppc405_uc.o ppc405_boards.o
 obj-y += ppc440_bamboo.o
@@ -27,4 +24,9 @@ obj-y += xilinx_ethlite.o
 
 obj-y := $(addprefix ../,$(obj-y))
 
+# OldWorld PowerMac
+obj-y += mac_oldworld.o
+# NewWorld PowerMac
+obj-y += mac_newworld.o
+# e500
 obj-$(CONFIG_FDT) += e500.o mpc8544ds.o e500plat.o
diff --git a/hw/ppc/e500-ccsr.h b/hw/ppc/e500-ccsr.h
new file mode 100644
index 00000000000..f20f51bcd28
--- /dev/null
+++ b/hw/ppc/e500-ccsr.h
@@ -0,0 +1,17 @@
+#ifndef E500_CCSR_H
+#define E500_CCSR_H
+
+#include "../sysbus.h"
+
+typedef struct PPCE500CCSRState {
+    /*< private >*/
+    SysBusDevice parent;
+    /*< public >*/
+
+    MemoryRegion ccsr_space;
+} PPCE500CCSRState;
+
+#define TYPE_CCSR "e500-ccsr"
+#define CCSR(obj) OBJECT_CHECK(PPCE500CCSRState, (obj), TYPE_CCSR)
+
+#endif /* E500_CCSR_H */
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index 6f0de6d9590..b7474c05f9a 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -17,27 +17,31 @@
 #include "config.h"
 #include "qemu-common.h"
 #include "e500.h"
-#include "net.h"
+#include "e500-ccsr.h"
+#include "net/net.h"
+#include "qemu/config-file.h"
 #include "hw/hw.h"
-#include "hw/pc.h"
-#include "hw/pci.h"
+#include "hw/serial.h"
+#include "hw/pci/pci.h"
 #include "hw/boards.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "hw/openpic.h"
 #include "hw/ppc.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "hw/sysbus.h"
-#include "exec-memory.h"
-#include "host-utils.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
+#include "hw/ppce500_pci.h"
 
 #define BINARY_DEVICE_TREE_FILE    "mpc8544ds.dtb"
 #define UIMAGE_LOAD_BASE           0
-#define DTC_LOAD_PAD               0x500000
+#define DTC_LOAD_PAD               0x1800000
 #define DTC_PAD_MASK               0xFFFFF
+#define DTB_MAX_SIZE               (8 * 1024 * 1024)
 #define INITRD_LOAD_PAD            0x2000000
 #define INITRD_PAD_MASK            0xFFFFFF
 
@@ -46,14 +50,16 @@
 /* TODO: parameterize */
 #define MPC8544_CCSRBAR_BASE       0xE0000000ULL
 #define MPC8544_CCSRBAR_SIZE       0x00100000ULL
-#define MPC8544_MPIC_REGS_BASE     (MPC8544_CCSRBAR_BASE + 0x40000ULL)
-#define MPC8544_SERIAL0_REGS_BASE  (MPC8544_CCSRBAR_BASE + 0x4500ULL)
-#define MPC8544_SERIAL1_REGS_BASE  (MPC8544_CCSRBAR_BASE + 0x4600ULL)
-#define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + 0x8000ULL)
+#define MPC8544_MPIC_REGS_OFFSET   0x40000ULL
+#define MPC8544_MSI_REGS_OFFSET   0x41600ULL
+#define MPC8544_SERIAL0_REGS_OFFSET 0x4500ULL
+#define MPC8544_SERIAL1_REGS_OFFSET 0x4600ULL
+#define MPC8544_PCI_REGS_OFFSET    0x8000ULL
+#define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + \
+                                    MPC8544_PCI_REGS_OFFSET)
 #define MPC8544_PCI_REGS_SIZE      0x1000ULL
 #define MPC8544_PCI_IO             0xE1000000ULL
-#define MPC8544_PCI_IOLEN          0x10000ULL
-#define MPC8544_UTIL_BASE          (MPC8544_CCSRBAR_BASE + 0xe0000ULL)
+#define MPC8544_UTIL_OFFSET        0xe0000ULL
 #define MPC8544_SPIN_BASE          0xEF000000ULL
 
 struct boot_info
@@ -63,25 +69,35 @@ struct boot_info
     uint32_t entry;
 };
 
-static void pci_map_create(void *fdt, uint32_t *pci_map, uint32_t mpic)
+static uint32_t *pci_map_create(void *fdt, uint32_t mpic, int first_slot,
+                                int nr_slots, int *len)
 {
-    int i;
-    const uint32_t tmp[] = {
-                             /* IDSEL 0x11 J17 Slot 1 */
-                             0x8800, 0x0, 0x0, 0x1, mpic, 0x2, 0x1,
-                             0x8800, 0x0, 0x0, 0x2, mpic, 0x3, 0x1,
-                             0x8800, 0x0, 0x0, 0x3, mpic, 0x4, 0x1,
-                             0x8800, 0x0, 0x0, 0x4, mpic, 0x1, 0x1,
-
-                             /* IDSEL 0x12 J16 Slot 2 */
-                             0x9000, 0x0, 0x0, 0x1, mpic, 0x3, 0x1,
-                             0x9000, 0x0, 0x0, 0x2, mpic, 0x4, 0x1,
-                             0x9000, 0x0, 0x0, 0x3, mpic, 0x2, 0x1,
-                             0x9000, 0x0, 0x0, 0x4, mpic, 0x1, 0x1,
-                           };
-    for (i = 0; i < (7 * 8); i++) {
-        pci_map[i] = cpu_to_be32(tmp[i]);
+    int i = 0;
+    int slot;
+    int pci_irq;
+    int host_irq;
+    int last_slot = first_slot + nr_slots;
+    uint32_t *pci_map;
+
+    *len = nr_slots * 4 * 7 * sizeof(uint32_t);
+    pci_map = g_malloc(*len);
+
+    for (slot = first_slot; slot < last_slot; slot++) {
+        for (pci_irq = 0; pci_irq < 4; pci_irq++) {
+            pci_map[i++] = cpu_to_be32(slot << 11);
+            pci_map[i++] = cpu_to_be32(0x0);
+            pci_map[i++] = cpu_to_be32(0x0);
+            pci_map[i++] = cpu_to_be32(pci_irq + 1);
+            pci_map[i++] = cpu_to_be32(mpic);
+            host_irq = ppce500_pci_map_irq_slot(slot, pci_irq);
+            pci_map[i++] = cpu_to_be32(host_irq + 1);
+            pci_map[i++] = cpu_to_be32(0x1);
+        }
     }
+
+    assert((i * sizeof(uint32_t)) == *len);
+
+    return pci_map;
 }
 
 static void dt_serial_create(void *fdt, unsigned long long offset,
@@ -108,9 +124,9 @@ static void dt_serial_create(void *fdt, unsigned long long offset,
 
 static int ppce500_load_device_tree(CPUPPCState *env,
                                     PPCE500Params *params,
-                                    target_phys_addr_t addr,
-                                    target_phys_addr_t initrd_base,
-                                    target_phys_addr_t initrd_size)
+                                    hwaddr addr,
+                                    hwaddr initrd_base,
+                                    hwaddr initrd_size)
 {
     int ret = -1;
     uint64_t mem_reg_property[] = { 0, cpu_to_be64(params->ram_size) };
@@ -125,9 +141,12 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     char soc[128];
     char mpic[128];
     uint32_t mpic_ph;
+    uint32_t msi_ph;
     char gutil[128];
     char pci[128];
-    uint32_t pci_map[7 * 8];
+    char msi[128];
+    uint32_t *pci_map = NULL;
+    int len;
     uint32_t pci_ranges[14] =
         {
             0x2000000, 0x0, 0xc0000000,
@@ -139,12 +158,10 @@ static int ppce500_load_device_tree(CPUPPCState *env,
             0x0, 0x10000,
         };
     QemuOpts *machine_opts;
-    const char *dumpdtb = NULL;
     const char *dtb_file = NULL;
 
     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
     if (machine_opts) {
-        dumpdtb = qemu_opt_get(machine_opts, "dumpdtb");
         dtb_file = qemu_opt_get(machine_opts, "dtb");
         toplevel_compat = qemu_opt_get(machine_opts, "dt_compatible");
     }
@@ -209,6 +226,10 @@ static int ppce500_load_device_tree(CPUPPCState *env,
         kvmppc_get_hypercall(env, hypercall, sizeof(hypercall));
         qemu_devtree_setprop(fdt, "/hypervisor", "hcall-instructions",
                              hypercall, sizeof(hypercall));
+        /* if KVM supports the idle hcall, set property indicating this */
+        if (kvmppc_get_hasidle(env)) {
+            qemu_devtree_setprop(fdt, "/hypervisor", "has-idle", NULL, 0);
+        }
     }
 
     /* Create CPU nodes */
@@ -219,25 +240,28 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     /* We need to generate the cpu nodes in reverse order, so Linux can pick
        the first node as boot node and be happy */
     for (i = smp_cpus - 1; i >= 0; i--) {
+        CPUState *cpu = NULL;
         char cpu_name[128];
         uint64_t cpu_release_addr = MPC8544_SPIN_BASE + (i * 0x20);
 
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            if (env->cpu_index == i) {
+            cpu = ENV_GET_CPU(env);
+            if (cpu->cpu_index == i) {
                 break;
             }
         }
 
-        if (!env) {
+        if (cpu == NULL) {
             continue;
         }
 
-        snprintf(cpu_name, sizeof(cpu_name), "/cpus/PowerPC,8544@%x", env->cpu_index);
+        snprintf(cpu_name, sizeof(cpu_name), "/cpus/PowerPC,8544@%x",
+                 cpu->cpu_index);
         qemu_devtree_add_subnode(fdt, cpu_name);
         qemu_devtree_setprop_cell(fdt, cpu_name, "clock-frequency", clock_freq);
         qemu_devtree_setprop_cell(fdt, cpu_name, "timebase-frequency", tb_freq);
         qemu_devtree_setprop_string(fdt, cpu_name, "device_type", "cpu");
-        qemu_devtree_setprop_cell(fdt, cpu_name, "reg", env->cpu_index);
+        qemu_devtree_setprop_cell(fdt, cpu_name, "reg", cpu->cpu_index);
         qemu_devtree_setprop_cell(fdt, cpu_name, "d-cache-line-size",
                                   env->dcache_line_size);
         qemu_devtree_setprop_cell(fdt, cpu_name, "i-cache-line-size",
@@ -245,7 +269,7 @@ static int ppce500_load_device_tree(CPUPPCState *env,
         qemu_devtree_setprop_cell(fdt, cpu_name, "d-cache-size", 0x8000);
         qemu_devtree_setprop_cell(fdt, cpu_name, "i-cache-size", 0x8000);
         qemu_devtree_setprop_cell(fdt, cpu_name, "bus-frequency", 0);
-        if (env->cpu_index) {
+        if (cpu->cpu_index) {
             qemu_devtree_setprop_string(fdt, cpu_name, "status", "disabled");
             qemu_devtree_setprop_string(fdt, cpu_name, "enable-method", "spin-table");
             qemu_devtree_setprop_u64(fdt, cpu_name, "cpu-release-addr",
@@ -270,13 +294,12 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     /* XXX should contain a reasonable value */
     qemu_devtree_setprop_cell(fdt, soc, "bus-frequency", 0);
 
-    snprintf(mpic, sizeof(mpic), "%s/pic@%llx", soc,
-             MPC8544_MPIC_REGS_BASE - MPC8544_CCSRBAR_BASE);
+    snprintf(mpic, sizeof(mpic), "%s/pic@%llx", soc, MPC8544_MPIC_REGS_OFFSET);
     qemu_devtree_add_subnode(fdt, mpic);
     qemu_devtree_setprop_string(fdt, mpic, "device_type", "open-pic");
-    qemu_devtree_setprop_string(fdt, mpic, "compatible", "chrp,open-pic");
-    qemu_devtree_setprop_cells(fdt, mpic, "reg", MPC8544_MPIC_REGS_BASE -
-                               MPC8544_CCSRBAR_BASE, 0x40000);
+    qemu_devtree_setprop_string(fdt, mpic, "compatible", "fsl,mpic");
+    qemu_devtree_setprop_cells(fdt, mpic, "reg", MPC8544_MPIC_REGS_OFFSET,
+                               0x40000);
     qemu_devtree_setprop_cell(fdt, mpic, "#address-cells", 0);
     qemu_devtree_setprop_cell(fdt, mpic, "#interrupt-cells", 2);
     mpic_ph = qemu_devtree_alloc_phandle(fdt);
@@ -289,19 +312,37 @@ static int ppce500_load_device_tree(CPUPPCState *env,
      * device it finds in the dt as serial output device. And we generate
      * devices in reverse order to the dt.
      */
-    dt_serial_create(fdt, MPC8544_SERIAL1_REGS_BASE - MPC8544_CCSRBAR_BASE,
+    dt_serial_create(fdt, MPC8544_SERIAL1_REGS_OFFSET,
                      soc, mpic, "serial1", 1, false);
-    dt_serial_create(fdt, MPC8544_SERIAL0_REGS_BASE - MPC8544_CCSRBAR_BASE,
+    dt_serial_create(fdt, MPC8544_SERIAL0_REGS_OFFSET,
                      soc, mpic, "serial0", 0, true);
 
     snprintf(gutil, sizeof(gutil), "%s/global-utilities@%llx", soc,
-             MPC8544_UTIL_BASE - MPC8544_CCSRBAR_BASE);
+             MPC8544_UTIL_OFFSET);
     qemu_devtree_add_subnode(fdt, gutil);
     qemu_devtree_setprop_string(fdt, gutil, "compatible", "fsl,mpc8544-guts");
-    qemu_devtree_setprop_cells(fdt, gutil, "reg", MPC8544_UTIL_BASE -
-                               MPC8544_CCSRBAR_BASE, 0x1000);
+    qemu_devtree_setprop_cells(fdt, gutil, "reg", MPC8544_UTIL_OFFSET, 0x1000);
     qemu_devtree_setprop(fdt, gutil, "fsl,has-rstcr", NULL, 0);
 
+    snprintf(msi, sizeof(msi), "/%s/msi@%llx", soc, MPC8544_MSI_REGS_OFFSET);
+    qemu_devtree_add_subnode(fdt, msi);
+    qemu_devtree_setprop_string(fdt, msi, "compatible", "fsl,mpic-msi");
+    qemu_devtree_setprop_cells(fdt, msi, "reg", MPC8544_MSI_REGS_OFFSET, 0x200);
+    msi_ph = qemu_devtree_alloc_phandle(fdt);
+    qemu_devtree_setprop_cells(fdt, msi, "msi-available-ranges", 0x0, 0x100);
+    qemu_devtree_setprop_phandle(fdt, msi, "interrupt-parent", mpic);
+    qemu_devtree_setprop_cells(fdt, msi, "interrupts",
+        0xe0, 0x0,
+        0xe1, 0x0,
+        0xe2, 0x0,
+        0xe3, 0x0,
+        0xe4, 0x0,
+        0xe5, 0x0,
+        0xe6, 0x0,
+        0xe7, 0x0);
+    qemu_devtree_setprop_cell(fdt, msi, "phandle", msi_ph);
+    qemu_devtree_setprop_cell(fdt, msi, "linux,phandle", msi_ph);
+
     snprintf(pci, sizeof(pci), "/pci@%llx", MPC8544_PCI_REGS_BASE);
     qemu_devtree_add_subnode(fdt, pci);
     qemu_devtree_setprop_cell(fdt, pci, "cell-index", 0);
@@ -309,14 +350,17 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     qemu_devtree_setprop_string(fdt, pci, "device_type", "pci");
     qemu_devtree_setprop_cells(fdt, pci, "interrupt-map-mask", 0xf800, 0x0,
                                0x0, 0x7);
-    pci_map_create(fdt, pci_map, qemu_devtree_get_phandle(fdt, mpic));
-    qemu_devtree_setprop(fdt, pci, "interrupt-map", pci_map, sizeof(pci_map));
+    pci_map = pci_map_create(fdt, qemu_devtree_get_phandle(fdt, mpic),
+                             params->pci_first_slot, params->pci_nr_slots,
+                             &len);
+    qemu_devtree_setprop(fdt, pci, "interrupt-map", pci_map, len);
     qemu_devtree_setprop_phandle(fdt, pci, "interrupt-parent", mpic);
     qemu_devtree_setprop_cells(fdt, pci, "interrupts", 24, 2);
     qemu_devtree_setprop_cells(fdt, pci, "bus-range", 0, 255);
     for (i = 0; i < 14; i++) {
         pci_ranges[i] = cpu_to_be32(pci_ranges[i]);
     }
+    qemu_devtree_setprop_cell(fdt, pci, "fsl,msi", msi_ph);
     qemu_devtree_setprop(fdt, pci, "ranges", pci_ranges, sizeof(pci_ranges));
     qemu_devtree_setprop_cells(fdt, pci, "reg", MPC8544_PCI_REGS_BASE >> 32,
                                MPC8544_PCI_REGS_BASE, 0, 0x1000);
@@ -334,18 +378,7 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     }
 
 done:
-    if (dumpdtb) {
-        /* Dump the dtb to a file and quit */
-        FILE *f = fopen(dumpdtb, "wb");
-        size_t len;
-        len = fwrite(fdt, fdt_size, 1, f);
-        fclose(f);
-        if (len != fdt_size) {
-            exit(1);
-        }
-        exit(0);
-    }
-
+    qemu_devtree_dumpdtb(fdt, fdt_size);
     ret = rom_add_blob_fixed(BINARY_DEVICE_TREE_FILE, fdt, fdt_size, addr);
     if (ret < 0) {
         goto out;
@@ -354,12 +387,13 @@ done:
     ret = fdt_size;
 
 out:
+    g_free(pci_map);
 
     return ret;
 }
 
 /* Create -kernel TLB entries for BookE.  */
-static inline target_phys_addr_t booke206_page_size_to_tlb(uint64_t size)
+static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
 {
     return 63 - clz64(size >> 10);
 }
@@ -368,13 +402,17 @@ static void mmubooke_create_initial_mapping(CPUPPCState *env)
 {
     struct boot_info *bi = env->load_info;
     ppcmas_tlb_t *tlb = booke206_get_tlbm(env, 1, 0, 0);
-    target_phys_addr_t size, dt_end;
+    hwaddr size, dt_end;
     int ps;
 
     /* Our initial TLB entry needs to cover everything from 0 to
        the device tree top */
     dt_end = bi->dt_base + bi->dt_size;
     ps = booke206_page_size_to_tlb(dt_end) + 1;
+    if (ps & 1) {
+        /* e500v2 can only do even TLB size bits */
+        ps++;
+    }
     size = (ps << MAS1_TSIZE_SHIFT);
     tlb->mas1 = MAS1_VALID | size;
     tlb->mas2 = 0;
@@ -421,17 +459,21 @@ void ppce500_init(PPCE500Params *params)
     CPUPPCState *env = NULL;
     uint64_t elf_entry;
     uint64_t elf_lowaddr;
-    target_phys_addr_t entry=0;
-    target_phys_addr_t loadaddr=UIMAGE_LOAD_BASE;
+    hwaddr entry=0;
+    hwaddr loadaddr=UIMAGE_LOAD_BASE;
     target_long kernel_size=0;
     target_ulong dt_base = 0;
     target_ulong initrd_base = 0;
-    target_long initrd_size=0;
-    int i=0;
+    target_long initrd_size = 0;
+    target_ulong cur_base = 0;
+    int i = 0, j, k;
     unsigned int pci_irq_nrs[4] = {1, 2, 3, 4};
     qemu_irq **irqs, *mpic;
     DeviceState *dev;
     CPUPPCState *firstenv = NULL;
+    MemoryRegion *ccsr_addr_space;
+    SysBusDevice *s;
+    PPCE500CCSRState *ccsr;
 
     /* Setup CPUs */
     if (params->cpu_model == NULL) {
@@ -442,6 +484,7 @@ void ppce500_init(PPCE500Params *params)
     irqs[0] = g_malloc0(smp_cpus * sizeof(qemu_irq) * OPENPIC_OUTPUT_NB);
     for (i = 0; i < smp_cpus; i++) {
         PowerPCCPU *cpu;
+        CPUState *cs;
         qemu_irq *input;
 
         cpu = cpu_ppc_init(params->cpu_model);
@@ -450,6 +493,7 @@ void ppce500_init(PPCE500Params *params)
             exit(1);
         }
         env = &cpu->env;
+        cs = CPU(cpu);
 
         if (!firstenv) {
             firstenv = env;
@@ -459,10 +503,11 @@ void ppce500_init(PPCE500Params *params)
         input = (qemu_irq *)env->irq_inputs;
         irqs[i][OPENPIC_OUTPUT_INT] = input[PPCE500_INPUT_INT];
         irqs[i][OPENPIC_OUTPUT_CINT] = input[PPCE500_INPUT_CINT];
-        env->spr[SPR_BOOKE_PIR] = env->cpu_index = i;
-        env->mpic_cpu_base = MPC8544_MPIC_REGS_BASE + 0x20000;
+        env->spr[SPR_BOOKE_PIR] = cs->cpu_index = i;
+        env->mpic_iack = MPC8544_CCSRBAR_BASE +
+                         MPC8544_MPIC_REGS_OFFSET + 0xa0;
 
-        ppc_booke_timers_init(env, 400000000, PPC_TIMER_E500);
+        ppc_booke_timers_init(cpu, 400000000, PPC_TIMER_E500);
 
         /* Register reset handler */
         if (!i) {
@@ -487,40 +532,74 @@ void ppce500_init(PPCE500Params *params)
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
+    dev = qdev_create(NULL, "e500-ccsr");
+    object_property_add_child(qdev_get_machine(), "e500-ccsr",
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+    ccsr = CCSR(dev);
+    ccsr_addr_space = &ccsr->ccsr_space;
+    memory_region_add_subregion(address_space_mem, MPC8544_CCSRBAR_BASE,
+                                ccsr_addr_space);
+
     /* MPIC */
-    mpic = mpic_init(address_space_mem, MPC8544_MPIC_REGS_BASE,
-                     smp_cpus, irqs, NULL);
+    mpic = g_new(qemu_irq, 256);
+    dev = qdev_create(NULL, "openpic");
+    qdev_prop_set_uint32(dev, "nb_cpus", smp_cpus);
+    qdev_prop_set_uint32(dev, "model", params->mpic_version);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+
+    k = 0;
+    for (i = 0; i < smp_cpus; i++) {
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_connect_irq(s, k++, irqs[i][j]);
+        }
+    }
 
-    if (!mpic) {
-        cpu_abort(env, "MPIC failed to initialize\n");
+    for (i = 0; i < 256; i++) {
+        mpic[i] = qdev_get_gpio_in(dev, i);
     }
 
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_MPIC_REGS_OFFSET,
+                                s->mmio[0].memory);
+
     /* Serial */
     if (serial_hds[0]) {
-        serial_mm_init(address_space_mem, MPC8544_SERIAL0_REGS_BASE,
-                       0, mpic[12+26], 399193,
+        serial_mm_init(ccsr_addr_space, MPC8544_SERIAL0_REGS_OFFSET,
+                       0, mpic[42], 399193,
                        serial_hds[0], DEVICE_BIG_ENDIAN);
     }
 
     if (serial_hds[1]) {
-        serial_mm_init(address_space_mem, MPC8544_SERIAL1_REGS_BASE,
-                       0, mpic[12+26], 399193,
-                       serial_hds[0], DEVICE_BIG_ENDIAN);
+        serial_mm_init(ccsr_addr_space, MPC8544_SERIAL1_REGS_OFFSET,
+                       0, mpic[42], 399193,
+                       serial_hds[1], DEVICE_BIG_ENDIAN);
     }
 
     /* General Utility device */
-    sysbus_create_simple("mpc8544-guts", MPC8544_UTIL_BASE, NULL);
+    dev = qdev_create(NULL, "mpc8544-guts");
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_UTIL_OFFSET,
+                                sysbus_mmio_get_region(s, 0));
 
     /* PCI */
-    dev = sysbus_create_varargs("e500-pcihost", MPC8544_PCI_REGS_BASE,
-                                mpic[pci_irq_nrs[0]], mpic[pci_irq_nrs[1]],
-                                mpic[pci_irq_nrs[2]], mpic[pci_irq_nrs[3]],
-                                NULL);
+    dev = qdev_create(NULL, "e500-pcihost");
+    qdev_prop_set_uint32(dev, "first_slot", params->pci_first_slot);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    sysbus_connect_irq(s, 0, mpic[pci_irq_nrs[0]]);
+    sysbus_connect_irq(s, 1, mpic[pci_irq_nrs[1]]);
+    sysbus_connect_irq(s, 2, mpic[pci_irq_nrs[2]]);
+    sysbus_connect_irq(s, 3, mpic[pci_irq_nrs[3]]);
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_PCI_REGS_OFFSET,
+                                sysbus_mmio_get_region(s, 0));
+
     pci_bus = (PCIBus *)qdev_get_child_bus(dev, "pci.0");
     if (!pci_bus)
         printf("couldn't create PCI controller!\n");
 
-    isa_mmio_init(MPC8544_PCI_IO, MPC8544_PCI_IOLEN);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 1, MPC8544_PCI_IO);
 
     if (pci_bus) {
         /* Register network interfaces. */
@@ -549,11 +628,17 @@ void ppce500_init(PPCE500Params *params)
                     params->kernel_filename);
             exit(1);
         }
+
+        cur_base = loadaddr + kernel_size;
+
+        /* Reserve space for dtb */
+        dt_base = (cur_base + DTC_LOAD_PAD) & ~DTC_PAD_MASK;
+        cur_base += DTB_MAX_SIZE;
     }
 
     /* Load initrd. */
     if (params->initrd_filename) {
-        initrd_base = (kernel_size + INITRD_LOAD_PAD) & ~INITRD_PAD_MASK;
+        initrd_base = (cur_base + INITRD_LOAD_PAD) & ~INITRD_PAD_MASK;
         initrd_size = load_image_targphys(params->initrd_filename, initrd_base,
                                           ram_size - initrd_base);
 
@@ -562,6 +647,8 @@ void ppce500_init(PPCE500Params *params)
                     params->initrd_filename);
             exit(1);
         }
+
+        cur_base = initrd_base + initrd_size;
     }
 
     /* If we're loading a kernel directly, we must load the device tree too. */
@@ -569,13 +656,13 @@ void ppce500_init(PPCE500Params *params)
         struct boot_info *boot_info;
         int dt_size;
 
-        dt_base = (loadaddr + kernel_size + DTC_LOAD_PAD) & ~DTC_PAD_MASK;
         dt_size = ppce500_load_device_tree(env, params, dt_base, initrd_base,
                                            initrd_size);
         if (dt_size < 0) {
             fprintf(stderr, "couldn't load device tree\n");
             exit(1);
         }
+        assert(dt_size < DTB_MAX_SIZE);
 
         boot_info = env->load_info;
         boot_info->entry = entry;
@@ -587,3 +674,33 @@ void ppce500_init(PPCE500Params *params)
         kvmppc_init();
     }
 }
+
+static int e500_ccsr_initfn(SysBusDevice *dev)
+{
+    PPCE500CCSRState *ccsr;
+
+    ccsr = CCSR(dev);
+    memory_region_init(&ccsr->ccsr_space, "e500-ccsr",
+                       MPC8544_CCSRBAR_SIZE);
+    return 0;
+}
+
+static void e500_ccsr_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    k->init = e500_ccsr_initfn;
+}
+
+static const TypeInfo e500_ccsr_info = {
+    .name          = TYPE_CCSR,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PPCE500CCSRState),
+    .class_init    = e500_ccsr_class_init,
+};
+
+static void e500_register_types(void)
+{
+    type_register_static(&e500_ccsr_info);
+}
+
+type_init(e500_register_types)
diff --git a/hw/ppc/e500.h b/hw/ppc/e500.h
index 7ae87f4e214..226c93d248d 100644
--- a/hw/ppc/e500.h
+++ b/hw/ppc/e500.h
@@ -9,11 +9,15 @@ typedef struct PPCE500Params {
     const char *kernel_cmdline;
     const char *initrd_filename;
     const char *cpu_model;
+    int pci_first_slot;
+    int pci_nr_slots;
 
     /* e500-specific params */
 
     /* required -- must at least add toplevel board compatible */
     void (*fixup_devtree)(struct PPCE500Params *params, void *fdt);
+
+    int mpic_version;
 } PPCE500Params;
 
 void ppce500_init(PPCE500Params *params);
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 60a5cb3bd09..25ac4b1daeb 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -13,7 +13,9 @@
 #include "qemu-common.h"
 #include "e500.h"
 #include "../boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "hw/pci/pci.h"
+#include "hw/openpic.h"
 
 static void e500plat_fixup_devtree(PPCE500Params *params, void *fdt)
 {
@@ -25,13 +27,14 @@ static void e500plat_fixup_devtree(PPCE500Params *params, void *fdt)
                          sizeof(compatible));
 }
 
-static void e500plat_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void e500plat_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *boot_device = args->boot_device;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     PPCE500Params params = {
         .ram_size = ram_size,
         .boot_device = boot_device,
@@ -39,7 +42,10 @@ static void e500plat_init(ram_addr_t ram_size,
         .kernel_cmdline = kernel_cmdline,
         .initrd_filename = initrd_filename,
         .cpu_model = cpu_model,
+        .pci_first_slot = 0x1,
+        .pci_nr_slots = PCI_SLOT_MAX - 1,
         .fixup_devtree = e500plat_fixup_devtree,
+        .mpic_version = OPENPIC_MODEL_FSL_MPIC_42,
     };
 
     ppce500_init(&params);
@@ -49,7 +55,8 @@ static QEMUMachine e500plat_machine = {
     .name = "ppce500",
     .desc = "generic paravirt e500 platform",
     .init = e500plat_init,
-    .max_cpus = 15,
+    .max_cpus = 32,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void e500plat_machine_init(void)
diff --git a/hw/ppc/mac.h b/hw/ppc/mac.h
new file mode 100644
index 00000000000..b17107b7973
--- /dev/null
+++ b/hw/ppc/mac.h
@@ -0,0 +1,181 @@
+/*
+ * QEMU PowerMac emulation shared definitions and prototypes
+ *
+ * Copyright (c) 2004-2007 Fabrice Bellard
+ * Copyright (c) 2007 Jocelyn Mayer
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#if !defined(__PPC_MAC_H__)
+#define __PPC_MAC_H__
+
+#include "exec/memory.h"
+#include "hw/sysbus.h"
+#include "hw/ide/internal.h"
+#include "hw/adb.h"
+
+/* SMP is not enabled, for now */
+#define MAX_CPUS 1
+
+#define BIOS_SIZE     (1024 * 1024)
+#define BIOS_FILENAME "ppc_rom.bin"
+#define NVRAM_SIZE        0x2000
+#define PROM_FILENAME    "openbios-ppc"
+#define PROM_ADDR         0xfff00000
+
+#define KERNEL_LOAD_ADDR 0x01000000
+#define KERNEL_GAP       0x00100000
+
+#define ESCC_CLOCK 3686400
+
+/* Cuda */
+#define TYPE_CUDA "cuda"
+#define CUDA(obj) OBJECT_CHECK(CUDAState, (obj), TYPE_CUDA)
+
+/**
+ * CUDATimer:
+ * @counter_value: counter value at load time
+ */
+typedef struct CUDATimer {
+    int index;
+    uint16_t latch;
+    uint16_t counter_value;
+    int64_t load_time;
+    int64_t next_irq_time;
+    QEMUTimer *timer;
+} CUDATimer;
+
+/**
+ * CUDAState:
+ * @b: B-side data
+ * @a: A-side data
+ * @dirb: B-side direction (1=output)
+ * @dira: A-side direction (1=output)
+ * @sr: Shift register
+ * @acr: Auxiliary control register
+ * @pcr: Peripheral control register
+ * @ifr: Interrupt flag register
+ * @ier: Interrupt enable register
+ * @anh: A-side data, no handshake
+ * @last_b: last value of B register
+ * @last_acr: last value of ACR register
+ */
+typedef struct CUDAState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion mem;
+    /* cuda registers */
+    uint8_t b;
+    uint8_t a;
+    uint8_t dirb;
+    uint8_t dira;
+    uint8_t sr;
+    uint8_t acr;
+    uint8_t pcr;
+    uint8_t ifr;
+    uint8_t ier;
+    uint8_t anh;
+
+    ADBBusState adb_bus;
+    CUDATimer timers[2];
+
+    uint32_t tick_offset;
+
+    uint8_t last_b;
+    uint8_t last_acr;
+
+    int data_in_size;
+    int data_in_index;
+    int data_out_index;
+
+    qemu_irq irq;
+    uint8_t autopoll;
+    uint8_t data_in[128];
+    uint8_t data_out[16];
+    QEMUTimer *adb_poll_timer;
+} CUDAState;
+
+/* MacIO */
+#define TYPE_OLDWORLD_MACIO "macio-oldworld"
+#define TYPE_NEWWORLD_MACIO "macio-newworld"
+
+#define TYPE_MACIO_IDE "macio-ide"
+#define MACIO_IDE(obj) OBJECT_CHECK(MACIOIDEState, (obj), TYPE_MACIO_IDE)
+
+typedef struct MACIOIDEState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    qemu_irq irq;
+    qemu_irq dma_irq;
+
+    MemoryRegion mem;
+    IDEBus bus;
+    BlockDriverAIOCB *aiocb;
+} MACIOIDEState;
+
+void macio_ide_init_drives(MACIOIDEState *ide, DriveInfo **hd_table);
+void macio_ide_register_dma(MACIOIDEState *ide, void *dbdma, int channel);
+
+void macio_init(PCIDevice *dev,
+                MemoryRegion *pic_mem,
+                MemoryRegion *escc_mem);
+
+/* Heathrow PIC */
+qemu_irq *heathrow_pic_init(MemoryRegion **pmem,
+                            int nb_cpus, qemu_irq **irqs);
+
+/* Grackle PCI */
+#define TYPE_GRACKLE_PCI_HOST_BRIDGE "grackle-pcihost"
+PCIBus *pci_grackle_init(uint32_t base, qemu_irq *pic,
+                         MemoryRegion *address_space_mem,
+                         MemoryRegion *address_space_io);
+
+/* UniNorth PCI */
+PCIBus *pci_pmac_init(qemu_irq *pic,
+                      MemoryRegion *address_space_mem,
+                      MemoryRegion *address_space_io);
+PCIBus *pci_pmac_u3_init(qemu_irq *pic,
+                         MemoryRegion *address_space_mem,
+                         MemoryRegion *address_space_io);
+
+/* Mac NVRAM */
+#define TYPE_MACIO_NVRAM "macio-nvram"
+#define MACIO_NVRAM(obj) \
+    OBJECT_CHECK(MacIONVRAMState, (obj), TYPE_MACIO_NVRAM)
+
+typedef struct MacIONVRAMState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    uint32_t size;
+    uint32_t it_shift;
+
+    MemoryRegion mem;
+    uint8_t *data;
+} MacIONVRAMState;
+
+void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
+uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr);
+void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val);
+#endif /* !defined(__PPC_MAC_H__) */
diff --git a/hw/ppc_newworld.c b/hw/ppc/mac_newworld.c
index e95cfe831e2..065ea871b37 100644
--- a/hw/ppc_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -46,28 +46,28 @@
  * 0001:05:0c.0 IDE interface [0101]: Broadcom K2 SATA [1166:0240]
  *
  */
-#include "hw.h"
-#include "ppc.h"
-#include "ppc_mac.h"
-#include "adb.h"
-#include "mac_dbdma.h"
-#include "nvram.h"
-#include "pci.h"
-#include "net.h"
-#include "sysemu.h"
-#include "boards.h"
-#include "fw_cfg.h"
-#include "escc.h"
-#include "openpic.h"
-#include "ide.h"
-#include "loader.h"
+#include "hw/hw.h"
+#include "hw/ppc.h"
+#include "hw/ppc/mac.h"
+#include "hw/adb.h"
+#include "hw/mac_dbdma.h"
+#include "hw/nvram.h"
+#include "hw/pci/pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/boards.h"
+#include "hw/fw_cfg.h"
+#include "hw/escc.h"
+#include "hw/openpic.h"
+#include "hw/ide.h"
+#include "hw/loader.h"
 #include "elf.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "hw/usb.h"
-#include "blockdev.h"
-#include "exec-memory.h"
-#include "vga-pci.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -83,13 +83,13 @@
 #endif
 
 /* UniN device */
-static void unin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void unin_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     UNIN_DPRINTF("write addr " TARGET_FMT_plx " val %"PRIx64"\n", addr, value);
 }
 
-static uint64_t unin_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t unin_read(void *opaque, hwaddr addr, unsigned size)
 {
     uint32_t value;
 
@@ -116,7 +116,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static target_phys_addr_t round_page(target_phys_addr_t addr)
+static hwaddr round_page(hwaddr addr)
 {
     return (addr + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
 }
@@ -129,33 +129,37 @@ static void ppc_core99_reset(void *opaque)
 }
 
 /* PowerPC Mac99 hardware initialisation */
-static void ppc_core99_init (ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void ppc_core99_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
     char *filename;
     qemu_irq *pic, **openpic_irqs;
     MemoryRegion *unin_memory = g_new(MemoryRegion, 1);
-    int linux_boot, i;
+    int linux_boot, i, j, k;
     MemoryRegion *ram = g_new(MemoryRegion, 1), *bios = g_new(MemoryRegion, 1);
-    target_phys_addr_t kernel_base, initrd_base, cmdline_base = 0;
+    hwaddr kernel_base, initrd_base, cmdline_base = 0;
     long kernel_size, initrd_size;
     PCIBus *pci_bus;
+    PCIDevice *macio;
+    MACIOIDEState *macio_ide;
+    BusState *adb_bus;
     MacIONVRAMState *nvr;
     int bios_size;
-    MemoryRegion *pic_mem, *dbdma_mem, *cuda_mem, *escc_mem;
+    MemoryRegion *pic_mem, *escc_mem;
     MemoryRegion *escc_bar = g_new(MemoryRegion, 1);
-    MemoryRegion *ide_mem[3];
     int ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     void *fw_cfg;
-    void *dbdma;
     int machine_arch;
+    SysBusDevice *s;
+    DeviceState *dev;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -320,7 +324,25 @@ static void ppc_core99_init (ram_addr_t ram_size,
             exit(1);
         }
     }
-    pic = openpic_init(&pic_mem, smp_cpus, openpic_irqs, NULL);
+
+    pic = g_new(qemu_irq, 64);
+
+    dev = qdev_create(NULL, "openpic");
+    qdev_prop_set_uint32(dev, "model", OPENPIC_MODEL_RAVEN);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    pic_mem = s->mmio[0].memory;
+    k = 0;
+    for (i = 0; i < smp_cpus; i++) {
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_connect_irq(s, k++, openpic_irqs[i][j]);
+        }
+    }
+
+    for (i = 0; i < 64; i++) {
+        pic[i] = qdev_get_gpio_in(dev, i);
+    }
+
     if (PPC_INPUT(env) == PPC_FLAGS_INPUT_970) {
         /* 970 gets a U3 bus */
         pci_bus = pci_pmac_u3_init(pic, get_system_memory(), get_system_io());
@@ -341,46 +363,57 @@ static void ppc_core99_init (ram_addr_t ram_size,
         pci_nic_init_nofail(&nd_table[i], "ne2k_pci", NULL);
 
     ide_drive_get(hd, MAX_IDE_BUS);
-    dbdma = DBDMA_init(&dbdma_mem);
 
-    /* We only emulate 2 out of 3 IDE controllers for now */
-    ide_mem[0] = NULL;
-    ide_mem[1] = pmac_ide_init(hd, pic[0x0d], dbdma, 0x16, pic[0x02]);
-    ide_mem[2] = pmac_ide_init(&hd[MAX_IDE_DEVS], pic[0x0e], dbdma, 0x1a, pic[0x02]);
+    macio = pci_create(pci_bus, -1, TYPE_NEWWORLD_MACIO);
+    dev = DEVICE(macio);
+    qdev_connect_gpio_out(dev, 0, pic[0x19]); /* CUDA */
+    qdev_connect_gpio_out(dev, 1, pic[0x0d]); /* IDE */
+    qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE DMA */
+    qdev_connect_gpio_out(dev, 3, pic[0x0e]); /* IDE */
+    qdev_connect_gpio_out(dev, 4, pic[0x02]); /* IDE DMA */
+    macio_init(macio, pic_mem, escc_bar);
 
-    /* cuda also initialize ADB */
-    if (machine_arch == ARCH_MAC99_U3) {
-        usb_enabled = 1;
-    }
-    cuda_init(&cuda_mem, pic[0x19]);
-
-    adb_kbd_init(&adb_bus);
-    adb_mouse_init(&adb_bus);
-
-    macio_init(pci_bus, PCI_DEVICE_ID_APPLE_UNI_N_KEYL, 0, pic_mem,
-               dbdma_mem, cuda_mem, NULL, 3, ide_mem, escc_bar);
-
-    if (usb_enabled) {
+    /* We only emulate 2 out of 3 IDE controllers for now */
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide[0]"));
+    macio_ide_init_drives(macio_ide, hd);
+
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide[1]"));
+    macio_ide_init_drives(macio_ide, &hd[MAX_IDE_DEVS]);
+
+    dev = DEVICE(object_resolve_path_component(OBJECT(macio), "cuda"));
+    adb_bus = qdev_get_child_bus(dev, "adb.0");
+    dev = qdev_create(adb_bus, TYPE_ADB_KEYBOARD);
+    qdev_init_nofail(dev);
+    dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
+    qdev_init_nofail(dev);
+
+    if (usb_enabled(machine_arch == ARCH_MAC99_U3)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
-    }
-
-    /* U3 needs to use USB for input because Linux doesn't support via-cuda
-       on PPC64 */
-    if (machine_arch == ARCH_MAC99_U3) {
-        usbdevice_create("keyboard");
-        usbdevice_create("mouse");
+        /* U3 needs to use USB for input because Linux doesn't support via-cuda
+        on PPC64 */
+        if (machine_arch == ARCH_MAC99_U3) {
+            usbdevice_create("keyboard");
+            usbdevice_create("mouse");
+        }
     }
 
     if (graphic_depth != 15 && graphic_depth != 32 && graphic_depth != 8)
         graphic_depth = 15;
 
     /* The NewWorld NVRAM is not located in the MacIO device */
-    nvr = macio_nvram_init(0x2000, 1);
+    dev = qdev_create(NULL, TYPE_MACIO_NVRAM);
+    qdev_prop_set_uint32(dev, "size", 0x2000);
+    qdev_prop_set_uint32(dev, "it_shift", 1);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xFFF04000);
+    nvr = MACIO_NVRAM(dev);
     pmac_format_nvram_partition(nvr, 0x2000);
-    macio_nvram_setup_bar(nvr, get_system_memory(), 0xFFF04000);
     /* No PCI init: the BIOS will do it */
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, machine_arch);
@@ -426,6 +459,7 @@ static QEMUMachine core99_machine = {
 #ifdef TARGET_PPC64
     .is_default = 1,
 #endif
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void core99_machine_init(void)
diff --git a/hw/ppc_oldworld.c b/hw/ppc/mac_oldworld.c
index 1dcd8a6c36d..2778e45879d 100644
--- a/hw/ppc_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -23,27 +23,25 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "ppc.h"
-#include "ppc_mac.h"
-#include "adb.h"
-#include "mac_dbdma.h"
-#include "nvram.h"
-#include "sysemu.h"
-#include "net.h"
-#include "isa.h"
-#include "pci.h"
-#include "boards.h"
-#include "fw_cfg.h"
-#include "escc.h"
-#include "ide.h"
-#include "loader.h"
+#include "hw/hw.h"
+#include "hw/ppc.h"
+#include "mac.h"
+#include "hw/adb.h"
+#include "hw/nvram.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
+#include "hw/isa.h"
+#include "hw/pci/pci.h"
+#include "hw/boards.h"
+#include "hw/fw_cfg.h"
+#include "hw/escc.h"
+#include "hw/ide.h"
+#include "hw/loader.h"
 #include "elf.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "blockdev.h"
-#include "exec-memory.h"
-#include "vga-pci.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -60,7 +58,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static target_phys_addr_t round_page(target_phys_addr_t addr)
+static hwaddr round_page(hwaddr addr)
 {
     return (addr + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
 }
@@ -72,13 +70,14 @@ static void ppc_heathrow_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void ppc_heathrow_init (ram_addr_t ram_size,
-                               const char *boot_device,
-                               const char *kernel_filename,
-                               const char *kernel_cmdline,
-                               const char *initrd_filename,
-                               const char *cpu_model)
+static void ppc_heathrow_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
@@ -90,14 +89,16 @@ static void ppc_heathrow_init (ram_addr_t ram_size,
     uint32_t kernel_base, initrd_base, cmdline_base = 0;
     int32_t kernel_size, initrd_size;
     PCIBus *pci_bus;
-    MacIONVRAMState *nvr;
+    PCIDevice *macio;
+    MACIOIDEState *macio_ide;
+    DeviceState *dev;
+    BusState *adb_bus;
     int bios_size;
-    MemoryRegion *pic_mem, *dbdma_mem, *cuda_mem;
-    MemoryRegion *escc_mem, *escc_bar = g_new(MemoryRegion, 1), *ide_mem[2];
+    MemoryRegion *pic_mem;
+    MemoryRegion *escc_mem, *escc_bar = g_new(MemoryRegion, 1);
     uint16_t ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     void *fw_cfg;
-    void *dbdma;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -263,10 +264,17 @@ static void ppc_heathrow_init (ram_addr_t ram_size,
 
     ide_drive_get(hd, MAX_IDE_BUS);
 
+    macio = pci_create(pci_bus, -1, TYPE_OLDWORLD_MACIO);
+    dev = DEVICE(macio);
+    qdev_connect_gpio_out(dev, 0, pic[0x12]); /* CUDA */
+    qdev_connect_gpio_out(dev, 1, pic[0x0D]); /* IDE */
+    qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE DMA */
+    macio_init(macio, pic_mem, escc_bar);
+
     /* First IDE channel is a MAC IDE on the MacIO bus */
-    dbdma = DBDMA_init(&dbdma_mem);
-    ide_mem[0] = NULL;
-    ide_mem[1] = pmac_ide_init(hd, pic[0x0D], dbdma, 0x16, pic[0x02]);
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide"));
+    macio_ide_init_drives(macio_ide, hd);
 
     /* Second IDE channel is a CMD646 on the PCI bus */
     hd[0] = hd[MAX_IDE_DEVS];
@@ -274,19 +282,14 @@ static void ppc_heathrow_init (ram_addr_t ram_size,
     hd[3] = hd[2] = NULL;
     pci_cmd646_ide_init(pci_bus, hd, 0);
 
-    /* cuda also initialize ADB */
-    cuda_init(&cuda_mem, pic[0x12]);
-
-    adb_kbd_init(&adb_bus);
-    adb_mouse_init(&adb_bus);
-
-    nvr = macio_nvram_init(0x2000, 4);
-    pmac_format_nvram_partition(nvr, 0x2000);
-
-    macio_init(pci_bus, PCI_DEVICE_ID_APPLE_343S1201, 1, pic_mem,
-               dbdma_mem, cuda_mem, nvr, 2, ide_mem, escc_bar);
+    dev = DEVICE(object_resolve_path_component(OBJECT(macio), "cuda"));
+    adb_bus = qdev_get_child_bus(dev, "adb.0");
+    dev = qdev_create(adb_bus, TYPE_ADB_KEYBOARD);
+    qdev_init_nofail(dev);
+    dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
+    qdev_init_nofail(dev);
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
@@ -296,6 +299,7 @@ static void ppc_heathrow_init (ram_addr_t ram_size,
     /* No PCI init: the BIOS will do it */
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, ARCH_HEATHROW);
@@ -341,6 +345,7 @@ static QEMUMachine heathrow_machine = {
 #ifndef TARGET_PPC64
     .is_default = 1,
 #endif
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void heathrow_machine_init(void)
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index 984d21cbf5f..e25c70b1f31 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -13,7 +13,8 @@
 #include "qemu-common.h"
 #include "e500.h"
 #include "../boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "hw/openpic.h"
 
 static void mpc8544ds_fixup_devtree(PPCE500Params *params, void *fdt)
 {
@@ -25,13 +26,14 @@ static void mpc8544ds_fixup_devtree(PPCE500Params *params, void *fdt)
                          sizeof(compatible));
 }
 
-static void mpc8544ds_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void mpc8544ds_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *boot_device = args->boot_device;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     PPCE500Params params = {
         .ram_size = ram_size,
         .boot_device = boot_device,
@@ -39,7 +41,10 @@ static void mpc8544ds_init(ram_addr_t ram_size,
         .kernel_cmdline = kernel_cmdline,
         .initrd_filename = initrd_filename,
         .cpu_model = cpu_model,
+        .pci_first_slot = 0x11,
+        .pci_nr_slots = 2,
         .fixup_devtree = mpc8544ds_fixup_devtree,
+        .mpic_version = OPENPIC_MODEL_FSL_MPIC_20,
     };
 
     ppce500_init(&params);
@@ -51,6 +56,7 @@ static QEMUMachine ppce500_machine = {
     .desc = "mpc8544ds",
     .init = mpc8544ds_init,
     .max_cpus = 15,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void ppce500_machine_init(void)
diff --git a/hw/ppc405.h b/hw/ppc405.h
index 1f5dc5fd362..535cbfb3390 100644
--- a/hw/ppc405.h
+++ b/hw/ppc405.h
@@ -61,20 +61,20 @@ ram_addr_t ppc405_set_bootinfo (CPUPPCState *env, ppc4xx_bd_info_t *bd,
 
 CPUPPCState *ppc405cr_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[4],
-                        target_phys_addr_t ram_bases[4],
-                        target_phys_addr_t ram_sizes[4],
+                        hwaddr ram_bases[4],
+                        hwaddr ram_sizes[4],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init);
 CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[2],
-                        target_phys_addr_t ram_bases[2],
-                        target_phys_addr_t ram_sizes[2],
+                        hwaddr ram_bases[2],
+                        hwaddr ram_sizes[2],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init);
 /* IBM STBxxx microcontrollers */
 CPUPPCState *ppc_stb025_init (MemoryRegion ram_memories[2],
-                           target_phys_addr_t ram_bases[2],
-                           target_phys_addr_t ram_sizes[2],
+                           hwaddr ram_bases[2],
+                           hwaddr ram_sizes[2],
                            uint32_t sysclk, qemu_irq **picp,
                            ram_addr_t *offsetp);
 
diff --git a/hw/ppc405_boards.c b/hw/ppc405_boards.c
index 476775d05b3..cf371db053a 100644
--- a/hw/ppc405_boards.c
+++ b/hw/ppc405_boards.c
@@ -26,13 +26,13 @@
 #include "ppc405.h"
 #include "nvram.h"
 #include "flash.h"
-#include "sysemu.h"
-#include "block.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
 #include "boards.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME "ppc405_rom.bin"
 #define BIOS_SIZE (2048 * 1024)
@@ -60,7 +60,7 @@ struct ref405ep_fpga_t {
     uint8_t reg1;
 };
 
-static uint32_t ref405ep_fpga_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readb (void *opaque, hwaddr addr)
 {
     ref405ep_fpga_t *fpga;
     uint32_t ret;
@@ -82,7 +82,7 @@ static uint32_t ref405ep_fpga_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writeb (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_t *fpga;
 
@@ -99,7 +99,7 @@ static void ref405ep_fpga_writeb (void *opaque,
     }
 }
 
-static uint32_t ref405ep_fpga_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -110,13 +110,13 @@ static uint32_t ref405ep_fpga_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writew (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_writeb(opaque, addr, (value >> 8) & 0xFF);
     ref405ep_fpga_writeb(opaque, addr + 1, value & 0xFF);
 }
 
-static uint32_t ref405ep_fpga_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -129,7 +129,7 @@ static uint32_t ref405ep_fpga_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writel (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_writeb(opaque, addr, (value >> 24) & 0xFF);
     ref405ep_fpga_writeb(opaque, addr + 1, (value >> 16) & 0xFF);
@@ -158,7 +158,7 @@ static void ref405ep_fpga_reset (void *opaque)
     fpga->reg1 = 0x0F;
 }
 
-static void ref405ep_fpga_init (MemoryRegion *sysmem, uint32_t base)
+static void ref405ep_fpga_init(MemoryRegion *sysmem, uint32_t base)
 {
     ref405ep_fpga_t *fpga;
     MemoryRegion *fpga_memory = g_new(MemoryRegion, 1);
@@ -170,13 +170,12 @@ static void ref405ep_fpga_init (MemoryRegion *sysmem, uint32_t base)
     qemu_register_reset(&ref405ep_fpga_reset, fpga);
 }
 
-static void ref405ep_init (ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ref405ep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     ppc4xx_bd_info_t bd;
     CPUPPCState *env;
@@ -185,7 +184,7 @@ static void ref405ep_init (ram_addr_t ram_size,
     MemoryRegion *sram = g_new(MemoryRegion, 1);
     ram_addr_t bdloc;
     MemoryRegion *ram_memories = g_malloc(2 * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[2], ram_sizes[2];
+    hwaddr ram_bases[2], ram_sizes[2];
     target_ulong sram_size;
     long bios_size;
     //int phy_addr = 0;
@@ -363,6 +362,7 @@ static QEMUMachine ref405ep_machine = {
     .name = "ref405ep",
     .desc = "ref405ep",
     .init = ref405ep_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 /*****************************************************************************/
@@ -390,7 +390,7 @@ struct taihu_cpld_t {
     uint8_t reg1;
 };
 
-static uint32_t taihu_cpld_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
 {
     taihu_cpld_t *cpld;
     uint32_t ret;
@@ -412,7 +412,7 @@ static uint32_t taihu_cpld_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_t *cpld;
 
@@ -429,7 +429,7 @@ static void taihu_cpld_writeb (void *opaque,
     }
 }
 
-static uint32_t taihu_cpld_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -440,13 +440,13 @@ static uint32_t taihu_cpld_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_writeb(opaque, addr, (value >> 8) & 0xFF);
     taihu_cpld_writeb(opaque, addr + 1, value & 0xFF);
 }
 
-static uint32_t taihu_cpld_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -459,7 +459,7 @@ static uint32_t taihu_cpld_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_writel(opaque, addr, (value >> 24) & 0xFF);
     taihu_cpld_writel(opaque, addr + 1, (value >> 16) & 0xFF);
@@ -484,7 +484,7 @@ static void taihu_cpld_reset (void *opaque)
     cpld->reg1 = 0x80;
 }
 
-static void taihu_cpld_init (MemoryRegion *sysmem, uint32_t base)
+static void taihu_cpld_init(MemoryRegion *sysmem, uint32_t base)
 {
     taihu_cpld_t *cpld;
     MemoryRegion *cpld_memory = g_new(MemoryRegion, 1);
@@ -495,19 +495,17 @@ static void taihu_cpld_init (MemoryRegion *sysmem, uint32_t base)
     qemu_register_reset(&taihu_cpld_reset, cpld);
 }
 
-static void taihu_405ep_init(ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void taihu_405ep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *kernel_filename = args->kernel_filename;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     qemu_irq *pic;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *bios;
     MemoryRegion *ram_memories = g_malloc(2 * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[2], ram_sizes[2];
+    hwaddr ram_bases[2], ram_sizes[2];
     long bios_size;
     target_ulong kernel_base, initrd_base;
     long kernel_size, initrd_size;
@@ -652,6 +650,7 @@ static QEMUMachine taihu_machine = {
     .name = "taihu",
     .desc = "taihu",
     .init = taihu_405ep_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void ppc405_machine_init(void)
diff --git a/hw/ppc405_uc.c b/hw/ppc405_uc.c
index 89e5013b577..c96d103d1c4 100644
--- a/hw/ppc405_uc.c
+++ b/hw/ppc405_uc.c
@@ -24,11 +24,11 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc405.h"
-#include "pc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "serial.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 #define DEBUG_OPBA
 #define DEBUG_SDRAM
@@ -191,7 +191,8 @@ enum {
 typedef struct ppc4xx_pob_t ppc4xx_pob_t;
 struct ppc4xx_pob_t {
     uint32_t bear;
-    uint32_t besr[2];
+    uint32_t besr0;
+    uint32_t besr1;
 };
 
 static uint32_t dcr_read_pob (void *opaque, int dcrn)
@@ -205,8 +206,10 @@ static uint32_t dcr_read_pob (void *opaque, int dcrn)
         ret = pob->bear;
         break;
     case POB0_BESR0:
+        ret = pob->besr0;
+        break;
     case POB0_BESR1:
-        ret = pob->besr[dcrn - POB0_BESR0];
+        ret = pob->besr1;
         break;
     default:
         /* Avoid gcc warning */
@@ -227,9 +230,12 @@ static void dcr_write_pob (void *opaque, int dcrn, uint32_t val)
         /* Read only */
         break;
     case POB0_BESR0:
+        /* Write-clear */
+        pob->besr0 &= ~val;
+        break;
     case POB0_BESR1:
         /* Write-clear */
-        pob->besr[dcrn - POB0_BESR0] &= ~val;
+        pob->besr1 &= ~val;
         break;
     }
 }
@@ -241,8 +247,8 @@ static void ppc4xx_pob_reset (void *opaque)
     pob = opaque;
     /* No error */
     pob->bear = 0x00000000;
-    pob->besr[0] = 0x0000000;
-    pob->besr[1] = 0x0000000;
+    pob->besr0 = 0x0000000;
+    pob->besr1 = 0x0000000;
 }
 
 static void ppc4xx_pob_init(CPUPPCState *env)
@@ -265,7 +271,7 @@ struct ppc4xx_opba_t {
     uint8_t pr;
 };
 
-static uint32_t opba_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readb (void *opaque, hwaddr addr)
 {
     ppc4xx_opba_t *opba;
     uint32_t ret;
@@ -290,7 +296,7 @@ static uint32_t opba_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writeb (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
     ppc4xx_opba_t *opba;
 
@@ -311,7 +317,7 @@ static void opba_writeb (void *opaque,
     }
 }
 
-static uint32_t opba_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -325,7 +331,7 @@ static uint32_t opba_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writew (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_OPBA
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -335,7 +341,7 @@ static void opba_writew (void *opaque,
     opba_writeb(opaque, addr + 1, value);
 }
 
-static uint32_t opba_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -349,7 +355,7 @@ static uint32_t opba_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writel (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_OPBA
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -376,7 +382,7 @@ static void ppc4xx_opba_reset (void *opaque)
     opba->pr = 0x11;
 }
 
-static void ppc4xx_opba_init(target_phys_addr_t base)
+static void ppc4xx_opba_init(hwaddr base)
 {
     ppc4xx_opba_t *opba;
 
@@ -732,7 +738,7 @@ struct ppc405_gpio_t {
     uint32_t isr1l;
 };
 
-static uint32_t ppc405_gpio_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readb (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -742,7 +748,7 @@ static uint32_t ppc405_gpio_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writeb (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -750,7 +756,7 @@ static void ppc405_gpio_writeb (void *opaque,
 #endif
 }
 
-static uint32_t ppc405_gpio_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readw (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -760,7 +766,7 @@ static uint32_t ppc405_gpio_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writew (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -768,7 +774,7 @@ static void ppc405_gpio_writew (void *opaque,
 #endif
 }
 
-static uint32_t ppc405_gpio_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readl (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -778,7 +784,7 @@ static uint32_t ppc405_gpio_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writel (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -798,7 +804,7 @@ static void ppc405_gpio_reset (void *opaque)
 {
 }
 
-static void ppc405_gpio_init(target_phys_addr_t base)
+static void ppc405_gpio_init(hwaddr base)
 {
     ppc405_gpio_t *gpio;
 
@@ -1004,7 +1010,7 @@ struct ppc4xx_i2c_t {
     uint8_t directcntl;
 };
 
-static uint32_t ppc4xx_i2c_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readb (void *opaque, hwaddr addr)
 {
     ppc4xx_i2c_t *i2c;
     uint32_t ret;
@@ -1072,7 +1078,7 @@ static uint32_t ppc4xx_i2c_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     ppc4xx_i2c_t *i2c;
 
@@ -1131,7 +1137,7 @@ static void ppc4xx_i2c_writeb (void *opaque,
     }
 }
 
-static uint32_t ppc4xx_i2c_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -1145,7 +1151,7 @@ static uint32_t ppc4xx_i2c_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1155,7 +1161,7 @@ static void ppc4xx_i2c_writew (void *opaque,
     ppc4xx_i2c_writeb(opaque, addr + 1, value);
 }
 
-static uint32_t ppc4xx_i2c_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -1171,7 +1177,7 @@ static uint32_t ppc4xx_i2c_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1207,7 +1213,7 @@ static void ppc4xx_i2c_reset (void *opaque)
     i2c->directcntl = 0x0F;
 }
 
-static void ppc405_i2c_init(target_phys_addr_t base, qemu_irq irq)
+static void ppc405_i2c_init(hwaddr base, qemu_irq irq)
 {
     ppc4xx_i2c_t *i2c;
 
@@ -1239,7 +1245,7 @@ struct ppc4xx_gpt_t {
     uint32_t mask[5];
 };
 
-static uint32_t ppc4xx_gpt_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readb (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPT
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -1249,7 +1255,7 @@ static uint32_t ppc4xx_gpt_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1258,7 +1264,7 @@ static void ppc4xx_gpt_writeb (void *opaque,
     /* XXX: generate a bus fault */
 }
 
-static uint32_t ppc4xx_gpt_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readw (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPT
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -1268,7 +1274,7 @@ static uint32_t ppc4xx_gpt_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1329,7 +1335,7 @@ static void ppc4xx_gpt_compute_timer (ppc4xx_gpt_t *gpt)
     /* XXX: TODO */
 }
 
-static uint32_t ppc4xx_gpt_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readl (void *opaque, hwaddr addr)
 {
     ppc4xx_gpt_t *gpt;
     uint32_t ret;
@@ -1385,7 +1391,7 @@ static uint32_t ppc4xx_gpt_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     ppc4xx_gpt_t *gpt;
     int idx;
@@ -1482,7 +1488,7 @@ static void ppc4xx_gpt_reset (void *opaque)
     }
 }
 
-static void ppc4xx_gpt_init(target_phys_addr_t base, qemu_irq irqs[5])
+static void ppc4xx_gpt_init(hwaddr base, qemu_irq irqs[5])
 {
     ppc4xx_gpt_t *gpt;
     int i;
@@ -2098,19 +2104,21 @@ static void ppc405cr_cpc_init (CPUPPCState *env, clk_setup_t clk_setup[7],
 
 CPUPPCState *ppc405cr_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[4],
-                        target_phys_addr_t ram_bases[4],
-                        target_phys_addr_t ram_sizes[4],
+                        hwaddr ram_bases[4],
+                        hwaddr ram_sizes[4],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init)
 {
     clk_setup_t clk_setup[PPC405CR_CLK_NB];
     qemu_irq dma_irqs[4];
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     qemu_irq *pic, *irqs;
 
     memset(clk_setup, 0, sizeof(clk_setup));
-    env = ppc4xx_init("405cr", &clk_setup[PPC405CR_CPU_CLK],
+    cpu = ppc4xx_init("405cr", &clk_setup[PPC405CR_CPU_CLK],
                       &clk_setup[PPC405CR_TMR_CLK], sysclk);
+    env = &cpu->env;
     /* Memory mapped devices registers */
     /* PLB arbitrer */
     ppc4xx_plb_init(env);
@@ -2447,20 +2455,22 @@ static void ppc405ep_cpc_init (CPUPPCState *env, clk_setup_t clk_setup[8],
 
 CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[2],
-                        target_phys_addr_t ram_bases[2],
-                        target_phys_addr_t ram_sizes[2],
+                        hwaddr ram_bases[2],
+                        hwaddr ram_sizes[2],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init)
 {
     clk_setup_t clk_setup[PPC405EP_CLK_NB], tlb_clk_setup;
     qemu_irq dma_irqs[4], gpt_irqs[5], mal_irqs[4];
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     qemu_irq *pic, *irqs;
 
     memset(clk_setup, 0, sizeof(clk_setup));
     /* init CPUs */
-    env = ppc4xx_init("405ep", &clk_setup[PPC405EP_CPU_CLK],
+    cpu = ppc4xx_init("405ep", &clk_setup[PPC405EP_CPU_CLK],
                       &tlb_clk_setup, sysclk);
+    env = &cpu->env;
     clk_setup[PPC405EP_CPU_CLK].cb = tlb_clk_setup.cb;
     clk_setup[PPC405EP_CPU_CLK].opaque = tlb_clk_setup.opaque;
     /* Internal devices init */
@@ -2472,7 +2482,7 @@ CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
     /* OBP arbitrer */
     ppc4xx_opba_init(0xef600600);
     /* Initialize timers */
-    ppc_booke_timers_init(env, sysclk, 0);
+    ppc_booke_timers_init(cpu, sysclk, 0);
     /* Universal interrupt controller */
     irqs = g_malloc0(sizeof(qemu_irq) * PPCUIC_OUTPUT_NB);
     irqs[PPCUIC_OUTPUT_INT] =
diff --git a/hw/ppc440_bamboo.c b/hw/ppc440_bamboo.c
index c198071170a..73b5ac725c1 100644
--- a/hw/ppc440_bamboo.c
+++ b/hw/ppc440_bamboo.c
@@ -13,20 +13,20 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "net.h"
+#include "net/net.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "boards.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
-#include "pc.h"
+#include "exec/address-spaces.h"
+#include "serial.h"
 #include "ppc.h"
 #include "ppc405.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 
 #define BINARY_DEVICE_TREE_FILE "bamboo.dtb"
@@ -49,17 +49,17 @@ static const unsigned int ppc440ep_sdram_bank_sizes[] = {
     256<<20, 128<<20, 64<<20, 32<<20, 16<<20, 8<<20, 0
 };
 
-static target_phys_addr_t entry;
+static hwaddr entry;
 
-static int bamboo_load_device_tree(target_phys_addr_t addr,
+static int bamboo_load_device_tree(hwaddr addr,
                                      uint32_t ramsize,
-                                     target_phys_addr_t initrd_base,
-                                     target_phys_addr_t initrd_size,
+                                     hwaddr initrd_base,
+                                     hwaddr initrd_size,
                                      const char *kernel_cmdline)
 {
     int ret = -1;
 #ifdef CONFIG_FDT
-    uint32_t mem_reg_property[] = { 0, 0, ramsize };
+    uint32_t mem_reg_property[] = { 0, 0, cpu_to_be32(ramsize) };
     char *filename;
     int fdt_size;
     void *fdt;
@@ -123,7 +123,7 @@ out:
 /* Create reset TLB entries for BookE, spanning the 32bit addr space.  */
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa)
+                                     hwaddr pa)
 {
     ppcemb_tlb_t *tlb = &env->tlb.tlbe[0];
 
@@ -157,19 +157,19 @@ static void main_cpu_reset(void *opaque)
     mmubooke_create_initial_mapping(env, 0, 0);
 }
 
-static void bamboo_init(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void bamboo_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     unsigned int pci_irq_nrs[4] = { 28, 27, 26, 25 };
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram_memories
         = g_malloc(PPC440EP_SDRAM_NR_BANKS * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[PPC440EP_SDRAM_NR_BANKS];
-    target_phys_addr_t ram_sizes[PPC440EP_SDRAM_NR_BANKS];
+    hwaddr ram_bases[PPC440EP_SDRAM_NR_BANKS];
+    hwaddr ram_sizes[PPC440EP_SDRAM_NR_BANKS];
     qemu_irq *pic;
     qemu_irq *irqs;
     PCIBus *pcibus;
@@ -177,7 +177,7 @@ static void bamboo_init(ram_addr_t ram_size,
     CPUPPCState *env;
     uint64_t elf_entry;
     uint64_t elf_lowaddr;
-    target_phys_addr_t loadaddr = 0;
+    hwaddr loadaddr = 0;
     target_long initrd_size = 0;
     DeviceState *dev;
     int success;
@@ -195,7 +195,7 @@ static void bamboo_init(ram_addr_t ram_size,
     env = &cpu->env;
 
     qemu_register_reset(main_cpu_reset, cpu);
-    ppc_booke_timers_init(env, 400000000, 0);
+    ppc_booke_timers_init(cpu, 400000000, 0);
     ppc_dcr_init(env, NULL, NULL);
 
     /* interrupt controller */
@@ -295,6 +295,7 @@ static QEMUMachine bamboo_machine = {
     .name = "bamboo",
     .desc = "bamboo",
     .init = bamboo_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void bamboo_machine_init(void)
diff --git a/hw/ppc4xx.h b/hw/ppc4xx.h
index 5cd78b62911..59dba9e2926 100644
--- a/hw/ppc4xx.h
+++ b/hw/ppc4xx.h
@@ -25,12 +25,12 @@
 #if !defined(PPC_4XX_H)
 #define PPC_4XX_H
 
-#include "pci.h"
+#include "pci/pci.h"
 
 /* PowerPC 4xx core initialization */
-CPUPPCState *ppc4xx_init (const char *cpu_model,
-                       clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
-                       uint32_t sysclk);
+PowerPCCPU *ppc4xx_init(const char *cpu_model,
+                        clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
+                        uint32_t sysclk);
 
 /* PowerPC 4xx universal interrupt controller */
 enum {
@@ -43,22 +43,22 @@ qemu_irq *ppcuic_init (CPUPPCState *env, qemu_irq *irqs,
 
 ram_addr_t ppc4xx_sdram_adjust(ram_addr_t ram_size, int nr_banks,
                                MemoryRegion ram_memories[],
-                               target_phys_addr_t ram_bases[],
-                               target_phys_addr_t ram_sizes[],
+                               hwaddr ram_bases[],
+                               hwaddr ram_sizes[],
                                const unsigned int sdram_bank_sizes[]);
 
 void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
                         MemoryRegion ram_memories[],
-                        target_phys_addr_t *ram_bases,
-                        target_phys_addr_t *ram_sizes,
+                        hwaddr *ram_bases,
+                        hwaddr *ram_sizes,
                         int do_init);
 
 #define TYPE_PPC4xx_PCI_HOST_BRIDGE "ppc4xx-pcihost"
 
 PCIBus *ppc4xx_pci_init(CPUPPCState *env, qemu_irq pci_irqs[4],
-                        target_phys_addr_t config_space,
-                        target_phys_addr_t int_ack,
-                        target_phys_addr_t special_cycle,
-                        target_phys_addr_t registers);
+                        hwaddr config_space,
+                        hwaddr int_ack,
+                        hwaddr special_cycle,
+                        hwaddr registers);
 
 #endif /* !defined(PPC_4XX_H) */
diff --git a/hw/ppc4xx_devs.c b/hw/ppc4xx_devs.c
index 41163e607de..5e491bc0b4c 100644
--- a/hw/ppc4xx_devs.c
+++ b/hw/ppc4xx_devs.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc4xx.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_MMIO
 //#define DEBUG_UNASSIGNED
@@ -47,9 +47,9 @@ static void ppc4xx_reset(void *opaque)
 
 /*****************************************************************************/
 /* Generic PowerPC 4xx processor instantiation */
-CPUPPCState *ppc4xx_init (const char *cpu_model,
-                       clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
-                       uint32_t sysclk)
+PowerPCCPU *ppc4xx_init(const char *cpu_model,
+                        clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
+                        uint32_t sysclk)
 {
     PowerPCCPU *cpu;
     CPUPPCState *env;
@@ -72,7 +72,7 @@ CPUPPCState *ppc4xx_init (const char *cpu_model,
     /* Register qemu callbacks */
     qemu_register_reset(ppc4xx_reset, cpu);
 
-    return env;
+    return cpu;
 }
 
 /*****************************************************************************/
@@ -326,8 +326,8 @@ struct ppc4xx_sdram_t {
     int nbanks;
     MemoryRegion containers[4]; /* used for clipping */
     MemoryRegion *ram_memories;
-    target_phys_addr_t ram_bases[4];
-    target_phys_addr_t ram_sizes[4];
+    hwaddr ram_bases[4];
+    hwaddr ram_sizes[4];
     uint32_t besr0;
     uint32_t besr1;
     uint32_t bear;
@@ -348,11 +348,11 @@ enum {
 };
 
 /* XXX: TOFIX: some patches have made this code become inconsistent:
- *      there are type inconsistencies, mixing target_phys_addr_t, target_ulong
+ *      there are type inconsistencies, mixing hwaddr, target_ulong
  *      and uint32_t
  */
-static uint32_t sdram_bcr (target_phys_addr_t ram_base,
-                           target_phys_addr_t ram_size)
+static uint32_t sdram_bcr (hwaddr ram_base,
+                           hwaddr ram_size)
 {
     uint32_t bcr;
 
@@ -389,7 +389,7 @@ static uint32_t sdram_bcr (target_phys_addr_t ram_base,
     return bcr;
 }
 
-static inline target_phys_addr_t sdram_base(uint32_t bcr)
+static inline hwaddr sdram_base(uint32_t bcr)
 {
     return bcr & 0xFF800000;
 }
@@ -646,8 +646,8 @@ static void sdram_reset (void *opaque)
 
 void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
                         MemoryRegion *ram_memories,
-                        target_phys_addr_t *ram_bases,
-                        target_phys_addr_t *ram_sizes,
+                        hwaddr *ram_bases,
+                        hwaddr *ram_sizes,
                         int do_init)
 {
     ppc4xx_sdram_t *sdram;
@@ -656,12 +656,12 @@ void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
     sdram->irq = irq;
     sdram->nbanks = nbanks;
     sdram->ram_memories = ram_memories;
-    memset(sdram->ram_bases, 0, 4 * sizeof(target_phys_addr_t));
+    memset(sdram->ram_bases, 0, 4 * sizeof(hwaddr));
     memcpy(sdram->ram_bases, ram_bases,
-           nbanks * sizeof(target_phys_addr_t));
-    memset(sdram->ram_sizes, 0, 4 * sizeof(target_phys_addr_t));
+           nbanks * sizeof(hwaddr));
+    memset(sdram->ram_sizes, 0, 4 * sizeof(hwaddr));
     memcpy(sdram->ram_sizes, ram_sizes,
-           nbanks * sizeof(target_phys_addr_t));
+           nbanks * sizeof(hwaddr));
     qemu_register_reset(&sdram_reset, sdram);
     ppc_dcr_register(env, SDRAM0_CFGADDR,
                      sdram, &dcr_read_sdram, &dcr_write_sdram);
@@ -680,8 +680,8 @@ void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
  * sizes varies by SoC. */
 ram_addr_t ppc4xx_sdram_adjust(ram_addr_t ram_size, int nr_banks,
                                MemoryRegion ram_memories[],
-                               target_phys_addr_t ram_bases[],
-                               target_phys_addr_t ram_sizes[],
+                               hwaddr ram_bases[],
+                               hwaddr ram_sizes[],
                                const unsigned int sdram_bank_sizes[])
 {
     ram_addr_t size_left = ram_size;
diff --git a/hw/ppc4xx_pci.c b/hw/ppc4xx_pci.c
index a14fd426c19..ba2d669b834 100644
--- a/hw/ppc4xx_pci.c
+++ b/hw/ppc4xx_pci.c
@@ -22,9 +22,9 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc4xx.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "exec/address-spaces.h"
 
 #undef DEBUG
 #ifdef DEBUG
@@ -92,7 +92,7 @@ typedef struct PPC4xxPCIState PPC4xxPCIState;
 
 #define PCI_ALL_SIZE        (PCI_REG_BASE + PCI_REG_SIZE)
 
-static uint64_t pci4xx_cfgaddr_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci4xx_cfgaddr_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     PPC4xxPCIState *ppc4xx_pci = opaque;
@@ -101,7 +101,7 @@ static uint64_t pci4xx_cfgaddr_read(void *opaque, target_phys_addr_t addr,
     return phb->config_reg;
 }
 
-static void pci4xx_cfgaddr_write(void *opaque, target_phys_addr_t addr,
+static void pci4xx_cfgaddr_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     PPC4xxPCIState *ppc4xx_pci = opaque;
@@ -116,7 +116,7 @@ static const MemoryRegionOps pci4xx_cfgaddr_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void ppc4xx_pci_reg_write4(void *opaque, target_phys_addr_t offset,
+static void ppc4xx_pci_reg_write4(void *opaque, hwaddr offset,
                                   uint64_t value, unsigned size)
 {
     struct PPC4xxPCIState *pci = opaque;
@@ -184,7 +184,7 @@ static void ppc4xx_pci_reg_write4(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t ppc4xx_pci_reg_read4(void *opaque, target_phys_addr_t offset,
+static uint64_t ppc4xx_pci_reg_read4(void *opaque, hwaddr offset,
                                      unsigned size)
 {
     struct PPC4xxPCIState *pci = opaque;
diff --git a/hw/ppc_booke.c b/hw/ppc_booke.c
index d51e7fad678..25a4e91b696 100644
--- a/hw/ppc_booke.c
+++ b/hw/ppc_booke.c
@@ -23,10 +23,10 @@
  */
 #include "hw.h"
 #include "ppc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "nvram.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
 
 
@@ -71,17 +71,19 @@ struct booke_timer_t {
     uint32_t flags;
 };
 
-static void booke_update_irq(CPUPPCState *env)
+static void booke_update_irq(PowerPCCPU *cpu)
 {
-    ppc_set_irq(env, PPC_INTERRUPT_DECR,
+    CPUPPCState *env = &cpu->env;
+
+    ppc_set_irq(cpu, PPC_INTERRUPT_DECR,
                 (env->spr[SPR_BOOKE_TSR] & TSR_DIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_DIE));
 
-    ppc_set_irq(env, PPC_INTERRUPT_WDT,
+    ppc_set_irq(cpu, PPC_INTERRUPT_WDT,
                 (env->spr[SPR_BOOKE_TSR] & TSR_WIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_WIE));
 
-    ppc_set_irq(env, PPC_INTERRUPT_FIT,
+    ppc_set_irq(cpu, PPC_INTERRUPT_FIT,
                 (env->spr[SPR_BOOKE_TSR] & TSR_FIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_FIE));
 }
@@ -153,10 +155,11 @@ static void booke_update_fixed_timer(CPUPPCState         *env,
 
 static void booke_decr_cb(void *opaque)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     env->spr[SPR_BOOKE_TSR] |= TSR_DIS;
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     if (env->spr[SPR_BOOKE_TCR] & TCR_ARE) {
         /* Auto Reload */
@@ -166,16 +169,16 @@ static void booke_decr_cb(void *opaque)
 
 static void booke_fit_cb(void *opaque)
 {
-    CPUPPCState *env;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
 
-    env = opaque;
     tb_env = env->tb_env;
     booke_timer = tb_env->opaque;
     env->spr[SPR_BOOKE_TSR] |= TSR_FIS;
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_fit_target(env, tb_env),
@@ -185,17 +188,17 @@ static void booke_fit_cb(void *opaque)
 
 static void booke_wdt_cb(void *opaque)
 {
-    CPUPPCState *env;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
 
-    env = opaque;
     tb_env = env->tb_env;
     booke_timer = tb_env->opaque;
 
     /* TODO: There's lots of complicated stuff to do here */
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_wdt_target(env, tb_env),
@@ -205,19 +208,22 @@ static void booke_wdt_cb(void *opaque)
 
 void store_booke_tsr(CPUPPCState *env, target_ulong val)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->spr[SPR_BOOKE_TSR] &= ~val;
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 }
 
 void store_booke_tcr(CPUPPCState *env, target_ulong val)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env = env->tb_env;
     booke_timer_t *booke_timer = tb_env->opaque;
 
     tb_env = env->tb_env;
     env->spr[SPR_BOOKE_TCR] = val;
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_fit_target(env, tb_env),
@@ -231,7 +237,18 @@ void store_booke_tcr(CPUPPCState *env, target_ulong val)
 
 }
 
-void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags)
+static void ppc_booke_timer_reset_handle(void *opaque)
+{
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
+
+    env->spr[SPR_BOOKE_TSR] = 0;
+    env->spr[SPR_BOOKE_TCR] = 0;
+
+    booke_update_irq(cpu);
+}
+
+void ppc_booke_timers_init(PowerPCCPU *cpu, uint32_t freq, uint32_t flags)
 {
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
@@ -239,16 +256,18 @@ void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags)
     tb_env      = g_malloc0(sizeof(ppc_tb_t));
     booke_timer = g_malloc0(sizeof(booke_timer_t));
 
-    env->tb_env = tb_env;
+    cpu->env.tb_env = tb_env;
     tb_env->flags = flags | PPC_TIMER_BOOKE | PPC_DECR_ZERO_TRIGGERED;
 
     tb_env->tb_freq    = freq;
     tb_env->decr_freq  = freq;
     tb_env->opaque     = booke_timer;
-    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &booke_decr_cb, env);
+    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &booke_decr_cb, cpu);
 
     booke_timer->fit_timer =
-        qemu_new_timer_ns(vm_clock, &booke_fit_cb, env);
+        qemu_new_timer_ns(vm_clock, &booke_fit_cb, cpu);
     booke_timer->wdt_timer =
-        qemu_new_timer_ns(vm_clock, &booke_wdt_cb, env);
+        qemu_new_timer_ns(vm_clock, &booke_wdt_cb, cpu);
+
+    qemu_register_reset(ppc_booke_timer_reset_handle, cpu);
 }
diff --git a/hw/ppc_mac.h b/hw/ppc_mac.h
deleted file mode 100644
index 7d084184fc5..00000000000
--- a/hw/ppc_mac.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * QEMU PowerMac emulation shared definitions and prototypes
- *
- * Copyright (c) 2004-2007 Fabrice Bellard
- * Copyright (c) 2007 Jocelyn Mayer
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#if !defined(__PPC_MAC_H__)
-#define __PPC_MAC_H__
-
-#include "memory.h"
-
-/* SMP is not enabled, for now */
-#define MAX_CPUS 1
-
-#define BIOS_SIZE     (1024 * 1024)
-#define BIOS_FILENAME "ppc_rom.bin"
-#define NVRAM_SIZE        0x2000
-#define PROM_FILENAME    "openbios-ppc"
-#define PROM_ADDR         0xfff00000
-
-#define KERNEL_LOAD_ADDR 0x01000000
-#define KERNEL_GAP       0x00100000
-
-#define ESCC_CLOCK 3686400
-
-/* Cuda */
-void cuda_init (MemoryRegion **cuda_mem, qemu_irq irq);
-
-/* MacIO */
-void macio_init (PCIBus *bus, int device_id, int is_oldworld,
-                 MemoryRegion *pic_mem, MemoryRegion *dbdma_mem,
-                 MemoryRegion *cuda_mem, void *nvram,
-                 int nb_ide, MemoryRegion **ide_mem, MemoryRegion *escc_mem);
-
-/* Heathrow PIC */
-qemu_irq *heathrow_pic_init(MemoryRegion **pmem,
-                            int nb_cpus, qemu_irq **irqs);
-
-/* Grackle PCI */
-#define TYPE_GRACKLE_PCI_HOST_BRIDGE "grackle-pcihost"
-PCIBus *pci_grackle_init(uint32_t base, qemu_irq *pic,
-                         MemoryRegion *address_space_mem,
-                         MemoryRegion *address_space_io);
-
-/* UniNorth PCI */
-PCIBus *pci_pmac_init(qemu_irq *pic,
-                      MemoryRegion *address_space_mem,
-                      MemoryRegion *address_space_io);
-PCIBus *pci_pmac_u3_init(qemu_irq *pic,
-                         MemoryRegion *address_space_mem,
-                         MemoryRegion *address_space_io);
-
-/* Mac NVRAM */
-typedef struct MacIONVRAMState MacIONVRAMState;
-
-MacIONVRAMState *macio_nvram_init (target_phys_addr_t size,
-                                   unsigned int it_shift);
-void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           target_phys_addr_t mem_base);
-void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
-uint32_t macio_nvram_read (void *opaque, uint32_t addr);
-void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val);
-#endif /* !defined(__PPC_MAC_H__) */
diff --git a/hw/ppc_prep.c b/hw/ppc_prep.c
index 592b7b255f8..a35fbedbdce 100644
--- a/hw/ppc_prep.c
+++ b/hw/ppc_prep.c
@@ -24,22 +24,23 @@
 #include "hw.h"
 #include "nvram.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "isa.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "ppc.h"
 #include "boards.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "ide.h"
 #include "loader.h"
 #include "mc146818rtc.h"
-#include "blockdev.h"
-#include "arch_init.h"
-#include "exec-memory.h"
-#include "vga-pci.h"
+#include "pc87312.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/arch_init.h"
+#include "exec/address-spaces.h"
 
 //#define HARD_DEBUG_PPC_IO
 //#define DEBUG_PPC_IO
@@ -115,27 +116,27 @@ static struct {
 } XCSR;
 
 static void PPC_XCSR_writeb (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
 static void PPC_XCSR_writew (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
 static void PPC_XCSR_writel (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
-static uint32_t PPC_XCSR_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readb (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -145,7 +146,7 @@ static uint32_t PPC_XCSR_readb (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t PPC_XCSR_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readw (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -155,7 +156,7 @@ static uint32_t PPC_XCSR_readw (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t PPC_XCSR_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readl (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -181,7 +182,6 @@ typedef struct sysctrl_t {
     M48t59State *nvram;
     uint8_t state;
     uint8_t syscontrol;
-    uint8_t fake_io[2];
     int contiguous_map;
     int endian;
 } sysctrl_t;
@@ -192,24 +192,6 @@ enum {
 
 static sysctrl_t *sysctrl;
 
-static void PREP_io_write (void *opaque, uint32_t addr, uint32_t val)
-{
-    sysctrl_t *sysctrl = opaque;
-
-    PPC_IO_DPRINTF("0x%08" PRIx32 " => 0x%02" PRIx32 "\n", addr - PPC_IO_BASE,
-                   val);
-    sysctrl->fake_io[addr - 0x0398] = val;
-}
-
-static uint32_t PREP_io_read (void *opaque, uint32_t addr)
-{
-    sysctrl_t *sysctrl = opaque;
-
-    PPC_IO_DPRINTF("0x%08" PRIx32 " <= 0x%02" PRIx32 "\n", addr - PPC_IO_BASE,
-                   sysctrl->fake_io[addr - 0x0398]);
-    return sysctrl->fake_io[addr - 0x0398];
-}
-
 static void PREP_io_800_writeb (void *opaque, uint32_t addr, uint32_t val)
 {
     sysctrl_t *sysctrl = opaque;
@@ -343,8 +325,8 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
     return retval;
 }
 
-static inline target_phys_addr_t prep_IO_address(sysctrl_t *sysctrl,
-                                                 target_phys_addr_t addr)
+static inline hwaddr prep_IO_address(sysctrl_t *sysctrl,
+                                                 hwaddr addr)
 {
     if (sysctrl->contiguous_map == 0) {
         /* 64 KB contiguous space for IOs */
@@ -357,7 +339,7 @@ static inline target_phys_addr_t prep_IO_address(sysctrl_t *sysctrl,
     return addr;
 }
 
-static void PPC_prep_io_writeb (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writeb (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -366,7 +348,7 @@ static void PPC_prep_io_writeb (void *opaque, target_phys_addr_t addr,
     cpu_outb(addr, value);
 }
 
-static uint32_t PPC_prep_io_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readb (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -377,7 +359,7 @@ static uint32_t PPC_prep_io_readb (void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-static void PPC_prep_io_writew (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writew (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -387,7 +369,7 @@ static void PPC_prep_io_writew (void *opaque, target_phys_addr_t addr,
     cpu_outw(addr, value);
 }
 
-static uint32_t PPC_prep_io_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readw (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -399,7 +381,7 @@ static uint32_t PPC_prep_io_readw (void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-static void PPC_prep_io_writel (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writel (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -409,7 +391,7 @@ static void PPC_prep_io_writel (void *opaque, target_phys_addr_t addr,
     cpu_outl(addr, value);
 }
 
-static uint32_t PPC_prep_io_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readl (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -448,13 +430,14 @@ static void ppc_prep_reset(void *opaque)
 }
 
 /* PowerPC PREP hardware initialisation */
-static void ppc_prep_init (ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ppc_prep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
@@ -475,10 +458,10 @@ static void ppc_prep_init (ram_addr_t ram_size,
     PCIBus *pci_bus;
     PCIDevice *pci;
     ISABus *isa_bus;
+    ISADevice *isa;
     qemu_irq *cpu_exit_irq;
     int ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
-    DriveInfo *fd[MAX_FD];
 
     sysctrl = g_malloc0(sizeof(sysctrl_t));
 
@@ -524,7 +507,7 @@ static void ppc_prep_init (ram_addr_t ram_size,
         bios_size = -1;
     }
     if (bios_size > 0 && bios_size <= BIOS_SIZE) {
-        target_phys_addr_t bios_addr;
+        hwaddr bios_addr;
         bios_size = (bios_size + 0xfff) & ~0xfff;
         bios_addr = (uint32_t)(-bios_size);
         bios_size = load_image_targphys(filename, bios_addr, bios_size);
@@ -605,6 +588,11 @@ static void ppc_prep_init (ram_addr_t ram_size,
     sysbus_connect_irq(&pcihost->busdev, 3, qdev_get_gpio_in(&pci->qdev, 11));
     isa_bus = DO_UPCAST(ISABus, qbus, qdev_get_child_bus(&pci->qdev, "isa.0"));
 
+    /* Super I/O (parallel + serial ports) */
+    isa = isa_create(isa_bus, TYPE_PC87312);
+    qdev_prop_set_uint8(&isa->qdev, "config", 13); /* fdc, ser0, ser1, par0 */
+    qdev_init_nofail(&isa->qdev);
+
     /* Register 8 MB of ISA IO space (needed for non-contiguous map) */
     memory_region_init_io(PPC_io_memory, &PPC_prep_io_ops, sysctrl,
                           "ppc-io", 0x00800000);
@@ -613,8 +601,6 @@ static void ppc_prep_init (ram_addr_t ram_size,
     /* init basic PC hardware */
     pci_vga_init(pci_bus);
 
-    if (serial_hds[0])
-        serial_isa_init(isa_bus, 0, serial_hds[0]);
     nb_nics1 = nb_nics;
     if (nb_nics1 > NE2000_NB_MAX)
         nb_nics1 = NE2000_NB_MAX;
@@ -638,17 +624,7 @@ static void ppc_prep_init (ram_addr_t ram_size,
     }
     isa_create_simple(isa_bus, "i8042");
 
-    //    SB16_init();
-
-    for(i = 0; i < MAX_FD; i++) {
-        fd[i] = drive_get(IF_FLOPPY, 0, i);
-    }
-    fdctrl_init_isa(isa_bus, fd);
-
-    /* Register fake IO ports for PREP */
     sysctrl->reset_irq = first_cpu->irq_inputs[PPC6xx_INPUT_HRESET];
-    register_ioport_read(0x398, 2, 1, &PREP_io_read, sysctrl);
-    register_ioport_write(0x398, 2, 1, &PREP_io_write, sysctrl);
     /* System control ports */
     register_ioport_read(0x0092, 0x01, 1, &PREP_io_800_readb, sysctrl);
     register_ioport_write(0x0092, 0x01, 1, &PREP_io_800_writeb, sysctrl);
@@ -660,7 +636,7 @@ static void ppc_prep_init (ram_addr_t ram_size,
     memory_region_add_subregion(sysmem, 0xFEFF0000, xcsr);
 #endif
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
@@ -693,6 +669,7 @@ static QEMUMachine prep_machine = {
     .desc = "PowerPC PREP platform",
     .init = ppc_prep_init,
     .max_cpus = MAX_CPUS,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void prep_machine_init(void)
diff --git a/hw/ppce500_pci.c b/hw/ppce500_pci.c
index 92b1dc05345..1e1ade3d2ea 100644
--- a/hw/ppce500_pci.c
+++ b/hw/ppce500_pci.c
@@ -15,9 +15,11 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "bswap.h"
+#include "hw/ppc/e500-ccsr.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "qemu/bswap.h"
+#include "ppce500_pci.h"
 
 #ifdef DEBUG_PCI
 #define pci_debug(fmt, ...) fprintf(stderr, fmt, ## __VA_ARGS__)
@@ -31,6 +33,8 @@
 #define PCIE500_ALL_SIZE      0x1000
 #define PCIE500_REG_SIZE      (PCIE500_ALL_SIZE - PCIE500_REG_BASE)
 
+#define PCIE500_PCI_IOLEN     0x10000ULL
+
 #define PPCE500_PCI_CONFIG_ADDR         0x0
 #define PPCE500_PCI_CONFIG_DATA         0x4
 #define PPCE500_PCI_INTACK              0x8
@@ -84,14 +88,29 @@ struct PPCE500PCIState {
     struct pci_inbound pib[PPCE500_PCI_NR_PIBS];
     uint32_t gasket_time;
     qemu_irq irq[4];
+    uint32_t first_slot;
     /* mmio maps */
     MemoryRegion container;
     MemoryRegion iomem;
+    MemoryRegion pio;
+};
+
+#define TYPE_PPC_E500_PCI_BRIDGE "e500-host-bridge"
+#define PPC_E500_PCI_BRIDGE(obj) \
+    OBJECT_CHECK(PPCE500PCIBridgeState, (obj), TYPE_PPC_E500_PCI_BRIDGE)
+
+struct PPCE500PCIBridgeState {
+    /*< private >*/
+    PCIDevice parent;
+    /*< public >*/
+
+    MemoryRegion bar0;
 };
 
+typedef struct PPCE500PCIBridgeState PPCE500PCIBridgeState;
 typedef struct PPCE500PCIState PPCE500PCIState;
 
-static uint64_t pci_reg_read4(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PPCE500PCIState *pci = opaque;
@@ -160,7 +179,7 @@ static uint64_t pci_reg_read4(void *opaque, target_phys_addr_t addr,
     return value;
 }
 
-static void pci_reg_write4(void *opaque, target_phys_addr_t addr,
+static void pci_reg_write4(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     PPCE500PCIState *pci = opaque;
@@ -235,17 +254,10 @@ static const MemoryRegionOps e500_pci_reg_ops = {
 
 static int mpc85xx_pci_map_irq(PCIDevice *pci_dev, int irq_num)
 {
-    int devno = pci_dev->devfn >> 3, ret = 0;
+    int devno = pci_dev->devfn >> 3;
+    int ret;
 
-    switch (devno) {
-        /* Two PCI slot */
-        case 0x11:
-        case 0x12:
-            ret = (irq_num + devno - 0x10) % 4;
-            break;
-        default:
-            printf("Error:%s:unknown dev number\n", __func__);
-    }
+    ret = ppce500_pci_map_irq_slot(devno, irq_num);
 
     pci_debug("%s: devfn %x irq %d -> %d  devno:%x\n", __func__,
            pci_dev->devfn, irq_num, ret, devno);
@@ -305,7 +317,25 @@ static const VMStateDescription vmstate_ppce500_pci = {
     }
 };
 
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
+
+static int e500_pcihost_bridge_initfn(PCIDevice *d)
+{
+    PPCE500PCIBridgeState *b = PPC_E500_PCI_BRIDGE(d);
+    PPCE500CCSRState *ccsr = CCSR(container_get(qdev_get_machine(),
+                                  "/e500-ccsr"));
+
+    pci_config_set_class(d->config, PCI_CLASS_BRIDGE_PCI);
+    d->config[PCI_HEADER_TYPE] =
+        (d->config[PCI_HEADER_TYPE] & PCI_HEADER_TYPE_MULTI_FUNCTION) |
+        PCI_HEADER_TYPE_BRIDGE;
+
+    memory_region_init_alias(&b->bar0, "e500-pci-bar0", &ccsr->ccsr_space,
+                             0, int128_get64(ccsr->ccsr_space.size));
+    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &b->bar0);
+
+    return 0;
+}
 
 static int e500_pcihost_initfn(SysBusDevice *dev)
 {
@@ -314,7 +344,6 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
     PCIBus *b;
     int i;
     MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *address_space_io = get_system_io();
 
     h = PCI_HOST_BRIDGE(dev);
     s = PPC_E500_PCI_HOST_BRIDGE(dev);
@@ -323,9 +352,11 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
         sysbus_init_irq(dev, &s->irq[i]);
     }
 
+    memory_region_init(&s->pio, "pci-pio", PCIE500_PCI_IOLEN);
+
     b = pci_register_bus(DEVICE(dev), NULL, mpc85xx_pci_set_irq,
                          mpc85xx_pci_map_irq, s->irq, address_space_mem,
-                         address_space_io, PCI_DEVFN(0x11, 0), 4);
+                         &s->pio, PCI_DEVFN(s->first_slot, 0), 4);
     h->bus = b;
 
     pci_create_simple(b, 0, "e500-host-bridge");
@@ -341,6 +372,7 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
     memory_region_add_subregion(&s->container, PCIE500_CFGDATA, &h->data_mem);
     memory_region_add_subregion(&s->container, PCIE500_REG_BASE, &s->iomem);
     sysbus_init_mmio(dev, &s->container);
+    sysbus_init_mmio(dev, &s->pio);
 
     return 0;
 }
@@ -350,6 +382,7 @@ static void e500_host_bridge_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
+    k->init = e500_pcihost_bridge_initfn;
     k->vendor_id = PCI_VENDOR_ID_FREESCALE;
     k->device_id = PCI_DEVICE_ID_MPC8533E;
     k->class_id = PCI_CLASS_PROCESSOR_POWERPC;
@@ -359,16 +392,22 @@ static void e500_host_bridge_class_init(ObjectClass *klass, void *data)
 static const TypeInfo e500_host_bridge_info = {
     .name          = "e500-host-bridge",
     .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIDevice),
+    .instance_size = sizeof(PPCE500PCIBridgeState),
     .class_init    = e500_host_bridge_class_init,
 };
 
+static Property pcihost_properties[] = {
+    DEFINE_PROP_UINT32("first_slot", PPCE500PCIState, first_slot, 0x11),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void e500_pcihost_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
     k->init = e500_pcihost_initfn;
+    dc->props = pcihost_properties;
     dc->vmsd = &vmstate_ppce500_pci;
 }
 
diff --git a/hw/ppce500_pci.h b/hw/ppce500_pci.h
new file mode 100644
index 00000000000..61f773ef30f
--- /dev/null
+++ b/hw/ppce500_pci.h
@@ -0,0 +1,9 @@
+#ifndef PPCE500_PCI_H
+#define PPCE500_PCI_H
+
+static inline int ppce500_pci_map_irq_slot(int devno, int irq_num)
+{
+    return (devno + irq_num) % 4;
+}
+
+#endif
diff --git a/hw/ppce500_spin.c b/hw/ppce500_spin.c
index c5b8e051ec4..7e90fb98249 100644
--- a/hw/ppce500_spin.c
+++ b/hw/ppce500_spin.c
@@ -28,9 +28,9 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define MAX_CPUS 32
 
@@ -49,7 +49,7 @@ typedef struct spin_state {
 } SpinState;
 
 typedef struct spin_kick {
-    CPUPPCState *env;
+    PowerPCCPU *cpu;
     SpinInfo *spin;
 } SpinKick;
 
@@ -68,18 +68,18 @@ static void spin_reset(void *opaque)
 }
 
 /* Create -kernel TLB entries for BookE, linearly spanning 256MB.  */
-static inline target_phys_addr_t booke206_page_size_to_tlb(uint64_t size)
+static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
 {
     return (ffs(size >> 10) - 1) >> 1;
 }
 
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa,
-                                     target_phys_addr_t len)
+                                     hwaddr pa,
+                                     hwaddr len)
 {
     ppcmas_tlb_t *tlb = booke206_get_tlbm(env, 1, 0, 1);
-    target_phys_addr_t size;
+    hwaddr size;
 
     size = (booke206_page_size_to_tlb(len) << MAS1_TSIZE_SHIFT);
     tlb->mas1 = MAS1_VALID | size;
@@ -92,10 +92,11 @@ static void mmubooke_create_initial_mapping(CPUPPCState *env,
 static void spin_kick(void *data)
 {
     SpinKick *kick = data;
-    CPUPPCState *env = kick->env;
+    CPUState *cpu = CPU(kick->cpu);
+    CPUPPCState *env = &kick->cpu->env;
     SpinInfo *curspin = kick->spin;
-    target_phys_addr_t map_size = 64 * 1024 * 1024;
-    target_phys_addr_t map_start;
+    hwaddr map_size = 64 * 1024 * 1024;
+    hwaddr map_start;
 
     cpu_synchronize_state(env);
     stl_p(&curspin->pir, env->spr[SPR_PIR]);
@@ -113,31 +114,33 @@ static void spin_kick(void *data)
 
     env->halted = 0;
     env->exception_index = -1;
-    env->stopped = 0;
-    qemu_cpu_kick(env);
+    cpu->stopped = false;
+    qemu_cpu_kick(cpu);
 }
 
-static void spin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void spin_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned len)
 {
     SpinState *s = opaque;
     int env_idx = addr / sizeof(SpinInfo);
     CPUPPCState *env;
+    CPUState *cpu = NULL;
     SpinInfo *curspin = &s->spin[env_idx];
     uint8_t *curspin_p = (uint8_t*)curspin;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index == env_idx) {
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index == env_idx) {
             break;
         }
     }
 
-    if (!env) {
+    if (cpu == NULL) {
         /* Unknown CPU */
         return;
     }
 
-    if (!env->cpu_index) {
+    if (cpu->cpu_index == 0) {
         /* primary CPU doesn't spin */
         return;
     }
@@ -158,15 +161,15 @@ static void spin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
     if (!(ldq_p(&curspin->addr) & 1)) {
         /* run CPU */
         SpinKick kick = {
-            .env = env,
+            .cpu = ppc_env_get_cpu(env),
             .spin = curspin,
         };
 
-        run_on_cpu(env, spin_kick, &kick);
+        run_on_cpu(CPU(kick.cpu), spin_kick, &kick);
     }
 }
 
-static uint64_t spin_read(void *opaque, target_phys_addr_t addr, unsigned len)
+static uint64_t spin_read(void *opaque, hwaddr addr, unsigned len)
 {
     SpinState *s = opaque;
     uint8_t *spin_p = &((uint8_t*)s->spin)[addr];
@@ -193,7 +196,7 @@ static int ppce500_spin_initfn(SysBusDevice *dev)
 {
     SpinState *s;
 
-    s = FROM_SYSBUS(SpinState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(SpinState, SYS_BUS_DEVICE(dev));
 
     memory_region_init_io(&s->iomem, &spin_rw_ops, s, "e500 spin pv device",
                           sizeof(SpinInfo) * MAX_CPUS);
@@ -211,7 +214,7 @@ static void ppce500_spin_class_init(ObjectClass *klass, void *data)
     k->init = ppce500_spin_initfn;
 }
 
-static TypeInfo ppce500_spin_info = {
+static const TypeInfo ppce500_spin_info = {
     .name          = "e500-spin",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SpinState),
diff --git a/hw/prep_pci.c b/hw/prep_pci.c
index cc44e619872..212a2ac4f1e 100644
--- a/hw/prep_pci.c
+++ b/hw/prep_pci.c
@@ -23,10 +23,10 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "pc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define TYPE_RAVEN_PCI_HOST_BRIDGE "raven-pcihost"
 
@@ -44,7 +44,7 @@ typedef struct RavenPCIState {
     PCIDevice dev;
 } RavenPCIState;
 
-static inline uint32_t PPC_PCIIO_config(target_phys_addr_t addr)
+static inline uint32_t PPC_PCIIO_config(hwaddr addr)
 {
     int i;
 
@@ -56,7 +56,7 @@ static inline uint32_t PPC_PCIIO_config(target_phys_addr_t addr)
     return (addr & 0x7ff) |  (i << 11);
 }
 
-static void ppc_pci_io_write(void *opaque, target_phys_addr_t addr,
+static void ppc_pci_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned int size)
 {
     PREPPCIState *s = opaque;
@@ -64,7 +64,7 @@ static void ppc_pci_io_write(void *opaque, target_phys_addr_t addr,
     pci_data_write(phb->bus, PPC_PCIIO_config(addr), val, size);
 }
 
-static uint64_t ppc_pci_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ppc_pci_io_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     PREPPCIState *s = opaque;
@@ -78,7 +78,7 @@ static const MemoryRegionOps PPC_PCIIO_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t ppc_intack_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ppc_intack_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     return pic_read_irq(isa_pic);
diff --git a/hw/ps2.c b/hw/ps2.c
index f93cd24d94d..15cfd5bb76e 100644
--- a/hw/ps2.c
+++ b/hw/ps2.c
@@ -23,8 +23,8 @@
  */
 #include "hw.h"
 #include "ps2.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 /* debug PC keyboard */
 //#define DEBUG_KBD
diff --git a/hw/ptimer.c b/hw/ptimer.c
index bc0b3f802f8..24af6a2afe3 100644
--- a/hw/ptimer.c
+++ b/hw/ptimer.c
@@ -6,9 +6,9 @@
  * This code is licensed under the GNU LGPL.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 struct ptimer_state
 {
diff --git a/hw/ptimer.h b/hw/ptimer.h
index 6638f613222..28fcaf17f83 100644
--- a/hw/ptimer.h
+++ b/hw/ptimer.h
@@ -9,8 +9,8 @@
 #define PTIMER_H
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "vmstate.h"
+#include "qemu/timer.h"
+#include "migration/vmstate.h"
 
 /* ptimer.c */
 typedef struct ptimer_state ptimer_state;
diff --git a/hw/puv3.c b/hw/puv3.c
index 43f7216e4e0..c722510d7e7 100644
--- a/hw/puv3.c
+++ b/hw/puv3.c
@@ -8,9 +8,11 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
-#include "console.h"
+
+#include "qemu-common.h"
+#include "ui/console.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"
 #include "boards.h"
 #include "loader.h"
@@ -91,10 +93,12 @@ static void puv3_load_kernel(const char *kernel_filename)
     graphic_console_init(NULL, NULL, NULL, NULL, NULL);
 }
 
-static void puv3_init(ram_addr_t ram_size, const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void puv3_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *initrd_filename = args->initrd_filename;
     CPUUniCore32State *env;
 
     if (initrd_filename) {
@@ -120,7 +124,7 @@ static QEMUMachine puv3_machine = {
     .desc = "PKUnity Version-3 based on UniCore32",
     .init = puv3_init,
     .is_default = 1,
-    .use_scsi = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void puv3_machine_init(void)
diff --git a/hw/puv3_dma.c b/hw/puv3_dma.c
index 85b97bfdeb9..9de63b4c344 100644
--- a/hw/puv3_dma.c
+++ b/hw/puv3_dma.c
@@ -24,7 +24,7 @@ typedef struct {
     uint32_t reg_CFG[PUV3_DMA_CH_NR];
 } PUV3DMAState;
 
-static uint64_t puv3_dma_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_dma_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3DMAState *s = opaque;
@@ -44,7 +44,7 @@ static uint64_t puv3_dma_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_dma_write(void *opaque, target_phys_addr_t offset,
+static void puv3_dma_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3DMAState *s = opaque;
diff --git a/hw/puv3_gpio.c b/hw/puv3_gpio.c
index 9436e6c62cb..152248d2919 100644
--- a/hw/puv3_gpio.c
+++ b/hw/puv3_gpio.c
@@ -24,7 +24,7 @@ typedef struct {
     uint32_t reg_GPIR;
 } PUV3GPIOState;
 
-static uint64_t puv3_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_gpio_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3GPIOState *s = opaque;
@@ -48,7 +48,7 @@ static uint64_t puv3_gpio_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_gpio_write(void *opaque, target_phys_addr_t offset,
+static void puv3_gpio_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3GPIOState *s = opaque;
diff --git a/hw/puv3_intc.c b/hw/puv3_intc.c
index 9e0b975ea2f..07f56490656 100644
--- a/hw/puv3_intc.c
+++ b/hw/puv3_intc.c
@@ -46,7 +46,7 @@ static void puv3_intc_handler(void *opaque, int irq, int level)
     puv3_intc_update(s);
 }
 
-static uint64_t puv3_intc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_intc_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3INTCState *s = opaque;
@@ -66,7 +66,7 @@ static uint64_t puv3_intc_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_intc_write(void *opaque, target_phys_addr_t offset,
+static void puv3_intc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3INTCState *s = opaque;
diff --git a/hw/puv3_ost.c b/hw/puv3_ost.c
index dd30cad0e23..14c6f21a756 100644
--- a/hw/puv3_ost.c
+++ b/hw/puv3_ost.c
@@ -28,7 +28,7 @@ typedef struct {
     uint32_t reg_OIER;
 } PUV3OSTState;
 
-static uint64_t puv3_ost_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_ost_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3OSTState *s = opaque;
@@ -51,7 +51,7 @@ static uint64_t puv3_ost_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_ost_write(void *opaque, target_phys_addr_t offset,
+static void puv3_ost_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3OSTState *s = opaque;
diff --git a/hw/puv3_pm.c b/hw/puv3_pm.c
index 621c96875c9..87a687afaea 100644
--- a/hw/puv3_pm.c
+++ b/hw/puv3_pm.c
@@ -26,7 +26,7 @@ typedef struct {
     uint32_t reg_DIVCFG;
 } PUV3PMState;
 
-static uint64_t puv3_pm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_pm_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3PMState *s = opaque;
@@ -74,7 +74,7 @@ static uint64_t puv3_pm_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_pm_write(void *opaque, target_phys_addr_t offset,
+static void puv3_pm_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3PMState *s = opaque;
diff --git a/hw/pxa.h b/hw/pxa.h
index 6a2120555f1..668232cead9 100644
--- a/hw/pxa.h
+++ b/hw/pxa.h
@@ -9,7 +9,7 @@
 #ifndef PXA_H
 # define PXA_H			"pxa.h"
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Interrupt numbers */
 # define PXA2XX_PIC_SSP3	0
@@ -65,28 +65,28 @@
 # define PXA2XX_INTERNAL_SIZE	0x40000
 
 /* pxa2xx_pic.c */
-DeviceState *pxa2xx_pic_init(target_phys_addr_t base, ARMCPU *cpu);
+DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu);
 
 /* pxa2xx_gpio.c */
-DeviceState *pxa2xx_gpio_init(target_phys_addr_t base,
-                CPUARMState *env, DeviceState *pic, int lines);
+DeviceState *pxa2xx_gpio_init(hwaddr base,
+                              ARMCPU *cpu, DeviceState *pic, int lines);
 void pxa2xx_gpio_read_notifier(DeviceState *dev, qemu_irq handler);
 
 /* pxa2xx_dma.c */
-DeviceState *pxa255_dma_init(target_phys_addr_t base, qemu_irq irq);
-DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq);
+DeviceState *pxa255_dma_init(hwaddr base, qemu_irq irq);
+DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq);
 
 /* pxa2xx_lcd.c */
 typedef struct PXA2xxLCDState PXA2xxLCDState;
 PXA2xxLCDState *pxa2xx_lcdc_init(MemoryRegion *sysmem,
-                target_phys_addr_t base, qemu_irq irq);
+                hwaddr base, qemu_irq irq);
 void pxa2xx_lcd_vsync_notifier(PXA2xxLCDState *s, qemu_irq handler);
 void pxa2xx_lcdc_oritentation(void *opaque, int angle);
 
 /* pxa2xx_mmci.c */
 typedef struct PXA2xxMMCIState PXA2xxMMCIState;
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 BlockDriverState *bd, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma);
 void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
@@ -95,7 +95,7 @@ void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
 /* pxa2xx_pcmcia.c */
 typedef struct PXA2xxPCMCIAState PXA2xxPCMCIAState;
 PXA2xxPCMCIAState *pxa2xx_pcmcia_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base);
+                                      hwaddr base);
 int pxa2xx_pcmcia_attach(void *opaque, PCMCIACardState *card);
 int pxa2xx_pcmcia_dettach(void *opaque);
 void pxa2xx_pcmcia_set_irq_cb(void *opaque, qemu_irq irq, qemu_irq cd_irq);
@@ -107,14 +107,14 @@ struct  keymap {
 };
 typedef struct PXA2xxKeyPadState PXA2xxKeyPadState;
 PXA2xxKeyPadState *pxa27x_keypad_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base,
+                                      hwaddr base,
                                       qemu_irq irq);
 void pxa27x_register_keypad(PXA2xxKeyPadState *kp, struct keymap *map,
                 int size);
 
 /* pxa2xx.c */
 typedef struct PXA2xxI2CState PXA2xxI2CState;
-PXA2xxI2CState *pxa2xx_i2c_init(target_phys_addr_t base,
+PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
                 qemu_irq irq, uint32_t page_size);
 i2c_bus *pxa2xx_i2c_bus(PXA2xxI2CState *s);
 
@@ -142,16 +142,16 @@ typedef struct {
     PXA2xxKeyPadState *kp;
 
     /* Power management */
-    target_phys_addr_t pm_base;
+    hwaddr pm_base;
     uint32_t pm_regs[0x40];
 
     /* Clock management */
-    target_phys_addr_t cm_base;
+    hwaddr cm_base;
     uint32_t cm_regs[4];
     uint32_t clkcfg;
 
     /* Memory management */
-    target_phys_addr_t mm_base;
+    hwaddr mm_base;
     uint32_t mm_regs[0x1a];
 
     /* Performance monitoring */
diff --git a/hw/pxa2xx.c b/hw/pxa2xx.c
index d5f1420ed95..2367c6a4a4d 100644
--- a/hw/pxa2xx.c
+++ b/hw/pxa2xx.c
@@ -9,15 +9,15 @@
 
 #include "sysbus.h"
 #include "pxa.h"
-#include "sysemu.h"
-#include "pc.h"
+#include "sysemu/sysemu.h"
+#include "serial.h"
 #include "i2c.h"
 #include "ssi.h"
-#include "qemu-char.h"
-#include "blockdev.h"
+#include "char/char.h"
+#include "sysemu/blockdev.h"
 
 static struct {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irqn;
 } pxa255_serial[] = {
     { 0x40100000, PXA2XX_PIC_FFUART },
@@ -33,7 +33,7 @@ static struct {
 };
 
 typedef struct PXASSPDef {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irqn;
 } PXASSPDef;
 
@@ -88,7 +88,7 @@ static PXASSPDef pxa27x_ssp[] = {
 #define PCMD0	0x80	/* Power Manager I2C Command register File 0 */
 #define PCMD31	0xfc	/* Power Manager I2C Command register File 31 */
 
-static uint64_t pxa2xx_pm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_pm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -107,7 +107,7 @@ static uint64_t pxa2xx_pm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_pm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_pm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -160,7 +160,7 @@ static const VMStateDescription vmstate_pxa2xx_pm = {
 #define OSCC	0x08	/* Oscillator Configuration register */
 #define CCSR	0x0c	/* Core Clock Status register */
 
-static uint64_t pxa2xx_cm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_cm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -181,7 +181,7 @@ static uint64_t pxa2xx_cm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_cm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_cm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -405,7 +405,7 @@ static void pxa2xx_setup_cp14(PXA2xxState *s)
 #define BSCNTR3		0x60	/* Memory Buffer Strength Control register 3 */
 #define SA1110		0x64	/* SA-1110 Memory Compatibility register */
 
-static uint64_t pxa2xx_mm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_mm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -422,7 +422,7 @@ static uint64_t pxa2xx_mm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_mm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_mm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -567,7 +567,7 @@ static void pxa2xx_ssp_fifo_update(PXA2xxSSPState *s)
     pxa2xx_ssp_int_update(s);
 }
 
-static uint64_t pxa2xx_ssp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_ssp_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
@@ -613,7 +613,7 @@ static uint64_t pxa2xx_ssp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_ssp_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_ssp_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
@@ -943,7 +943,7 @@ static inline void pxa2xx_rtc_pi_tick(void *opaque)
     pxa2xx_rtc_int_update(s);
 }
 
-static uint64_t pxa2xx_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_rtc_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxRTCState *s = (PXA2xxRTCState *) opaque;
@@ -989,7 +989,7 @@ static uint64_t pxa2xx_rtc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_rtc_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_rtc_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxRTCState *s = (PXA2xxRTCState *) opaque;
@@ -1194,7 +1194,7 @@ static void pxa2xx_rtc_sysbus_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pxa2xx_rtc_regs;
 }
 
-static TypeInfo pxa2xx_rtc_sysbus_info = {
+static const TypeInfo pxa2xx_rtc_sysbus_info = {
     .name          = "pxa2xx_rtc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxRTCState),
@@ -1294,7 +1294,7 @@ static int pxa2xx_i2c_tx(I2CSlave *i2c, uint8_t data)
     return 1;
 }
 
-static uint64_t pxa2xx_i2c_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_i2c_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxI2CState *s = (PXA2xxI2CState *) opaque;
@@ -1322,7 +1322,7 @@ static uint64_t pxa2xx_i2c_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_i2c_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_i2c_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxI2CState *s = (PXA2xxI2CState *) opaque;
@@ -1442,21 +1442,21 @@ static void pxa2xx_i2c_slave_class_init(ObjectClass *klass, void *data)
     k->send = pxa2xx_i2c_tx;
 }
 
-static TypeInfo pxa2xx_i2c_slave_info = {
+static const TypeInfo pxa2xx_i2c_slave_info = {
     .name          = "pxa2xx-i2c-slave",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(PXA2xxI2CSlaveState),
     .class_init    = pxa2xx_i2c_slave_class_init,
 };
 
-PXA2xxI2CState *pxa2xx_i2c_init(target_phys_addr_t base,
+PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
                 qemu_irq irq, uint32_t region_size)
 {
     DeviceState *dev;
     SysBusDevice *i2c_dev;
     PXA2xxI2CState *s;
 
-    i2c_dev = sysbus_from_qdev(qdev_create(NULL, "pxa2xx_i2c"));
+    i2c_dev = SYS_BUS_DEVICE(qdev_create(NULL, "pxa2xx_i2c"));
     qdev_prop_set_uint32(&i2c_dev->qdev, "size", region_size + 1);
     qdev_prop_set_uint32(&i2c_dev->qdev, "offset", base & region_size);
 
@@ -1510,7 +1510,7 @@ static void pxa2xx_i2c_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_i2c_properties;
 }
 
-static TypeInfo pxa2xx_i2c_info = {
+static const TypeInfo pxa2xx_i2c_info = {
     .name          = "pxa2xx_i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxI2CState),
@@ -1572,7 +1572,7 @@ static inline void pxa2xx_i2s_update(PXA2xxI2SState *i2s)
 #define SADIV	0x60	/* Serial Audio Clock Divider register */
 #define SADR	0x80	/* Serial Audio Data register */
 
-static uint64_t pxa2xx_i2s_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_i2s_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *) opaque;
@@ -1604,7 +1604,7 @@ static uint64_t pxa2xx_i2s_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_i2s_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_i2s_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *) opaque;
@@ -1706,7 +1706,7 @@ static void pxa2xx_i2s_data_req(void *opaque, int tx, int rx)
 }
 
 static PXA2xxI2SState *pxa2xx_i2s_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *)
@@ -1801,7 +1801,7 @@ static inline void pxa2xx_fir_update(PXA2xxFIrState *s)
 #define ICSR1	0x18	/* FICP Status register 1 */
 #define ICFOR	0x1c	/* FICP FIFO Occupancy Status register */
 
-static uint64_t pxa2xx_fir_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_fir_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
@@ -1839,7 +1839,7 @@ static uint64_t pxa2xx_fir_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_fir_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_fir_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
@@ -1963,7 +1963,7 @@ static int pxa2xx_fir_load(QEMUFile *f, void *opaque, int version_id)
 }
 
 static PXA2xxFIrState *pxa2xx_fir_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma,
                 CharDriverState *chr)
 {
@@ -2045,7 +2045,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
                     qdev_get_gpio_in(s->pic, PXA27X_PIC_OST_4_11),
                     NULL);
 
-    s->gpio = pxa2xx_gpio_init(0x40e00000, &s->cpu->env, s->pic, 121);
+    s->gpio = pxa2xx_gpio_init(0x40e00000, s->cpu, s->pic, 121);
 
     dinfo = drive_get(IF_SD, 0, 0);
     if (!dinfo) {
@@ -2108,7 +2108,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         sysbus_create_simple("sysbus-ohci", 0x4c000000,
                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
     }
@@ -2176,7 +2176,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
                     qdev_get_gpio_in(s->pic, PXA2XX_PIC_OST_0 + 3),
                     NULL);
 
-    s->gpio = pxa2xx_gpio_init(0x40e00000, &s->cpu->env, s->pic, 85);
+    s->gpio = pxa2xx_gpio_init(0x40e00000, s->cpu, s->pic, 85);
 
     dinfo = drive_get(IF_SD, 0, 0);
     if (!dinfo) {
@@ -2239,7 +2239,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         sysbus_create_simple("sysbus-ohci", 0x4c000000,
                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
     }
@@ -2273,7 +2273,7 @@ static void pxa2xx_ssp_class_init(ObjectClass *klass, void *data)
     sdc->init = pxa2xx_ssp_init;
 }
 
-static TypeInfo pxa2xx_ssp_info = {
+static const TypeInfo pxa2xx_ssp_info = {
     .name          = "pxa2xx-ssp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxSSPState),
diff --git a/hw/pxa2xx_dma.c b/hw/pxa2xx_dma.c
index 031015400ba..c0dba45752c 100644
--- a/hw/pxa2xx_dma.c
+++ b/hw/pxa2xx_dma.c
@@ -147,7 +147,7 @@ static inline void pxa2xx_dma_descriptor_fetch(
                 PXA2xxDMAState *s, int ch)
 {
     uint32_t desc[4];
-    target_phys_addr_t daddr = s->chan[ch].descr & ~0xf;
+    hwaddr daddr = s->chan[ch].descr & ~0xf;
     if ((s->chan[ch].descr & DDADR_BREN) && (s->chan[ch].state & DCSR_CMPST))
         daddr += 32;
 
@@ -251,7 +251,7 @@ static void pxa2xx_dma_run(PXA2xxDMAState *s)
     }
 }
 
-static uint64_t pxa2xx_dma_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_dma_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     PXA2xxDMAState *s = (PXA2xxDMAState *) opaque;
@@ -310,7 +310,7 @@ static uint64_t pxa2xx_dma_read(void *opaque, target_phys_addr_t offset,
     return 7;
 }
 
-static void pxa2xx_dma_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_dma_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     PXA2xxDMAState *s = (PXA2xxDMAState *) opaque;
@@ -473,7 +473,7 @@ static int pxa2xx_dma_init(SysBusDevice *dev)
     return 0;
 }
 
-DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq)
+DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq)
 {
     DeviceState *dev;
 
@@ -481,13 +481,13 @@ DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq)
     qdev_prop_set_int32(dev, "channels", PXA27X_DMA_NUM_CHANNELS);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
 
-DeviceState *pxa255_dma_init(target_phys_addr_t base, qemu_irq irq)
+DeviceState *pxa255_dma_init(hwaddr base, qemu_irq irq)
 {
     DeviceState *dev;
 
@@ -495,8 +495,8 @@ DeviceState *pxa255_dma_init(target_phys_addr_t base, qemu_irq irq)
     qdev_prop_set_int32(dev, "channels", PXA27X_DMA_NUM_CHANNELS);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
@@ -559,7 +559,7 @@ static void pxa2xx_dma_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_dma_properties;
 }
 
-static TypeInfo pxa2xx_dma_info = {
+static const TypeInfo pxa2xx_dma_info = {
     .name          = "pxa2xx-dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxDMAState),
diff --git a/hw/pxa2xx_gpio.c b/hw/pxa2xx_gpio.c
index 3c90c9c4e00..05d2ad2add6 100644
--- a/hw/pxa2xx_gpio.c
+++ b/hw/pxa2xx_gpio.c
@@ -139,7 +139,7 @@ static void pxa2xx_gpio_handler_update(PXA2xxGPIOInfo *s) {
     }
 }
 
-static uint64_t pxa2xx_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_gpio_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     PXA2xxGPIOInfo *s = (PXA2xxGPIOInfo *) opaque;
@@ -191,7 +191,7 @@ static uint64_t pxa2xx_gpio_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_gpio_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_gpio_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     PXA2xxGPIOInfo *s = (PXA2xxGPIOInfo *) opaque;
@@ -249,22 +249,23 @@ static const MemoryRegionOps pxa_gpio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-DeviceState *pxa2xx_gpio_init(target_phys_addr_t base,
-                CPUARMState *env, DeviceState *pic, int lines)
+DeviceState *pxa2xx_gpio_init(hwaddr base,
+                              ARMCPU *cpu, DeviceState *pic, int lines)
 {
+    CPUState *cs = CPU(cpu);
     DeviceState *dev;
 
     dev = qdev_create(NULL, "pxa2xx-gpio");
     qdev_prop_set_int32(dev, "lines", lines);
-    qdev_prop_set_int32(dev, "ncpu", env->cpu_index);
+    qdev_prop_set_int32(dev, "ncpu", cs->cpu_index);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0,
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_0));
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1,
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_1));
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2,
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_X));
 
     return dev;
@@ -276,7 +277,7 @@ static int pxa2xx_gpio_initfn(SysBusDevice *dev)
 
     s = FROM_SYSBUS(PXA2xxGPIOInfo, dev);
 
-    s->cpu = arm_env_get_cpu(qemu_get_cpu(s->ncpu));
+    s->cpu = ARM_CPU(qemu_get_cpu(s->ncpu));
 
     qdev_init_gpio_in(&dev->qdev, pxa2xx_gpio_set, s->lines);
     qdev_init_gpio_out(&dev->qdev, s->handler, s->lines);
@@ -296,7 +297,7 @@ static int pxa2xx_gpio_initfn(SysBusDevice *dev)
  */
 void pxa2xx_gpio_read_notifier(DeviceState *dev, qemu_irq handler)
 {
-    PXA2xxGPIOInfo *s = FROM_SYSBUS(PXA2xxGPIOInfo, sysbus_from_qdev(dev));
+    PXA2xxGPIOInfo *s = FROM_SYSBUS(PXA2xxGPIOInfo, SYS_BUS_DEVICE(dev));
     s->read_notify = handler;
 }
 
@@ -334,7 +335,7 @@ static void pxa2xx_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_gpio_properties;
 }
 
-static TypeInfo pxa2xx_gpio_info = {
+static const TypeInfo pxa2xx_gpio_info = {
     .name          = "pxa2xx-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxGPIOInfo),
diff --git a/hw/pxa2xx_keypad.c b/hw/pxa2xx_keypad.c
index 59db02584e0..4ff04ad63b3 100644
--- a/hw/pxa2xx_keypad.c
+++ b/hw/pxa2xx_keypad.c
@@ -13,7 +13,7 @@
 
 #include "hw.h"
 #include "pxa.h"
-#include "console.h"
+#include "ui/console.h"
 
 /*
  * Keypad
@@ -172,10 +172,9 @@ static void pxa27x_keyboard_event (PXA2xxKeyPadState *kp, int keycode)
         kp->kpc |= KPC_MI;
         qemu_irq_raise(kp->irq);
     }
-    return;
 }
 
-static uint64_t pxa2xx_keypad_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_keypad_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     PXA2xxKeyPadState *s = (PXA2xxKeyPadState *) opaque;
@@ -237,7 +236,7 @@ static uint64_t pxa2xx_keypad_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_keypad_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_keypad_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     PXA2xxKeyPadState *s = (PXA2xxKeyPadState *) opaque;
@@ -306,7 +305,7 @@ static const VMStateDescription vmstate_pxa2xx_keypad = {
 };
 
 PXA2xxKeyPadState *pxa27x_keypad_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base,
+                                      hwaddr base,
                                       qemu_irq irq)
 {
     PXA2xxKeyPadState *s;
diff --git a/hw/pxa2xx_lcd.c b/hw/pxa2xx_lcd.c
index ee8bf577cb8..512a27e7029 100644
--- a/hw/pxa2xx_lcd.c
+++ b/hw/pxa2xx_lcd.c
@@ -11,11 +11,11 @@
  */
 
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pxa.h"
-#include "pixel_ops.h"
+#include "ui/pixel_ops.h"
 /* FIXME: For graphic_rotate. Should probably be done in common code.  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "framebuffer.h"
 
 struct DMAChannel {
@@ -23,7 +23,7 @@ struct DMAChannel {
     uint8_t up;
     uint8_t palette[1024];
     uint8_t pbuffer[1024];
-    void (*redraw)(PXA2xxLCDState *s, target_phys_addr_t addr,
+    void (*redraw)(PXA2xxLCDState *s, hwaddr addr,
                    int *miny, int *maxy);
 
     uint32_t descriptor;
@@ -291,7 +291,7 @@ static inline void pxa2xx_dma_rdst_set(PXA2xxLCDState *s)
 static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
 {
     PXAFrameDescriptor desc;
-    target_phys_addr_t descptr;
+    hwaddr descptr;
     int i;
 
     for (i = 0; i < PXA_LCDDMA_CHANS; i ++) {
@@ -323,7 +323,7 @@ static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
     }
 }
 
-static uint64_t pxa2xx_lcdc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_lcdc_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
@@ -417,7 +417,7 @@ static uint64_t pxa2xx_lcdc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_lcdc_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_lcdc_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
@@ -674,7 +674,7 @@ static void pxa2xx_palette_parse(PXA2xxLCDState *s, int ch, int bpp)
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot0(PXA2xxLCDState *s,
-                target_phys_addr_t addr, int *miny, int *maxy)
+                hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -701,7 +701,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot0(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot90(PXA2xxLCDState *s,
-               target_phys_addr_t addr, int *miny, int *maxy)
+               hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -729,7 +729,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot90(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot180(PXA2xxLCDState *s,
-                target_phys_addr_t addr, int *miny, int *maxy)
+                hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -759,7 +759,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot180(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot270(PXA2xxLCDState *s,
-               target_phys_addr_t addr, int *miny, int *maxy)
+               hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -813,7 +813,7 @@ static void pxa2xx_lcdc_resize(PXA2xxLCDState *s)
 static void pxa2xx_update_display(void *opaque)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
-    target_phys_addr_t fbptr;
+    hwaddr fbptr;
     int miny, maxy;
     int ch;
     if (!(s->control[0] & LCCR0_ENB))
@@ -871,20 +871,20 @@ static void pxa2xx_update_display(void *opaque)
     if (miny >= 0) {
         switch (s->orientation) {
         case 0:
-            dpy_update(s->ds, 0, miny, s->xres, maxy - miny + 1);
+            dpy_gfx_update(s->ds, 0, miny, s->xres, maxy - miny + 1);
             break;
         case 90:
-            dpy_update(s->ds, miny, 0, maxy - miny + 1, s->xres);
+            dpy_gfx_update(s->ds, miny, 0, maxy - miny + 1, s->xres);
             break;
         case 180:
             maxy = s->yres - maxy - 1;
             miny = s->yres - miny - 1;
-            dpy_update(s->ds, 0, maxy, s->xres, miny - maxy + 1);
+            dpy_gfx_update(s->ds, 0, maxy, s->xres, miny - maxy + 1);
             break;
         case 270:
             maxy = s->yres - maxy - 1;
             miny = s->yres - miny - 1;
-            dpy_update(s->ds, maxy, 0, miny - maxy + 1, s->xres);
+            dpy_gfx_update(s->ds, maxy, 0, miny - maxy + 1, s->xres);
             break;
         }
     }
@@ -987,7 +987,7 @@ static const VMStateDescription vmstate_pxa2xx_lcdc = {
 #include "pxa2xx_template.h"
 
 PXA2xxLCDState *pxa2xx_lcdc_init(MemoryRegion *sysmem,
-                                 target_phys_addr_t base, qemu_irq irq)
+                                 hwaddr base, qemu_irq irq)
 {
     PXA2xxLCDState *s;
 
diff --git a/hw/pxa2xx_mmci.c b/hw/pxa2xx_mmci.c
index b505a4cc987..35899687125 100644
--- a/hw/pxa2xx_mmci.c
+++ b/hw/pxa2xx_mmci.c
@@ -215,7 +215,7 @@ static void pxa2xx_mmci_wakequeues(PXA2xxMMCIState *s)
     pxa2xx_mmci_fifo_update(s);
 }
 
-static uint32_t pxa2xx_mmci_read(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_read(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     uint32_t ret;
@@ -277,7 +277,7 @@ static uint32_t pxa2xx_mmci_read(void *opaque, target_phys_addr_t offset)
 }
 
 static void pxa2xx_mmci_write(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
 
@@ -386,21 +386,21 @@ static void pxa2xx_mmci_write(void *opaque,
     }
 }
 
-static uint32_t pxa2xx_mmci_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readb(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 1;
     return pxa2xx_mmci_read(opaque, offset);
 }
 
-static uint32_t pxa2xx_mmci_readh(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readh(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 2;
     return pxa2xx_mmci_read(opaque, offset);
 }
 
-static uint32_t pxa2xx_mmci_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readw(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 4;
@@ -408,7 +408,7 @@ static uint32_t pxa2xx_mmci_readw(void *opaque, target_phys_addr_t offset)
 }
 
 static void pxa2xx_mmci_writeb(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 1;
@@ -416,7 +416,7 @@ static void pxa2xx_mmci_writeb(void *opaque,
 }
 
 static void pxa2xx_mmci_writeh(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 2;
@@ -424,7 +424,7 @@ static void pxa2xx_mmci_writeh(void *opaque,
 }
 
 static void pxa2xx_mmci_writew(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 4;
@@ -522,7 +522,7 @@ static int pxa2xx_mmci_load(QEMUFile *f, void *opaque, int version_id)
 }
 
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 BlockDriverState *bd, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma)
 {
diff --git a/hw/pxa2xx_pcmcia.c b/hw/pxa2xx_pcmcia.c
index b15872a9d90..3a79c728ab2 100644
--- a/hw/pxa2xx_pcmcia.c
+++ b/hw/pxa2xx_pcmcia.c
@@ -27,7 +27,7 @@ struct PXA2xxPCMCIAState {
 };
 
 static uint64_t pxa2xx_pcmcia_common_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -38,7 +38,7 @@ static uint64_t pxa2xx_pcmcia_common_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_common_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_common_write(void *opaque, hwaddr offset,
                                        uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -49,7 +49,7 @@ static void pxa2xx_pcmcia_common_write(void *opaque, target_phys_addr_t offset,
 }
 
 static uint64_t pxa2xx_pcmcia_attr_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -60,7 +60,7 @@ static uint64_t pxa2xx_pcmcia_attr_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_attr_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_attr_write(void *opaque, hwaddr offset,
                                      uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -71,7 +71,7 @@ static void pxa2xx_pcmcia_attr_write(void *opaque, target_phys_addr_t offset,
 }
 
 static uint64_t pxa2xx_pcmcia_io_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -82,7 +82,7 @@ static uint64_t pxa2xx_pcmcia_io_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_io_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_io_write(void *opaque, hwaddr offset,
                                    uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -120,7 +120,7 @@ static void pxa2xx_pcmcia_set_irq(void *opaque, int line, int level)
 }
 
 PXA2xxPCMCIAState *pxa2xx_pcmcia_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base)
+                                      hwaddr base)
 {
     PXA2xxPCMCIAState *s;
 
diff --git a/hw/pxa2xx_pic.c b/hw/pxa2xx_pic.c
index e1e8830ff09..90b8fef3f9e 100644
--- a/hw/pxa2xx_pic.c
+++ b/hw/pxa2xx_pic.c
@@ -119,7 +119,7 @@ static inline uint32_t pxa2xx_pic_highest(PXA2xxPICState *s) {
     return ichp;
 }
 
-static uint64_t pxa2xx_pic_mem_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_pic_mem_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     PXA2xxPICState *s = (PXA2xxPICState *) opaque;
@@ -159,7 +159,7 @@ static uint64_t pxa2xx_pic_mem_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void pxa2xx_pic_mem_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pic_mem_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     PXA2xxPICState *s = (PXA2xxPICState *) opaque;
@@ -257,11 +257,11 @@ static int pxa2xx_pic_post_load(void *opaque, int version_id)
     return 0;
 }
 
-DeviceState *pxa2xx_pic_init(target_phys_addr_t base, ARMCPU *cpu)
+DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu)
 {
     CPUARMState *env = &cpu->env;
     DeviceState *dev = qdev_create(NULL, "pxa2xx_pic");
-    PXA2xxPICState *s = FROM_SYSBUS(PXA2xxPICState, sysbus_from_qdev(dev));
+    PXA2xxPICState *s = FROM_SYSBUS(PXA2xxPICState, SYS_BUS_DEVICE(dev));
 
     s->cpu = cpu;
 
@@ -279,8 +279,8 @@ DeviceState *pxa2xx_pic_init(target_phys_addr_t base, ARMCPU *cpu)
     /* Enable IC memory-mapped registers access.  */
     memory_region_init_io(&s->iomem, &pxa2xx_pic_ops, s,
                           "pxa2xx-pic", 0x00100000);
-    sysbus_init_mmio(sysbus_from_qdev(dev), &s->iomem);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     /* Enable IC coprocessor access.  */
     define_arm_cp_regs_with_opaque(arm_env_get_cpu(env), pxa_pic_cp_reginfo, s);
@@ -319,7 +319,7 @@ static void pxa2xx_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pxa2xx_pic_regs;
 }
 
-static TypeInfo pxa2xx_pic_info = {
+static const TypeInfo pxa2xx_pic_info = {
     .name          = "pxa2xx_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxPICState),
diff --git a/hw/pxa2xx_timer.c b/hw/pxa2xx_timer.c
index 77b033b5412..5c9d2e8bc6b 100644
--- a/hw/pxa2xx_timer.c
+++ b/hw/pxa2xx_timer.c
@@ -8,8 +8,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "pxa.h"
 #include "sysbus.h"
 
@@ -149,7 +149,7 @@ static void pxa2xx_timer_update4(void *opaque, uint64_t now_qemu, int n)
     qemu_mod_timer(s->tm4[n].tm.qtimer, new_qemu);
 }
 
-static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     PXA2xxTimerInfo *s = (PXA2xxTimerInfo *) opaque;
@@ -157,17 +157,27 @@ static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
 
     switch (offset) {
     case OSMR3:  tm ++;
+        /* fall through */
     case OSMR2:  tm ++;
+        /* fall through */
     case OSMR1:  tm ++;
+        /* fall through */
     case OSMR0:
         return s->timer[tm].value;
     case OSMR11: tm ++;
+        /* fall through */
     case OSMR10: tm ++;
+        /* fall through */
     case OSMR9:  tm ++;
+        /* fall through */
     case OSMR8:  tm ++;
+        /* fall through */
     case OSMR7:  tm ++;
+        /* fall through */
     case OSMR6:  tm ++;
+        /* fall through */
     case OSMR5:  tm ++;
+        /* fall through */
     case OSMR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -176,12 +186,19 @@ static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
         return s->clock + muldiv64(qemu_get_clock_ns(vm_clock) -
                         s->lastload, s->freq, get_ticks_per_sec());
     case OSCR11: tm ++;
+        /* fall through */
     case OSCR10: tm ++;
+        /* fall through */
     case OSCR9:  tm ++;
+        /* fall through */
     case OSCR8:  tm ++;
+        /* fall through */
     case OSCR7:  tm ++;
+        /* fall through */
     case OSCR6:  tm ++;
+        /* fall through */
     case OSCR5:  tm ++;
+        /* fall through */
     case OSCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -207,12 +224,19 @@ static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
     case OWER:
         return s->reset3;
     case OMCR11: tm ++;
+        /* fall through */
     case OMCR10: tm ++;
+        /* fall through */
     case OMCR9:  tm ++;
+        /* fall through */
     case OMCR8:  tm ++;
+        /* fall through */
     case OMCR7:  tm ++;
+        /* fall through */
     case OMCR6:  tm ++;
+        /* fall through */
     case OMCR5:  tm ++;
+        /* fall through */
     case OMCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -227,7 +251,7 @@ static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_timer_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     int i, tm = 0;
@@ -235,19 +259,29 @@ static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
 
     switch (offset) {
     case OSMR3:  tm ++;
+        /* fall through */
     case OSMR2:  tm ++;
+        /* fall through */
     case OSMR1:  tm ++;
+        /* fall through */
     case OSMR0:
         s->timer[tm].value = value;
         pxa2xx_timer_update(s, qemu_get_clock_ns(vm_clock));
         break;
     case OSMR11: tm ++;
+        /* fall through */
     case OSMR10: tm ++;
+        /* fall through */
     case OSMR9:  tm ++;
+        /* fall through */
     case OSMR8:  tm ++;
+        /* fall through */
     case OSMR7:  tm ++;
+        /* fall through */
     case OSMR6:  tm ++;
+        /* fall through */
     case OSMR5:  tm ++;
+        /* fall through */
     case OSMR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -261,12 +295,19 @@ static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
         pxa2xx_timer_update(s, s->lastload);
         break;
     case OSCR11: tm ++;
+        /* fall through */
     case OSCR10: tm ++;
+        /* fall through */
     case OSCR9:  tm ++;
+        /* fall through */
     case OSCR8:  tm ++;
+        /* fall through */
     case OSCR7:  tm ++;
+        /* fall through */
     case OSCR6:  tm ++;
+        /* fall through */
     case OSCR5:  tm ++;
+        /* fall through */
     case OSCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -291,8 +332,11 @@ static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
         s->reset3 = value;
         break;
     case OMCR7:  tm ++;
+        /* fall through */
     case OMCR6:  tm ++;
+        /* fall through */
     case OMCR5:  tm ++;
+        /* fall through */
     case OMCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -306,8 +350,11 @@ static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
         }
         break;
     case OMCR11: tm ++;
+        /* fall through */
     case OMCR10: tm ++;
+        /* fall through */
     case OMCR9:  tm ++;
+        /* fall through */
     case OMCR8:  tm += 4;
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -495,7 +542,7 @@ static void pxa25x_timer_dev_class_init(ObjectClass *klass, void *data)
     dc->props = pxa25x_timer_dev_properties;
 }
 
-static TypeInfo pxa25x_timer_dev_info = {
+static const TypeInfo pxa25x_timer_dev_info = {
     .name          = "pxa25x-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
@@ -520,7 +567,7 @@ static void pxa27x_timer_dev_class_init(ObjectClass *klass, void *data)
     dc->props = pxa27x_timer_dev_properties;
 }
 
-static TypeInfo pxa27x_timer_dev_info = {
+static const TypeInfo pxa27x_timer_dev_info = {
     .name          = "pxa27x-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
diff --git a/hw/q35.c b/hw/q35.c
new file mode 100644
index 00000000000..efebc2786ac
--- /dev/null
+++ b/hw/q35.c
@@ -0,0 +1,309 @@
+/*
+ * QEMU MCH/ICH9 PCI Bridge Emulation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009, 2010, 2011
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on piix_pci.c, but heavily modified.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "hw.h"
+#include "q35.h"
+
+/****************************************************************************
+ * Q35 host
+ */
+
+static int q35_host_init(SysBusDevice *dev)
+{
+    PCIBus *b;
+    PCIHostState *pci = FROM_SYSBUS(PCIHostState, dev);
+    Q35PCIHost *s = Q35_HOST_DEVICE(&dev->qdev);
+
+    memory_region_init_io(&pci->conf_mem, &pci_host_conf_le_ops, pci,
+                          "pci-conf-idx", 4);
+    sysbus_add_io(dev, MCH_HOST_BRIDGE_CONFIG_ADDR, &pci->conf_mem);
+    sysbus_init_ioports(&pci->busdev, MCH_HOST_BRIDGE_CONFIG_ADDR, 4);
+
+    memory_region_init_io(&pci->data_mem, &pci_host_data_le_ops, pci,
+                          "pci-conf-data", 4);
+    sysbus_add_io(dev, MCH_HOST_BRIDGE_CONFIG_DATA, &pci->data_mem);
+    sysbus_init_ioports(&pci->busdev, MCH_HOST_BRIDGE_CONFIG_DATA, 4);
+
+    if (pcie_host_init(&s->host) < 0) {
+        return -1;
+    }
+    b = pci_bus_new(&s->host.pci.busdev.qdev, "pcie.0",
+                    s->mch.pci_address_space, s->mch.address_space_io, 0);
+    s->host.pci.bus = b;
+    qdev_set_parent_bus(DEVICE(&s->mch), BUS(b));
+    qdev_init_nofail(DEVICE(&s->mch));
+
+    return 0;
+}
+
+static Property mch_props[] = {
+    DEFINE_PROP_UINT64("MCFG", Q35PCIHost, host.base_addr,
+                        MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void q35_host_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = q35_host_init;
+    dc->props = mch_props;
+}
+
+static void q35_host_initfn(Object *obj)
+{
+    Q35PCIHost *s = Q35_HOST_DEVICE(obj);
+
+    object_initialize(&s->mch, TYPE_MCH_PCI_DEVICE);
+    object_property_add_child(OBJECT(s), "mch", OBJECT(&s->mch), NULL);
+    qdev_prop_set_uint32(DEVICE(&s->mch), "addr", PCI_DEVFN(0, 0));
+    qdev_prop_set_bit(DEVICE(&s->mch), "multifunction", false);
+}
+
+static const TypeInfo q35_host_info = {
+    .name       = TYPE_Q35_HOST_DEVICE,
+    .parent     = TYPE_PCIE_HOST_BRIDGE,
+    .instance_size = sizeof(Q35PCIHost),
+    .instance_init = q35_host_initfn,
+    .class_init = q35_host_class_init,
+};
+
+/****************************************************************************
+ * MCH D0:F0
+ */
+
+/* PCIe MMCFG */
+static void mch_update_pciexbar(MCHPCIState *mch)
+{
+    PCIDevice *pci_dev = &mch->d;
+    BusState *bus = qdev_get_parent_bus(&pci_dev->qdev);
+    DeviceState *qdev = bus->parent;
+    Q35PCIHost *s = Q35_HOST_DEVICE(qdev);
+
+    uint64_t pciexbar;
+    int enable;
+    uint64_t addr;
+    uint64_t addr_mask;
+    uint32_t length;
+
+    pciexbar = pci_get_quad(pci_dev->config + MCH_HOST_BRIDGE_PCIEXBAR);
+    enable = pciexbar & MCH_HOST_BRIDGE_PCIEXBAREN;
+    addr_mask = MCH_HOST_BRIDGE_PCIEXBAR_ADMSK;
+    switch (pciexbar & MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_MASK) {
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_256M:
+        length = 256 * 1024 * 1024;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M:
+        length = 128 * 1024 * 1024;
+        addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK |
+            MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_64M:
+        length = 64 * 1024 * 1024;
+        addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD:
+    default:
+        enable = 0;
+        length = 0;
+        abort();
+        break;
+    }
+    addr = pciexbar & addr_mask;
+    pcie_host_mmcfg_update(&s->host, enable, addr, length);
+}
+
+/* PAM */
+static void mch_update_pam(MCHPCIState *mch)
+{
+    int i;
+
+    memory_region_transaction_begin();
+    for (i = 0; i < 13; i++) {
+        pam_update(&mch->pam_regions[i], i,
+                   mch->d.config[MCH_HOST_BRIDGE_PAM0 + ((i + 1) / 2)]);
+    }
+    memory_region_transaction_commit();
+}
+
+/* SMRAM */
+static void mch_update_smram(MCHPCIState *mch)
+{
+    memory_region_transaction_begin();
+    smram_update(&mch->smram_region, mch->d.config[MCH_HOST_BRDIGE_SMRAM],
+                    mch->smm_enabled);
+    memory_region_transaction_commit();
+}
+
+static void mch_set_smm(int smm, void *arg)
+{
+    MCHPCIState *mch = arg;
+
+    memory_region_transaction_begin();
+    smram_set_smm(&mch->smm_enabled, smm, mch->d.config[MCH_HOST_BRDIGE_SMRAM],
+                    &mch->smram_region);
+    memory_region_transaction_commit();
+}
+
+static void mch_write_config(PCIDevice *d,
+                              uint32_t address, uint32_t val, int len)
+{
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    /* XXX: implement SMRAM.D_LOCK */
+    pci_default_write_config(d, address, val, len);
+
+    if (ranges_overlap(address, len, MCH_HOST_BRIDGE_PAM0,
+                       MCH_HOST_BRIDGE_PAM_SIZE)) {
+        mch_update_pam(mch);
+    }
+
+    if (ranges_overlap(address, len, MCH_HOST_BRIDGE_PCIEXBAR,
+                       MCH_HOST_BRIDGE_PCIEXBAR_SIZE)) {
+        mch_update_pciexbar(mch);
+    }
+
+    if (ranges_overlap(address, len, MCH_HOST_BRDIGE_SMRAM,
+                       MCH_HOST_BRDIGE_SMRAM_SIZE)) {
+        mch_update_smram(mch);
+    }
+}
+
+static void mch_update(MCHPCIState *mch)
+{
+    mch_update_pciexbar(mch);
+    mch_update_pam(mch);
+    mch_update_smram(mch);
+}
+
+static int mch_post_load(void *opaque, int version_id)
+{
+    MCHPCIState *mch = opaque;
+    mch_update(mch);
+    return 0;
+}
+
+static const VMStateDescription vmstate_mch = {
+    .name = "mch",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = mch_post_load,
+    .fields = (VMStateField []) {
+        VMSTATE_PCI_DEVICE(d, MCHPCIState),
+        VMSTATE_UINT8(smm_enabled, MCHPCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void mch_reset(DeviceState *qdev)
+{
+    PCIDevice *d = PCI_DEVICE(qdev);
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    pci_set_quad(d->config + MCH_HOST_BRIDGE_PCIEXBAR,
+                 MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT);
+
+    d->config[MCH_HOST_BRDIGE_SMRAM] = MCH_HOST_BRIDGE_SMRAM_DEFAULT;
+
+    mch_update(mch);
+}
+
+static int mch_init(PCIDevice *d)
+{
+    int i;
+    hwaddr pci_hole64_size;
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    /* setup pci memory regions */
+    memory_region_init_alias(&mch->pci_hole, "pci-hole",
+                             mch->pci_address_space,
+                             mch->below_4g_mem_size,
+                             0x100000000ULL - mch->below_4g_mem_size);
+    memory_region_add_subregion(mch->system_memory, mch->below_4g_mem_size,
+                                &mch->pci_hole);
+    pci_hole64_size = (sizeof(hwaddr) == 4 ? 0 :
+                       ((uint64_t)1 << 62));
+    memory_region_init_alias(&mch->pci_hole_64bit, "pci-hole64",
+                             mch->pci_address_space,
+                             0x100000000ULL + mch->above_4g_mem_size,
+                             pci_hole64_size);
+    if (pci_hole64_size) {
+        memory_region_add_subregion(mch->system_memory,
+                                    0x100000000ULL + mch->above_4g_mem_size,
+                                    &mch->pci_hole_64bit);
+    }
+    /* smram */
+    cpu_smm_register(&mch_set_smm, mch);
+    memory_region_init_alias(&mch->smram_region, "smram-region",
+                             mch->pci_address_space, 0xa0000, 0x20000);
+    memory_region_add_subregion_overlap(mch->system_memory, 0xa0000,
+                                        &mch->smram_region, 1);
+    memory_region_set_enabled(&mch->smram_region, false);
+    init_pam(mch->ram_memory, mch->system_memory, mch->pci_address_space,
+             &mch->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
+    for (i = 0; i < 12; ++i) {
+        init_pam(mch->ram_memory, mch->system_memory, mch->pci_address_space,
+                 &mch->pam_regions[i+1], PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE,
+                 PAM_EXPAN_SIZE);
+    }
+    return 0;
+}
+
+static void mch_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    k->init = mch_init;
+    k->config_write = mch_write_config;
+    dc->reset = mch_reset;
+    dc->desc = "Host bridge";
+    dc->vmsd = &vmstate_mch;
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_Q35_MCH;
+    k->revision = MCH_HOST_BRIDGE_REVISION_DEFUALT;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+}
+
+static const TypeInfo mch_info = {
+    .name = TYPE_MCH_PCI_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(MCHPCIState),
+    .class_init = mch_class_init,
+};
+
+static void q35_register(void)
+{
+    type_register_static(&mch_info);
+    type_register_static(&q35_host_info);
+}
+
+type_init(q35_register);
diff --git a/hw/q35.h b/hw/q35.h
new file mode 100644
index 00000000000..246c12cb044
--- /dev/null
+++ b/hw/q35.h
@@ -0,0 +1,150 @@
+/*
+ * q35.h
+ *
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef HW_Q35_H
+#define HW_Q35_H
+
+#include "hw.h"
+#include "qemu/range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "apic.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+
+#define TYPE_Q35_HOST_DEVICE "q35-pcihost"
+#define Q35_HOST_DEVICE(obj) \
+     OBJECT_CHECK(Q35PCIHost, (obj), TYPE_Q35_HOST_DEVICE)
+
+#define TYPE_MCH_PCI_DEVICE "mch"
+#define MCH_PCI_DEVICE(obj) \
+     OBJECT_CHECK(MCHPCIState, (obj), TYPE_MCH_PCI_DEVICE)
+
+typedef struct MCHPCIState {
+    PCIDevice d;
+    MemoryRegion *ram_memory;
+    MemoryRegion *pci_address_space;
+    MemoryRegion *system_memory;
+    MemoryRegion *address_space_io;
+    PAMMemoryRegion pam_regions[13];
+    MemoryRegion smram_region;
+    MemoryRegion pci_hole;
+    MemoryRegion pci_hole_64bit;
+    uint8_t smm_enabled;
+    ram_addr_t below_4g_mem_size;
+    ram_addr_t above_4g_mem_size;
+} MCHPCIState;
+
+typedef struct Q35PCIHost {
+    PCIExpressHost host;
+    MCHPCIState mch;
+} Q35PCIHost;
+
+#define Q35_MASK(bit, ms_bit, ls_bit) \
+((uint##bit##_t)(((1ULL << ((ms_bit) + 1)) - 1) & ~((1ULL << ls_bit) - 1)))
+
+/*
+ * gmch part
+ */
+
+/* PCI configuration */
+#define MCH_HOST_BRIDGE                        "MCH"
+
+#define MCH_HOST_BRIDGE_CONFIG_ADDR            0xcf8
+#define MCH_HOST_BRIDGE_CONFIG_DATA            0xcfc
+
+/* D0:F0 configuration space */
+#define MCH_HOST_BRIDGE_REVISION_DEFUALT       0x0
+
+#define MCH_HOST_BRIDGE_PCIEXBAR               0x60    /* 64bit register */
+#define MCH_HOST_BRIDGE_PCIEXBAR_SIZE          8       /* 64bit register */
+#define MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT       0xb0000000
+#define MCH_HOST_BRIDGE_PCIEXBAR_ADMSK         Q35_MASK(64, 35, 28)
+#define MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK      ((uint64_t)(1 << 26))
+#define MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK       ((uint64_t)(1 << 25))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_MASK   ((uint64_t)(0x3 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_256M   ((uint64_t)(0x0 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M   ((uint64_t)(0x1 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_64M    ((uint64_t)(0x2 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD    ((uint64_t)(0x3 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAREN             ((uint64_t)1)
+
+#define MCH_HOST_BRIDGE_PAM_NB                 7
+#define MCH_HOST_BRIDGE_PAM_SIZE               7
+#define MCH_HOST_BRIDGE_PAM0                   0x90
+#define MCH_HOST_BRIDGE_PAM_BIOS_AREA          0xf0000
+#define MCH_HOST_BRIDGE_PAM_AREA_SIZE          0x10000 /* 16KB */
+#define MCH_HOST_BRIDGE_PAM1                   0x91
+#define MCH_HOST_BRIDGE_PAM_EXPAN_AREA         0xc0000
+#define MCH_HOST_BRIDGE_PAM_EXPAN_SIZE         0x04000
+#define MCH_HOST_BRIDGE_PAM2                   0x92
+#define MCH_HOST_BRIDGE_PAM3                   0x93
+#define MCH_HOST_BRIDGE_PAM4                   0x94
+#define MCH_HOST_BRIDGE_PAM_EXBIOS_AREA        0xe0000
+#define MCH_HOST_BRIDGE_PAM_EXBIOS_SIZE        0x04000
+#define MCH_HOST_BRIDGE_PAM5                   0x95
+#define MCH_HOST_BRIDGE_PAM6                   0x96
+#define MCH_HOST_BRIDGE_PAM_WE_HI              ((uint8_t)(0x2 << 4))
+#define MCH_HOST_BRIDGE_PAM_RE_HI              ((uint8_t)(0x1 << 4))
+#define MCH_HOST_BRIDGE_PAM_HI_MASK            ((uint8_t)(0x3 << 4))
+#define MCH_HOST_BRIDGE_PAM_WE_LO              ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_PAM_RE_LO              ((uint8_t)0x1)
+#define MCH_HOST_BRIDGE_PAM_LO_MASK            ((uint8_t)0x3)
+#define MCH_HOST_BRIDGE_PAM_WE                 ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_PAM_RE                 ((uint8_t)0x1)
+#define MCH_HOST_BRIDGE_PAM_MASK               ((uint8_t)0x3)
+
+#define MCH_HOST_BRDIGE_SMRAM                  0x9d
+#define MCH_HOST_BRDIGE_SMRAM_SIZE             1
+#define MCH_HOST_BRIDGE_SMRAM_DEFAULT          ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_SMRAM_D_OPEN           ((uint8_t)(1 << 6))
+#define MCH_HOST_BRIDGE_SMRAM_D_CLS            ((uint8_t)(1 << 5))
+#define MCH_HOST_BRIDGE_SMRAM_D_LCK            ((uint8_t)(1 << 4))
+#define MCH_HOST_BRIDGE_SMRAM_G_SMRAME         ((uint8_t)(1 << 3))
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE_SEG_MASK  ((uint8_t)0x7)
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE_SEG       ((uint8_t)0x2)  /* hardwired to b010 */
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE           0xa0000
+#define MCH_HOST_BRIDGE_SMRAM_C_END            0xc0000
+#define MCH_HOST_BRIDGE_SMRAM_C_SIZE           0x20000
+#define MCH_HOST_BRIDGE_UPPER_SYSTEM_BIOS_END  0x100000
+
+#define MCH_HOST_BRIDGE_ESMRAMC                0x9e
+#define MCH_HOST_BRDIGE_ESMRAMC_H_SMRAME       ((uint8_t)(1 << 6))
+#define MCH_HOST_BRDIGE_ESMRAMC_E_SMERR        ((uint8_t)(1 << 5))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_CACHE       ((uint8_t)(1 << 4))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_L1          ((uint8_t)(1 << 3))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_L2          ((uint8_t)(1 << 2))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_MASK   ((uint8_t)(0x3 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_1MB    ((uint8_t)(0x0 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_2MB    ((uint8_t)(0x1 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_8MB    ((uint8_t)(0x2 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_T_EN           ((uint8_t)1)
+
+/* D1:F0 PCIE* port*/
+#define MCH_PCIE_DEV                           1
+#define MCH_PCIE_FUNC                          0
+
+#endif /* HW_Q35_H */
diff --git a/hw/qdev-addr.c b/hw/qdev-addr.c
index b711b6bf962..b4388f6a663 100644
--- a/hw/qdev-addr.c
+++ b/hw/qdev-addr.c
@@ -1,12 +1,13 @@
 #include "qdev.h"
 #include "qdev-addr.h"
-#include "targphys.h"
+#include "exec/hwaddr.h"
+#include "qapi/visitor.h"
 
 /* --- target physical address --- */
 
 static int parse_taddr(DeviceState *dev, Property *prop, const char *str)
 {
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
 
     *ptr = strtoull(str, NULL, 16);
     return 0;
@@ -14,7 +15,7 @@ static int parse_taddr(DeviceState *dev, Property *prop, const char *str)
 
 static int print_taddr(DeviceState *dev, Property *prop, char *dest, size_t len)
 {
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     return snprintf(dest, len, "0x" TARGET_FMT_plx, *ptr);
 }
 
@@ -23,7 +24,7 @@ static void get_taddr(Object *obj, Visitor *v, void *opaque,
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     int64_t value;
 
     value = *ptr;
@@ -35,11 +36,11 @@ static void set_taddr(Object *obj, Visitor *v, void *opaque,
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     Error *local_err = NULL;
     int64_t value;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -49,12 +50,12 @@ static void set_taddr(Object *obj, Visitor *v, void *opaque,
         error_propagate(errp, local_err);
         return;
     }
-    if ((uint64_t)value <= (uint64_t) ~(target_phys_addr_t)0) {
+    if ((uint64_t)value <= (uint64_t) ~(hwaddr)0) {
         *ptr = value;
     } else {
         error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
                   dev->id?:"", name, value, (uint64_t) 0,
-                  (uint64_t) ~(target_phys_addr_t)0);
+                  (uint64_t) ~(hwaddr)0);
     }
 }
 
@@ -67,7 +68,7 @@ PropertyInfo qdev_prop_taddr = {
     .set   = set_taddr,
 };
 
-void qdev_prop_set_taddr(DeviceState *dev, const char *name, target_phys_addr_t value)
+void qdev_prop_set_taddr(DeviceState *dev, const char *name, hwaddr value)
 {
     Error *errp = NULL;
     object_property_set_int(OBJECT(dev), value, name, &errp);
diff --git a/hw/qdev-addr.h b/hw/qdev-addr.h
index a0ddf3863c0..79708e67512 100644
--- a/hw/qdev-addr.h
+++ b/hw/qdev-addr.h
@@ -1,5 +1,10 @@
+#ifndef HW_QDEV_ADDR_H
+#define HW_QDEV_ADDR_H 1
+
 #define DEFINE_PROP_TADDR(_n, _s, _f, _d)                               \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_taddr, target_phys_addr_t)
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_taddr, hwaddr)
 
 extern PropertyInfo qdev_prop_taddr;
-void qdev_prop_set_taddr(DeviceState *dev, const char *name, target_phys_addr_t value);
+void qdev_prop_set_taddr(DeviceState *dev, const char *name, hwaddr value);
+
+#endif
diff --git a/hw/qdev-core.h b/hw/qdev-core.h
new file mode 100644
index 00000000000..d1b8e37d802
--- /dev/null
+++ b/hw/qdev-core.h
@@ -0,0 +1,296 @@
+#ifndef QDEV_CORE_H
+#define QDEV_CORE_H
+
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/typedefs.h"
+#include "qom/object.h"
+#include "hw/irq.h"
+#include "qapi/error.h"
+
+enum {
+    DEV_NVECTORS_UNSPECIFIED = -1,
+};
+
+#define TYPE_DEVICE "device"
+#define DEVICE(obj) OBJECT_CHECK(DeviceState, (obj), TYPE_DEVICE)
+#define DEVICE_CLASS(klass) OBJECT_CLASS_CHECK(DeviceClass, (klass), TYPE_DEVICE)
+#define DEVICE_GET_CLASS(obj) OBJECT_GET_CLASS(DeviceClass, (obj), TYPE_DEVICE)
+
+typedef int (*qdev_initfn)(DeviceState *dev);
+typedef int (*qdev_event)(DeviceState *dev);
+typedef void (*qdev_resetfn)(DeviceState *dev);
+typedef void (*DeviceRealize)(DeviceState *dev, Error **errp);
+typedef void (*DeviceUnrealize)(DeviceState *dev, Error **errp);
+
+struct VMStateDescription;
+
+/**
+ * DeviceClass:
+ * @props: Properties accessing state fields.
+ * @realize: Callback function invoked when the #DeviceState:realized
+ * property is changed to %true. The default invokes @init if not %NULL.
+ * @unrealize: Callback function invoked when the #DeviceState:realized
+ * property is changed to %false.
+ * @init: Callback function invoked when the #DeviceState::realized property
+ * is changed to %true. Deprecated, new types inheriting directly from
+ * TYPE_DEVICE should use @realize instead, new leaf types should consult
+ * their respective parent type.
+ *
+ * # Realization #
+ * Devices are constructed in two stages,
+ * 1) object instantiation via object_initialize() and
+ * 2) device realization via #DeviceState:realized property.
+ * The former may not fail (it might assert or exit), the latter may return
+ * error information to the caller and must be re-entrant.
+ * Trivial field initializations should go into #TypeInfo.instance_init.
+ * Operations depending on @props static properties should go into @realize.
+ * After successful realization, setting static properties will fail.
+ *
+ * As an interim step, the #DeviceState:realized property is set by deprecated
+ * functions qdev_init() and qdev_init_nofail().
+ * In the future, devices will propagate this state change to their children
+ * and along busses they expose.
+ * The point in time will be deferred to machine creation, so that values
+ * set in @realize will not be introspectable beforehand. Therefore devices
+ * must not create children during @realize; they should initialize them via
+ * object_initialize() in their own #TypeInfo.instance_init and forward the
+ * realization events appropriately.
+ *
+ * The @init callback is considered private to a particular bus implementation
+ * (immediate abstract child types of TYPE_DEVICE). Derived leaf types set an
+ * "init" callback on their parent class instead.
+ *
+ * Any type may override the @realize and/or @unrealize callbacks but needs
+ * to call the parent type's implementation if keeping their functionality
+ * is desired. Refer to QOM documentation for further discussion and examples.
+ *
+ * <note>
+ *   <para>
+ * If a type derived directly from TYPE_DEVICE implements @realize, it does
+ * not need to implement @init and therefore does not need to store and call
+ * #DeviceClass' default @realize callback.
+ * For other types consult the documentation and implementation of the
+ * respective parent types.
+ *   </para>
+ * </note>
+ */
+typedef struct DeviceClass {
+    /*< private >*/
+    ObjectClass parent_class;
+    /*< public >*/
+
+    const char *fw_name;
+    const char *desc;
+    Property *props;
+    int no_user;
+
+    /* callbacks */
+    void (*reset)(DeviceState *dev);
+    DeviceRealize realize;
+    DeviceUnrealize unrealize;
+
+    /* device state */
+    const struct VMStateDescription *vmsd;
+
+    /* Private to qdev / bus.  */
+    qdev_initfn init; /* TODO remove, once users are converted to realize */
+    qdev_event unplug;
+    qdev_event exit;
+    const char *bus_type;
+} DeviceClass;
+
+/**
+ * DeviceState:
+ * @realized: Indicates whether the device has been fully constructed.
+ *
+ * This structure should not be accessed directly.  We declare it here
+ * so that it can be embedded in individual device state structures.
+ */
+struct DeviceState {
+    /*< private >*/
+    Object parent_obj;
+    /*< public >*/
+
+    const char *id;
+    bool realized;
+    QemuOpts *opts;
+    int hotplugged;
+    BusState *parent_bus;
+    int num_gpio_out;
+    qemu_irq *gpio_out;
+    int num_gpio_in;
+    qemu_irq *gpio_in;
+    QLIST_HEAD(, BusState) child_bus;
+    int num_child_bus;
+    int instance_id_alias;
+    int alias_required_for_version;
+};
+
+#define TYPE_BUS "bus"
+#define BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_BUS)
+#define BUS_CLASS(klass) OBJECT_CLASS_CHECK(BusClass, (klass), TYPE_BUS)
+#define BUS_GET_CLASS(obj) OBJECT_GET_CLASS(BusClass, (obj), TYPE_BUS)
+
+struct BusClass {
+    ObjectClass parent_class;
+
+    /* FIXME first arg should be BusState */
+    void (*print_dev)(Monitor *mon, DeviceState *dev, int indent);
+    char *(*get_dev_path)(DeviceState *dev);
+    /*
+     * This callback is used to create Open Firmware device path in accordance
+     * with OF spec http://forthworks.com/standards/of1275.pdf. Individual bus
+     * bindings can be found at http://playground.sun.com/1275/bindings/.
+     */
+    char *(*get_fw_dev_path)(DeviceState *dev);
+    int (*reset)(BusState *bus);
+    /* maximum devices allowed on the bus, 0: no limit. */
+    int max_dev;
+};
+
+typedef struct BusChild {
+    DeviceState *child;
+    int index;
+    QTAILQ_ENTRY(BusChild) sibling;
+} BusChild;
+
+/**
+ * BusState:
+ */
+struct BusState {
+    Object obj;
+    DeviceState *parent;
+    const char *name;
+    int allow_hotplug;
+    int max_index;
+    QTAILQ_HEAD(ChildrenHead, BusChild) children;
+    QLIST_ENTRY(BusState) sibling;
+};
+
+struct Property {
+    const char   *name;
+    PropertyInfo *info;
+    int          offset;
+    uint8_t      bitnr;
+    uint8_t      qtype;
+    int64_t      defval;
+};
+
+struct PropertyInfo {
+    const char *name;
+    const char *legacy_name;
+    const char **enum_table;
+    int (*parse)(DeviceState *dev, Property *prop, const char *str);
+    int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
+    ObjectPropertyAccessor *get;
+    ObjectPropertyAccessor *set;
+    ObjectPropertyRelease *release;
+};
+
+typedef struct GlobalProperty {
+    const char *driver;
+    const char *property;
+    const char *value;
+    QTAILQ_ENTRY(GlobalProperty) next;
+} GlobalProperty;
+
+/*** Board API.  This should go away once we have a machine config file.  ***/
+
+DeviceState *qdev_create(BusState *bus, const char *name);
+DeviceState *qdev_try_create(BusState *bus, const char *name);
+int qdev_init(DeviceState *dev) QEMU_WARN_UNUSED_RESULT;
+void qdev_init_nofail(DeviceState *dev);
+void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
+                                 int required_for_version);
+void qdev_unplug(DeviceState *dev, Error **errp);
+void qdev_free(DeviceState *dev);
+int qdev_simple_unplug_cb(DeviceState *dev);
+void qdev_machine_creation_done(void);
+bool qdev_machine_modified(void);
+
+qemu_irq qdev_get_gpio_in(DeviceState *dev, int n);
+void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
+
+BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
+
+/*** Device API.  ***/
+
+/* Register device properties.  */
+/* GPIO inputs also double as IRQ sinks.  */
+void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n);
+void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n);
+
+BusState *qdev_get_parent_bus(DeviceState *dev);
+
+/*** BUS API. ***/
+
+DeviceState *qdev_find_recursive(BusState *bus, const char *id);
+
+/* Returns 0 to walk children, > 0 to skip walk, < 0 to terminate walk. */
+typedef int (qbus_walkerfn)(BusState *bus, void *opaque);
+typedef int (qdev_walkerfn)(DeviceState *dev, void *opaque);
+
+void qbus_create_inplace(BusState *bus, const char *typename,
+                         DeviceState *parent, const char *name);
+BusState *qbus_create(const char *typename, DeviceState *parent, const char *name);
+/* Returns > 0 if either devfn or busfn skip walk somewhere in cursion,
+ *         < 0 if either devfn or busfn terminate walk somewhere in cursion,
+ *           0 otherwise. */
+int qbus_walk_children(BusState *bus, qdev_walkerfn *devfn,
+                       qbus_walkerfn *busfn, void *opaque);
+int qdev_walk_children(DeviceState *dev, qdev_walkerfn *devfn,
+                       qbus_walkerfn *busfn, void *opaque);
+void qdev_reset_all(DeviceState *dev);
+
+/**
+ * @qbus_reset_all:
+ * @bus: Bus to be reset.
+ *
+ * Reset @bus and perform a bus-level ("hard") reset of all devices connected
+ * to it, including recursive processing of all buses below @bus itself.  A
+ * hard reset means that qbus_reset_all will reset all state of the device.
+ * For PCI devices, for example, this will include the base address registers
+ * or configuration space.
+ */
+void qbus_reset_all(BusState *bus);
+void qbus_reset_all_fn(void *opaque);
+
+void qbus_free(BusState *bus);
+
+#define FROM_QBUS(type, dev) DO_UPCAST(type, qbus, dev)
+
+/* This should go away once we get rid of the NULL bus hack */
+BusState *sysbus_get_default(void);
+
+char *qdev_get_fw_dev_path(DeviceState *dev);
+
+/**
+ * @qdev_machine_init
+ *
+ * Initialize platform devices before machine init.  This is a hack until full
+ * support for composition is added.
+ */
+void qdev_machine_init(void);
+
+/**
+ * @device_reset
+ *
+ * Reset a single device (by calling the reset method).
+ */
+void device_reset(DeviceState *dev);
+
+const struct VMStateDescription *qdev_get_vmsd(DeviceState *dev);
+
+const char *qdev_fw_name(DeviceState *dev);
+
+Object *qdev_get_machine(void);
+
+/* FIXME: make this a link<> */
+void qdev_set_parent_bus(DeviceState *dev, BusState *bus);
+
+extern int qdev_hotplug;
+
+char *qdev_get_dev_path(DeviceState *dev);
+
+#endif
diff --git a/hw/qdev-monitor.c b/hw/qdev-monitor.c
index 33b7f79a942..4e2a92b9dd0 100644
--- a/hw/qdev-monitor.c
+++ b/hw/qdev-monitor.c
@@ -18,9 +18,10 @@
  */
 
 #include "qdev.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "qmp-commands.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
+#include "qemu/config-file.h"
 
 /*
  * Aliases were a bad idea from the start.  Let's keep them
@@ -44,6 +45,7 @@ static const QDevAlias qdev_alias_table[] = {
     { "virtio-serial-s390", "virtio-serial", QEMU_ARCH_S390X },
     { "lsi53c895a", "lsi" },
     { "ich9-ahci", "ahci" },
+    { "kvm-pci-assign", "pci-assign" },
     { }
 };
 
@@ -281,6 +283,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
 static BusState *qbus_find_recursive(BusState *bus, const char *name,
                                      const char *bus_typename)
 {
+    BusClass *bus_class = BUS_GET_CLASS(bus);
     BusChild *kid;
     BusState *child, *ret;
     int match = 1;
@@ -288,10 +291,20 @@ static BusState *qbus_find_recursive(BusState *bus, const char *name,
     if (name && (strcmp(bus->name, name) != 0)) {
         match = 0;
     }
-    if (bus_typename &&
-        (strcmp(object_get_typename(OBJECT(bus)), bus_typename) != 0)) {
+    if (bus_typename && !object_dynamic_cast(OBJECT(bus), bus_typename)) {
         match = 0;
     }
+    if ((bus_class->max_dev != 0) && (bus_class->max_dev <= bus->max_index)) {
+        if (name != NULL) {
+            /* bus was explicitly specified: return an error. */
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Bus '%s' is full",
+                          bus->name);
+            return NULL;
+        } else {
+            /* bus was not specified: try to find another one. */
+            match = 0;
+        }
+    }
     if (match) {
         return bus;
     }
@@ -434,7 +447,7 @@ DeviceState *qdev_device_add(QemuOpts *opts)
         if (!bus) {
             return NULL;
         }
-        if (strcmp(object_get_typename(OBJECT(bus)), k->bus_type) != 0) {
+        if (!object_dynamic_cast(OBJECT(bus), k->bus_type)) {
             qerror_report(QERR_BAD_BUS_FOR_DEVICE,
                           driver, object_get_typename(OBJECT(bus)));
             return NULL;
@@ -563,13 +576,13 @@ static void qbus_print(Monitor *mon, BusState *bus, int indent)
 }
 #undef qdev_printf
 
-void do_info_qtree(Monitor *mon)
+void do_info_qtree(Monitor *mon, const QDict *qdict)
 {
     if (sysbus_get_default())
         qbus_print(mon, sysbus_get_default(), 0);
 }
 
-void do_info_qdm(Monitor *mon)
+void do_info_qdm(Monitor *mon, const QDict *qdict)
 {
     object_class_foreach(qdev_print_devinfo, TYPE_DEVICE, false, NULL);
 }
@@ -614,3 +627,54 @@ void qdev_machine_init(void)
     qdev_get_peripheral_anon();
     qdev_get_peripheral();
 }
+
+QemuOptsList qemu_device_opts = {
+    .name = "device",
+    .implied_opt_name = "driver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_device_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any
+         * sanity checking will happen later
+         * when setting device properties
+         */
+        { /* end of list */ }
+    },
+};
+
+QemuOptsList qemu_global_opts = {
+    .name = "global",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_global_opts.head),
+    .desc = {
+        {
+            .name = "driver",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "property",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "value",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+int qemu_global_option(const char *str)
+{
+    char driver[64], property[64];
+    QemuOpts *opts;
+    int rc, offset;
+
+    rc = sscanf(str, "%63[^.].%63[^=]%n", driver, property, &offset);
+    if (rc < 2 || str[offset] != '=') {
+        error_report("can't parse: \"%s\"", str);
+        return -1;
+    }
+
+    opts = qemu_opts_create_nofail(&qemu_global_opts);
+    qemu_opt_set(opts, "driver", driver);
+    qemu_opt_set(opts, "property", property);
+    qemu_opt_set(opts, "value", str+offset+1);
+    return 0;
+}
diff --git a/hw/qdev-monitor.h b/hw/qdev-monitor.h
new file mode 100644
index 00000000000..9ec485028e8
--- /dev/null
+++ b/hw/qdev-monitor.h
@@ -0,0 +1,16 @@
+#ifndef QEMU_QDEV_MONITOR_H
+#define QEMU_QDEV_MONITOR_H
+
+#include "qdev-core.h"
+#include "monitor/monitor.h"
+
+/*** monitor commands ***/
+
+void do_info_qtree(Monitor *mon, const QDict *qdict);
+void do_info_qdm(Monitor *mon, const QDict *qdict);
+int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
+int do_device_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
+int qdev_device_help(QemuOpts *opts);
+DeviceState *qdev_device_add(QemuOpts *opts);
+
+#endif
diff --git a/hw/qdev-properties-system.c b/hw/qdev-properties-system.c
new file mode 100644
index 00000000000..ce0f7933e60
--- /dev/null
+++ b/hw/qdev-properties-system.c
@@ -0,0 +1,358 @@
+/*
+ * qdev property parsing and global properties
+ * (parts specific for qemu-system-*)
+ *
+ * This file is based on code from hw/qdev-properties.c from
+ * commit 074a86fccd185616469dfcdc0e157f438aebba18,
+ * Copyright (c) Gerd Hoffmann <kraxel@redhat.com> and other contributors.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "net/net.h"
+#include "qdev.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/blockdev.h"
+#include "hw/block-common.h"
+#include "net/hub.h"
+#include "qapi/visitor.h"
+#include "char/char.h"
+
+static void get_pointer(Object *obj, Visitor *v, Property *prop,
+                        const char *(*print)(void *ptr),
+                        const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    void **ptr = qdev_get_prop_ptr(dev, prop);
+    char *p;
+
+    p = (char *) (*ptr ? print(*ptr) : "");
+    visit_type_str(v, &p, name, errp);
+}
+
+static void set_pointer(Object *obj, Visitor *v, Property *prop,
+                        int (*parse)(DeviceState *dev, const char *str,
+                                     void **ptr),
+                        const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Error *local_err = NULL;
+    void **ptr = qdev_get_prop_ptr(dev, prop);
+    char *str;
+    int ret;
+
+    if (dev->realized) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    visit_type_str(v, &str, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    if (!*str) {
+        g_free(str);
+        *ptr = NULL;
+        return;
+    }
+    ret = parse(dev, str, ptr);
+    error_set_from_qdev_prop_error(errp, ret, dev, prop, str);
+    g_free(str);
+}
+
+/* --- drive --- */
+
+static int parse_drive(DeviceState *dev, const char *str, void **ptr)
+{
+    BlockDriverState *bs;
+
+    bs = bdrv_find(str);
+    if (bs == NULL) {
+        return -ENOENT;
+    }
+    if (bdrv_attach_dev(bs, dev) < 0) {
+        return -EEXIST;
+    }
+    *ptr = bs;
+    return 0;
+}
+
+static void release_drive(Object *obj, const char *name, void *opaque)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    BlockDriverState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        bdrv_detach_dev(*ptr, dev);
+        blockdev_auto_del(*ptr);
+    }
+}
+
+static const char *print_drive(void *ptr)
+{
+    return bdrv_get_device_name(ptr);
+}
+
+static void get_drive(Object *obj, Visitor *v, void *opaque,
+                      const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_drive, name, errp);
+}
+
+static void set_drive(Object *obj, Visitor *v, void *opaque,
+                      const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_drive, name, errp);
+}
+
+PropertyInfo qdev_prop_drive = {
+    .name  = "drive",
+    .get   = get_drive,
+    .set   = set_drive,
+    .release = release_drive,
+};
+
+/* --- character device --- */
+
+static int parse_chr(DeviceState *dev, const char *str, void **ptr)
+{
+    CharDriverState *chr = qemu_chr_find(str);
+    if (chr == NULL) {
+        return -ENOENT;
+    }
+    if (chr->avail_connections < 1) {
+        return -EEXIST;
+    }
+    *ptr = chr;
+    --chr->avail_connections;
+    return 0;
+}
+
+static void release_chr(Object *obj, const char *name, void *opaque)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    CharDriverState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        qemu_chr_add_handlers(*ptr, NULL, NULL, NULL, NULL);
+    }
+}
+
+
+static const char *print_chr(void *ptr)
+{
+    CharDriverState *chr = ptr;
+
+    return chr->label ? chr->label : "";
+}
+
+static void get_chr(Object *obj, Visitor *v, void *opaque,
+                    const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_chr, name, errp);
+}
+
+static void set_chr(Object *obj, Visitor *v, void *opaque,
+                    const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_chr, name, errp);
+}
+
+PropertyInfo qdev_prop_chr = {
+    .name  = "chr",
+    .get   = get_chr,
+    .set   = set_chr,
+    .release = release_chr,
+};
+
+/* --- netdev device --- */
+
+static int parse_netdev(DeviceState *dev, const char *str, void **ptr)
+{
+    NetClientState *netdev = qemu_find_netdev(str);
+
+    if (netdev == NULL) {
+        return -ENOENT;
+    }
+    if (netdev->peer) {
+        return -EEXIST;
+    }
+    *ptr = netdev;
+    return 0;
+}
+
+static const char *print_netdev(void *ptr)
+{
+    NetClientState *netdev = ptr;
+
+    return netdev->name ? netdev->name : "";
+}
+
+static void get_netdev(Object *obj, Visitor *v, void *opaque,
+                       const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_netdev, name, errp);
+}
+
+static void set_netdev(Object *obj, Visitor *v, void *opaque,
+                       const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_netdev, name, errp);
+}
+
+PropertyInfo qdev_prop_netdev = {
+    .name  = "netdev",
+    .get   = get_netdev,
+    .set   = set_netdev,
+};
+
+/* --- vlan --- */
+
+static int print_vlan(DeviceState *dev, Property *prop, char *dest, size_t len)
+{
+    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        int id;
+        if (!net_hub_id_for_client(*ptr, &id)) {
+            return snprintf(dest, len, "%d", id);
+        }
+    }
+
+    return snprintf(dest, len, "<null>");
+}
+
+static void get_vlan(Object *obj, Visitor *v, void *opaque,
+                     const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
+    int32_t id = -1;
+
+    if (*ptr) {
+        int hub_id;
+        if (!net_hub_id_for_client(*ptr, &hub_id)) {
+            id = hub_id;
+        }
+    }
+
+    visit_type_int32(v, &id, name, errp);
+}
+
+static void set_vlan(Object *obj, Visitor *v, void *opaque,
+                     const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
+    Error *local_err = NULL;
+    int32_t id;
+    NetClientState *hubport;
+
+    if (dev->realized) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    visit_type_int32(v, &id, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    if (id == -1) {
+        *ptr = NULL;
+        return;
+    }
+
+    hubport = net_hub_port_find(id);
+    if (!hubport) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  name, prop->info->name);
+        return;
+    }
+    *ptr = hubport;
+}
+
+PropertyInfo qdev_prop_vlan = {
+    .name  = "vlan",
+    .print = print_vlan,
+    .get   = get_vlan,
+    .set   = set_vlan,
+};
+
+int qdev_prop_set_drive(DeviceState *dev, const char *name,
+                        BlockDriverState *value)
+{
+    Error *errp = NULL;
+    const char *bdrv_name = value ? bdrv_get_device_name(value) : "";
+    object_property_set_str(OBJECT(dev), bdrv_name,
+                            name, &errp);
+    if (errp) {
+        qerror_report_err(errp);
+        error_free(errp);
+        return -1;
+    }
+    return 0;
+}
+
+void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
+                                BlockDriverState *value)
+{
+    if (qdev_prop_set_drive(dev, name, value) < 0) {
+        exit(1);
+    }
+}
+void qdev_prop_set_chr(DeviceState *dev, const char *name,
+                       CharDriverState *value)
+{
+    Error *errp = NULL;
+    assert(!value || value->label);
+    object_property_set_str(OBJECT(dev),
+                            value ? value->label : "", name, &errp);
+    assert_no_error(errp);
+}
+
+void qdev_prop_set_netdev(DeviceState *dev, const char *name,
+                          NetClientState *value)
+{
+    Error *errp = NULL;
+    assert(!value || value->name);
+    object_property_set_str(OBJECT(dev),
+                            value ? value->name : "", name, &errp);
+    assert_no_error(errp);
+}
+
+void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd)
+{
+    qdev_prop_set_macaddr(dev, "mac", nd->macaddr.a);
+    if (nd->netdev) {
+        qdev_prop_set_netdev(dev, "netdev", nd->netdev);
+    }
+    if (nd->nvectors != DEV_NVECTORS_UNSPECIFIED &&
+        object_property_find(OBJECT(dev), "vectors", NULL)) {
+        qdev_prop_set_uint32(dev, "vectors", nd->nvectors);
+    }
+    nd->instantiated = 1;
+}
+
+static int qdev_add_one_global(QemuOpts *opts, void *opaque)
+{
+    GlobalProperty *g;
+
+    g = g_malloc0(sizeof(*g));
+    g->driver   = qemu_opt_get(opts, "driver");
+    g->property = qemu_opt_get(opts, "property");
+    g->value    = qemu_opt_get(opts, "value");
+    qdev_prop_register_global(g);
+    return 0;
+}
+
+void qemu_add_globals(void)
+{
+    qemu_opts_foreach(qemu_find_opts("global"), qdev_add_one_global, NULL, 0);
+}
diff --git a/hw/qdev-properties.c b/hw/qdev-properties.c
index 8aca0d43fe2..a8a31f56e42 100644
--- a/hw/qdev-properties.c
+++ b/hw/qdev-properties.c
@@ -1,9 +1,11 @@
-#include "net.h"
+#include "net/net.h"
 #include "qdev.h"
-#include "qerror.h"
-#include "blockdev.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
 #include "net/hub.h"
+#include "qapi/visitor.h"
+#include "char/char.h"
 
 void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
 {
@@ -12,49 +14,6 @@ void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
     return ptr;
 }
 
-static void get_pointer(Object *obj, Visitor *v, Property *prop,
-                        const char *(*print)(void *ptr),
-                        const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    void **ptr = qdev_get_prop_ptr(dev, prop);
-    char *p;
-
-    p = (char *) (*ptr ? print(*ptr) : "");
-    visit_type_str(v, &p, name, errp);
-}
-
-static void set_pointer(Object *obj, Visitor *v, Property *prop,
-                        int (*parse)(DeviceState *dev, const char *str,
-                                     void **ptr),
-                        const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Error *local_err = NULL;
-    void **ptr = qdev_get_prop_ptr(dev, prop);
-    char *str;
-    int ret;
-
-    if (dev->state != DEV_STATE_CREATED) {
-        error_set(errp, QERR_PERMISSION_DENIED);
-        return;
-    }
-
-    visit_type_str(v, &str, name, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-    if (!*str) {
-        g_free(str);
-        *ptr = NULL;
-        return;
-    }
-    ret = parse(dev, str, ptr);
-    error_set_from_qdev_prop_error(errp, ret, dev, prop, str);
-    g_free(str);
-}
-
 static void get_enum(Object *obj, Visitor *v, void *opaque,
                      const char *name, Error **errp)
 {
@@ -73,7 +32,7 @@ static void set_enum(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     int *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -94,10 +53,11 @@ static void bit_prop_set(DeviceState *dev, Property *props, bool val)
 {
     uint32_t *p = qdev_get_prop_ptr(dev, props);
     uint32_t mask = qdev_get_prop_mask(props);
-    if (val)
+    if (val) {
         *p |= mask;
-    else
+    } else {
         *p &= ~mask;
+    }
 }
 
 static int print_bit(DeviceState *dev, Property *prop, char *dest, size_t len)
@@ -125,7 +85,7 @@ static void set_bit(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     bool value;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -165,7 +125,7 @@ static void set_uint8(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint8_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -232,7 +192,7 @@ static void set_uint16(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint16_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -265,7 +225,7 @@ static void set_uint32(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -290,7 +250,7 @@ static void set_int32(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -363,7 +323,7 @@ static void set_uint64(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -419,11 +379,13 @@ static void release_string(Object *obj, const char *name, void *opaque)
     g_free(*(char **)qdev_get_prop_ptr(DEVICE(obj), prop));
 }
 
-static int print_string(DeviceState *dev, Property *prop, char *dest, size_t len)
+static int print_string(DeviceState *dev, Property *prop, char *dest,
+                        size_t len)
 {
     char **ptr = qdev_get_prop_ptr(dev, prop);
-    if (!*ptr)
+    if (!*ptr) {
         return snprintf(dest, len, "<null>");
+    }
     return snprintf(dest, len, "\"%s\"", *ptr);
 }
 
@@ -451,7 +413,7 @@ static void set_string(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     char *str;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -475,227 +437,6 @@ PropertyInfo qdev_prop_string = {
     .set   = set_string,
 };
 
-/* --- drive --- */
-
-static int parse_drive(DeviceState *dev, const char *str, void **ptr)
-{
-    BlockDriverState *bs;
-
-    bs = bdrv_find(str);
-    if (bs == NULL)
-        return -ENOENT;
-    if (bdrv_attach_dev(bs, dev) < 0)
-        return -EEXIST;
-    *ptr = bs;
-    return 0;
-}
-
-static void release_drive(Object *obj, const char *name, void *opaque)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    BlockDriverState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        bdrv_detach_dev(*ptr, dev);
-        blockdev_auto_del(*ptr);
-    }
-}
-
-static const char *print_drive(void *ptr)
-{
-    return bdrv_get_device_name(ptr);
-}
-
-static void get_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_drive, name, errp);
-}
-
-static void set_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_drive, name, errp);
-}
-
-PropertyInfo qdev_prop_drive = {
-    .name  = "drive",
-    .get   = get_drive,
-    .set   = set_drive,
-    .release = release_drive,
-};
-
-/* --- character device --- */
-
-static int parse_chr(DeviceState *dev, const char *str, void **ptr)
-{
-    CharDriverState *chr = qemu_chr_find(str);
-    if (chr == NULL) {
-        return -ENOENT;
-    }
-    if (chr->avail_connections < 1) {
-        return -EEXIST;
-    }
-    *ptr = chr;
-    --chr->avail_connections;
-    return 0;
-}
-
-static void release_chr(Object *obj, const char *name, void *opaque)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    CharDriverState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        qemu_chr_add_handlers(*ptr, NULL, NULL, NULL, NULL);
-    }
-}
-
-
-static const char *print_chr(void *ptr)
-{
-    CharDriverState *chr = ptr;
-
-    return chr->label ? chr->label : "";
-}
-
-static void get_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_chr, name, errp);
-}
-
-static void set_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_chr, name, errp);
-}
-
-PropertyInfo qdev_prop_chr = {
-    .name  = "chr",
-    .get   = get_chr,
-    .set   = set_chr,
-    .release = release_chr,
-};
-
-/* --- netdev device --- */
-
-static int parse_netdev(DeviceState *dev, const char *str, void **ptr)
-{
-    NetClientState *netdev = qemu_find_netdev(str);
-
-    if (netdev == NULL) {
-        return -ENOENT;
-    }
-    if (netdev->peer) {
-        return -EEXIST;
-    }
-    *ptr = netdev;
-    return 0;
-}
-
-static const char *print_netdev(void *ptr)
-{
-    NetClientState *netdev = ptr;
-
-    return netdev->name ? netdev->name : "";
-}
-
-static void get_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_netdev, name, errp);
-}
-
-static void set_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_netdev, name, errp);
-}
-
-PropertyInfo qdev_prop_netdev = {
-    .name  = "netdev",
-    .get   = get_netdev,
-    .set   = set_netdev,
-};
-
-/* --- vlan --- */
-
-static int print_vlan(DeviceState *dev, Property *prop, char *dest, size_t len)
-{
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        int id;
-        if (!net_hub_id_for_client(*ptr, &id)) {
-            return snprintf(dest, len, "%d", id);
-        }
-    }
-
-    return snprintf(dest, len, "<null>");
-}
-
-static void get_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-    int32_t id = -1;
-
-    if (*ptr) {
-        int hub_id;
-        if (!net_hub_id_for_client(*ptr, &hub_id)) {
-            id = hub_id;
-        }
-    }
-
-    visit_type_int32(v, &id, name, errp);
-}
-
-static void set_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-    Error *local_err = NULL;
-    int32_t id;
-    NetClientState *hubport;
-
-    if (dev->state != DEV_STATE_CREATED) {
-        error_set(errp, QERR_PERMISSION_DENIED);
-        return;
-    }
-
-    visit_type_int32(v, &id, name, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-    if (id == -1) {
-        *ptr = NULL;
-        return;
-    }
-
-    hubport = net_hub_port_find(id);
-    if (!hubport) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  name, prop->info->name);
-        return;
-    }
-    *ptr = hubport;
-}
-
-PropertyInfo qdev_prop_vlan = {
-    .name  = "vlan",
-    .print = print_vlan,
-    .get   = get_vlan,
-    .set   = set_vlan,
-};
-
 /* --- pointer --- */
 
 /* Not a proper property, just for dirty hacks.  TODO Remove it!  */
@@ -736,7 +477,7 @@ static void set_mac(Object *obj, Visitor *v, void *opaque,
     int i, pos;
     char *str, *p;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -748,16 +489,20 @@ static void set_mac(Object *obj, Visitor *v, void *opaque,
     }
 
     for (i = 0, pos = 0; i < 6; i++, pos += 3) {
-        if (!qemu_isxdigit(str[pos]))
+        if (!qemu_isxdigit(str[pos])) {
             goto inval;
-        if (!qemu_isxdigit(str[pos+1]))
+        }
+        if (!qemu_isxdigit(str[pos+1])) {
             goto inval;
+        }
         if (i == 5) {
-            if (str[pos+2] != '\0')
+            if (str[pos+2] != '\0') {
                 goto inval;
+            }
         } else {
-            if (str[pos+2] != ':' && str[pos+2] != '-')
+            if (str[pos+2] != ':' && str[pos+2] != '-') {
                 goto inval;
+            }
         }
         mac->a[i] = strtol(str+pos, &p, 16);
     }
@@ -824,7 +569,7 @@ static void set_pci_devfn(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     char *str;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -863,7 +608,8 @@ invalid:
     g_free(str);
 }
 
-static int print_pci_devfn(DeviceState *dev, Property *prop, char *dest, size_t len)
+static int print_pci_devfn(DeviceState *dev, Property *prop, char *dest,
+                           size_t len)
 {
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
@@ -894,7 +640,7 @@ static void set_blocksize(Object *obj, Visitor *v, void *opaque,
     const int64_t min = 512;
     const int64_t max = 32768;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -962,7 +708,7 @@ static void set_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
     unsigned long dom = 0, bus = 0;
     unsigned int slot = 0, func = 0;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -1037,11 +783,13 @@ PropertyInfo qdev_prop_pci_host_devaddr = {
 
 static Property *qdev_prop_walk(Property *props, const char *name)
 {
-    if (!props)
+    if (!props) {
         return NULL;
+    }
     while (props->name) {
-        if (strcmp(props->name, name) == 0)
+        if (strcmp(props->name, name) == 0) {
             return props;
+        }
         props++;
     }
     return NULL;
@@ -1157,44 +905,6 @@ void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value)
     assert_no_error(errp);
 }
 
-int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value)
-{
-    Error *errp = NULL;
-    const char *bdrv_name = value ? bdrv_get_device_name(value) : "";
-    object_property_set_str(OBJECT(dev), bdrv_name,
-                            name, &errp);
-    if (errp) {
-        qerror_report_err(errp);
-        error_free(errp);
-        return -1;
-    }
-    return 0;
-}
-
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value)
-{
-    if (qdev_prop_set_drive(dev, name, value) < 0) {
-        exit(1);
-    }
-}
-void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value)
-{
-    Error *errp = NULL;
-    assert(!value || value->label);
-    object_property_set_str(OBJECT(dev),
-                            value ? value->label : "", name, &errp);
-    assert_no_error(errp);
-}
-
-void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value)
-{
-    Error *errp = NULL;
-    assert(!value || value->name);
-    object_property_set_str(OBJECT(dev),
-                            value ? value->name : "", name, &errp);
-    assert_no_error(errp);
-}
-
 void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value)
 {
     Error *errp = NULL;
@@ -1228,9 +938,10 @@ void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value)
     *ptr = value;
 }
 
-static QTAILQ_HEAD(, GlobalProperty) global_props = QTAILQ_HEAD_INITIALIZER(global_props);
+static QTAILQ_HEAD(, GlobalProperty) global_props =
+        QTAILQ_HEAD_INITIALIZER(global_props);
 
-static void qdev_prop_register_global(GlobalProperty *prop)
+void qdev_prop_register_global(GlobalProperty *prop)
 {
     QTAILQ_INSERT_TAIL(&global_props, prop, next);
 }
@@ -1261,20 +972,3 @@ void qdev_prop_set_globals(DeviceState *dev)
         class = object_class_get_parent(class);
     } while (class);
 }
-
-static int qdev_add_one_global(QemuOpts *opts, void *opaque)
-{
-    GlobalProperty *g;
-
-    g = g_malloc0(sizeof(*g));
-    g->driver   = qemu_opt_get(opts, "driver");
-    g->property = qemu_opt_get(opts, "property");
-    g->value    = qemu_opt_get(opts, "value");
-    qdev_prop_register_global(g);
-    return 0;
-}
-
-void qemu_add_globals(void)
-{
-    qemu_opts_foreach(qemu_find_opts("global"), qdev_add_one_global, NULL, 0);
-}
diff --git a/hw/qdev-properties.h b/hw/qdev-properties.h
new file mode 100644
index 00000000000..ddcf7745061
--- /dev/null
+++ b/hw/qdev-properties.h
@@ -0,0 +1,131 @@
+#ifndef QEMU_QDEV_PROPERTIES_H
+#define QEMU_QDEV_PROPERTIES_H
+
+#include "qdev-core.h"
+
+/*** qdev-properties.c ***/
+
+extern PropertyInfo qdev_prop_bit;
+extern PropertyInfo qdev_prop_uint8;
+extern PropertyInfo qdev_prop_uint16;
+extern PropertyInfo qdev_prop_uint32;
+extern PropertyInfo qdev_prop_int32;
+extern PropertyInfo qdev_prop_uint64;
+extern PropertyInfo qdev_prop_hex8;
+extern PropertyInfo qdev_prop_hex32;
+extern PropertyInfo qdev_prop_hex64;
+extern PropertyInfo qdev_prop_string;
+extern PropertyInfo qdev_prop_chr;
+extern PropertyInfo qdev_prop_ptr;
+extern PropertyInfo qdev_prop_macaddr;
+extern PropertyInfo qdev_prop_losttickpolicy;
+extern PropertyInfo qdev_prop_bios_chs_trans;
+extern PropertyInfo qdev_prop_drive;
+extern PropertyInfo qdev_prop_netdev;
+extern PropertyInfo qdev_prop_vlan;
+extern PropertyInfo qdev_prop_pci_devfn;
+extern PropertyInfo qdev_prop_blocksize;
+extern PropertyInfo qdev_prop_pci_host_devaddr;
+
+#define DEFINE_PROP(_name, _state, _field, _prop, _type) { \
+        .name      = (_name),                                    \
+        .info      = &(_prop),                                   \
+        .offset    = offsetof(_state, _field)                    \
+            + type_check(_type,typeof_field(_state, _field)),    \
+        }
+#define DEFINE_PROP_DEFAULT(_name, _state, _field, _defval, _prop, _type) { \
+        .name      = (_name),                                           \
+        .info      = &(_prop),                                          \
+        .offset    = offsetof(_state, _field)                           \
+            + type_check(_type,typeof_field(_state, _field)),           \
+        .qtype     = QTYPE_QINT,                                        \
+        .defval    = (_type)_defval,                                    \
+        }
+#define DEFINE_PROP_BIT(_name, _state, _field, _bit, _defval) {  \
+        .name      = (_name),                                    \
+        .info      = &(qdev_prop_bit),                           \
+        .bitnr    = (_bit),                                      \
+        .offset    = offsetof(_state, _field)                    \
+            + type_check(uint32_t,typeof_field(_state, _field)), \
+        .qtype     = QTYPE_QBOOL,                                \
+        .defval    = (bool)_defval,                              \
+        }
+
+#define DEFINE_PROP_UINT8(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint8, uint8_t)
+#define DEFINE_PROP_UINT16(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint16, uint16_t)
+#define DEFINE_PROP_UINT32(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint32, uint32_t)
+#define DEFINE_PROP_INT32(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_int32, int32_t)
+#define DEFINE_PROP_UINT64(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint64, uint64_t)
+#define DEFINE_PROP_HEX8(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex8, uint8_t)
+#define DEFINE_PROP_HEX32(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex32, uint32_t)
+#define DEFINE_PROP_HEX64(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex64, uint64_t)
+#define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d)                   \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t)
+
+#define DEFINE_PROP_PTR(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_ptr, void*)
+#define DEFINE_PROP_CHR(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_chr, CharDriverState*)
+#define DEFINE_PROP_STRING(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_string, char*)
+#define DEFINE_PROP_NETDEV(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NetClientState*)
+#define DEFINE_PROP_VLAN(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NetClientState*)
+#define DEFINE_PROP_DRIVE(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
+#define DEFINE_PROP_MACADDR(_n, _s, _f)         \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
+                        LostTickPolicy)
+#define DEFINE_PROP_BIOS_CHS_TRANS(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_bios_chs_trans, int)
+#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blocksize, uint16_t)
+#define DEFINE_PROP_PCI_HOST_DEVADDR(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_pci_host_devaddr, PCIHostDeviceAddress)
+
+#define DEFINE_PROP_END_OF_LIST()               \
+    {}
+
+/* Set properties between creation and init.  */
+void *qdev_get_prop_ptr(DeviceState *dev, Property *prop);
+int qdev_prop_parse(DeviceState *dev, const char *name, const char *value);
+void qdev_prop_set_bit(DeviceState *dev, const char *name, bool value);
+void qdev_prop_set_uint8(DeviceState *dev, const char *name, uint8_t value);
+void qdev_prop_set_uint16(DeviceState *dev, const char *name, uint16_t value);
+void qdev_prop_set_uint32(DeviceState *dev, const char *name, uint32_t value);
+void qdev_prop_set_int32(DeviceState *dev, const char *name, int32_t value);
+void qdev_prop_set_uint64(DeviceState *dev, const char *name, uint64_t value);
+void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
+void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
+void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
+int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value) QEMU_WARN_UNUSED_RESULT;
+void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value);
+void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
+void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
+/* FIXME: Remove opaque pointer properties.  */
+void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
+
+void qdev_prop_register_global(GlobalProperty *prop);
+void qdev_prop_register_global_list(GlobalProperty *props);
+void qdev_prop_set_globals(DeviceState *dev);
+void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
+                                    Property *prop, const char *value);
+
+/**
+ * @qdev_property_add_static - add a @Property to a device referencing a
+ * field in a struct.
+ */
+void qdev_property_add_static(DeviceState *dev, Property *prop, Error **errp);
+
+#endif
diff --git a/hw/qdev.c b/hw/qdev.c
index b5a52ac503a..97610167c2d 100644
--- a/hw/qdev.c
+++ b/hw/qdev.c
@@ -25,16 +25,15 @@
    inherit from a particular bus (e.g. PCI or I2C) rather than
    this API directly.  */
 
-#include "net.h"
 #include "qdev.h"
-#include "sysemu.h"
-#include "error.h"
+#include "sysemu/sysemu.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
 
 int qdev_hotplug = 0;
 static bool qdev_hot_added = false;
 static bool qdev_hot_removed = false;
 
-/* Register a new device type.  */
 const VMStateDescription *qdev_get_vmsd(DeviceState *dev)
 {
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
@@ -52,11 +51,6 @@ const char *qdev_fw_name(DeviceState *dev)
     return object_get_typename(OBJECT(dev));
 }
 
-bool qdev_exists(const char *name)
-{
-    return !!object_class_by_name(name);
-}
-
 static void qdev_property_add_legacy(DeviceState *dev, Property *prop,
                                      Error **errp);
 
@@ -114,10 +108,12 @@ DeviceState *qdev_create(BusState *bus, const char *name)
     dev = qdev_try_create(bus, name);
     if (!dev) {
         if (bus) {
-            hw_error("Unknown device '%s' for bus '%s'\n", name,
-                     object_get_typename(OBJECT(bus)));
+            error_report("Unknown device '%s' for bus '%s'\n", name,
+                         object_get_typename(OBJECT(bus)));
+            abort();
         } else {
-            hw_error("Unknown device '%s' for default sysbus\n", name);
+            error_report("Unknown device '%s' for default sysbus\n", name);
+            abort();
         }
     }
 
@@ -152,43 +148,36 @@ DeviceState *qdev_try_create(BusState *bus, const char *type)
    Return 0 on success.  */
 int qdev_init(DeviceState *dev)
 {
-    DeviceClass *dc = DEVICE_GET_CLASS(dev);
-    int rc;
+    Error *local_err = NULL;
 
-    assert(dev->state == DEV_STATE_CREATED);
+    assert(!dev->realized);
 
-    rc = dc->init(dev);
-    if (rc < 0) {
+    object_property_set_bool(OBJECT(dev), true, "realized", &local_err);
+    if (local_err != NULL) {
+        error_free(local_err);
         qdev_free(dev);
-        return rc;
+        return -1;
     }
+    return 0;
+}
 
-    if (!OBJECT(dev)->parent) {
-        static int unattached_count = 0;
-        gchar *name = g_strdup_printf("device[%d]", unattached_count++);
-
-        object_property_add_child(container_get(qdev_get_machine(),
-                                                "/unattached"),
-                                  name, OBJECT(dev), NULL);
-        g_free(name);
-    }
+static void device_realize(DeviceState *dev, Error **err)
+{
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
 
-    if (qdev_get_vmsd(dev)) {
-        vmstate_register_with_alias_id(dev, -1, qdev_get_vmsd(dev), dev,
-                                       dev->instance_id_alias,
-                                       dev->alias_required_for_version);
-    }
-    dev->state = DEV_STATE_INITIALIZED;
-    if (dev->hotplugged) {
-        device_reset(dev);
+    if (dc->init) {
+        int rc = dc->init(dev);
+        if (rc < 0) {
+            error_setg(err, "Device initialization failed.");
+            return;
+        }
     }
-    return 0;
 }
 
 void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
                                  int required_for_version)
 {
-    assert(dev->state == DEV_STATE_CREATED);
+    assert(!dev->realized);
     dev->instance_id_alias = alias_id;
     dev->alias_required_for_version = required_for_version;
 }
@@ -232,10 +221,15 @@ void qdev_reset_all(DeviceState *dev)
     qdev_walk_children(dev, qdev_reset_one, qbus_reset_one, NULL);
 }
 
+void qbus_reset_all(BusState *bus)
+{
+    qbus_walk_children(bus, qdev_reset_one, qbus_reset_one, NULL);
+}
+
 void qbus_reset_all_fn(void *opaque)
 {
     BusState *bus = opaque;
-    qbus_walk_children(bus, qdev_reset_one, qbus_reset_one, NULL);
+    qbus_reset_all(bus);
 }
 
 /* can be used as ->unplug() callback for the simple cases */
@@ -291,9 +285,9 @@ BusState *qdev_get_parent_bus(DeviceState *dev)
 
 void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n)
 {
-    assert(dev->num_gpio_in == 0);
-    dev->num_gpio_in = n;
-    dev->gpio_in = qemu_allocate_irqs(handler, dev, n);
+    dev->gpio_in = qemu_extend_irqs(dev->gpio_in, dev->num_gpio_in, handler,
+                                        dev, n);
+    dev->num_gpio_in += n;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -315,18 +309,6 @@ void qdev_connect_gpio_out(DeviceState * dev, int n, qemu_irq pin)
     dev->gpio_out[n] = pin;
 }
 
-void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd)
-{
-    qdev_prop_set_macaddr(dev, "mac", nd->macaddr.a);
-    if (nd->netdev)
-        qdev_prop_set_netdev(dev, "netdev", nd->netdev);
-    if (nd->nvectors != DEV_NVECTORS_UNSPECIFIED &&
-        object_property_find(OBJECT(dev), "vectors", NULL)) {
-        qdev_prop_set_uint32(dev, "vectors", nd->nvectors);
-    }
-    nd->instantiated = 1;
-}
-
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
 {
     BusState *bus;
@@ -459,7 +441,6 @@ BusState *qbus_create(const char *typename, DeviceState *parent, const char *nam
     BusState *bus;
 
     bus = BUS(object_new(typename));
-    bus->qom_allocated = true;
 
     bus->parent = parent;
     bus->name = name ? g_strdup(name) : NULL;
@@ -470,14 +451,7 @@ BusState *qbus_create(const char *typename, DeviceState *parent, const char *nam
 
 void qbus_free(BusState *bus)
 {
-    if (bus->qom_allocated) {
-        object_delete(OBJECT(bus));
-    } else {
-        object_finalize(OBJECT(bus));
-        if (bus->glib_allocated) {
-            g_free(bus);
-        }
-    }
+    object_delete(OBJECT(bus));
 }
 
 static char *bus_get_fw_dev_path(BusState *bus, DeviceState *dev)
@@ -520,7 +494,7 @@ char* qdev_get_fw_dev_path(DeviceState *dev)
 
     path[l-1] = '\0';
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 char *qdev_get_dev_path(DeviceState *dev)
@@ -565,7 +539,7 @@ static void qdev_set_legacy_property(Object *obj, Visitor *v, void *opaque,
     char *ptr = NULL;
     int ret;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -660,6 +634,55 @@ void qdev_property_add_static(DeviceState *dev, Property *prop,
     assert_no_error(local_err);
 }
 
+static bool device_get_realized(Object *obj, Error **err)
+{
+    DeviceState *dev = DEVICE(obj);
+    return dev->realized;
+}
+
+static void device_set_realized(Object *obj, bool value, Error **err)
+{
+    DeviceState *dev = DEVICE(obj);
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
+    Error *local_err = NULL;
+
+    if (value && !dev->realized) {
+        if (dc->realize) {
+            dc->realize(dev, &local_err);
+        }
+
+        if (!obj->parent && local_err == NULL) {
+            static int unattached_count;
+            gchar *name = g_strdup_printf("device[%d]", unattached_count++);
+
+            object_property_add_child(container_get(qdev_get_machine(),
+                                                    "/unattached"),
+                                      name, obj, &local_err);
+            g_free(name);
+        }
+
+        if (qdev_get_vmsd(dev) && local_err == NULL) {
+            vmstate_register_with_alias_id(dev, -1, qdev_get_vmsd(dev), dev,
+                                           dev->instance_id_alias,
+                                           dev->alias_required_for_version);
+        }
+        if (dev->hotplugged && local_err == NULL) {
+            device_reset(dev);
+        }
+    } else if (!value && dev->realized) {
+        if (dc->unrealize) {
+            dc->unrealize(dev, &local_err);
+        }
+    }
+
+    if (local_err != NULL) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    dev->realized = value;
+}
+
 static void device_initfn(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
@@ -672,7 +695,10 @@ static void device_initfn(Object *obj)
     }
 
     dev->instance_id_alias = -1;
-    dev->state = DEV_STATE_CREATED;
+    dev->realized = false;
+
+    object_property_add_bool(obj, "realized",
+                             device_get_realized, device_set_realized, NULL);
 
     class = object_get_class(OBJECT(dev));
     do {
@@ -695,7 +721,7 @@ static void device_finalize(Object *obj)
     BusState *bus;
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
 
-    if (dev->state == DEV_STATE_INITIALIZED) {
+    if (dev->realized) {
         while (dev->num_child_bus) {
             bus = QLIST_FIRST(&dev->child_bus);
             qbus_free(bus);
@@ -710,9 +736,6 @@ static void device_finalize(Object *obj)
             qemu_opts_del(dev->opts);
         }
     }
-    if (dev->parent_bus) {
-        bus_remove_child(dev->parent_bus, dev);
-    }
 }
 
 static void device_class_base_init(ObjectClass *class, void *data)
@@ -725,6 +748,23 @@ static void device_class_base_init(ObjectClass *class, void *data)
     klass->props = NULL;
 }
 
+static void device_unparent(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+
+    if (dev->parent_bus != NULL) {
+        bus_remove_child(dev->parent_bus, dev);
+    }
+}
+
+static void device_class_init(ObjectClass *class, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(class);
+
+    class->unparent = device_unparent;
+    dc->realize = device_realize;
+}
+
 void device_reset(DeviceState *dev)
 {
     DeviceClass *klass = DEVICE_GET_CLASS(dev);
@@ -745,13 +785,14 @@ Object *qdev_get_machine(void)
     return dev;
 }
 
-static TypeInfo device_type_info = {
+static const TypeInfo device_type_info = {
     .name = TYPE_DEVICE,
     .parent = TYPE_OBJECT,
     .instance_size = sizeof(DeviceState),
     .instance_init = device_initfn,
     .instance_finalize = device_finalize,
     .class_base_init = device_class_base_init,
+    .class_init = device_class_init,
     .abstract = true,
     .class_size = sizeof(DeviceClass),
 };
diff --git a/hw/qdev.h b/hw/qdev.h
index d699194418e..365b8d6ca2a 100644
--- a/hw/qdev.h
+++ b/hw/qdev.h
@@ -1,372 +1,9 @@
 #ifndef QDEV_H
 #define QDEV_H
 
-#include "hw.h"
-#include "qemu-queue.h"
-#include "qemu-char.h"
-#include "qemu-option.h"
-#include "qapi/qapi-visit-core.h"
-#include "qemu/object.h"
-#include "error.h"
-
-typedef struct Property Property;
-
-typedef struct PropertyInfo PropertyInfo;
-
-typedef struct CompatProperty CompatProperty;
-
-typedef struct BusState BusState;
-
-typedef struct BusClass BusClass;
-
-enum DevState {
-    DEV_STATE_CREATED = 1,
-    DEV_STATE_INITIALIZED,
-};
-
-enum {
-    DEV_NVECTORS_UNSPECIFIED = -1,
-};
-
-#define TYPE_DEVICE "device"
-#define DEVICE(obj) OBJECT_CHECK(DeviceState, (obj), TYPE_DEVICE)
-#define DEVICE_CLASS(klass) OBJECT_CLASS_CHECK(DeviceClass, (klass), TYPE_DEVICE)
-#define DEVICE_GET_CLASS(obj) OBJECT_GET_CLASS(DeviceClass, (obj), TYPE_DEVICE)
-
-typedef int (*qdev_initfn)(DeviceState *dev);
-typedef int (*qdev_event)(DeviceState *dev);
-typedef void (*qdev_resetfn)(DeviceState *dev);
-
-typedef struct DeviceClass {
-    ObjectClass parent_class;
-
-    const char *fw_name;
-    const char *desc;
-    Property *props;
-    int no_user;
-
-    /* callbacks */
-    void (*reset)(DeviceState *dev);
-
-    /* device state */
-    const VMStateDescription *vmsd;
-
-    /* Private to qdev / bus.  */
-    qdev_initfn init;
-    qdev_event unplug;
-    qdev_event exit;
-    const char *bus_type;
-} DeviceClass;
-
-/* This structure should not be accessed directly.  We declare it here
-   so that it can be embedded in individual device state structures.  */
-struct DeviceState {
-    Object parent_obj;
-
-    const char *id;
-    enum DevState state;
-    QemuOpts *opts;
-    int hotplugged;
-    BusState *parent_bus;
-    int num_gpio_out;
-    qemu_irq *gpio_out;
-    int num_gpio_in;
-    qemu_irq *gpio_in;
-    QLIST_HEAD(, BusState) child_bus;
-    int num_child_bus;
-    int instance_id_alias;
-    int alias_required_for_version;
-};
-
-#define TYPE_BUS "bus"
-#define BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_BUS)
-#define BUS_CLASS(klass) OBJECT_CLASS_CHECK(BusClass, (klass), TYPE_BUS)
-#define BUS_GET_CLASS(obj) OBJECT_GET_CLASS(BusClass, (obj), TYPE_BUS)
-
-struct BusClass {
-    ObjectClass parent_class;
-
-    /* FIXME first arg should be BusState */
-    void (*print_dev)(Monitor *mon, DeviceState *dev, int indent);
-    char *(*get_dev_path)(DeviceState *dev);
-    /*
-     * This callback is used to create Open Firmware device path in accordance
-     * with OF spec http://forthworks.com/standards/of1275.pdf. Individual bus
-     * bindings can be found at http://playground.sun.com/1275/bindings/.
-     */
-    char *(*get_fw_dev_path)(DeviceState *dev);
-    int (*reset)(BusState *bus);
-};
-
-typedef struct BusChild {
-    DeviceState *child;
-    int index;
-    QTAILQ_ENTRY(BusChild) sibling;
-} BusChild;
-
-/**
- * BusState:
- * @qom_allocated: Indicates whether the object was allocated by QOM.
- * @glib_allocated: Indicates whether the object was initialized in-place
- * yet is expected to be freed with g_free().
- */
-struct BusState {
-    Object obj;
-    DeviceState *parent;
-    const char *name;
-    int allow_hotplug;
-    bool qom_allocated;
-    bool glib_allocated;
-    int max_index;
-    QTAILQ_HEAD(ChildrenHead, BusChild) children;
-    QLIST_ENTRY(BusState) sibling;
-};
-
-struct Property {
-    const char   *name;
-    PropertyInfo *info;
-    int          offset;
-    uint8_t      bitnr;
-    uint8_t      qtype;
-    int64_t      defval;
-};
-
-struct PropertyInfo {
-    const char *name;
-    const char *legacy_name;
-    const char **enum_table;
-    int (*parse)(DeviceState *dev, Property *prop, const char *str);
-    int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
-    ObjectPropertyAccessor *get;
-    ObjectPropertyAccessor *set;
-    ObjectPropertyRelease *release;
-};
-
-typedef struct GlobalProperty {
-    const char *driver;
-    const char *property;
-    const char *value;
-    QTAILQ_ENTRY(GlobalProperty) next;
-} GlobalProperty;
-
-/*** Board API.  This should go away once we have a machine config file.  ***/
-
-DeviceState *qdev_create(BusState *bus, const char *name);
-DeviceState *qdev_try_create(BusState *bus, const char *name);
-bool qdev_exists(const char *name);
-int qdev_device_help(QemuOpts *opts);
-DeviceState *qdev_device_add(QemuOpts *opts);
-int qdev_init(DeviceState *dev) QEMU_WARN_UNUSED_RESULT;
-void qdev_init_nofail(DeviceState *dev);
-void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
-                                 int required_for_version);
-void qdev_unplug(DeviceState *dev, Error **errp);
-void qdev_free(DeviceState *dev);
-int qdev_simple_unplug_cb(DeviceState *dev);
-void qdev_machine_creation_done(void);
-bool qdev_machine_modified(void);
-
-qemu_irq qdev_get_gpio_in(DeviceState *dev, int n);
-void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
-
-BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
-
-/*** Device API.  ***/
-
-/* Register device properties.  */
-/* GPIO inputs also double as IRQ sinks.  */
-void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n);
-void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n);
-
-BusState *qdev_get_parent_bus(DeviceState *dev);
-
-/*** BUS API. ***/
-
-DeviceState *qdev_find_recursive(BusState *bus, const char *id);
-
-/* Returns 0 to walk children, > 0 to skip walk, < 0 to terminate walk. */
-typedef int (qbus_walkerfn)(BusState *bus, void *opaque);
-typedef int (qdev_walkerfn)(DeviceState *dev, void *opaque);
-
-void qbus_create_inplace(BusState *bus, const char *typename,
-                         DeviceState *parent, const char *name);
-BusState *qbus_create(const char *typename, DeviceState *parent, const char *name);
-/* Returns > 0 if either devfn or busfn skip walk somewhere in cursion,
- *         < 0 if either devfn or busfn terminate walk somewhere in cursion,
- *           0 otherwise. */
-int qbus_walk_children(BusState *bus, qdev_walkerfn *devfn,
-                       qbus_walkerfn *busfn, void *opaque);
-int qdev_walk_children(DeviceState *dev, qdev_walkerfn *devfn,
-                       qbus_walkerfn *busfn, void *opaque);
-void qdev_reset_all(DeviceState *dev);
-void qbus_reset_all_fn(void *opaque);
-
-void qbus_free(BusState *bus);
-
-#define FROM_QBUS(type, dev) DO_UPCAST(type, qbus, dev)
-
-/* This should go away once we get rid of the NULL bus hack */
-BusState *sysbus_get_default(void);
-
-/*** monitor commands ***/
-
-void do_info_qtree(Monitor *mon);
-void do_info_qdm(Monitor *mon);
-int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
-int do_device_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
-
-/*** qdev-properties.c ***/
-
-extern PropertyInfo qdev_prop_bit;
-extern PropertyInfo qdev_prop_uint8;
-extern PropertyInfo qdev_prop_uint16;
-extern PropertyInfo qdev_prop_uint32;
-extern PropertyInfo qdev_prop_int32;
-extern PropertyInfo qdev_prop_uint64;
-extern PropertyInfo qdev_prop_hex8;
-extern PropertyInfo qdev_prop_hex32;
-extern PropertyInfo qdev_prop_hex64;
-extern PropertyInfo qdev_prop_string;
-extern PropertyInfo qdev_prop_chr;
-extern PropertyInfo qdev_prop_ptr;
-extern PropertyInfo qdev_prop_macaddr;
-extern PropertyInfo qdev_prop_losttickpolicy;
-extern PropertyInfo qdev_prop_bios_chs_trans;
-extern PropertyInfo qdev_prop_drive;
-extern PropertyInfo qdev_prop_netdev;
-extern PropertyInfo qdev_prop_vlan;
-extern PropertyInfo qdev_prop_pci_devfn;
-extern PropertyInfo qdev_prop_blocksize;
-extern PropertyInfo qdev_prop_pci_host_devaddr;
-
-#define DEFINE_PROP(_name, _state, _field, _prop, _type) { \
-        .name      = (_name),                                    \
-        .info      = &(_prop),                                   \
-        .offset    = offsetof(_state, _field)                    \
-            + type_check(_type,typeof_field(_state, _field)),    \
-        }
-#define DEFINE_PROP_DEFAULT(_name, _state, _field, _defval, _prop, _type) { \
-        .name      = (_name),                                           \
-        .info      = &(_prop),                                          \
-        .offset    = offsetof(_state, _field)                           \
-            + type_check(_type,typeof_field(_state, _field)),           \
-        .qtype     = QTYPE_QINT,                                        \
-        .defval    = (_type)_defval,                                    \
-        }
-#define DEFINE_PROP_BIT(_name, _state, _field, _bit, _defval) {  \
-        .name      = (_name),                                    \
-        .info      = &(qdev_prop_bit),                           \
-        .bitnr    = (_bit),                                      \
-        .offset    = offsetof(_state, _field)                    \
-            + type_check(uint32_t,typeof_field(_state, _field)), \
-        .qtype     = QTYPE_QBOOL,                                \
-        .defval    = (bool)_defval,                              \
-        }
-
-#define DEFINE_PROP_UINT8(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint8, uint8_t)
-#define DEFINE_PROP_UINT16(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint16, uint16_t)
-#define DEFINE_PROP_UINT32(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint32, uint32_t)
-#define DEFINE_PROP_INT32(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_int32, int32_t)
-#define DEFINE_PROP_UINT64(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint64, uint64_t)
-#define DEFINE_PROP_HEX8(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex8, uint8_t)
-#define DEFINE_PROP_HEX32(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex32, uint32_t)
-#define DEFINE_PROP_HEX64(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex64, uint64_t)
-#define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d)                   \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t)
-
-#define DEFINE_PROP_PTR(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_ptr, void*)
-#define DEFINE_PROP_CHR(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_chr, CharDriverState*)
-#define DEFINE_PROP_STRING(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_string, char*)
-#define DEFINE_PROP_NETDEV(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NetClientState*)
-#define DEFINE_PROP_VLAN(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NetClientState*)
-#define DEFINE_PROP_DRIVE(_n, _s, _f) \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
-#define DEFINE_PROP_MACADDR(_n, _s, _f)         \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
-#define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
-                        LostTickPolicy)
-#define DEFINE_PROP_BIOS_CHS_TRANS(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_bios_chs_trans, int)
-#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blocksize, uint16_t)
-#define DEFINE_PROP_PCI_HOST_DEVADDR(_n, _s, _f) \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_pci_host_devaddr, PCIHostDeviceAddress)
-
-#define DEFINE_PROP_END_OF_LIST()               \
-    {}
-
-/* Set properties between creation and init.  */
-void *qdev_get_prop_ptr(DeviceState *dev, Property *prop);
-int qdev_prop_parse(DeviceState *dev, const char *name, const char *value);
-void qdev_prop_set_bit(DeviceState *dev, const char *name, bool value);
-void qdev_prop_set_uint8(DeviceState *dev, const char *name, uint8_t value);
-void qdev_prop_set_uint16(DeviceState *dev, const char *name, uint16_t value);
-void qdev_prop_set_uint32(DeviceState *dev, const char *name, uint32_t value);
-void qdev_prop_set_int32(DeviceState *dev, const char *name, int32_t value);
-void qdev_prop_set_uint64(DeviceState *dev, const char *name, uint64_t value);
-void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
-void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
-void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
-int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value) QEMU_WARN_UNUSED_RESULT;
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value);
-void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
-void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
-/* FIXME: Remove opaque pointer properties.  */
-void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
-
-void qdev_prop_register_global_list(GlobalProperty *props);
-void qdev_prop_set_globals(DeviceState *dev);
-void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
-                                    Property *prop, const char *value);
-
-char *qdev_get_fw_dev_path(DeviceState *dev);
-
-/**
- * @qdev_property_add_static - add a @Property to a device referencing a
- * field in a struct.
- */
-void qdev_property_add_static(DeviceState *dev, Property *prop, Error **errp);
-
-/**
- * @qdev_machine_init
- *
- * Initialize platform devices before machine init.  This is a hack until full
- * support for composition is added.
- */
-void qdev_machine_init(void);
-
-/**
- * @device_reset
- *
- * Reset a single device (by calling the reset method).
- */
-void device_reset(DeviceState *dev);
-
-const VMStateDescription *qdev_get_vmsd(DeviceState *dev);
-
-const char *qdev_fw_name(DeviceState *dev);
-
-Object *qdev_get_machine(void);
-
-/* FIXME: make this a link<> */
-void qdev_set_parent_bus(DeviceState *dev, BusState *bus);
-
-extern int qdev_hotplug;
-
-char *qdev_get_dev_path(DeviceState *dev);
+#include "hw/hw.h"
+#include "qdev-core.h"
+#include "qdev-properties.h"
+#include "qdev-monitor.h"
 
 #endif
diff --git a/hw/qxl-logger.c b/hw/qxl-logger.c
index fe2878c8366..3cd85d9b978 100644
--- a/hw/qxl-logger.c
+++ b/hw/qxl-logger.c
@@ -19,7 +19,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qxl.h"
 
 static const char *qxl_type[] = {
diff --git a/hw/qxl-render.c b/hw/qxl-render.c
index e2e3fe2d370..88e63f80854 100644
--- a/hw/qxl-render.c
+++ b/hw/qxl-render.c
@@ -24,7 +24,7 @@
 static void qxl_blit(PCIQXLDevice *qxl, QXLRect *rect)
 {
     uint8_t *src;
-    uint8_t *dst = qxl->vga.ds->surface->data;
+    uint8_t *dst = ds_get_data(qxl->vga.ds);
     int len, i;
 
     if (is_buffer_shared(qxl->vga.ds->surface)) {
@@ -99,7 +99,6 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
 {
     VGACommonState *vga = &qxl->vga;
     int i;
-    DisplaySurface *surface = vga->ds->surface;
 
     if (qxl->guest_primary.resized) {
         qxl->guest_primary.resized = 0;
@@ -112,32 +111,30 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                qxl->guest_primary.qxl_stride,
                qxl->guest_primary.bytes_pp,
                qxl->guest_primary.bits_pp);
-    }
-    if (surface->width != qxl->guest_primary.surface.width ||
-        surface->height != qxl->guest_primary.surface.height) {
         if (qxl->guest_primary.qxl_stride > 0) {
             qemu_free_displaysurface(vga->ds);
-            qemu_create_displaysurface_from(qxl->guest_primary.surface.width,
-                                            qxl->guest_primary.surface.height,
-                                            qxl->guest_primary.bits_pp,
-                                            qxl->guest_primary.abs_stride,
-                                            qxl->guest_primary.data);
+            vga->ds->surface = qemu_create_displaysurface_from
+                (qxl->guest_primary.surface.width,
+                 qxl->guest_primary.surface.height,
+                 qxl->guest_primary.bits_pp,
+                 qxl->guest_primary.abs_stride,
+                 qxl->guest_primary.data);
         } else {
             qemu_resize_displaysurface(vga->ds,
                     qxl->guest_primary.surface.width,
                     qxl->guest_primary.surface.height);
         }
-        dpy_resize(vga->ds);
+        dpy_gfx_resize(vga->ds);
     }
     for (i = 0; i < qxl->num_dirty_rects; i++) {
         if (qemu_spice_rect_is_empty(qxl->dirty+i)) {
             break;
         }
         qxl_blit(qxl, qxl->dirty+i);
-        dpy_update(vga->ds,
-                   qxl->dirty[i].left, qxl->dirty[i].top,
-                   qxl->dirty[i].right - qxl->dirty[i].left,
-                   qxl->dirty[i].bottom - qxl->dirty[i].top);
+        dpy_gfx_update(vga->ds,
+                       qxl->dirty[i].left, qxl->dirty[i].top,
+                       qxl->dirty[i].right - qxl->dirty[i].left,
+                       qxl->dirty[i].bottom - qxl->dirty[i].top);
     }
     qxl->num_dirty_rects = 0;
 }
@@ -238,7 +235,7 @@ int qxl_render_cursor(PCIQXLDevice *qxl, QXLCommandExt *ext)
         return 1;
     }
 
-    if (!qxl->ssd.ds->mouse_set || !qxl->ssd.ds->cursor_define) {
+    if (!dpy_cursor_define_supported(qxl->ssd.ds)) {
         return 0;
     }
 
diff --git a/hw/qxl.c b/hw/qxl.c
index c2dd3b471bf..a125e294aa4 100644
--- a/hw/qxl.c
+++ b/hw/qxl.c
@@ -18,11 +18,13 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <zlib.h>
+
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 #include "qxl.h"
@@ -35,33 +37,25 @@
  */
 #undef SPICE_RING_PROD_ITEM
 #define SPICE_RING_PROD_ITEM(qxl, r, ret) {                             \
-        typeof(r) start = r;                                            \
-        typeof(r) end = r + 1;                                          \
         uint32_t prod = (r)->prod & SPICE_RING_INDEX_MASK(r);           \
-        typeof(&(r)->items[prod]) m_item = &(r)->items[prod];           \
-        if (!((uint8_t*)m_item >= (uint8_t*)(start) && (uint8_t*)(m_item + 1) <= (uint8_t*)(end))) { \
+        if (prod >= ARRAY_SIZE((r)->items)) {                           \
             qxl_set_guest_bug(qxl, "SPICE_RING_PROD_ITEM indices mismatch " \
-                          "! %p <= %p < %p", (uint8_t *)start,          \
-                          (uint8_t *)m_item, (uint8_t *)end);           \
+                          "%u >= %zu", prod, ARRAY_SIZE((r)->items));   \
             ret = NULL;                                                 \
         } else {                                                        \
-            ret = &m_item->el;                                          \
+            ret = &(r)->items[prod].el;                                 \
         }                                                               \
     }
 
 #undef SPICE_RING_CONS_ITEM
 #define SPICE_RING_CONS_ITEM(qxl, r, ret) {                             \
-        typeof(r) start = r;                                            \
-        typeof(r) end = r + 1;                                          \
         uint32_t cons = (r)->cons & SPICE_RING_INDEX_MASK(r);           \
-        typeof(&(r)->items[cons]) m_item = &(r)->items[cons];           \
-        if (!((uint8_t*)m_item >= (uint8_t*)(start) && (uint8_t*)(m_item + 1) <= (uint8_t*)(end))) { \
+        if (cons >= ARRAY_SIZE((r)->items)) {                           \
             qxl_set_guest_bug(qxl, "SPICE_RING_CONS_ITEM indices mismatch " \
-                          "! %p <= %p < %p", (uint8_t *)start,          \
-                          (uint8_t *)m_item, (uint8_t *)end);           \
+                          "%u >= %zu", cons, ARRAY_SIZE((r)->items));   \
             ret = NULL;                                                 \
         } else {                                                        \
-            ret = &m_item->el;                                          \
+            ret = &(r)->items[cons].el;                                 \
         }                                                               \
     }
 
@@ -86,9 +80,7 @@
 
 #define QXL_MODE_EX(x_res, y_res)                 \
     QXL_MODE_16_32(x_res, y_res, 0),              \
-    QXL_MODE_16_32(y_res, x_res, 1),              \
-    QXL_MODE_16_32(x_res, y_res, 2),              \
-    QXL_MODE_16_32(y_res, x_res, 3)
+    QXL_MODE_16_32(x_res, y_res, 1)
 
 static QXLMode qxl_modes[] = {
     QXL_MODE_EX(640, 480),
@@ -136,6 +128,7 @@ static void qxl_ring_set_dirty(PCIQXLDevice *qxl);
 
 void qxl_set_guest_bug(PCIQXLDevice *qxl, const char *msg, ...)
 {
+    trace_qxl_set_guest_bug(qxl->id);
     qxl_send_events(qxl, QXL_INTERRUPT_ERROR);
     qxl->guest_bug = 1;
     if (qxl->guestdebug) {
@@ -196,6 +189,7 @@ static void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id,
         spice_qxl_destroy_surface_async(&qxl->ssd.qxl, id, (uintptr_t)cookie);
     } else {
         qxl->ssd.worker->destroy_surface_wait(qxl->ssd.worker, id);
+        qxl_spice_destroy_surface_wait_complete(qxl, id);
     }
 }
 
@@ -231,7 +225,8 @@ static void qxl_spice_destroy_surfaces_complete(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_destroy_surfaces_complete(qxl->id);
     qemu_mutex_lock(&qxl->track_lock);
-    memset(&qxl->guest_surfaces.cmds, 0, sizeof(qxl->guest_surfaces.cmds));
+    memset(qxl->guest_surfaces.cmds, 0,
+           sizeof(qxl->guest_surfaces.cmds) * qxl->ssd.num_surfaces);
     qxl->guest_surfaces.count = 0;
     qemu_mutex_unlock(&qxl->track_lock);
 }
@@ -249,6 +244,32 @@ static void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl, qxl_async_io async)
     }
 }
 
+static void qxl_spice_monitors_config_async(PCIQXLDevice *qxl, int replay)
+{
+    trace_qxl_spice_monitors_config(qxl->id);
+    if (replay) {
+        /*
+         * don't use QXL_COOKIE_TYPE_IO:
+         *  - we are not running yet (post_load), we will assert
+         *    in send_events
+         *  - this is not a guest io, but a reply, so async_io isn't set.
+         */
+        spice_qxl_monitors_config_async(&qxl->ssd.qxl,
+                qxl->guest_monitors_config,
+                MEMSLOT_GROUP_GUEST,
+                (uintptr_t)qxl_cookie_new(
+                    QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
+                    0));
+    } else {
+        qxl->guest_monitors_config = qxl->ram->monitors_config;
+        spice_qxl_monitors_config_async(&qxl->ssd.qxl,
+                qxl->ram->monitors_config,
+                MEMSLOT_GROUP_GUEST,
+                (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
+                                          QXL_IO_MONITORS_CONFIG_ASYNC));
+    }
+}
+
 void qxl_spice_reset_image_cache(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_reset_image_cache(qxl->id);
@@ -262,6 +283,10 @@ void qxl_spice_reset_cursor(PCIQXLDevice *qxl)
     qemu_mutex_lock(&qxl->track_lock);
     qxl->guest_cursor = 0;
     qemu_mutex_unlock(&qxl->track_lock);
+    if (qxl->ssd.cursor) {
+        cursor_put(qxl->ssd.cursor);
+    }
+    qxl->ssd.cursor = cursor_builtin_hidden();
 }
 
 
@@ -279,10 +304,13 @@ static inline uint32_t msb_mask(uint32_t val)
 
 static ram_addr_t qxl_rom_size(void)
 {
-    uint32_t rom_size = sizeof(QXLRom) + sizeof(QXLModes) + sizeof(qxl_modes);
+    uint32_t required_rom_size = sizeof(QXLRom) + sizeof(QXLModes) +
+                                 sizeof(qxl_modes);
+    uint32_t rom_size = 8192; /* two pages */
 
-    rom_size = MAX(rom_size, TARGET_PAGE_SIZE);
-    rom_size = msb_mask(rom_size * 2 - 1);
+    required_rom_size = MAX(required_rom_size, TARGET_PAGE_SIZE);
+    required_rom_size = msb_mask(required_rom_size * 2 - 1);
+    assert(required_rom_size <= rom_size);
     return rom_size;
 }
 
@@ -307,7 +335,7 @@ static void init_qxl_rom(PCIQXLDevice *d)
     rom->slot_id_bits  = MEMSLOT_SLOT_BITS;
     rom->slots_start   = 1;
     rom->slots_end     = NUM_MEMSLOTS - 1;
-    rom->n_surfaces    = cpu_to_le32(NUM_SURFACES);
+    rom->n_surfaces    = cpu_to_le32(d->ssd.num_surfaces);
 
     for (i = 0, n = 0; i < ARRAY_SIZE(qxl_modes); i++) {
         fb = qxl_modes[i].y_res * qxl_modes[i].stride;
@@ -411,9 +439,15 @@ static int qxl_track_command(PCIQXLDevice *qxl, struct QXLCommandExt *ext)
         }
         uint32_t id = le32_to_cpu(cmd->surface_id);
 
-        if (id >= NUM_SURFACES) {
+        if (id >= qxl->ssd.num_surfaces) {
             qxl_set_guest_bug(qxl, "QXL_CMD_SURFACE id %d >= %d", id,
-                              NUM_SURFACES);
+                              qxl->ssd.num_surfaces);
+            return 1;
+        }
+        if (cmd->type == QXL_SURFACE_CMD_CREATE &&
+            (cmd->u.surface_create.stride & 0x03) != 0) {
+            qxl_set_guest_bug(qxl, "QXL_CMD_SURFACE stride = %d %% 4 != 0\n",
+                              cmd->u.surface_create.stride);
             return 1;
         }
         qemu_mutex_lock(&qxl->track_lock);
@@ -489,7 +523,7 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
     info->qxl_ram_size = le32_to_cpu(qxl->shadow_rom.num_pages) << TARGET_PAGE_BITS;
-    info->n_surfaces = NUM_SURFACES;
+    info->n_surfaces = qxl->ssd.num_surfaces;
 }
 
 static const char *qxl_mode_to_string(int mode)
@@ -538,6 +572,7 @@ static const char *io_port_to_string(uint32_t io_port)
                                         = "QXL_IO_DESTROY_ALL_SURFACES_ASYNC",
         [QXL_IO_FLUSH_SURFACES_ASYNC]   = "QXL_IO_FLUSH_SURFACES_ASYNC",
         [QXL_IO_FLUSH_RELEASE]          = "QXL_IO_FLUSH_RELEASE",
+        [QXL_IO_MONITORS_CONFIG_ASYNC]  = "QXL_IO_MONITORS_CONFIG_ASYNC",
     };
     return io_port_to_string[io_port];
 }
@@ -557,9 +592,9 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
     case QXL_MODE_VGA:
         ret = false;
         qemu_mutex_lock(&qxl->ssd.lock);
-        if (qxl->ssd.update != NULL) {
-            update = qxl->ssd.update;
-            qxl->ssd.update = NULL;
+        update = QTAILQ_FIRST(&qxl->ssd.updates);
+        if (update != NULL) {
+            QTAILQ_REMOVE(&qxl->ssd.updates, update, next);
             *ext = update->ext;
             ret = true;
         }
@@ -819,6 +854,7 @@ static void interface_async_complete_io(PCIQXLDevice *qxl, QXLCookie *cookie)
     case QXL_IO_DESTROY_PRIMARY_ASYNC:
     case QXL_IO_UPDATE_AREA_ASYNC:
     case QXL_IO_FLUSH_SURFACES_ASYNC:
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
         break;
     case QXL_IO_CREATE_PRIMARY_ASYNC:
         qxl_create_guest_primary_complete(qxl);
@@ -894,6 +930,8 @@ static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
     case QXL_COOKIE_TYPE_RENDER_UPDATE_AREA:
         qxl_render_update_area_done(qxl, cookie);
         break;
+    case QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG:
+        break;
     default:
         fprintf(stderr, "qxl: %s: unexpected cookie type %d\n",
                 __func__, cookie->type);
@@ -901,6 +939,109 @@ static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
     }
 }
 
+/* called from spice server thread context only */
+static void interface_set_client_capabilities(QXLInstance *sin,
+                                              uint8_t client_present,
+                                              uint8_t caps[58])
+{
+    PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
+
+    if (qxl->revision < 4) {
+        trace_qxl_set_client_capabilities_unsupported_by_revision(qxl->id,
+                                                              qxl->revision);
+        return;
+    }
+
+    if (runstate_check(RUN_STATE_INMIGRATE) ||
+        runstate_check(RUN_STATE_POSTMIGRATE)) {
+        return;
+    }
+
+    qxl->shadow_rom.client_present = client_present;
+    memcpy(qxl->shadow_rom.client_capabilities, caps,
+           sizeof(qxl->shadow_rom.client_capabilities));
+    qxl->rom->client_present = client_present;
+    memcpy(qxl->rom->client_capabilities, caps,
+           sizeof(qxl->rom->client_capabilities));
+    qxl_rom_set_dirty(qxl);
+
+    qxl_send_events(qxl, QXL_INTERRUPT_CLIENT);
+}
+
+static uint32_t qxl_crc32(const uint8_t *p, unsigned len)
+{
+    /*
+     * zlib xors the seed with 0xffffffff, and xors the result
+     * again with 0xffffffff; Both are not done with linux's crc32,
+     * which we want to be compatible with, so undo that.
+     */
+    return crc32(0xffffffff, p, len) ^ 0xffffffff;
+}
+
+/* called from main context only */
+static int interface_client_monitors_config(QXLInstance *sin,
+                                        VDAgentMonitorsConfig *monitors_config)
+{
+    PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
+    QXLRom *rom = memory_region_get_ram_ptr(&qxl->rom_bar);
+    int i;
+
+    if (qxl->revision < 4) {
+        trace_qxl_client_monitors_config_unsupported_by_device(qxl->id,
+                                                               qxl->revision);
+        return 0;
+    }
+    /*
+     * Older windows drivers set int_mask to 0 when their ISR is called,
+     * then later set it to ~0. So it doesn't relate to the actual interrupts
+     * handled. However, they are old, so clearly they don't support this
+     * interrupt
+     */
+    if (qxl->ram->int_mask == 0 || qxl->ram->int_mask == ~0 ||
+        !(qxl->ram->int_mask & QXL_INTERRUPT_CLIENT_MONITORS_CONFIG)) {
+        trace_qxl_client_monitors_config_unsupported_by_guest(qxl->id,
+                                                            qxl->ram->int_mask,
+                                                            monitors_config);
+        return 0;
+    }
+    if (!monitors_config) {
+        return 1;
+    }
+    memset(&rom->client_monitors_config, 0,
+           sizeof(rom->client_monitors_config));
+    rom->client_monitors_config.count = monitors_config->num_of_monitors;
+    /* monitors_config->flags ignored */
+    if (rom->client_monitors_config.count >=
+            ARRAY_SIZE(rom->client_monitors_config.heads)) {
+        trace_qxl_client_monitors_config_capped(qxl->id,
+                                monitors_config->num_of_monitors,
+                                ARRAY_SIZE(rom->client_monitors_config.heads));
+        rom->client_monitors_config.count =
+            ARRAY_SIZE(rom->client_monitors_config.heads);
+    }
+    for (i = 0 ; i < rom->client_monitors_config.count ; ++i) {
+        VDAgentMonConfig *monitor = &monitors_config->monitors[i];
+        QXLURect *rect = &rom->client_monitors_config.heads[i];
+        /* monitor->depth ignored */
+        rect->left = monitor->x;
+        rect->top = monitor->y;
+        rect->right = monitor->x + monitor->width;
+        rect->bottom = monitor->y + monitor->height;
+    }
+    rom->client_monitors_config_crc = qxl_crc32(
+            (const uint8_t *)&rom->client_monitors_config,
+            sizeof(rom->client_monitors_config));
+    trace_qxl_client_monitors_config_crc(qxl->id,
+            sizeof(rom->client_monitors_config),
+            rom->client_monitors_config_crc);
+
+    trace_qxl_interrupt_client_monitors_config(qxl->id,
+                        rom->client_monitors_config.count,
+                        rom->client_monitors_config.heads);
+    qxl_send_events(qxl, QXL_INTERRUPT_CLIENT_MONITORS_CONFIG);
+    return 1;
+}
+
 static const QXLInterface qxl_interface = {
     .base.type               = SPICE_INTERFACE_QXL,
     .base.description        = "qxl gpu",
@@ -922,6 +1063,8 @@ static const QXLInterface qxl_interface = {
     .flush_resources         = interface_flush_resources,
     .async_complete          = interface_async_complete,
     .update_area_complete    = interface_update_area_complete,
+    .set_client_capabilities = interface_set_client_capabilities,
+    .client_monitors_config = interface_client_monitors_config,
 };
 
 static void qxl_enter_vga_mode(PCIQXLDevice *d)
@@ -932,7 +1075,7 @@ static void qxl_enter_vga_mode(PCIQXLDevice *d)
     trace_qxl_enter_vga_mode(d->id);
     qemu_spice_create_host_primary(&d->ssd);
     d->mode = QXL_MODE_VGA;
-    memset(&d->ssd.dirty, 0, sizeof(d->ssd.dirty));
+    dpy_gfx_resize(d->ssd.ds);
     vga_dirty_log_start(&d->vga);
 }
 
@@ -958,9 +1101,10 @@ static void qxl_update_irq(PCIQXLDevice *d)
 static void qxl_check_state(PCIQXLDevice *d)
 {
     QXLRam *ram = d->ram;
+    int spice_display_running = qemu_spice_display_is_running(&d->ssd);
 
-    assert(!d->ssd.running || SPICE_RING_IS_EMPTY(&ram->cmd_ring));
-    assert(!d->ssd.running || SPICE_RING_IS_EMPTY(&ram->cursor_ring));
+    assert(!spice_display_running || SPICE_RING_IS_EMPTY(&ram->cmd_ring));
+    assert(!spice_display_running || SPICE_RING_IS_EMPTY(&ram->cursor_ring));
 }
 
 static void qxl_reset_state(PCIQXLDevice *d)
@@ -1229,6 +1373,12 @@ static void qxl_create_guest_primary(PCIQXLDevice *qxl, int loadvm,
     trace_qxl_create_guest_primary_rest(qxl->id, sc->stride, sc->type,
                                         sc->flags);
 
+    if ((surface.stride & 0x3) != 0) {
+        qxl_set_guest_bug(qxl, "primary surface stride = %d %% 4 != 0",
+                          surface.stride);
+        return;
+    }
+
     surface.mouse_mode = true;
     surface.group_id   = MEMSLOT_GROUP_GUEST;
     if (loadvm) {
@@ -1292,17 +1442,15 @@ static void qxl_set_mode(PCIQXLDevice *d, int modenr, int loadvm)
 
     d->mode = QXL_MODE_COMPAT;
     d->cmdflags = QXL_COMMAND_FLAG_COMPAT;
-#ifdef QXL_COMMAND_FLAG_COMPAT_16BPP /* new in spice 0.6.1 */
     if (mode->bits == 16) {
         d->cmdflags |= QXL_COMMAND_FLAG_COMPAT_16BPP;
     }
-#endif
     d->shadow_rom.mode = cpu_to_le32(modenr);
     d->rom->mode = cpu_to_le32(modenr);
     qxl_rom_set_dirty(d);
 }
 
-static void ioport_write(void *opaque, target_phys_addr_t addr,
+static void ioport_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     PCIQXLDevice *d = opaque;
@@ -1310,7 +1458,14 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
     qxl_async_io async = QXL_SYNC;
     uint32_t orig_io_port = io_port;
 
-    if (d->guest_bug && !io_port == QXL_IO_RESET) {
+    if (d->guest_bug && io_port != QXL_IO_RESET) {
+        return;
+    }
+
+    if (d->revision <= QXL_REVISION_STABLE_V10 &&
+        io_port > QXL_IO_FLUSH_RELEASE) {
+        qxl_set_guest_bug(d, "unsupported io %d for revision %d\n",
+            io_port, d->revision);
         return;
     }
 
@@ -1330,10 +1485,10 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
             break;
         }
         trace_qxl_io_unexpected_vga_mode(d->id,
-            io_port, io_port_to_string(io_port));
+            addr, val, io_port_to_string(io_port));
         /* be nice to buggy guest drivers */
         if (io_port >= QXL_IO_UPDATE_AREA_ASYNC &&
-            io_port <= QXL_IO_DESTROY_ALL_SURFACES_ASYNC) {
+            io_port < QXL_IO_RANGE_SIZE) {
             qxl_send_events(d, QXL_INTERRUPT_IO_CMD);
         }
         return;
@@ -1361,6 +1516,7 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
         io_port = QXL_IO_DESTROY_ALL_SURFACES;
         goto async_common;
     case QXL_IO_FLUSH_SURFACES_ASYNC:
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
 async_common:
         async = QXL_ASYNC;
         qemu_mutex_lock(&d->async_lock);
@@ -1385,6 +1541,18 @@ async_common:
         QXLCookie *cookie = NULL;
         QXLRect update = d->ram->update_area;
 
+        if (d->ram->update_surface > d->ssd.num_surfaces) {
+            qxl_set_guest_bug(d, "QXL_IO_UPDATE_AREA: invalid surface id %d\n",
+                              d->ram->update_surface);
+            break;
+        }
+        if (update.left >= update.right || update.top >= update.bottom ||
+            update.left < 0 || update.top < 0) {
+            qxl_set_guest_bug(d,
+                    "QXL_IO_UPDATE_AREA: invalid area (%ux%u)x(%ux%u)\n",
+                    update.left, update.top, update.right, update.bottom);
+            break;
+        }
         if (async == QXL_ASYNC) {
             cookie = qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                     QXL_IO_UPDATE_AREA_ASYNC);
@@ -1416,6 +1584,7 @@ async_common:
         qxl_set_mode(d, val, 0);
         break;
     case QXL_IO_LOG:
+        trace_qxl_io_log(d->id, d->ram->log_buf);
         if (d->guestdebug) {
             fprintf(stderr, "qxl/guest-%d: %" PRId64 ": %s", d->id,
                     qemu_get_clock_ns(vm_clock), d->ram->log_buf);
@@ -1466,7 +1635,7 @@ async_common:
         }
         break;
     case QXL_IO_DESTROY_SURFACE_WAIT:
-        if (val >= NUM_SURFACES) {
+        if (val >= d->ssd.num_surfaces) {
             qxl_set_guest_bug(d, "QXL_IO_DESTROY_SURFACE (async=%d):"
                              "%" PRIu64 " >= NUM_SURFACES", async, val);
             goto cancel_async;
@@ -1490,6 +1659,9 @@ async_common:
         d->mode = QXL_MODE_UNDEFINED;
         qxl_spice_destroy_surfaces(d, async);
         break;
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
+        qxl_spice_monitors_config_async(d, 0);
+        break;
     default:
         qxl_set_guest_bug(d, "%s: unexpected ioport=0x%x\n", __func__, io_port);
     }
@@ -1503,12 +1675,12 @@ cancel_async:
     }
 }
 
-static uint64_t ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ioport_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
-    PCIQXLDevice *d = opaque;
+    PCIQXLDevice *qxl = opaque;
 
-    trace_qxl_io_read_unexpected(d->id);
+    trace_qxl_io_read_unexpected(qxl->id);
     return 0xff;
 }
 
@@ -1538,7 +1710,14 @@ static void qxl_send_events(PCIQXLDevice *d, uint32_t events)
     uint32_t old_pending;
     uint32_t le_events = cpu_to_le32(events);
 
-    assert(d->ssd.running);
+    trace_qxl_send_events(d->id, events);
+    if (!qemu_spice_display_is_running(&d->ssd)) {
+        /* spice-server tracks guest running state and should not do this */
+        fprintf(stderr, "%s: spice-server bug: guest stopped, ignoring\n",
+                __func__);
+        trace_qxl_send_events_vm_stopped(d->id, events);
+        return;
+    }
     old_pending = __sync_fetch_and_or(&d->ram->int_pending, le_events);
     if ((old_pending & le_events) == le_events) {
         return;
@@ -1595,7 +1774,8 @@ static void qxl_hw_invalidate(void *opaque)
     vga->invalidate(vga);
 }
 
-static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     PCIQXLDevice *qxl = opaque;
     VGACommonState *vga = &qxl->vga;
@@ -1604,10 +1784,10 @@ static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch)
     case QXL_MODE_COMPAT:
     case QXL_MODE_NATIVE:
         qxl_render_update(qxl);
-        ppm_save(filename, qxl->ssd.ds->surface);
+        ppm_save(filename, qxl->ssd.ds->surface, errp);
         break;
     case QXL_MODE_VGA:
-        vga->screen_dump(vga, filename, cswitch);
+        vga->screen_dump(vga, filename, cswitch, errp);
         break;
     default:
         break;
@@ -1627,7 +1807,7 @@ static void qxl_hw_text_update(void *opaque, console_ch_t *chardata)
 
 static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
 {
-    intptr_t vram_start;
+    uintptr_t vram_start;
     int i;
 
     if (qxl->mode != QXL_MODE_NATIVE && qxl->mode != QXL_MODE_COMPAT) {
@@ -1638,10 +1818,10 @@ static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
     qxl_set_dirty(&qxl->vga.vram, qxl->shadow_rom.draw_area_offset,
                   qxl->shadow_rom.surface0_area_size);
 
-    vram_start =  (intptr_t)memory_region_get_ram_ptr(&qxl->vram_bar);
+    vram_start = (uintptr_t)memory_region_get_ram_ptr(&qxl->vram_bar);
 
     /* dirty the off-screen surfaces */
-    for (i = 0; i < NUM_SURFACES; i++) {
+    for (i = 0; i < qxl->ssd.num_surfaces; i++) {
         QXLSurfaceCmd *cmd;
         intptr_t surface_offset;
         int surface_size;
@@ -1670,7 +1850,6 @@ static void qxl_vm_change_state_handler(void *opaque, int running,
                                         RunState state)
 {
     PCIQXLDevice *qxl = opaque;
-    qemu_spice_vm_change_state_handler(&qxl->ssd, running, state);
 
     if (running) {
         /*
@@ -1713,8 +1892,8 @@ static void display_refresh(struct DisplayState *ds)
 }
 
 static DisplayChangeListener display_listener = {
-    .dpy_update  = display_update,
-    .dpy_resize  = display_resize,
+    .dpy_gfx_update  = display_update,
+    .dpy_gfx_resize  = display_resize,
     .dpy_refresh = display_refresh,
 };
 
@@ -1769,7 +1948,6 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     qxl->mode = QXL_MODE_UNDEFINED;
     qxl->generation = 1;
     qxl->num_memslots = NUM_MEMSLOTS;
-    qxl->num_surfaces = NUM_SURFACES;
     qemu_mutex_init(&qxl->track_lock);
     qemu_mutex_init(&qxl->async_lock);
     qxl->current_async = QXL_UNDEFINED_IO;
@@ -1785,10 +1963,17 @@ static int qxl_init_common(PCIQXLDevice *qxl)
         io_size = 16;
         break;
     case 3: /* qxl-3 */
-    default:
-        pci_device_rev = QXL_DEFAULT_REVISION;
+        pci_device_rev = QXL_REVISION_STABLE_V10;
+        io_size = 32; /* PCI region size must be pow2 */
+        break;
+    case 4: /* qxl-4 */
+        pci_device_rev = QXL_REVISION_STABLE_V12;
         io_size = msb_mask(QXL_IO_RANGE_SIZE * 2 - 1);
         break;
+    default:
+        error_report("Invalid revision %d for qxl device (max %d)",
+                     qxl->revision, QXL_DEFAULT_REVISION);
+        return -1;
     }
 
     pci_set_byte(&config[PCI_REVISION_ID], pci_device_rev);
@@ -1800,6 +1985,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
+    qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, "qxl.vram", qxl->vram_size);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, "qxl.vram32", &qxl->vram_bar,
@@ -1810,6 +1996,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     if (qxl->id == 0) {
         vga_dirty_log_start(&qxl->vga);
     }
+    memory_region_set_flush_coalesced(&qxl->io_bar);
 
 
     pci_register_bar(&qxl->pci, QXL_IO_RANGE_INDEX,
@@ -1848,7 +2035,11 @@ static int qxl_init_common(PCIQXLDevice *qxl)
 
     qxl->ssd.qxl.base.sif = &qxl_interface.base;
     qxl->ssd.qxl.id = qxl->id;
-    qemu_spice_add_interface(&qxl->ssd.qxl.base);
+    if (qemu_spice_add_interface(&qxl->ssd.qxl.base) != 0) {
+        error_report("qxl interface %d.%d not supported by spice-server\n",
+                     SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
+        return -1;
+    }
     qemu_add_vm_change_state_handler(qxl_vm_change_state_handler, qxl);
 
     init_pipe_signaling(qxl);
@@ -1864,6 +2055,7 @@ static int qxl_init_primary(PCIDevice *dev)
     PCIQXLDevice *qxl = DO_UPCAST(PCIQXLDevice, pci, dev);
     VGACommonState *vga = &qxl->vga;
     PortioList *qxl_vga_port_list = g_new(PortioList, 1);
+    int rc;
 
     qxl->id = 0;
     qxl_init_ramsize(qxl);
@@ -1878,9 +2070,14 @@ static int qxl_init_primary(PCIDevice *dev)
     qemu_spice_display_init_common(&qxl->ssd, vga->ds);
 
     qxl0 = qxl;
-    register_displaychangelistener(vga->ds, &display_listener);
 
-    return qxl_init_common(qxl);
+    rc = qxl_init_common(qxl);
+    if (rc != 0) {
+        return rc;
+    }
+
+    register_displaychangelistener(vga->ds, &display_listener);
+    return rc;
 }
 
 static int qxl_init_secondary(PCIDevice *dev)
@@ -1955,6 +2152,7 @@ static int qxl_post_load(void *opaque, int version)
 
     switch (newmode) {
     case QXL_MODE_UNDEFINED:
+        qxl_create_memslots(d);
         break;
     case QXL_MODE_VGA:
         qxl_create_memslots(d);
@@ -1965,8 +2163,8 @@ static int qxl_post_load(void *opaque, int version)
         qxl_create_guest_primary(d, 1, QXL_SYNC);
 
         /* replay surface-create and cursor-set commands */
-        cmds = g_malloc0(sizeof(QXLCommandExt) * (NUM_SURFACES + 1));
-        for (in = 0, out = 0; in < NUM_SURFACES; in++) {
+        cmds = g_malloc0(sizeof(QXLCommandExt) * (d->ssd.num_surfaces + 1));
+        for (in = 0, out = 0; in < d->ssd.num_surfaces; in++) {
             if (d->guest_surfaces.cmds[in] == 0) {
                 continue;
             }
@@ -1983,7 +2181,9 @@ static int qxl_post_load(void *opaque, int version)
         }
         qxl_spice_loadvm_commands(d, cmds, out);
         g_free(cmds);
-
+        if (d->guest_monitors_config) {
+            qxl_spice_monitors_config_async(d, 1);
+        }
         break;
     case QXL_MODE_COMPAT:
         /* note: no need to call qxl_create_memslots, qxl_set_mode
@@ -1996,6 +2196,14 @@ static int qxl_post_load(void *opaque, int version)
 
 #define QXL_SAVE_VERSION 21
 
+static bool qxl_monitors_config_needed(void *opaque)
+{
+    PCIQXLDevice *qxl = opaque;
+
+    return qxl->guest_monitors_config != 0;
+}
+
+
 static VMStateDescription qxl_memslot = {
     .name               = "qxl-memslot",
     .version_id         = QXL_SAVE_VERSION,
@@ -2026,6 +2234,16 @@ static VMStateDescription qxl_surface = {
     }
 };
 
+static VMStateDescription qxl_vmstate_monitors_config = {
+    .name               = "qxl/monitors-config",
+    .version_id         = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(guest_monitors_config, PCIQXLDevice),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static VMStateDescription qxl_vmstate = {
     .name               = "qxl",
     .version_id         = QXL_SAVE_VERSION,
@@ -2033,7 +2251,7 @@ static VMStateDescription qxl_vmstate = {
     .pre_save           = qxl_pre_save,
     .pre_load           = qxl_pre_load,
     .post_load          = qxl_post_load,
-    .fields = (VMStateField []) {
+    .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(pci, PCIQXLDevice),
         VMSTATE_STRUCT(vga, PCIQXLDevice, 0, vmstate_vga_common, VGACommonState),
         VMSTATE_UINT32(shadow_rom.mode, PCIQXLDevice),
@@ -2046,12 +2264,21 @@ static VMStateDescription qxl_vmstate = {
                              qxl_memslot, struct guest_slots),
         VMSTATE_STRUCT(guest_primary.surface, PCIQXLDevice, 0,
                        qxl_surface, QXLSurfaceCreate),
-        VMSTATE_INT32_EQUAL(num_surfaces, PCIQXLDevice),
-        VMSTATE_ARRAY(guest_surfaces.cmds, PCIQXLDevice, NUM_SURFACES, 0,
-                      vmstate_info_uint64, uint64_t),
+        VMSTATE_INT32_EQUAL(ssd.num_surfaces, PCIQXLDevice),
+        VMSTATE_VARRAY_INT32(guest_surfaces.cmds, PCIQXLDevice,
+                             ssd.num_surfaces, 0,
+                             vmstate_info_uint64, uint64_t),
         VMSTATE_UINT64(guest_cursor, PCIQXLDevice),
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &qxl_vmstate_monitors_config,
+            .needed = qxl_monitors_config_needed,
+        }, {
+            /* empty */
+        }
+    }
 };
 
 static Property qxl_properties[] = {
@@ -2068,6 +2295,7 @@ static Property qxl_properties[] = {
         DEFINE_PROP_UINT32("vram_size_mb", PCIQXLDevice, vram32_size_mb, -1),
         DEFINE_PROP_UINT32("vram64_size_mb", PCIQXLDevice, vram_size_mb, -1),
         DEFINE_PROP_UINT32("vgamem_mb", PCIQXLDevice, vgamem_size_mb, 16),
+        DEFINE_PROP_INT32("surfaces", PCIQXLDevice, ssd.num_surfaces, 1024),
         DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2088,7 +2316,7 @@ static void qxl_primary_class_init(ObjectClass *klass, void *data)
     dc->props = qxl_properties;
 }
 
-static TypeInfo qxl_primary_info = {
+static const TypeInfo qxl_primary_info = {
     .name          = "qxl-vga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIQXLDevice),
@@ -2110,7 +2338,7 @@ static void qxl_secondary_class_init(ObjectClass *klass, void *data)
     dc->props = qxl_properties;
 }
 
-static TypeInfo qxl_secondary_info = {
+static const TypeInfo qxl_secondary_info = {
     .name          = "qxl",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIQXLDevice),
diff --git a/hw/qxl.h b/hw/qxl.h
index 172baf6cc2d..f867a1d0ac6 100644
--- a/hw/qxl.h
+++ b/hw/qxl.h
@@ -1,10 +1,13 @@
+#ifndef HW_QXL_H
+#define HW_QXL_H 1
+
 #include "qemu-common.h"
 
-#include "console.h"
+#include "ui/console.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "ui/qemu-spice.h"
 #include "ui/spice-display.h"
@@ -40,7 +43,6 @@ typedef struct PCIQXLDevice {
     uint32_t           revision;
 
     int32_t            num_memslots;
-    int32_t            num_surfaces;
 
     uint32_t           current_async;
     QemuMutex          async_lock;
@@ -65,12 +67,14 @@ typedef struct PCIQXLDevice {
     } guest_primary;
 
     struct surfaces {
-        QXLPHYSICAL    cmds[NUM_SURFACES];
+        QXLPHYSICAL    *cmds;
         uint32_t       count;
         uint32_t       max;
     } guest_surfaces;
     QXLPHYSICAL        guest_cursor;
 
+    QXLPHYSICAL        guest_monitors_config;
+
     QemuMutex          track_lock;
 
     /* thread signaling */
@@ -128,7 +132,7 @@ typedef struct PCIQXLDevice {
         }                                                               \
     } while (0)
 
-#define QXL_DEFAULT_REVISION QXL_REVISION_STABLE_V10
+#define QXL_DEFAULT_REVISION QXL_REVISION_STABLE_V12
 
 /* qxl.c */
 void *qxl_phys2virt(PCIQXLDevice *qxl, QXLPHYSICAL phys, int group_id);
@@ -157,3 +161,5 @@ void qxl_render_update(PCIQXLDevice *qxl);
 int qxl_render_cursor(PCIQXLDevice *qxl, QXLCommandExt *ext);
 void qxl_render_update_area_done(PCIQXLDevice *qxl, QXLCookie *cookie);
 void qxl_render_update_area_bh(void *opaque);
+
+#endif
diff --git a/hw/r2d.c b/hw/r2d.c
index 0f16e81afe1..a2e3b6fe1c5 100644
--- a/hw/r2d.c
+++ b/hw/r2d.c
@@ -27,17 +27,17 @@
 #include "hw.h"
 #include "sh.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "sh7750_regs.h"
 #include "ide.h"
 #include "loader.h"
 #include "usb.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define FLASH_BASE 0x00000000
 #define FLASH_SIZE 0x02000000
@@ -127,7 +127,7 @@ static void r2d_fpga_irq_set(void *opaque, int n, int level)
     update_irl(fpga);
 }
 
-static uint32_t r2d_fpga_read(void *opaque, target_phys_addr_t addr)
+static uint32_t r2d_fpga_read(void *opaque, hwaddr addr)
 {
     r2d_fpga_t *s = opaque;
 
@@ -146,7 +146,7 @@ static uint32_t r2d_fpga_read(void *opaque, target_phys_addr_t addr)
 }
 
 static void
-r2d_fpga_write(void *opaque, target_phys_addr_t addr, uint32_t value)
+r2d_fpga_write(void *opaque, hwaddr addr, uint32_t value)
 {
     r2d_fpga_t *s = opaque;
 
@@ -178,7 +178,7 @@ static const MemoryRegionOps r2d_fpga_ops = {
 };
 
 static qemu_irq *r2d_fpga_init(MemoryRegion *sysmem,
-                               target_phys_addr_t base, qemu_irq irl)
+                               hwaddr base, qemu_irq irl)
 {
     r2d_fpga_t *s;
 
@@ -219,11 +219,12 @@ static struct QEMU_PACKED
     char kernel_cmdline[256];
 } boot_params;
 
-static void r2d_init(ram_addr_t ram_size,
-              const char *boot_device,
-	      const char *kernel_filename, const char *kernel_cmdline,
-	      const char *initrd_filename, const char *cpu_model)
+static void r2d_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     SuperHCPU *cpu;
     CPUSH4State *env;
     ResetData *reset_info;
@@ -261,7 +262,7 @@ static void r2d_init(ram_addr_t ram_size,
     irq = r2d_fpga_init(address_space_mem, 0x04000000, sh7750_irl(s));
 
     dev = qdev_create(NULL, "sh_pci");
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(busdev, 0, P4ADDR(0x1e200000));
     sysbus_mmio_map(busdev, 1, A7ADDR(0x1e200000));
@@ -332,6 +333,8 @@ static void r2d_init(ram_addr_t ram_size,
     }
 
     if (kernel_cmdline) {
+        /* I see no evidence that this .kernel_cmdline buffer requires
+           NUL-termination, so using strncpy should be ok. */
         strncpy(boot_params.kernel_cmdline, kernel_cmdline,
                 sizeof(boot_params.kernel_cmdline));
     }
@@ -344,6 +347,7 @@ static QEMUMachine r2d_machine = {
     .name = "r2d",
     .desc = "r2d-plus board",
     .init = r2d_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void r2d_machine_init(void)
diff --git a/hw/rc4030.c b/hw/rc4030.c
index 9f39b305054..a0358a319c9 100644
--- a/hw/rc4030.c
+++ b/hw/rc4030.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "mips.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /********************************************************/
 /* debug rc4030 */
@@ -112,7 +112,7 @@ static void set_next_tick(rc4030State *s)
 }
 
 /* called for accesses to rc4030 */
-static uint32_t rc4030_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readl(void *opaque, hwaddr addr)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -250,7 +250,7 @@ static uint32_t rc4030_readl(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t rc4030_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readw(void *opaque, hwaddr addr)
 {
     uint32_t v = rc4030_readl(opaque, addr & ~0x3);
     if (addr & 0x2)
@@ -259,13 +259,13 @@ static uint32_t rc4030_readw(void *opaque, target_phys_addr_t addr)
         return v & 0xffff;
 }
 
-static uint32_t rc4030_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readb(void *opaque, hwaddr addr)
 {
     uint32_t v = rc4030_readl(opaque, addr & ~0x3);
     return (v >> (8 * (addr & 0x3))) & 0xff;
 }
 
-static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     rc4030State *s = opaque;
     addr &= 0x3fff;
@@ -308,7 +308,7 @@ static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     case 0x0060:
         /* HACK */
         if (s->cache_ltag == 0x80000001 && s->cache_bmask == 0xf0f0f0f) {
-            target_phys_addr_t dest = s->cache_ptag & ~0x1;
+            hwaddr dest = s->cache_ptag & ~0x1;
             dest += (s->cache_maint & 0x3) << 3;
             cpu_physical_memory_write(dest, &val, 4);
         }
@@ -390,7 +390,7 @@ static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void rc4030_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
 
@@ -401,7 +401,7 @@ static void rc4030_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     rc4030_writel(opaque, addr & ~0x3, val);
 }
 
-static void rc4030_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
 
@@ -479,7 +479,7 @@ static void rc4030_periodic_timer(void *opaque)
     qemu_irq_raise(s->timer_irq);
 }
 
-static uint32_t jazzio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readw(void *opaque, hwaddr addr)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -517,14 +517,14 @@ static uint32_t jazzio_readw(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t jazzio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readb(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = jazzio_readw(opaque, addr & ~0x1);
     return (v >> (8 * (addr & 0x1))) & 0xff;
 }
 
-static uint32_t jazzio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readl(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = jazzio_readw(opaque, addr);
@@ -532,7 +532,7 @@ static uint32_t jazzio_readl(void *opaque, target_phys_addr_t addr)
     return v;
 }
 
-static void jazzio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     rc4030State *s = opaque;
     addr &= 0xfff;
@@ -551,7 +551,7 @@ static void jazzio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void jazzio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = jazzio_readw(opaque, addr & ~0x1);
 
@@ -566,7 +566,7 @@ static void jazzio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
     jazzio_writew(opaque, addr & ~0x1, val);
 }
 
-static void jazzio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     jazzio_writew(opaque, addr, val & 0xffff);
     jazzio_writew(opaque, addr + 2, (val >> 16) & 0xffff);
@@ -672,11 +672,11 @@ static void rc4030_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->itr);
 }
 
-void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write)
+void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write)
 {
     rc4030State *s = opaque;
-    target_phys_addr_t entry_addr;
-    target_phys_addr_t phys_addr;
+    hwaddr entry_addr;
+    hwaddr phys_addr;
     dma_pagetable_entry entry;
     int index;
     int ncpy, i;
@@ -713,7 +713,7 @@ void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, i
 static void rc4030_do_dma(void *opaque, int n, uint8_t *buf, int len, int is_write)
 {
     rc4030State *s = opaque;
-    target_phys_addr_t dma_addr;
+    hwaddr dma_addr;
     int dev_to_mem;
 
     s->dma_regs[n][DMA_REG_ENABLE] &= ~(DMA_FLAG_TC_INTR | DMA_FLAG_MEM_INTR | DMA_FLAG_ADDR_INTR);
diff --git a/hw/realview.c b/hw/realview.c
index 19db4d026be..78da7676c48 100644
--- a/hw/realview.c
+++ b/hw/realview.c
@@ -11,13 +11,13 @@
 #include "arm-misc.h"
 #include "primecell.h"
 #include "devices.h"
-#include "pci.h"
-#include "net.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "i2c.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define SMP_BOOT_ADDR 0xe0000000
 #define SMP_BOOTREG_ADDR 0x10000030
@@ -44,11 +44,8 @@ static const int realview_board_id[] = {
     0x76d
 };
 
-static void realview_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model,
-                     enum realview_board_type board_type)
+static void realview_init(QEMUMachineInitArgs *args,
+                          enum realview_board_type board_type)
 {
     ARMCPU *cpu = NULL;
     CPUARMState *env;
@@ -73,6 +70,7 @@ static void realview_init(ram_addr_t ram_size,
     uint32_t proc_id = 0;
     uint32_t sys_id;
     ram_addr_t low_ram_size;
+    ram_addr_t ram_size = args->ram_size;
 
     switch (board_type) {
     case BOARD_EB:
@@ -89,7 +87,7 @@ static void realview_init(ram_addr_t ram_size,
         break;
     }
     for (n = 0; n < smp_cpus; n++) {
-        cpu = cpu_arm_init(cpu_model);
+        cpu = cpu_arm_init(args->cpu_model);
         if (!cpu) {
             fprintf(stderr, "Unable to find CPU definition\n");
             exit(1);
@@ -142,14 +140,14 @@ static void realview_init(ram_addr_t ram_size,
     qdev_prop_set_uint32(sysctl, "sys_id", sys_id);
     qdev_prop_set_uint32(sysctl, "proc_id", proc_id);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, 0x10000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, 0x10000000);
 
     if (is_mpcore) {
-        target_phys_addr_t periphbase;
+        hwaddr periphbase;
         dev = qdev_create(NULL, is_pb ? "a9mpcore_priv": "realview_mpcore");
         qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
         qdev_init_nofail(dev);
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         if (is_pb) {
             periphbase = 0x1f000000;
         } else {
@@ -174,8 +172,8 @@ static void realview_init(ram_addr_t ram_size,
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, 0x10004000);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, pic[19]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, 0x10004000);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, pic[19]);
 
     sysbus_create_simple("pl050_keyboard", 0x10006000, pic[20]);
     sysbus_create_simple("pl050_mouse", 0x10007000, pic[21]);
@@ -217,7 +215,7 @@ static void realview_init(ram_addr_t ram_size,
 
     if (!is_pb) {
         dev = qdev_create(NULL, "realview_pci");
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         qdev_init_nofail(dev);
         sysbus_mmio_map(busdev, 0, 0x61000000); /* PCI self-config */
         sysbus_mmio_map(busdev, 1, 0x62000000); /* PCI config */
@@ -227,7 +225,7 @@ static void realview_init(ram_addr_t ram_size,
         sysbus_connect_irq(busdev, 2, pic[50]);
         sysbus_connect_irq(busdev, 3, pic[51]);
         pci_bus = (PCIBus *)qdev_get_child_bus(dev, "pci");
-        if (usb_enabled) {
+        if (usb_enabled(false)) {
             pci_create_simple(pci_bus, -1, "pci-ohci");
         }
         n = drive_get_max_bus(IF_SCSI);
@@ -321,90 +319,78 @@ static void realview_init(ram_addr_t ram_size,
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
     realview_binfo.ram_size = ram_size;
-    realview_binfo.kernel_filename = kernel_filename;
-    realview_binfo.kernel_cmdline = kernel_cmdline;
-    realview_binfo.initrd_filename = initrd_filename;
+    realview_binfo.kernel_filename = args->kernel_filename;
+    realview_binfo.kernel_cmdline = args->kernel_cmdline;
+    realview_binfo.initrd_filename = args->initrd_filename;
     realview_binfo.nb_cpus = smp_cpus;
     realview_binfo.board_id = realview_board_id[board_type];
     realview_binfo.loader_start = (board_type == BOARD_PB_A8 ? 0x70000000 : 0);
     arm_load_kernel(arm_env_get_cpu(first_cpu), &realview_binfo);
 }
 
-static void realview_eb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_eb_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "arm926";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm926";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_EB);
+    realview_init(args, BOARD_EB);
 }
 
-static void realview_eb_mpcore_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_eb_mpcore_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "arm11mpcore";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm11mpcore";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_EB_MPCORE);
+    realview_init(args, BOARD_EB_MPCORE);
 }
 
-static void realview_pb_a8_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_pb_a8_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "cortex-a8";
+    if (!args->cpu_model) {
+        args->cpu_model = "cortex-a8";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_PB_A8);
+    realview_init(args, BOARD_PB_A8);
 }
 
-static void realview_pbx_a9_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_pbx_a9_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "cortex-a9";
+    if (!args->cpu_model) {
+        args->cpu_model = "cortex-a9";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_PBX_A9);
+    realview_init(args, BOARD_PBX_A9);
 }
 
 static QEMUMachine realview_eb_machine = {
     .name = "realview-eb",
     .desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)",
     .init = realview_eb_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_eb_mpcore_machine = {
     .name = "realview-eb-mpcore",
     .desc = "ARM RealView Emulation Baseboard (ARM11MPCore)",
     .init = realview_eb_mpcore_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_pb_a8_machine = {
     .name = "realview-pb-a8",
     .desc = "ARM RealView Platform Baseboard for Cortex-A8",
     .init = realview_pb_a8_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_pbx_a9_machine = {
     .name = "realview-pbx-a9",
     .desc = "ARM RealView Platform Baseboard Explore for Cortex-A9",
     .init = realview_pbx_a9_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void realview_machine_init(void)
diff --git a/hw/realview_gic.c b/hw/realview_gic.c
index 5bc37a7120b..8f2a7e2f345 100644
--- a/hw/realview_gic.c
+++ b/hw/realview_gic.c
@@ -35,7 +35,7 @@ static int realview_gic_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", 1);
     qdev_prop_set_uint32(s->gic, "num-irq", numirq);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -59,7 +59,7 @@ static void realview_gic_class_init(ObjectClass *klass, void *data)
     sdc->init = realview_gic_init;
 }
 
-static TypeInfo realview_gic_info = {
+static const TypeInfo realview_gic_info = {
     .name          = "realview_gic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RealViewGICState),
diff --git a/hw/rtl8139.c b/hw/rtl8139.c
index 844f1b8c3f6..cfbf3f47c1c 100644
--- a/hw/rtl8139.c
+++ b/hw/rtl8139.c
@@ -52,13 +52,13 @@
 #include <zlib.h>
 
 #include "hw.h"
-#include "pci.h"
-#include "dma.h"
-#include "qemu-timer.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
+#include "qemu/timer.h"
+#include "net/net.h"
 #include "loader.h"
-#include "sysemu.h"
-#include "iov.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
 
 /* debug RTL8139 card */
 //#define DEBUG_RTL8139 1
@@ -167,7 +167,7 @@ enum IntrStatusBits {
     PCIErr = 0x8000,
     PCSTimeout = 0x4000,
     RxFIFOOver = 0x40,
-    RxUnderrun = 0x20,
+    RxUnderrun = 0x20, /* Packet Underrun / Link Change */
     RxOverflow = 0x10,
     TxErr = 0x08,
     TxOK = 0x04,
@@ -774,11 +774,7 @@ static void rtl8139_write_buffer(RTL8139State *s, const void *buf, int size)
 #define MIN_BUF_SIZE 60
 static inline dma_addr_t rtl8139_addr64(uint32_t low, uint32_t high)
 {
-#if TARGET_PHYS_ADDR_BITS > 32
-    return low | ((target_phys_addr_t)high << 32);
-#else
-    return low;
-#endif
+    return low | ((uint64_t)high << 32);
 }
 
 /* Workaround for buggy guest driver such as linux who allocates rx
@@ -1262,7 +1258,8 @@ static void rtl8139_reset(DeviceState *d)
     s->BasicModeStatus  = 0x7809;
     //s->BasicModeStatus |= 0x0040; /* UTP medium */
     s->BasicModeStatus |= 0x0020; /* autonegotiation completed */
-    s->BasicModeStatus |= 0x0004; /* link is up */
+    /* preserve link state */
+    s->BasicModeStatus |= s->nic->nc.link_down ? 0 : 0x04;
 
     s->NWayAdvert    = 0x05e1; /* all modes, full duplex */
     s->NWayLPAR      = 0x05e1; /* all modes, full duplex */
@@ -2459,7 +2456,7 @@ static void rtl8139_TxStatus_write(RTL8139State *s, uint32_t txRegOffset, uint32
 
         if (descriptor == 0 && (val & 0x8))
         {
-            target_phys_addr_t tc_addr = rtl8139_addr64(s->TxStatus[0] & ~0x3f, s->TxStatus[1]);
+            hwaddr tc_addr = rtl8139_addr64(s->TxStatus[0] & ~0x3f, s->TxStatus[1]);
 
             /* dump tally counters to specified memory location */
             RTL8139TallyCounters_dma_write(s, tc_addr);
@@ -3007,7 +3004,8 @@ static uint32_t rtl8139_io_readb(void *opaque, uint8_t addr)
             break;
 
         case MediaStatus:
-            ret = 0xd0;
+            /* The LinkDown bit of MediaStatus is inverse with link status */
+            ret = 0xd0 | (~s->BasicModeStatus & 0x04);
             DPRINTF("MediaStatus read 0x%x\n", ret);
             break;
 
@@ -3190,65 +3188,33 @@ static uint32_t rtl8139_io_readl(void *opaque, uint8_t addr)
 
 /* */
 
-static void rtl8139_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writeb(opaque, addr & 0xFF, val);
 }
 
-static void rtl8139_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writew(opaque, addr & 0xFF, val);
 }
 
-static void rtl8139_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writel(opaque, addr & 0xFF, val);
 }
 
-static uint32_t rtl8139_ioport_readb(void *opaque, uint32_t addr)
+static uint32_t rtl8139_mmio_readb(void *opaque, hwaddr addr)
 {
     return rtl8139_io_readb(opaque, addr & 0xFF);
 }
 
-static uint32_t rtl8139_ioport_readw(void *opaque, uint32_t addr)
-{
-    return rtl8139_io_readw(opaque, addr & 0xFF);
-}
-
-static uint32_t rtl8139_ioport_readl(void *opaque, uint32_t addr)
-{
-    return rtl8139_io_readl(opaque, addr & 0xFF);
-}
-
-/* */
-
-static void rtl8139_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writeb(opaque, addr & 0xFF, val);
-}
-
-static void rtl8139_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writew(opaque, addr & 0xFF, val);
-}
-
-static void rtl8139_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writel(opaque, addr & 0xFF, val);
-}
-
-static uint32_t rtl8139_mmio_readb(void *opaque, target_phys_addr_t addr)
-{
-    return rtl8139_io_readb(opaque, addr & 0xFF);
-}
-
-static uint32_t rtl8139_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t rtl8139_mmio_readw(void *opaque, hwaddr addr)
 {
     uint32_t val = rtl8139_io_readw(opaque, addr & 0xFF);
     return val;
 }
 
-static uint32_t rtl8139_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t rtl8139_mmio_readl(void *opaque, hwaddr addr)
 {
     uint32_t val = rtl8139_io_readl(opaque, addr & 0xFF);
     return val;
@@ -3262,6 +3228,10 @@ static int rtl8139_post_load(void *opaque, int version_id)
         s->cplus_enabled = s->CpCmd != 0;
     }
 
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in BasicModeStatus */
+    s->nic->nc.link_down = (s->BasicModeStatus & 0x04) == 0;
+
     return 0;
 }
 
@@ -3385,18 +3355,44 @@ static const VMStateDescription vmstate_rtl8139 = {
 /***********************************************************/
 /* PCI RTL8139 definitions */
 
-static const MemoryRegionPortio rtl8139_portio[] = {
-    { 0, 0x100, 1, .read = rtl8139_ioport_readb, },
-    { 0, 0x100, 1, .write = rtl8139_ioport_writeb, },
-    { 0, 0x100, 2, .read = rtl8139_ioport_readw, },
-    { 0, 0x100, 2, .write = rtl8139_ioport_writew, },
-    { 0, 0x100, 4, .read = rtl8139_ioport_readl, },
-    { 0, 0x100, 4, .write = rtl8139_ioport_writel, },
-    PORTIO_END_OF_LIST()
-};
+static void rtl8139_ioport_write(void *opaque, hwaddr addr,
+                                 uint64_t val, unsigned size)
+{
+    switch (size) {
+    case 1:
+        rtl8139_io_writeb(opaque, addr, val);
+        break;
+    case 2:
+        rtl8139_io_writew(opaque, addr, val);
+        break;
+    case 4:
+        rtl8139_io_writel(opaque, addr, val);
+        break;
+    }
+}
+
+static uint64_t rtl8139_ioport_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    switch (size) {
+    case 1:
+        return rtl8139_io_readb(opaque, addr);
+    case 2:
+        return rtl8139_io_readw(opaque, addr);
+    case 4:
+        return rtl8139_io_readl(opaque, addr);
+    }
+
+    return -1;
+}
 
 static const MemoryRegionOps rtl8139_io_ops = {
-    .old_portio = rtl8139_portio,
+    .read = rtl8139_ioport_read,
+    .write = rtl8139_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
@@ -3453,12 +3449,27 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
     qemu_del_net_client(&s->nic->nc);
 }
 
+static void rtl8139_set_link_status(NetClientState *nc)
+{
+    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+
+    if (nc->link_down) {
+        s->BasicModeStatus &= ~0x04;
+    } else {
+        s->BasicModeStatus |= 0x04;
+    }
+
+    s->IntrStatus |= RxUnderrun;
+    rtl8139_update_irq(s);
+}
+
 static NetClientInfo net_rtl8139_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
     .can_receive = rtl8139_can_receive,
     .receive = rtl8139_receive,
     .cleanup = rtl8139_cleanup,
+    .link_status_changed = rtl8139_set_link_status,
 };
 
 static int pci_rtl8139_init(PCIDevice *dev)
@@ -3529,7 +3540,7 @@ static void rtl8139_class_init(ObjectClass *klass, void *data)
     dc->props = rtl8139_properties;
 }
 
-static TypeInfo rtl8139_info = {
+static const TypeInfo rtl8139_info = {
     .name          = "rtl8139",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(RTL8139State),
diff --git a/hw/s390-virtio-bus.c b/hw/s390-virtio-bus.c
index a245684692e..4f5939e0312 100644
--- a/hw/s390-virtio-bus.c
+++ b/hw/s390-virtio-bus.c
@@ -18,20 +18,23 @@
  */
 
 #include "hw.h"
-#include "block.h"
-#include "sysemu.h"
-#include "net.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "loader.h"
 #include "elf.h"
 #include "hw/virtio.h"
+#include "hw/virtio-rng.h"
 #include "hw/virtio-serial.h"
 #include "hw/virtio-net.h"
+#include "hw/virtio-scsi.h"
+#include "hw/vhost-scsi.h"
 #include "hw/sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #include "hw/s390-virtio-bus.h"
+#include "hw/virtio-bus.h"
 
 /* #define DEBUG_S390 */
 
@@ -56,7 +59,7 @@ static const VirtIOBindings virtio_s390_bindings;
 static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev);
 
 /* length of VirtIO device pages */
-const target_phys_addr_t virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
+const hwaddr virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
 
 static void s390_virtio_bus_reset(void *opaque)
 {
@@ -67,7 +70,7 @@ static void s390_virtio_bus_reset(void *opaque)
 void s390_virtio_reset_idx(VirtIOS390Device *dev)
 {
     int i;
-    target_phys_addr_t idx_addr;
+    hwaddr idx_addr;
     uint8_t num_vq;
 
     num_vq = s390_virtio_device_num_vq(dev);
@@ -110,10 +113,12 @@ VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size)
     return bus;
 }
 
-static void s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token)
+static void s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
 {
+    CPUS390XState *env = &cpu->env;
+
     if (kvm_enabled()) {
-        kvm_s390_virtio_irq(env, config_change, token);
+        kvm_s390_virtio_irq(cpu, config_change, token);
     } else {
         cpu_inject_ext(env, VIRTIO_EXT_CODE, config_change, token);
     }
@@ -136,14 +141,13 @@ static int s390_virtio_device_init(VirtIOS390Device *dev, VirtIODevice *vdev)
 
     bus->dev_offs += dev_len;
 
-    virtio_bind_device(vdev, &virtio_s390_bindings, dev);
+    virtio_bind_device(vdev, &virtio_s390_bindings, DEVICE(dev));
     dev->host_features = vdev->get_features(vdev, dev->host_features);
     s390_virtio_device_sync(dev);
     s390_virtio_reset_idx(dev);
     if (dev->qdev.hotplugged) {
         S390CPU *cpu = s390_cpu_addr2state(0);
-        CPUS390XState *env = &cpu->env;
-        s390_virtio_irq(env, VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
+        s390_virtio_irq(cpu, VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
     }
 
     return 0;
@@ -206,6 +210,30 @@ static int s390_virtio_scsi_init(VirtIOS390Device *dev)
     return s390_virtio_device_init(dev, vdev);
 }
 
+static int s390_vhost_scsi_init(VirtIOS390Device *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = vhost_scsi_init((DeviceState *)dev, &dev->scsi);
+    if (!vdev) {
+        return -1;
+    }
+
+    return s390_virtio_device_init(dev, vdev);
+}
+
+static int s390_virtio_rng_init(VirtIOS390Device *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_rng_init((DeviceState *)dev, &dev->rng);
+    if (!vdev) {
+        return -1;
+    }
+
+    return s390_virtio_device_init(dev, vdev);
+}
+
 static uint64_t s390_virtio_device_vq_token(VirtIOS390Device *dev, int vq)
 {
     ram_addr_t token_off;
@@ -351,19 +379,32 @@ VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem)
     return NULL;
 }
 
-static void virtio_s390_notify(void *opaque, uint16_t vector)
+/* DeviceState to VirtIOS390Device. Note: used on datapath,
+ * be careful and test performance if you change this.
+ */
+static inline VirtIOS390Device *to_virtio_s390_device_fast(DeviceState *d)
+{
+    return container_of(d, VirtIOS390Device, qdev);
+}
+
+/* DeviceState to VirtIOS390Device. TODO: use QOM. */
+static inline VirtIOS390Device *to_virtio_s390_device(DeviceState *d)
 {
-    VirtIOS390Device *dev = (VirtIOS390Device*)opaque;
+    return container_of(d, VirtIOS390Device, qdev);
+}
+
+static void virtio_s390_notify(DeviceState *d, uint16_t vector)
+{
+    VirtIOS390Device *dev = to_virtio_s390_device_fast(d);
     uint64_t token = s390_virtio_device_vq_token(dev, vector);
     S390CPU *cpu = s390_cpu_addr2state(0);
-    CPUS390XState *env = &cpu->env;
 
-    s390_virtio_irq(env, 0, token);
+    s390_virtio_irq(cpu, 0, token);
 }
 
-static unsigned virtio_s390_get_features(void *opaque)
+static unsigned virtio_s390_get_features(DeviceState *d)
 {
-    VirtIOS390Device *dev = (VirtIOS390Device*)opaque;
+    VirtIOS390Device *dev = to_virtio_s390_device(d);
     return dev->host_features;
 }
 
@@ -393,7 +434,7 @@ static void s390_virtio_net_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_net_properties;
 }
 
-static TypeInfo s390_virtio_net = {
+static const TypeInfo s390_virtio_net = {
     .name          = "virtio-net-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -419,7 +460,7 @@ static void s390_virtio_blk_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_blk_properties;
 }
 
-static TypeInfo s390_virtio_blk = {
+static const TypeInfo s390_virtio_blk = {
     .name          = "virtio-blk-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -441,18 +482,43 @@ static void s390_virtio_serial_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_serial_properties;
 }
 
-static TypeInfo s390_virtio_serial = {
+static const TypeInfo s390_virtio_serial = {
     .name          = "virtio-serial-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
     .class_init    = s390_virtio_serial_class_init,
 };
 
+static void s390_virtio_rng_initfn(Object *obj)
+{
+    VirtIOS390Device *dev = VIRTIO_S390_DEVICE(obj);
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&dev->rng.rng, NULL);
+}
+
+static void s390_virtio_rng_class_init(ObjectClass *klass, void *data)
+{
+    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
+
+    k->init = s390_virtio_rng_init;
+}
+
+static const TypeInfo s390_virtio_rng = {
+    .name          = "virtio-rng-s390",
+    .parent        = TYPE_VIRTIO_S390_DEVICE,
+    .instance_size = sizeof(VirtIOS390Device),
+    .instance_init = s390_virtio_rng_initfn,
+    .class_init    = s390_virtio_rng_class_init,
+};
+
 static int s390_virtio_busdev_init(DeviceState *dev)
 {
     VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
     VirtIOS390DeviceClass *_info = VIRTIO_S390_DEVICE_GET_CLASS(dev);
 
+    virtio_s390_bus_new(&_dev->bus, _dev);
+
     return _info->init(_dev);
 }
 
@@ -465,7 +531,7 @@ static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
     dc->unplug = qdev_simple_unplug_cb;
 }
 
-static TypeInfo virtio_s390_device_info = {
+static const TypeInfo virtio_s390_device_info = {
     .name = TYPE_VIRTIO_S390_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -474,6 +540,27 @@ static TypeInfo virtio_s390_device_info = {
     .abstract = true,
 };
 
+static Property s390_vhost_scsi_properties[] = {
+    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOS390Device, host_features, scsi),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_vhost_scsi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
+
+    k->init = s390_vhost_scsi_init;
+    dc->props = s390_vhost_scsi_properties;
+}
+
+static const TypeInfo s390_vhost_scsi = {
+    .name          = "vhost-scsi-s390",
+    .parent        = TYPE_VIRTIO_S390_DEVICE,
+    .instance_size = sizeof(VirtIOS390Device),
+    .class_init    = s390_vhost_scsi_class_init,
+};
+
 static Property s390_virtio_scsi_properties[] = {
     DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOS390Device, host_features, scsi),
     DEFINE_PROP_END_OF_LIST(),
@@ -488,7 +575,7 @@ static void s390_virtio_scsi_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_scsi_properties;
 }
 
-static TypeInfo s390_virtio_scsi = {
+static const TypeInfo s390_virtio_scsi = {
     .name          = "virtio-scsi-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -513,21 +600,50 @@ static void s390_virtio_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo s390_virtio_bridge_info = {
+static const TypeInfo s390_virtio_bridge_info = {
     .name          = "s390-virtio-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
     .class_init    = s390_virtio_bridge_class_init,
 };
 
+/* virtio-s390-bus */
+
+void virtio_s390_bus_new(VirtioBusState *bus, VirtIOS390Device *dev)
+{
+    DeviceState *qdev = DEVICE(dev);
+    BusState *qbus;
+    qbus_create_inplace((BusState *)bus, TYPE_VIRTIO_S390_BUS, qdev, NULL);
+    qbus = BUS(bus);
+    qbus->allow_hotplug = 0;
+}
+
+static void virtio_s390_bus_class_init(ObjectClass *klass, void *data)
+{
+    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
+    BusClass *bus_class = BUS_CLASS(klass);
+    bus_class->max_dev = 1;
+    k->notify = virtio_s390_notify;
+    k->get_features = virtio_s390_get_features;
+}
+
+static const TypeInfo virtio_s390_bus_info = {
+    .name          = TYPE_VIRTIO_S390_BUS,
+    .parent        = TYPE_VIRTIO_BUS,
+    .instance_size = sizeof(VirtioS390BusState),
+    .class_init    = virtio_s390_bus_class_init,
+};
+
 static void s390_virtio_register_types(void)
 {
+    type_register_static(&virtio_s390_bus_info);
     type_register_static(&s390_virtio_bus_info);
     type_register_static(&virtio_s390_device_info);
     type_register_static(&s390_virtio_serial);
     type_register_static(&s390_virtio_blk);
     type_register_static(&s390_virtio_net);
     type_register_static(&s390_virtio_scsi);
+    type_register_static(&s390_virtio_rng);
     type_register_static(&s390_virtio_bridge_info);
 }
 
diff --git a/hw/s390-virtio-bus.h b/hw/s390-virtio-bus.h
index 4873134ae90..438b37fd828 100644
--- a/hw/s390-virtio-bus.h
+++ b/hw/s390-virtio-bus.h
@@ -16,11 +16,15 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef HW_S390_VIRTIO_BUS_H
+#define HW_S390_VIRTIO_BUS_H 1
 
 #include "virtio-blk.h"
 #include "virtio-net.h"
+#include "virtio-rng.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
+#include "virtio-bus.h"
 
 #define VIRTIO_DEV_OFFS_TYPE		0	/* 8 bits */
 #define VIRTIO_DEV_OFFS_NUM_VQ		1	/* 8 bits */
@@ -56,8 +60,24 @@
 #define S390_VIRTIO_BUS(obj) \
      OBJECT_CHECK(VirtIOS390Bus, (obj), TYPE_S390_VIRTIO_BUS)
 
+/* virtio-s390-bus */
+
+typedef struct VirtioBusState VirtioS390BusState;
+typedef struct VirtioBusClass VirtioS390BusClass;
+
+#define TYPE_VIRTIO_S390_BUS "virtio-s390-bus"
+#define VIRTIO_S390_BUS(obj) \
+        OBJECT_CHECK(VirtioS390BusState, (obj), TYPE_VIRTIO_S390_BUS)
+#define VIRTIO_S390_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioS390BusClass, obj, TYPE_VIRTIO_S390_BUS)
+#define VIRTIO_S390_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioS390BusClass, klass, TYPE_VIRTIO_S390_BUS)
+
+
 typedef struct VirtIOS390Device VirtIOS390Device;
 
+void virtio_s390_bus_new(VirtioBusState *bus, VirtIOS390Device *dev);
+
 typedef struct VirtIOS390DeviceClass {
     DeviceClass qdev;
     int (*init)(VirtIOS390Device *dev);
@@ -75,6 +95,8 @@ struct VirtIOS390Device {
     virtio_serial_conf serial;
     virtio_net_conf net;
     VirtIOSCSIConf scsi;
+    VirtIORNGConf rng;
+    VirtioBusState bus;
 };
 
 typedef struct VirtIOS390Bus {
@@ -98,3 +120,5 @@ VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem);
 void s390_virtio_device_sync(VirtIOS390Device *dev);
 void s390_virtio_reset_idx(VirtIOS390Device *dev);
 
+
+#endif
diff --git a/hw/s390-virtio.c b/hw/s390-virtio.c
index 47eed35da39..5edaabb7c4f 100644
--- a/hw/s390-virtio.c
+++ b/hw/s390-virtio.c
@@ -2,6 +2,7 @@
  * QEMU S390 virtio target
  *
  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ * Copyright IBM Corp 2012
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -13,25 +14,29 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
 #include "hw.h"
-#include "block.h"
-#include "blockdev.h"
-#include "sysemu.h"
-#include "net.h"
+#include "block/block.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 #include "boards.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "loader.h"
-#include "elf.h"
 #include "hw/virtio.h"
 #include "hw/sysbus.h"
-#include "kvm.h"
-#include "exec-memory.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
 
 #include "hw/s390-virtio-bus.h"
+#include "hw/s390x/sclp.h"
+#include "hw/s390-virtio.h"
 
 //#define DEBUG_S390
 
@@ -43,21 +48,6 @@
     do { } while (0)
 #endif
 
-#define KVM_S390_VIRTIO_NOTIFY          0
-#define KVM_S390_VIRTIO_RESET           1
-#define KVM_S390_VIRTIO_SET_STATUS      2
-
-#define KERN_IMAGE_START                0x010000UL
-#define KERN_PARM_AREA                  0x010480UL
-#define INITRD_START                    0x800000UL
-#define INITRD_PARM_START               0x010408UL
-#define INITRD_PARM_SIZE                0x010410UL
-#define PARMFILE_START                  0x001000UL
-
-#define ZIPL_START			0x009000UL
-#define ZIPL_LOAD_ADDR			0x009000UL
-#define ZIPL_FILENAME			"s390-zipl.rom"
-
 #define MAX_BLK_DEVS                    10
 
 static VirtIOS390Bus *s390_bus;
@@ -72,56 +62,63 @@ S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
     return ipi_states[cpu_addr];
 }
 
-int s390_virtio_hypercall(CPUS390XState *env, uint64_t mem, uint64_t hypercall)
+static int s390_virtio_hcall_notify(const uint64_t *args)
 {
+    uint64_t mem = args[0];
     int r = 0, i;
 
-    dprintf("KVM hypercall: %ld\n", hypercall);
-    switch (hypercall) {
-    case KVM_S390_VIRTIO_NOTIFY:
-        if (mem > ram_size) {
-            VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus,
-                                                               mem, &i);
-            if (dev) {
-                virtio_queue_notify(dev->vdev, i);
-            } else {
-                r = -EINVAL;
-            }
-        } else {
-            /* Early printk */
-        }
-        break;
-    case KVM_S390_VIRTIO_RESET:
-    {
-        VirtIOS390Device *dev;
-
-        dev = s390_virtio_bus_find_mem(s390_bus, mem);
-        virtio_reset(dev->vdev);
-        stb_phys(dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0);
-        s390_virtio_device_sync(dev);
-        s390_virtio_reset_idx(dev);
-        break;
-    }
-    case KVM_S390_VIRTIO_SET_STATUS:
-    {
-        VirtIOS390Device *dev;
-
-        dev = s390_virtio_bus_find_mem(s390_bus, mem);
+    if (mem > ram_size) {
+        VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus, mem, &i);
         if (dev) {
-            s390_virtio_device_update_status(dev);
+            virtio_queue_notify(dev->vdev, i);
         } else {
             r = -EINVAL;
         }
-        break;
+    } else {
+        /* Early printk */
     }
-    default:
+    return r;
+}
+
+static int s390_virtio_hcall_reset(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+    VirtIOS390Device *dev;
+
+    dev = s390_virtio_bus_find_mem(s390_bus, mem);
+    virtio_reset(dev->vdev);
+    stb_phys(dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0);
+    s390_virtio_device_sync(dev);
+    s390_virtio_reset_idx(dev);
+
+    return 0;
+}
+
+static int s390_virtio_hcall_set_status(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+    int r = 0;
+    VirtIOS390Device *dev;
+
+    dev = s390_virtio_bus_find_mem(s390_bus, mem);
+    if (dev) {
+        s390_virtio_device_update_status(dev);
+    } else {
         r = -EINVAL;
-        break;
     }
-
     return r;
 }
 
+static void s390_virtio_register_hcalls(void)
+{
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_NOTIFY,
+                                   s390_virtio_hcall_notify);
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_RESET,
+                                   s390_virtio_hcall_reset);
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_SET_STATUS,
+                                   s390_virtio_hcall_set_status);
+}
+
 /*
  * The number of running CPUs. On s390 a shutdown is the state of all CPUs
  * being either stopped or disabled (for interrupts) waiting. We have to
@@ -151,24 +148,19 @@ unsigned s390_del_running_cpu(CPUS390XState *env)
 }
 
 /* PC hardware initialisation */
-static void s390_init(ram_addr_t my_ram_size,
-                      const char *boot_device,
-                      const char *kernel_filename,
-                      const char *kernel_cmdline,
-                      const char *initrd_filename,
-                      const char *cpu_model)
+static void s390_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t my_ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     CPUS390XState *env = NULL;
+    DeviceState *dev;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    ram_addr_t kernel_size = 0;
-    ram_addr_t initrd_offset;
-    ram_addr_t initrd_size = 0;
     int shift = 0;
     uint8_t *storage_keys;
     void *virtio_region;
-    target_phys_addr_t virtio_region_len;
-    target_phys_addr_t virtio_region_start;
+    hwaddr virtio_region_len;
+    hwaddr virtio_region_start;
     int i;
 
     /* s390x ram size detection needs a 16bit multiplier + an increment. So
@@ -183,6 +175,19 @@ static void s390_init(ram_addr_t my_ram_size,
 
     /* get a BUS */
     s390_bus = s390_virtio_bus_init(&my_ram_size);
+    s390_sclp_init();
+    dev  = qdev_create(NULL, "s390-ipl");
+    if (args->kernel_filename) {
+        qdev_prop_set_string(dev, "kernel", args->kernel_filename);
+    }
+    if (args->initrd_filename) {
+        qdev_prop_set_string(dev, "initrd", args->initrd_filename);
+    }
+    qdev_prop_set_string(dev, "cmdline", args->kernel_cmdline);
+    qdev_init_nofail(dev);
+
+    /* register hypercalls */
+    s390_virtio_register_hcalls();
 
     /* allocate RAM */
     memory_region_init_ram(ram, "s390.ram", my_ram_size);
@@ -223,76 +228,6 @@ static void s390_init(ram_addr_t my_ram_size,
         tmp_env->storage_keys = storage_keys;
     }
 
-    /* One CPU has to run */
-    s390_add_running_cpu(env);
-
-    if (kernel_filename) {
-
-        kernel_size = load_elf(kernel_filename, NULL, NULL, NULL, NULL,
-                               NULL, 1, ELF_MACHINE, 0);
-        if (kernel_size == -1UL) {
-            kernel_size = load_image_targphys(kernel_filename, 0, ram_size);
-        }
-        if (kernel_size == -1UL) {
-            fprintf(stderr, "qemu: could not load kernel '%s'\n",
-                    kernel_filename);
-            exit(1);
-        }
-        /*
-         * we can not rely on the ELF entry point, since up to 3.2 this
-         * value was 0x800 (the SALIPL loader) and it wont work. For
-         * all (Linux) cases 0x10000 (KERN_IMAGE_START) should be fine.
-         */
-        env->psw.addr = KERN_IMAGE_START;
-        env->psw.mask = 0x0000000180000000ULL;
-    } else {
-        ram_addr_t bios_size = 0;
-        char *bios_filename;
-
-        /* Load zipl bootloader */
-        if (bios_name == NULL) {
-            bios_name = ZIPL_FILENAME;
-        }
-
-        bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
-        bios_size = load_image_targphys(bios_filename, ZIPL_LOAD_ADDR, 4096);
-        g_free(bios_filename);
-
-        if ((long)bios_size < 0) {
-            hw_error("could not load bootloader '%s'\n", bios_name);
-        }
-
-        if (bios_size > 4096) {
-            hw_error("stage1 bootloader is > 4k\n");
-        }
-
-        env->psw.addr = ZIPL_START;
-        env->psw.mask = 0x0000000180000000ULL;
-    }
-
-    if (initrd_filename) {
-        initrd_offset = INITRD_START;
-        while (kernel_size + 0x100000 > initrd_offset) {
-            initrd_offset += 0x100000;
-        }
-        initrd_size = load_image_targphys(initrd_filename, initrd_offset,
-                                          ram_size - initrd_offset);
-        if (initrd_size == -1UL) {
-            fprintf(stderr, "qemu: could not load initrd '%s'\n",
-                    initrd_filename);
-            exit(1);
-        }
-
-        /* we have to overwrite values in the kernel image, which are "rom" */
-        memcpy(rom_ptr(INITRD_PARM_START), &initrd_offset, 8);
-        memcpy(rom_ptr(INITRD_PARM_SIZE), &initrd_size, 8);
-    }
-
-    if (rom_ptr(KERN_PARM_AREA)) {
-        /* we have to overwrite values in the kernel image, which are "rom" */
-        memcpy(rom_ptr(KERN_PARM_AREA), kernel_cmdline,
-               strlen(kernel_cmdline) + 1);
-    }
 
     /* Create VirtIO network adapters */
     for(i = 0; i < nb_nics; i++) {
@@ -312,21 +247,6 @@ static void s390_init(ram_addr_t my_ram_size,
         qdev_set_nic_properties(dev, nd);
         qdev_init_nofail(dev);
     }
-
-    /* Create VirtIO disk drives */
-    for(i = 0; i < MAX_BLK_DEVS; i++) {
-        DriveInfo *dinfo;
-        DeviceState *dev;
-
-        dinfo = drive_get(IF_IDE, 0, i);
-        if (!dinfo) {
-            continue;
-        }
-
-        dev = qdev_create((BusState *)s390_bus, "virtio-blk-s390");
-        qdev_prop_set_drive_nofail(dev, "drive", dinfo->bdrv);
-        qdev_init_nofail(dev);
-    }
 }
 
 static QEMUMachine s390_machine = {
@@ -334,6 +254,7 @@ static QEMUMachine s390_machine = {
     .alias = "s390",
     .desc = "VirtIO based S390 machine",
     .init = s390_init,
+    .block_default_type = IF_VIRTIO,
     .no_cdrom = 1,
     .no_floppy = 1,
     .no_serial = 1,
@@ -342,6 +263,7 @@ static QEMUMachine s390_machine = {
     .use_virtcon = 1,
     .max_cpus = 255,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void s390_machine_init(void)
diff --git a/hw/s390-virtio.h b/hw/s390-virtio.h
new file mode 100644
index 00000000000..25bb610fd8a
--- /dev/null
+++ b/hw/s390-virtio.h
@@ -0,0 +1,22 @@
+/*
+ * Virtio interfaces for s390
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_VIRTIO_H
+#define HW_S390_VIRTIO_H 1
+
+#define KVM_S390_VIRTIO_NOTIFY          0
+#define KVM_S390_VIRTIO_RESET           1
+#define KVM_S390_VIRTIO_SET_STATUS      2
+
+typedef int (*s390_virtio_fn)(const uint64_t *args);
+void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn);
+
+#endif
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index dcdcac8a816..1b40c2e66ea 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -1,3 +1,8 @@
 obj-y = s390-virtio-bus.o s390-virtio.o
 
 obj-y := $(addprefix ../,$(obj-y))
+obj-y += s390-virtio-hcall.o
+obj-y += sclp.o
+obj-y += event-facility.o
+obj-y += sclpquiesce.o sclpconsole.o
+obj-y += ipl.o
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
new file mode 100644
index 00000000000..6b569951895
--- /dev/null
+++ b/hw/s390x/event-facility.c
@@ -0,0 +1,399 @@
+/*
+ * SCLP
+ *    Event Facility
+ *       handles SCLP event types
+ *          - Signal Quiesce - system power down
+ *          - ASCII Console Data - VT220 read and write
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+
+#include "sclp.h"
+#include "event-facility.h"
+
+typedef struct EventTypesBus {
+    BusState qbus;
+} EventTypesBus;
+
+struct SCLPEventFacility {
+    EventTypesBus sbus;
+    DeviceState *qdev;
+    /* guest' receive mask */
+    unsigned int receive_mask;
+};
+
+/* return true if any child has event pending set */
+static bool event_pending(SCLPEventFacility *ef)
+{
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *event_class;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = DO_UPCAST(SCLPEvent, qdev, qdev);
+        event_class = SCLP_EVENT_GET_CLASS(event);
+        if (event->event_pending &&
+            event_class->get_send_mask() & ef->receive_mask) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static unsigned int get_host_send_mask(SCLPEventFacility *ef)
+{
+    unsigned int mask;
+    BusChild *kid;
+    SCLPEventClass *child;
+
+    mask = 0;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        child = SCLP_EVENT_GET_CLASS((SCLPEvent *) qdev);
+        mask |= child->get_send_mask();
+    }
+    return mask;
+}
+
+static unsigned int get_host_receive_mask(SCLPEventFacility *ef)
+{
+    unsigned int mask;
+    BusChild *kid;
+    SCLPEventClass *child;
+
+    mask = 0;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        child = SCLP_EVENT_GET_CLASS((SCLPEvent *) qdev);
+        mask |= child->get_receive_mask();
+    }
+    return mask;
+}
+
+static uint16_t write_event_length_check(SCCB *sccb)
+{
+    int slen;
+    unsigned elen = 0;
+    EventBufferHeader *event;
+    WriteEventData *wed = (WriteEventData *) sccb;
+
+    event = (EventBufferHeader *) &wed->ebh;
+    for (slen = sccb_data_len(sccb); slen > 0; slen -= elen) {
+        elen = be16_to_cpu(event->length);
+        if (elen < sizeof(*event) || elen > slen) {
+            return SCLP_RC_EVENT_BUFFER_SYNTAX_ERROR;
+        }
+        event = (void *) event + elen;
+    }
+    if (slen) {
+        return SCLP_RC_INCONSISTENT_LENGTHS;
+    }
+    return SCLP_RC_NORMAL_COMPLETION;
+}
+
+static uint16_t handle_write_event_buf(SCLPEventFacility *ef,
+                                       EventBufferHeader *event_buf, SCCB *sccb)
+{
+    uint16_t rc;
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *ec;
+
+    rc = SCLP_RC_INVALID_FUNCTION;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = (SCLPEvent *) qdev;
+        ec = SCLP_EVENT_GET_CLASS(event);
+
+        if (ec->write_event_data &&
+            ec->event_type() == event_buf->type) {
+            rc = ec->write_event_data(event, event_buf);
+            break;
+        }
+    }
+    return rc;
+}
+
+static uint16_t handle_sccb_write_events(SCLPEventFacility *ef, SCCB *sccb)
+{
+    uint16_t rc;
+    int slen;
+    unsigned elen = 0;
+    EventBufferHeader *event_buf;
+    WriteEventData *wed = (WriteEventData *) sccb;
+
+    event_buf = &wed->ebh;
+    rc = SCLP_RC_NORMAL_COMPLETION;
+
+    /* loop over all contained event buffers */
+    for (slen = sccb_data_len(sccb); slen > 0; slen -= elen) {
+        elen = be16_to_cpu(event_buf->length);
+
+        /* in case of a previous error mark all trailing buffers
+         * as not accepted */
+        if (rc != SCLP_RC_NORMAL_COMPLETION) {
+            event_buf->flags &= ~(SCLP_EVENT_BUFFER_ACCEPTED);
+        } else {
+            rc = handle_write_event_buf(ef, event_buf, sccb);
+        }
+        event_buf = (void *) event_buf + elen;
+    }
+    return rc;
+}
+
+static void write_event_data(SCLPEventFacility *ef, SCCB *sccb)
+{
+    if (sccb->h.function_code != SCLP_FC_NORMAL_WRITE) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
+        goto out;
+    }
+    if (be16_to_cpu(sccb->h.length) < 8) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH);
+        goto out;
+    }
+    /* first do a sanity check of the write events */
+    sccb->h.response_code = cpu_to_be16(write_event_length_check(sccb));
+
+    /* if no early error, then execute */
+    if (sccb->h.response_code == be16_to_cpu(SCLP_RC_NORMAL_COMPLETION)) {
+        sccb->h.response_code =
+                cpu_to_be16(handle_sccb_write_events(ef, sccb));
+    }
+
+out:
+    return;
+}
+
+static uint16_t handle_sccb_read_events(SCLPEventFacility *ef, SCCB *sccb,
+                                        unsigned int mask)
+{
+    uint16_t rc;
+    int slen;
+    unsigned elen = 0;
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *ec;
+    EventBufferHeader *event_buf;
+    ReadEventData *red = (ReadEventData *) sccb;
+
+    event_buf = &red->ebh;
+    event_buf->length = 0;
+    slen = sizeof(sccb->data);
+
+    rc = SCLP_RC_NO_EVENT_BUFFERS_STORED;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = (SCLPEvent *) qdev;
+        ec = SCLP_EVENT_GET_CLASS(event);
+
+        if (mask & ec->get_send_mask()) {
+            if (ec->read_event_data(event, event_buf, &slen)) {
+                rc = SCLP_RC_NORMAL_COMPLETION;
+            }
+        }
+        elen = be16_to_cpu(event_buf->length);
+        event_buf = (void *) event_buf + elen;
+    }
+
+    if (sccb->h.control_mask[2] & SCLP_VARIABLE_LENGTH_RESPONSE) {
+        /* architecture suggests to reset variable-length-response bit */
+        sccb->h.control_mask[2] &= ~SCLP_VARIABLE_LENGTH_RESPONSE;
+        /* with a new length value */
+        sccb->h.length = cpu_to_be16(SCCB_SIZE - slen);
+    }
+    return rc;
+}
+
+static void read_event_data(SCLPEventFacility *ef, SCCB *sccb)
+{
+    unsigned int sclp_active_selection_mask;
+    unsigned int sclp_cp_receive_mask;
+
+    ReadEventData *red = (ReadEventData *) sccb;
+
+    if (be16_to_cpu(sccb->h.length) != SCCB_SIZE) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH);
+        goto out;
+    }
+
+    sclp_cp_receive_mask = ef->receive_mask;
+
+    /* get active selection mask */
+    switch (sccb->h.function_code) {
+    case SCLP_UNCONDITIONAL_READ:
+        sclp_active_selection_mask = sclp_cp_receive_mask;
+        break;
+    case SCLP_SELECTIVE_READ:
+        if (!(sclp_cp_receive_mask & be32_to_cpu(red->mask))) {
+            sccb->h.response_code =
+                    cpu_to_be16(SCLP_RC_INVALID_SELECTION_MASK);
+            goto out;
+        }
+        sclp_active_selection_mask = be32_to_cpu(red->mask);
+        break;
+    default:
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
+        goto out;
+    }
+    sccb->h.response_code = cpu_to_be16(
+            handle_sccb_read_events(ef, sccb, sclp_active_selection_mask));
+
+out:
+    return;
+}
+
+static void write_event_mask(SCLPEventFacility *ef, SCCB *sccb)
+{
+    WriteEventMask *we_mask = (WriteEventMask *) sccb;
+
+    /* Attention: We assume that Linux uses 4-byte masks, what it actually
+       does. Architecture allows for masks of variable size, though */
+    if (be16_to_cpu(we_mask->mask_length) != 4) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_MASK_LENGTH);
+        goto out;
+    }
+
+    /* keep track of the guest's capability masks */
+    ef->receive_mask = be32_to_cpu(we_mask->cp_receive_mask);
+
+    /* return the SCLP's capability masks to the guest */
+    we_mask->send_mask = cpu_to_be32(get_host_send_mask(ef));
+    we_mask->receive_mask = cpu_to_be32(get_host_receive_mask(ef));
+
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+
+out:
+    return;
+}
+
+/* qemu object creation and initialization functions */
+
+#define TYPE_SCLP_EVENTS_BUS "s390-sclp-events-bus"
+
+static void sclp_events_bus_class_init(ObjectClass *klass, void *data)
+{
+}
+
+static const TypeInfo s390_sclp_events_bus_info = {
+    .name = TYPE_SCLP_EVENTS_BUS,
+    .parent = TYPE_BUS,
+    .class_init = sclp_events_bus_class_init,
+};
+
+static void command_handler(SCLPEventFacility *ef, SCCB *sccb, uint64_t code)
+{
+    switch (code) {
+    case SCLP_CMD_READ_EVENT_DATA:
+        read_event_data(ef, sccb);
+        break;
+    case SCLP_CMD_WRITE_EVENT_DATA:
+        write_event_data(ef, sccb);
+        break;
+    case SCLP_CMD_WRITE_EVENT_MASK:
+        write_event_mask(ef, sccb);
+        break;
+    default:
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        break;
+    }
+}
+
+static int init_event_facility(S390SCLPDevice *sdev)
+{
+    SCLPEventFacility *event_facility;
+    DeviceState *quiesce;
+
+    event_facility = g_malloc0(sizeof(SCLPEventFacility));
+    sdev->ef = event_facility;
+    sdev->sclp_command_handler = command_handler;
+    sdev->event_pending = event_pending;
+
+    /* Spawn a new sclp-events facility */
+    qbus_create_inplace(&event_facility->sbus.qbus,
+                        TYPE_SCLP_EVENTS_BUS, (DeviceState *)sdev, NULL);
+    event_facility->sbus.qbus.allow_hotplug = 0;
+    event_facility->qdev = (DeviceState *) sdev;
+
+    quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
+    if (!quiesce) {
+        return -1;
+    }
+    qdev_init_nofail(quiesce);
+
+    return 0;
+}
+
+static void init_event_facility_class(ObjectClass *klass, void *data)
+{
+    S390SCLPDeviceClass *k = SCLP_S390_DEVICE_CLASS(klass);
+
+    k->init = init_event_facility;
+}
+
+static const TypeInfo s390_sclp_event_facility_info = {
+    .name          = "s390-sclp-event-facility",
+    .parent        = TYPE_DEVICE_S390_SCLP,
+    .instance_size = sizeof(S390SCLPDevice),
+    .class_init    = init_event_facility_class,
+};
+
+static int event_qdev_init(DeviceState *qdev)
+{
+    SCLPEvent *event = DO_UPCAST(SCLPEvent, qdev, qdev);
+    SCLPEventClass *child = SCLP_EVENT_GET_CLASS(event);
+
+    return child->init(event);
+}
+
+static int event_qdev_exit(DeviceState *qdev)
+{
+    SCLPEvent *event = DO_UPCAST(SCLPEvent, qdev, qdev);
+    SCLPEventClass *child = SCLP_EVENT_GET_CLASS(event);
+    if (child->exit) {
+        child->exit(event);
+    }
+    return 0;
+}
+
+static void event_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->bus_type = TYPE_SCLP_EVENTS_BUS;
+    dc->unplug = qdev_simple_unplug_cb;
+    dc->init = event_qdev_init;
+    dc->exit = event_qdev_exit;
+}
+
+static const TypeInfo s390_sclp_event_type_info = {
+    .name = TYPE_SCLP_EVENT,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(SCLPEvent),
+    .class_init = event_class_init,
+    .class_size = sizeof(SCLPEventClass),
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&s390_sclp_events_bus_info);
+    type_register_static(&s390_sclp_event_facility_info);
+    type_register_static(&s390_sclp_event_type_info);
+}
+
+type_init(register_types)
diff --git a/hw/s390x/event-facility.h b/hw/s390x/event-facility.h
new file mode 100644
index 00000000000..791ab2a6dec
--- /dev/null
+++ b/hw/s390x/event-facility.h
@@ -0,0 +1,96 @@
+/*
+ * SCLP
+ *    Event Facility definitions
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_S390_SCLP_EVENT_FACILITY_H
+#define HW_S390_SCLP_EVENT_FACILITY_H
+
+#include <hw/qdev.h>
+#include "qemu/thread.h"
+
+/* SCLP event types */
+#define SCLP_EVENT_ASCII_CONSOLE_DATA           0x1a
+#define SCLP_EVENT_SIGNAL_QUIESCE               0x1d
+
+/* SCLP event masks */
+#define SCLP_EVENT_MASK_SIGNAL_QUIESCE          0x00000008
+#define SCLP_EVENT_MASK_MSG_ASCII               0x00000040
+
+#define SCLP_UNCONDITIONAL_READ                 0x00
+#define SCLP_SELECTIVE_READ                     0x01
+
+#define TYPE_SCLP_EVENT "s390-sclp-event-type"
+#define SCLP_EVENT(obj) \
+     OBJECT_CHECK(SCLPEvent, (obj), TYPE_SCLP_EVENT)
+#define SCLP_EVENT_CLASS(klass) \
+     OBJECT_CLASS_CHECK(SCLPEventClass, (klass), TYPE_SCLP_EVENT)
+#define SCLP_EVENT_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(SCLPEventClass, (obj), TYPE_SCLP_EVENT)
+
+typedef struct WriteEventMask {
+    SCCBHeader h;
+    uint16_t _reserved;
+    uint16_t mask_length;
+    uint32_t cp_receive_mask;
+    uint32_t cp_send_mask;
+    uint32_t send_mask;
+    uint32_t receive_mask;
+} QEMU_PACKED WriteEventMask;
+
+typedef struct EventBufferHeader {
+    uint16_t length;
+    uint8_t  type;
+    uint8_t  flags;
+    uint16_t _reserved;
+} QEMU_PACKED EventBufferHeader;
+
+typedef struct WriteEventData {
+    SCCBHeader h;
+    EventBufferHeader ebh;
+} QEMU_PACKED WriteEventData;
+
+typedef struct ReadEventData {
+    SCCBHeader h;
+    EventBufferHeader ebh;
+    uint32_t mask;
+} QEMU_PACKED ReadEventData;
+
+typedef struct SCLPEvent {
+    DeviceState qdev;
+    bool event_pending;
+    uint32_t event_type;
+    char *name;
+} SCLPEvent;
+
+typedef struct SCLPEventClass {
+    DeviceClass parent_class;
+    int (*init)(SCLPEvent *event);
+    int (*exit)(SCLPEvent *event);
+
+    /* get SCLP's send mask */
+    unsigned int (*get_send_mask)(void);
+
+    /* get SCLP's receive mask */
+    unsigned int (*get_receive_mask)(void);
+
+    int (*read_event_data)(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen);
+
+    int (*write_event_data)(SCLPEvent *event, EventBufferHeader *evt_buf_hdr);
+
+    /* returns the supported event type */
+    int (*event_type)(void);
+
+} SCLPEventClass;
+
+#endif
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
new file mode 100644
index 00000000000..7cbbf99fde2
--- /dev/null
+++ b/hw/s390x/ipl.c
@@ -0,0 +1,174 @@
+/*
+ * bootloader support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "sysemu/sysemu.h"
+#include "cpu.h"
+#include "elf.h"
+#include "hw/loader.h"
+#include "hw/sysbus.h"
+
+#define KERN_IMAGE_START                0x010000UL
+#define KERN_PARM_AREA                  0x010480UL
+#define INITRD_START                    0x800000UL
+#define INITRD_PARM_START               0x010408UL
+#define INITRD_PARM_SIZE                0x010410UL
+#define PARMFILE_START                  0x001000UL
+#define ZIPL_FILENAME                   "s390-zipl.rom"
+#define ZIPL_IMAGE_START                0x009000UL
+#define IPL_PSW_MASK                    (PSW_MASK_32 | PSW_MASK_64)
+
+#define TYPE_S390_IPL "s390-ipl"
+#define S390_IPL(obj) \
+    OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
+#if 0
+#define S390_IPL_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390IPLState, (klass), TYPE_S390_IPL)
+#define S390_IPL_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390IPLState, (obj), TYPE_S390_IPL)
+#endif
+
+typedef struct S390IPLClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    void (*parent_reset) (SysBusDevice *dev);
+} S390IPLClass;
+
+typedef struct S390IPLState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    char *kernel;
+    char *initrd;
+    char *cmdline;
+} S390IPLState;
+
+
+static void s390_ipl_cpu(uint64_t pswaddr)
+{
+    CPUS390XState *env = &S390_CPU(qemu_get_cpu(0))->env;
+    env->psw.addr = pswaddr;
+    env->psw.mask = IPL_PSW_MASK;
+    s390_add_running_cpu(env);
+}
+
+static int s390_ipl_init(SysBusDevice *dev)
+{
+    S390IPLState *ipl = S390_IPL(dev);
+    ram_addr_t kernel_size = 0;
+
+    if (!ipl->kernel) {
+        ram_addr_t bios_size = 0;
+        char *bios_filename;
+
+        /* Load zipl bootloader */
+        if (bios_name == NULL) {
+            bios_name = ZIPL_FILENAME;
+        }
+
+        bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+        bios_size = load_image_targphys(bios_filename, ZIPL_IMAGE_START, 4096);
+        g_free(bios_filename);
+
+        if ((long)bios_size < 0) {
+            hw_error("could not load bootloader '%s'\n", bios_name);
+        }
+
+        if (bios_size > 4096) {
+            hw_error("stage1 bootloader is > 4k\n");
+        }
+        return 0;
+    } else {
+        kernel_size = load_elf(ipl->kernel, NULL, NULL, NULL, NULL,
+                               NULL, 1, ELF_MACHINE, 0);
+        if (kernel_size == -1UL) {
+            kernel_size = load_image_targphys(ipl->kernel, 0, ram_size);
+        }
+        if (kernel_size == -1UL) {
+            fprintf(stderr, "could not load kernel '%s'\n", ipl->kernel);
+            return -1;
+        }
+        /* we have to overwrite values in the kernel image, which are "rom" */
+        strcpy(rom_ptr(KERN_PARM_AREA), ipl->cmdline);
+    }
+    if (ipl->initrd) {
+        ram_addr_t initrd_offset, initrd_size;
+
+        initrd_offset = INITRD_START;
+        while (kernel_size + 0x100000 > initrd_offset) {
+            initrd_offset += 0x100000;
+        }
+        initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
+                                          ram_size - initrd_offset);
+        if (initrd_size == -1UL) {
+            fprintf(stderr, "qemu: could not load initrd '%s'\n", ipl->initrd);
+            exit(1);
+        }
+
+        /* we have to overwrite values in the kernel image, which are "rom" */
+        stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
+        stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
+    }
+
+    return 0;
+}
+
+static Property s390_ipl_properties[] = {
+    DEFINE_PROP_STRING("kernel", S390IPLState, kernel),
+    DEFINE_PROP_STRING("initrd", S390IPLState, initrd),
+    DEFINE_PROP_STRING("cmdline", S390IPLState, cmdline),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_ipl_reset(DeviceState *dev)
+{
+    S390IPLState *ipl = S390_IPL(dev);
+
+    if (ipl->kernel) {
+        /*
+         * we can not rely on the ELF entry point, since up to 3.2 this
+         * value was 0x800 (the SALIPL loader) and it wont work. For
+         * all (Linux) cases 0x10000 (KERN_IMAGE_START) should be fine.
+         */
+        return s390_ipl_cpu(KERN_IMAGE_START);
+    } else {
+        return s390_ipl_cpu(ZIPL_IMAGE_START);
+    }
+}
+
+static void s390_ipl_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = s390_ipl_init;
+    dc->props = s390_ipl_properties;
+    dc->reset = s390_ipl_reset;
+    dc->no_user = 1;
+}
+
+static TypeInfo s390_ipl_info = {
+    .class_init = s390_ipl_class_init,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .name  = "s390-ipl",
+    .instance_size  = sizeof(S390IPLState),
+};
+
+static void s390_ipl_register_types(void)
+{
+    type_register_static(&s390_ipl_info);
+}
+
+type_init(s390_ipl_register_types)
diff --git a/hw/s390x/s390-virtio-hcall.c b/hw/s390x/s390-virtio-hcall.c
new file mode 100644
index 00000000000..d7938c07347
--- /dev/null
+++ b/hw/s390x/s390-virtio-hcall.c
@@ -0,0 +1,36 @@
+/*
+ * Support for virtio hypercalls on s390
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "cpu.h"
+#include "hw/s390-virtio.h"
+
+#define MAX_DIAG_SUBCODES 255
+
+static s390_virtio_fn s390_diag500_table[MAX_DIAG_SUBCODES];
+
+void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn)
+{
+    assert(code < MAX_DIAG_SUBCODES);
+    assert(!s390_diag500_table[code]);
+
+    s390_diag500_table[code] = fn;
+}
+
+int s390_virtio_hypercall(CPUS390XState *env)
+{
+    s390_virtio_fn fn = s390_diag500_table[env->regs[1]];
+
+    if (!fn) {
+        return -EINVAL;
+    }
+
+    return fn(&env->regs[2]);
+}
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
new file mode 100644
index 00000000000..a9d3a6a91d2
--- /dev/null
+++ b/hw/s390x/sclp.c
@@ -0,0 +1,163 @@
+/*
+ * SCLP Support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *  Heinz Graalfs <graalfs@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "cpu.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
+
+#include "sclp.h"
+
+static inline S390SCLPDevice *get_event_facility(void)
+{
+    ObjectProperty *op = object_property_find(qdev_get_machine(),
+                                              "s390-sclp-event-facility",
+                                              NULL);
+    assert(op);
+    return op->opaque;
+}
+
+/* Provide information about the configuration, CPUs and storage */
+static void read_SCP_info(SCCB *sccb)
+{
+    ReadInfo *read_info = (ReadInfo *) sccb;
+    int shift = 0;
+
+    while ((ram_size >> (20 + shift)) > 65535) {
+        shift++;
+    }
+    read_info->rnmax = cpu_to_be16(ram_size >> (20 + shift));
+    read_info->rnsize = 1 << shift;
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
+}
+
+static void sclp_execute(SCCB *sccb, uint64_t code)
+{
+    S390SCLPDevice *sdev = get_event_facility();
+
+    switch (code) {
+    case SCLP_CMDW_READ_SCP_INFO:
+    case SCLP_CMDW_READ_SCP_INFO_FORCED:
+        read_SCP_info(sccb);
+        break;
+    default:
+        sdev->sclp_command_handler(sdev->ef, sccb, code);
+        break;
+    }
+}
+
+int sclp_service_call(uint32_t sccb, uint64_t code)
+{
+    int r = 0;
+    SCCB work_sccb;
+
+    hwaddr sccb_len = sizeof(SCCB);
+
+    /* first some basic checks on program checks */
+    if (cpu_physical_memory_is_io(sccb)) {
+        r = -PGM_ADDRESSING;
+        goto out;
+    }
+    if (sccb & ~0x7ffffff8ul) {
+        r = -PGM_SPECIFICATION;
+        goto out;
+    }
+
+    /*
+     * we want to work on a private copy of the sccb, to prevent guests
+     * from playing dirty tricks by modifying the memory content after
+     * the host has checked the values
+     */
+    cpu_physical_memory_read(sccb, &work_sccb, sccb_len);
+
+    /* Valid sccb sizes */
+    if (be16_to_cpu(work_sccb.h.length) < sizeof(SCCBHeader) ||
+        be16_to_cpu(work_sccb.h.length) > SCCB_SIZE) {
+        r = -PGM_SPECIFICATION;
+        goto out;
+    }
+
+    sclp_execute((SCCB *)&work_sccb, code);
+
+    cpu_physical_memory_write(sccb, &work_sccb,
+                              be16_to_cpu(work_sccb.h.length));
+
+    sclp_service_interrupt(sccb);
+
+out:
+    return r;
+}
+
+void sclp_service_interrupt(uint32_t sccb)
+{
+    S390SCLPDevice *sdev = get_event_facility();
+    uint32_t param = sccb & ~3;
+
+    /* Indicate whether an event is still pending */
+    param |= sdev->event_pending(sdev->ef) ? 1 : 0;
+
+    if (!param) {
+        /* No need to send an interrupt, there's nothing to be notified about */
+        return;
+    }
+    s390_sclp_extint(param);
+}
+
+/* qemu object creation and initialization functions */
+
+void s390_sclp_init(void)
+{
+    DeviceState *dev  = qdev_create(NULL, "s390-sclp-event-facility");
+
+    object_property_add_child(qdev_get_machine(), "s390-sclp-event-facility",
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+}
+
+static int s390_sclp_dev_init(SysBusDevice *dev)
+{
+    int r;
+    S390SCLPDevice *sdev = (S390SCLPDevice *)dev;
+    S390SCLPDeviceClass *sclp = SCLP_S390_DEVICE_GET_CLASS(dev);
+
+    r = sclp->init(sdev);
+    if (!r) {
+        assert(sdev->event_pending);
+        assert(sdev->sclp_command_handler);
+    }
+
+    return r;
+}
+
+static void s390_sclp_device_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *dc = SYS_BUS_DEVICE_CLASS(klass);
+
+    dc->init = s390_sclp_dev_init;
+}
+
+static const TypeInfo s390_sclp_device_info = {
+    .name = TYPE_DEVICE_S390_SCLP,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(S390SCLPDevice),
+    .class_init = s390_sclp_device_class_init,
+    .class_size = sizeof(S390SCLPDeviceClass),
+    .abstract = true,
+};
+
+static void s390_sclp_register_types(void)
+{
+    type_register_static(&s390_sclp_device_info);
+}
+
+type_init(s390_sclp_register_types)
diff --git a/hw/s390x/sclp.h b/hw/s390x/sclp.h
new file mode 100644
index 00000000000..231a38aa096
--- /dev/null
+++ b/hw/s390x/sclp.h
@@ -0,0 +1,118 @@
+/*
+ * SCLP Support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_S390_SCLP_H
+#define HW_S390_SCLP_H
+
+#include <hw/sysbus.h>
+#include <hw/qdev.h>
+
+/* SCLP command codes */
+#define SCLP_CMDW_READ_SCP_INFO                 0x00020001
+#define SCLP_CMDW_READ_SCP_INFO_FORCED          0x00120001
+#define SCLP_CMD_READ_EVENT_DATA                0x00770005
+#define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
+#define SCLP_CMD_READ_EVENT_DATA                0x00770005
+#define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
+#define SCLP_CMD_WRITE_EVENT_MASK               0x00780005
+
+/* SCLP response codes */
+#define SCLP_RC_NORMAL_READ_COMPLETION          0x0010
+#define SCLP_RC_NORMAL_COMPLETION               0x0020
+#define SCLP_RC_INVALID_SCLP_COMMAND            0x01f0
+#define SCLP_RC_CONTAINED_EQUIPMENT_CHECK       0x0340
+#define SCLP_RC_INSUFFICIENT_SCCB_LENGTH        0x0300
+#define SCLP_RC_INVALID_FUNCTION                0x40f0
+#define SCLP_RC_NO_EVENT_BUFFERS_STORED         0x60f0
+#define SCLP_RC_INVALID_SELECTION_MASK          0x70f0
+#define SCLP_RC_INCONSISTENT_LENGTHS            0x72f0
+#define SCLP_RC_EVENT_BUFFER_SYNTAX_ERROR       0x73f0
+#define SCLP_RC_INVALID_MASK_LENGTH             0x74f0
+
+
+/* Service Call Control Block (SCCB) and its elements */
+
+#define SCCB_SIZE 4096
+
+#define SCLP_VARIABLE_LENGTH_RESPONSE           0x80
+#define SCLP_EVENT_BUFFER_ACCEPTED              0x80
+
+#define SCLP_FC_NORMAL_WRITE                    0
+
+/*
+ * Normally packed structures are not the right thing to do, since all code
+ * must take care of endianness. We cannot use ldl_phys and friends for two
+ * reasons, though:
+ * - some of the embedded structures below the SCCB can appear multiple times
+ *   at different locations, so there is no fixed offset
+ * - we work on a private copy of the SCCB, since there are several length
+ *   fields, that would cause a security nightmare if we allow the guest to
+ *   alter the structure while we parse it. We cannot use ldl_p and friends
+ *   either without doing pointer arithmetics
+ * So we have to double check that all users of sclp data structures use the
+ * right endianness wrappers.
+ */
+typedef struct SCCBHeader {
+    uint16_t length;
+    uint8_t function_code;
+    uint8_t control_mask[3];
+    uint16_t response_code;
+} QEMU_PACKED SCCBHeader;
+
+#define SCCB_DATA_LEN (SCCB_SIZE - sizeof(SCCBHeader))
+
+typedef struct ReadInfo {
+    SCCBHeader h;
+    uint16_t rnmax;
+    uint8_t rnsize;
+} QEMU_PACKED ReadInfo;
+
+typedef struct SCCB {
+    SCCBHeader h;
+    char data[SCCB_DATA_LEN];
+ } QEMU_PACKED SCCB;
+
+static inline int sccb_data_len(SCCB *sccb)
+{
+    return be16_to_cpu(sccb->h.length) - sizeof(sccb->h);
+}
+
+#define TYPE_DEVICE_S390_SCLP "s390-sclp-device"
+#define SCLP_S390_DEVICE(obj) \
+     OBJECT_CHECK(S390SCLPDevice, (obj), TYPE_DEVICE_S390_SCLP)
+#define SCLP_S390_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(S390SCLPDeviceClass, (klass), \
+             TYPE_DEVICE_S390_SCLP)
+#define SCLP_S390_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(S390SCLPDeviceClass, (obj), \
+             TYPE_DEVICE_S390_SCLP)
+
+typedef struct SCLPEventFacility SCLPEventFacility;
+
+typedef struct S390SCLPDevice {
+    SysBusDevice busdev;
+    SCLPEventFacility *ef;
+    void (*sclp_command_handler)(SCLPEventFacility *ef, SCCB *sccb,
+                                 uint64_t code);
+    bool (*event_pending)(SCLPEventFacility *ef);
+} S390SCLPDevice;
+
+typedef struct S390SCLPDeviceClass {
+    DeviceClass qdev;
+    int (*init)(S390SCLPDevice *sdev);
+} S390SCLPDeviceClass;
+
+void s390_sclp_init(void);
+void sclp_service_interrupt(uint32_t sccb);
+
+#endif
diff --git a/hw/s390x/sclpconsole.c b/hw/s390x/sclpconsole.c
new file mode 100644
index 00000000000..adc0ee83f49
--- /dev/null
+++ b/hw/s390x/sclpconsole.c
@@ -0,0 +1,307 @@
+/*
+ * SCLP event type
+ *    Ascii Console Data (VT220 Console)
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <hw/qdev.h>
+#include "qemu/thread.h"
+
+#include "sclp.h"
+#include "event-facility.h"
+#include "char/char.h"
+
+typedef struct ASCIIConsoleData {
+    EventBufferHeader ebh;
+    char data[0];
+} QEMU_PACKED ASCIIConsoleData;
+
+/* max size for ASCII data in 4K SCCB page */
+#define SIZE_BUFFER_VT220 4080
+
+typedef struct SCLPConsole {
+    SCLPEvent event;
+    CharDriverState *chr;
+    /* io vector                                                       */
+    uint8_t *iov;           /* iov buffer pointer                      */
+    uint8_t *iov_sclp;      /* pointer to SCLP read offset             */
+    uint8_t *iov_bs;        /* pointer byte stream read offset         */
+    uint32_t iov_data_len;  /* length of byte stream in buffer         */
+    uint32_t iov_sclp_rest; /* length of byte stream not read via SCLP */
+    qemu_irq irq_read_vt220;
+} SCLPConsole;
+
+/* character layer call-back functions */
+
+/* Return number of bytes that fit into iov buffer */
+static int chr_can_read(void *opaque)
+{
+    int can_read;
+    SCLPConsole *scon = opaque;
+
+    can_read = SIZE_BUFFER_VT220 - scon->iov_data_len;
+
+    return can_read;
+}
+
+/* Receive n bytes from character layer, save in iov buffer,
+ * and set event pending */
+static void receive_from_chr_layer(SCLPConsole *scon, const uint8_t *buf,
+                                   int size)
+{
+    assert(scon->iov);
+
+    /* read data must fit into current buffer */
+    assert(size <= SIZE_BUFFER_VT220 - scon->iov_data_len);
+
+    /* put byte-stream from character layer into buffer */
+    memcpy(scon->iov_bs, buf, size);
+    scon->iov_data_len += size;
+    scon->iov_sclp_rest += size;
+    scon->iov_bs += size;
+    scon->event.event_pending = true;
+}
+
+/* Send data from a char device over to the guest */
+static void chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    SCLPConsole *scon = opaque;
+
+    assert(scon);
+
+    receive_from_chr_layer(scon, buf, size);
+    /* trigger SCLP read operation */
+    qemu_irq_raise(scon->irq_read_vt220);
+}
+
+static void chr_event(void *opaque, int event)
+{
+    SCLPConsole *scon = opaque;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (!scon->iov) {
+            scon->iov = g_malloc0(SIZE_BUFFER_VT220);
+            scon->iov_sclp = scon->iov;
+            scon->iov_bs = scon->iov;
+            scon->iov_data_len = 0;
+            scon->iov_sclp_rest = 0;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        if (scon->iov) {
+            g_free(scon->iov);
+            scon->iov = NULL;
+        }
+        break;
+    }
+}
+
+/* functions to be called by event facility */
+
+static int event_type(void)
+{
+    return SCLP_EVENT_ASCII_CONSOLE_DATA;
+}
+
+static unsigned int send_mask(void)
+{
+    return SCLP_EVENT_MASK_MSG_ASCII;
+}
+
+static unsigned int receive_mask(void)
+{
+    return SCLP_EVENT_MASK_MSG_ASCII;
+}
+
+/* triggered by SCLP's read_event_data -
+ * copy console data byte-stream into provided (SCLP) buffer
+ */
+static void get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
+                             int avail)
+{
+    SCLPConsole *cons = DO_UPCAST(SCLPConsole, event, event);
+
+    /* first byte is hex 0 saying an ascii string follows */
+    *buf++ = '\0';
+    avail--;
+    /* if all data fit into provided SCLP buffer */
+    if (avail >= cons->iov_sclp_rest) {
+        /* copy character byte-stream to SCLP buffer */
+        memcpy(buf, cons->iov_sclp, cons->iov_sclp_rest);
+        *size = cons->iov_sclp_rest + 1;
+        cons->iov_sclp = cons->iov;
+        cons->iov_bs = cons->iov;
+        cons->iov_data_len = 0;
+        cons->iov_sclp_rest = 0;
+        event->event_pending = false;
+        /* data provided and no more data pending */
+    } else {
+        /* if provided buffer is too small, just copy part */
+        memcpy(buf, cons->iov_sclp, avail);
+        *size = avail + 1;
+        cons->iov_sclp_rest -= avail;
+        cons->iov_sclp += avail;
+        /* more data pending */
+    }
+}
+
+static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen)
+{
+    int avail;
+    size_t src_len;
+    uint8_t *to;
+    ASCIIConsoleData *acd = (ASCIIConsoleData *) evt_buf_hdr;
+
+    if (!event->event_pending) {
+        /* no data pending */
+        return 0;
+    }
+
+    to = (uint8_t *)&acd->data;
+    avail = *slen - sizeof(ASCIIConsoleData);
+    get_console_data(event, to, &src_len, avail);
+
+    acd->ebh.length = cpu_to_be16(sizeof(ASCIIConsoleData) + src_len);
+    acd->ebh.type = SCLP_EVENT_ASCII_CONSOLE_DATA;
+    acd->ebh.flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+    *slen = avail - src_len;
+
+    return 1;
+}
+
+/* triggered by SCLP's write_event_data
+ *  - write console data to character layer
+ *  returns < 0 if an error occurred
+ */
+static ssize_t write_console_data(SCLPEvent *event, const uint8_t *buf,
+                                  size_t len)
+{
+    ssize_t ret = 0;
+    const uint8_t *iov_offset;
+    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+
+    if (!scon->chr) {
+        /* If there's no backend, we can just say we consumed all data. */
+        return len;
+    }
+
+    iov_offset = buf;
+    while (len > 0) {
+        ret = qemu_chr_fe_write(scon->chr, buf, len);
+        if (ret == 0) {
+            /* a pty doesn't seem to be connected - no error */
+            len = 0;
+        } else if (ret == -EAGAIN || (ret > 0 && ret < len)) {
+            len -= ret;
+            iov_offset += ret;
+        } else {
+            len = 0;
+        }
+    }
+
+    return ret;
+}
+
+static int write_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr)
+{
+    int rc;
+    int length;
+    ssize_t written;
+    ASCIIConsoleData *acd = (ASCIIConsoleData *) evt_buf_hdr;
+
+    length = be16_to_cpu(evt_buf_hdr->length) - sizeof(EventBufferHeader);
+    written = write_console_data(event, (uint8_t *)acd->data, length);
+
+    rc = SCLP_RC_NORMAL_COMPLETION;
+    /* set event buffer accepted flag */
+    evt_buf_hdr->flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+
+    /* written will be zero if a pty is not connected - don't treat as error */
+    if (written < 0) {
+        /* event buffer not accepted due to error in character layer */
+        evt_buf_hdr->flags &= ~(SCLP_EVENT_BUFFER_ACCEPTED);
+        rc = SCLP_RC_CONTAINED_EQUIPMENT_CHECK;
+    }
+
+    return rc;
+}
+
+static void trigger_ascii_console_data(void *env, int n, int level)
+{
+    sclp_service_interrupt(0);
+}
+
+/* qemu object creation and initialization functions */
+
+/* tell character layer our call-back functions */
+static int console_init(SCLPEvent *event)
+{
+    static bool console_available;
+
+    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+
+    if (console_available) {
+        error_report("Multiple VT220 operator consoles are not supported");
+        return -1;
+    }
+    console_available = true;
+    event->event_type = SCLP_EVENT_ASCII_CONSOLE_DATA;
+    if (scon->chr) {
+        qemu_chr_add_handlers(scon->chr, chr_can_read,
+                              chr_read, chr_event, scon);
+    }
+    scon->irq_read_vt220 = *qemu_allocate_irqs(trigger_ascii_console_data,
+                                               NULL, 1);
+
+    return 0;
+}
+
+static int console_exit(SCLPEvent *event)
+{
+    return 0;
+}
+
+static Property console_properties[] = {
+    DEFINE_PROP_CHR("chardev", SCLPConsole, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void console_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SCLPEventClass *ec = SCLP_EVENT_CLASS(klass);
+
+    dc->props = console_properties;
+    ec->init = console_init;
+    ec->exit = console_exit;
+    ec->get_send_mask = send_mask;
+    ec->get_receive_mask = receive_mask;
+    ec->event_type = event_type;
+    ec->read_event_data = read_event_data;
+    ec->write_event_data = write_event_data;
+}
+
+static const TypeInfo sclp_console_info = {
+    .name          = "sclpconsole",
+    .parent        = TYPE_SCLP_EVENT,
+    .instance_size = sizeof(SCLPConsole),
+    .class_init    = console_class_init,
+    .class_size    = sizeof(SCLPEventClass),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_console_info);
+}
+
+type_init(register_types)
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
new file mode 100644
index 00000000000..2538498959e
--- /dev/null
+++ b/hw/s390x/sclpquiesce.c
@@ -0,0 +1,123 @@
+/*
+ * SCLP event type
+ *    Signal Quiesce - trigger system powerdown request
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+#include <hw/qdev.h>
+#include "sysemu/sysemu.h"
+#include "sclp.h"
+#include "event-facility.h"
+
+typedef struct SignalQuiesce {
+    EventBufferHeader ebh;
+    uint16_t timeout;
+    uint8_t unit;
+} QEMU_PACKED SignalQuiesce;
+
+static int event_type(void)
+{
+    return SCLP_EVENT_SIGNAL_QUIESCE;
+}
+
+static unsigned int send_mask(void)
+{
+    return SCLP_EVENT_MASK_SIGNAL_QUIESCE;
+}
+
+static unsigned int receive_mask(void)
+{
+    return 0;
+}
+
+static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen)
+{
+    SignalQuiesce *sq = (SignalQuiesce *) evt_buf_hdr;
+
+    if (*slen < sizeof(SignalQuiesce)) {
+        return 0;
+    }
+
+    if (!event->event_pending) {
+        return 0;
+    }
+    event->event_pending = false;
+
+    sq->ebh.length = cpu_to_be16(sizeof(SignalQuiesce));
+    sq->ebh.type = SCLP_EVENT_SIGNAL_QUIESCE;
+    sq->ebh.flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+    /*
+     * system_powerdown does not have a timeout. Fortunately the
+     * timeout value is currently ignored by Linux, anyway
+     */
+    sq->timeout = cpu_to_be16(0);
+    sq->unit = cpu_to_be16(0);
+    *slen -= sizeof(SignalQuiesce);
+
+    return 1;
+}
+
+typedef struct QuiesceNotifier QuiesceNotifier;
+
+static struct QuiesceNotifier {
+    Notifier notifier;
+    SCLPEvent *event;
+} qn;
+
+static void quiesce_powerdown_req(Notifier *n, void *opaque)
+{
+    QuiesceNotifier *qn = container_of(n, QuiesceNotifier, notifier);
+    SCLPEvent *event = qn->event;
+
+    event->event_pending = true;
+    /* trigger SCLP read operation */
+    sclp_service_interrupt(0);
+}
+
+static int quiesce_init(SCLPEvent *event)
+{
+    event->event_type = SCLP_EVENT_SIGNAL_QUIESCE;
+
+    qn.notifier.notify = quiesce_powerdown_req;
+    qn.event = event;
+
+    qemu_register_powerdown_notifier(&qn.notifier);
+
+    return 0;
+}
+
+static void quiesce_class_init(ObjectClass *klass, void *data)
+{
+    SCLPEventClass *k = SCLP_EVENT_CLASS(klass);
+
+    k->init = quiesce_init;
+
+    k->get_send_mask = send_mask;
+    k->get_receive_mask = receive_mask;
+    k->event_type = event_type;
+    k->read_event_data = read_event_data;
+    k->write_event_data = NULL;
+}
+
+static const TypeInfo sclp_quiesce_info = {
+    .name          = "sclpquiesce",
+    .parent        = TYPE_SCLP_EVENT,
+    .instance_size = sizeof(SCLPEvent),
+    .class_init    = quiesce_class_init,
+    .class_size    = sizeof(SCLPEventClass),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_quiesce_info);
+}
+
+type_init(register_types)
diff --git a/hw/sb16.c b/hw/sb16.c
index c81455d7f32..52dfedf5f1b 100644
--- a/hw/sb16.c
+++ b/hw/sb16.c
@@ -26,8 +26,8 @@
 #include "audio/audio.h"
 #include "isa.h"
 #include "qdev.h"
-#include "qemu-timer.h"
-#include "host-utils.h"
+#include "qemu/timer.h"
+#include "qemu/host-utils.h"
 
 #define dolog(...) AUD_log ("sb16", __VA_ARGS__)
 
@@ -822,7 +822,6 @@ static void complete (SB16State *s)
 
     ldebug ("\n");
     s->cmd = -1;
-    return;
 }
 
 static void legacy_reset (SB16State *s)
@@ -1410,7 +1409,7 @@ static void sb16_class_initfn (ObjectClass *klass, void *data)
     dc->props = sb16_properties;
 }
 
-static TypeInfo sb16_info = {
+static const TypeInfo sb16_info = {
     .name          = "sb16",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (SB16State),
diff --git a/hw/sbi.c b/hw/sbi.c
index 52982a9a3e1..d58184a6aaa 100644
--- a/hw/sbi.c
+++ b/hw/sbi.c
@@ -52,7 +52,7 @@ static void sbi_set_irq(void *opaque, int irq, int level)
 {
 }
 
-static uint64_t sbi_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sbi_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     SBIState *s = opaque;
@@ -69,7 +69,7 @@ static uint64_t sbi_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sbi_mem_write(void *opaque, target_phys_addr_t addr,
+static void sbi_mem_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned dize)
 {
     SBIState *s = opaque;
@@ -141,7 +141,7 @@ static void sbi_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_sbi;
 }
 
-static TypeInfo sbi_info = {
+static const TypeInfo sbi_info = {
     .name          = "sbi",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SBIState),
diff --git a/hw/scsi-bus.c b/hw/scsi-bus.c
index 4981a024365..a97f1cdc1c7 100644
--- a/hw/scsi-bus.c
+++ b/hw/scsi-bus.c
@@ -1,11 +1,11 @@
 #include "hw.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
 #include "scsi-defs.h"
 #include "qdev.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "trace.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
@@ -282,7 +282,7 @@ static const struct SCSIReqOps reqops_invalid_opcode = {
 
 static int32_t scsi_unit_attention(SCSIRequest *req, uint8_t *buf)
 {
-    if (req->dev && req->dev->unit_attention.key == UNIT_ATTENTION) {
+    if (req->dev->unit_attention.key == UNIT_ATTENTION) {
         scsi_req_build_sense(req, req->dev->unit_attention);
     } else if (req->bus->unit_attention.key == UNIT_ATTENTION) {
         scsi_req_build_sense(req, req->bus->unit_attention);
@@ -801,26 +801,39 @@ static int ata_passthrough_16_xfer_size(SCSIDevice *dev, uint8_t *buf)
     return xfer * unit;
 }
 
-static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+uint32_t scsi_data_cdb_length(uint8_t *buf)
+{
+    if ((buf[0] >> 5) == 0 && buf[4] == 0) {
+        return 256;
+    } else {
+        return scsi_cdb_length(buf);
+    }
+}
+
+uint32_t scsi_cdb_length(uint8_t *buf)
 {
     switch (buf[0] >> 5) {
     case 0:
-        cmd->xfer = buf[4];
+        return buf[4];
         break;
     case 1:
     case 2:
-        cmd->xfer = lduw_be_p(&buf[7]);
+        return lduw_be_p(&buf[7]);
         break;
     case 4:
-        cmd->xfer = ldl_be_p(&buf[10]) & 0xffffffffULL;
+        return ldl_be_p(&buf[10]) & 0xffffffffULL;
         break;
     case 5:
-        cmd->xfer = ldl_be_p(&buf[6]) & 0xffffffffULL;
+        return ldl_be_p(&buf[6]) & 0xffffffffULL;
         break;
     default:
         return -1;
     }
+}
 
+static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+{
+    cmd->xfer = scsi_cdb_length(buf);
     switch (buf[0]) {
     case TEST_UNIT_READY:
     case REWIND:
@@ -1710,12 +1723,8 @@ static char *scsibus_get_dev_path(DeviceState *dev)
 static char *scsibus_get_fw_dev_path(DeviceState *dev)
 {
     SCSIDevice *d = SCSI_DEVICE(dev);
-    char path[100];
-
-    snprintf(path, sizeof(path), "channel@%x/%s@%x,%x", d->channel,
-             qdev_fw_name(dev), d->id, d->lun);
-
-    return strdup(path);
+    return g_strdup_printf("channel@%x/%s@%x,%x", d->channel,
+                           qdev_fw_name(dev), d->id, d->lun);
 }
 
 SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int id, int lun)
@@ -1854,7 +1863,7 @@ static void scsi_device_class_init(ObjectClass *klass, void *data)
     k->props    = scsi_props;
 }
 
-static TypeInfo scsi_device_type_info = {
+static const TypeInfo scsi_device_type_info = {
     .name = TYPE_SCSI_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(SCSIDevice),
diff --git a/hw/scsi-defs.h b/hw/scsi-defs.h
index d7a401912be..9ab045b6132 100644
--- a/hw/scsi-defs.h
+++ b/hw/scsi-defs.h
@@ -19,6 +19,8 @@
  * This header file contains public constants and structures used by
  * the scsi code for linux.
  */
+#ifndef HW_SCSI_DEFS_H
+#define HW_SCSI_DEFS_H 1
 
 /*
  *      SCSI opcodes
@@ -301,3 +303,5 @@
 #define MMC_PROFILE_HDDVD_R_DL          0x0058
 #define MMC_PROFILE_HDDVD_RW_DL         0x005A
 #define MMC_PROFILE_INVALID             0xFFFF
+
+#endif
diff --git a/hw/scsi-disk.c b/hw/scsi-disk.c
index 1585683bce3..28e75bbf5bc 100644
--- a/hw/scsi-disk.c
+++ b/hw/scsi-disk.c
@@ -29,13 +29,13 @@ do { printf("scsi-disk: " fmt , ## __VA_ARGS__); } while (0)
 #endif
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "sysemu.h"
-#include "blockdev.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 #ifdef __linux
 #include <scsi/sg.h>
@@ -85,9 +85,7 @@ static void scsi_free_request(SCSIRequest *req)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
 
-    if (r->iov.iov_base) {
-        qemu_vfree(r->iov.iov_base);
-    }
+    qemu_vfree(r->iov.iov_base);
 }
 
 /* Helper function for command completion with sense.  */
@@ -386,23 +384,11 @@ static void scsi_read_data(SCSIRequest *req)
  */
 static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
 {
-    int is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
+    bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-    BlockErrorAction action = bdrv_get_on_error(s->qdev.conf.bs, is_read);
-
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
+    BlockErrorAction action = bdrv_get_error_action(s->qdev.conf.bs, is_read, error);
 
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->qdev.conf.bs, error);
-        scsi_req_retry(&r->req);
-    } else {
+    if (action == BDRV_ACTION_REPORT) {
         switch (error) {
         case ENOMEDIUM:
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
@@ -417,9 +403,12 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
             scsi_check_condition(r, SENSE_CODE(IO_ERROR));
             break;
         }
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_REPORT, is_read);
     }
-    return 1;
+    bdrv_error_action(s->qdev.conf.bs, action, is_read, error);
+    if (action == BDRV_ACTION_STOP) {
+        scsi_req_retry(&r->req);
+    }
+    return action != BDRV_ACTION_IGNORE;
 }
 
 static void scsi_write_complete(void * opaque, int ret)
@@ -661,7 +650,6 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
     if (buflen > SCSI_MAX_INQUIRY_LEN) {
         buflen = SCSI_MAX_INQUIRY_LEN;
     }
-    memset(outbuf, 0, buflen);
 
     outbuf[0] = s->qdev.type & 0x1f;
     outbuf[1] = (s->features & (1 << SCSI_DISK_F_REMOVABLE)) ? 0x80 : 0;
@@ -678,7 +666,7 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
      * is actually implemented, but we're good enough.
      */
     outbuf[2] = 5;
-    outbuf[3] = 2; /* Format 2 */
+    outbuf[3] = 2 | 0x10; /* Format 2, HiSup */
 
     if (buflen > 36) {
         outbuf[4] = buflen - 5; /* Additional Length = (Len - 1) - 4 */
@@ -1397,6 +1385,7 @@ invalid_param_len:
 
 static void scsi_disk_emulate_mode_select(SCSIDiskReq *r, uint8_t *inbuf)
 {
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint8_t *p = inbuf;
     int cmd = r->req.cmd.buf[0];
     int len = r->req.cmd.xfer;
@@ -1433,6 +1422,14 @@ static void scsi_disk_emulate_mode_select(SCSIDiskReq *r, uint8_t *inbuf)
             return;
         }
     }
+    if (!bdrv_enable_write_cache(s->qdev.conf.bs)) {
+        /* The request is used as the AIO opaque value, so add a ref.  */
+        scsi_req_ref(&r->req);
+        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
+        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_aio_complete, r);
+        return;
+    }
+
     scsi_req_complete(&r->req, GOOD);
     return;
 
@@ -1446,7 +1443,22 @@ invalid_param_len:
 
 invalid_field:
     scsi_check_condition(r, SENSE_CODE(INVALID_FIELD));
-    return;
+}
+
+static inline bool check_lba_range(SCSIDiskState *s,
+                                   uint64_t sector_num, uint32_t nb_sectors)
+{
+    /*
+     * The first line tests that no overflow happens when computing the last
+     * sector.  The second line tests that the last accessed sector is in
+     * range.
+     *
+     * Careful, the computations should not underflow for nb_sectors == 0,
+     * and a 0-block read to the first LBA beyond the end of device is
+     * valid.
+     */
+    return (sector_num <= sector_num + nb_sectors &&
+            sector_num + nb_sectors <= s->qdev.max_lba + 1);
 }
 
 typedef struct UnmapCBData {
@@ -1473,8 +1485,7 @@ static void scsi_unmap_complete(void *opaque, int ret)
     if (data->count > 0 && !r->req.io_canceled) {
         sector_num = ldq_be_p(&data->inbuf[0]);
         nb_sectors = ldl_be_p(&data->inbuf[8]) & 0xffffffffULL;
-        if (sector_num > sector_num + nb_sectors ||
-            sector_num + nb_sectors - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, sector_num, nb_sectors)) {
             scsi_check_condition(r, SENSE_CODE(LBA_OUT_OF_RANGE));
             goto done;
         }
@@ -1529,7 +1540,6 @@ static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 
 invalid_param_len:
     scsi_check_condition(r, SENSE_CODE(INVALID_PARAM_LEN));
-    return;
 }
 
 static void scsi_disk_emulate_write_data(SCSIRequest *req)
@@ -1592,24 +1602,26 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         break;
     }
 
+    /*
+     * FIXME: we shouldn't return anything bigger than 4k, but the code
+     * requires the buffer to be as big as req->cmd.xfer in several
+     * places.  So, do not allow CDBs with a very large ALLOCATION
+     * LENGTH.  The real fix would be to modify scsi_read_data and
+     * dma_buf_read, so that they return data beyond the buflen
+     * as all zeros.
+     */
+    if (req->cmd.xfer > 65536) {
+        goto illegal_request;
+    }
+    r->buflen = MAX(4096, req->cmd.xfer);
+
     if (!r->iov.iov_base) {
-        /*
-         * FIXME: we shouldn't return anything bigger than 4k, but the code
-         * requires the buffer to be as big as req->cmd.xfer in several
-         * places.  So, do not allow CDBs with a very large ALLOCATION
-         * LENGTH.  The real fix would be to modify scsi_read_data and
-         * dma_buf_read, so that they return data beyond the buflen
-         * as all zeros.
-         */
-        if (req->cmd.xfer > 65536) {
-            goto illegal_request;
-        }
-        r->buflen = MAX(4096, req->cmd.xfer);
         r->iov.iov_base = qemu_blockalign(s->qdev.conf.bs, r->buflen);
     }
 
     buflen = req->cmd.xfer;
     outbuf = r->iov.iov_base;
+    memset(outbuf, 0, r->buflen);
     switch (req->cmd.buf[0]) {
     case TEST_UNIT_READY:
         assert(!s->tray_open && bdrv_is_inserted(s->qdev.conf.bs));
@@ -1668,7 +1680,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
         if (!nb_sectors) {
             scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
-            return -1;
+            return 0;
         }
         if ((req->cmd.buf[8] & 1) == 0 && req->cmd.lba) {
             goto illegal_request;
@@ -1690,12 +1702,14 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         outbuf[5] = 0;
         outbuf[6] = s->qdev.blocksize >> 8;
         outbuf[7] = 0;
-        buflen = 8;
         break;
     case REQUEST_SENSE:
         /* Just return "NO SENSE".  */
         buflen = scsi_build_sense(NULL, 0, outbuf, r->buflen,
                                   (req->cmd.buf[1] & 1) == 0);
+        if (buflen < 0) {
+            goto illegal_request;
+        }
         break;
     case MECHANISM_STATUS:
         buflen = scsi_emulate_mechanism_status(s, outbuf);
@@ -1735,7 +1749,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
             bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
             if (!nb_sectors) {
                 scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
-                return -1;
+                return 0;
             }
             if ((req->cmd.buf[14] & 1) == 0 && req->cmd.lba) {
                 goto illegal_request;
@@ -1766,7 +1780,6 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
             }
 
             /* Protection, exponent and lowest lba field left blank. */
-            buflen = req->cmd.xfer;
             break;
         }
         DPRINTF("Unsupported Service Action In\n");
@@ -1793,17 +1806,13 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         DPRINTF("Unmap (len %lu)\n", (long)r->req.cmd.xfer);
         break;
     case WRITE_SAME_10:
-        nb_sectors = lduw_be_p(&req->cmd.buf[7]);
-        goto write_same;
     case WRITE_SAME_16:
-        nb_sectors = ldl_be_p(&req->cmd.buf[10]) & 0xffffffffULL;
-    write_same:
+        nb_sectors = scsi_data_cdb_length(r->req.cmd.buf);
         if (bdrv_is_read_only(s->qdev.conf.bs)) {
             scsi_check_condition(r, SENSE_CODE(WRITE_PROTECTED));
             return 0;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + nb_sectors ||
-            r->req.cmd.lba + nb_sectors - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, nb_sectors)) {
             goto illegal_lba;
         }
 
@@ -1827,7 +1836,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         return 0;
     }
     assert(!r->req.aiocb);
-    r->iov.iov_len = MIN(buflen, req->cmd.xfer);
+    r->iov.iov_len = MIN(r->buflen, req->cmd.xfer);
     if (r->iov.iov_len == 0) {
         scsi_req_complete(&r->req, GOOD);
     }
@@ -1858,7 +1867,7 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
-    int32_t len;
+    uint32_t len;
     uint8_t command;
 
     command = buf[0];
@@ -1868,18 +1877,17 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
         return 0;
     }
 
+    len = scsi_data_cdb_length(r->req.cmd.buf);
     switch (command) {
     case READ_6:
     case READ_10:
     case READ_12:
     case READ_16:
-        len = r->req.cmd.xfer / s->qdev.blocksize;
-        DPRINTF("Read (sector %" PRId64 ", count %d)\n", r->req.cmd.lba, len);
+        DPRINTF("Read (sector %" PRId64 ", count %u)\n", r->req.cmd.lba, len);
         if (r->req.cmd.buf[1] & 0xe0) {
             goto illegal_request;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + len ||
-            r->req.cmd.lba + len - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, len)) {
             goto illegal_lba;
         }
         r->sector = r->req.cmd.lba * (s->qdev.blocksize / 512);
@@ -1900,15 +1908,13 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
     case VERIFY_10:
     case VERIFY_12:
     case VERIFY_16:
-        len = r->req.cmd.xfer / s->qdev.blocksize;
-        DPRINTF("Write %s(sector %" PRId64 ", count %d)\n",
+        DPRINTF("Write %s(sector %" PRId64 ", count %u)\n",
                 (command & 0xe) == 0xe ? "And Verify " : "",
                 r->req.cmd.lba, len);
         if (r->req.cmd.buf[1] & 0xe0) {
             goto illegal_request;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + len ||
-            r->req.cmd.lba + len - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, len)) {
             goto illegal_lba;
         }
         r->sector = r->req.cmd.lba * (s->qdev.blocksize / 512);
@@ -1965,7 +1971,6 @@ static void scsi_disk_resize_cb(void *opaque)
      * direct-access devices.
      */
     if (s->qdev.type == TYPE_DISK) {
-        scsi_device_set_ua(&s->qdev, SENSE_CODE(CAPACITY_CHANGED));
         scsi_device_report_change(&s->qdev, SENSE_CODE(CAPACITY_CHANGED));
     }
 }
@@ -2382,7 +2387,7 @@ static void scsi_hd_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_hd_info = {
+static const TypeInfo scsi_hd_info = {
     .name          = "scsi-hd",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2411,7 +2416,7 @@ static void scsi_cd_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_cd_info = {
+static const TypeInfo scsi_cd_info = {
     .name          = "scsi-cd",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2440,7 +2445,7 @@ static void scsi_block_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_block_info = {
+static const TypeInfo scsi_block_info = {
     .name          = "scsi-block",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2474,7 +2479,7 @@ static void scsi_disk_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_disk_info = {
+static const TypeInfo scsi_disk_info = {
     .name          = "scsi-disk",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
diff --git a/hw/scsi-generic.c b/hw/scsi-generic.c
index a5eb663ecfd..8175474a67c 100644
--- a/hw/scsi-generic.c
+++ b/hw/scsi-generic.c
@@ -12,9 +12,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 #ifdef __linux__
 
@@ -400,11 +400,11 @@ static int scsi_generic_initfn(SCSIDevice *s)
         return -1;
     }
 
-    if (bdrv_get_on_error(s->conf.bs, 0) != BLOCK_ERR_STOP_ENOSPC) {
+    if (bdrv_get_on_error(s->conf.bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
         error_report("Device doesn't support drive option werror");
         return -1;
     }
-    if (bdrv_get_on_error(s->conf.bs, 1) != BLOCK_ERR_REPORT) {
+    if (bdrv_get_on_error(s->conf.bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
         error_report("Device doesn't support drive option rerror");
         return -1;
     }
@@ -499,7 +499,7 @@ static void scsi_generic_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_device;
 }
 
-static TypeInfo scsi_generic_info = {
+static const TypeInfo scsi_generic_info = {
     .name          = "scsi-generic",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDevice),
diff --git a/hw/scsi.h b/hw/scsi.h
index 1aeee4659c6..a5b5b2ec0d4 100644
--- a/hw/scsi.h
+++ b/hw/scsi.h
@@ -2,9 +2,9 @@
 #define QEMU_HW_SCSI_H
 
 #include "qdev.h"
-#include "block.h"
+#include "block/block.h"
 #include "hw/block-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define MAX_SCSI_DEVS	255
 
@@ -218,6 +218,8 @@ extern const struct SCSISense sense_code_WRITE_PROTECTED;
 
 #define SENSE_CODE(x) sense_code_ ## x
 
+uint32_t scsi_data_cdb_length(uint8_t *buf);
+uint32_t scsi_cdb_length(uint8_t *buf);
 int scsi_sense_valid(SCSISense sense);
 int scsi_build_sense(uint8_t *in_buf, int in_len,
                      uint8_t *buf, int len, bool fixed);
diff --git a/hw/sd.c b/hw/sd.c
index ec264075430..428bd78e327 100644
--- a/hw/sd.c
+++ b/hw/sd.c
@@ -30,9 +30,9 @@
  */
 
 #include "hw.h"
-#include "block.h"
+#include "block/block.h"
 #include "sd.h"
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 
 //#define DEBUG_SD 1
 
@@ -55,24 +55,28 @@ typedef enum {
     sd_illegal = -2,
 } sd_rsp_type_t;
 
+enum SDCardModes {
+    sd_inactive,
+    sd_card_identification_mode,
+    sd_data_transfer_mode,
+};
+
+enum SDCardStates {
+    sd_inactive_state = -1,
+    sd_idle_state = 0,
+    sd_ready_state,
+    sd_identification_state,
+    sd_standby_state,
+    sd_transfer_state,
+    sd_sendingdata_state,
+    sd_receivingdata_state,
+    sd_programming_state,
+    sd_disconnect_state,
+};
+
 struct SDState {
-    enum {
-        sd_inactive,
-        sd_card_identification_mode,
-        sd_data_transfer_mode,
-    } mode;
-    enum {
-        sd_inactive_state = -1,
-        sd_idle_state = 0,
-        sd_ready_state,
-        sd_identification_state,
-        sd_standby_state,
-        sd_transfer_state,
-        sd_sendingdata_state,
-        sd_receivingdata_state,
-        sd_programming_state,
-        sd_disconnect_state,
-    } state;
+    uint32_t mode;    /* current card mode, one of SDCardModes */
+    int32_t state;    /* current card state, one of SDCardStates */
     uint32_t ocr;
     uint8_t scr[8];
     uint8_t cid[16];
@@ -83,21 +87,22 @@ struct SDState {
     uint32_t vhs;
     bool wp_switch;
     unsigned long *wp_groups;
+    int32_t wpgrps_size;
     uint64_t size;
-    int blk_len;
+    uint32_t blk_len;
     uint32_t erase_start;
     uint32_t erase_end;
     uint8_t pwd[16];
-    int pwd_len;
-    int function_group[6];
+    uint32_t pwd_len;
+    uint8_t function_group[6];
 
     bool spi;
-    int current_cmd;
+    uint8_t current_cmd;
     /* True if we will handle the next command as an ACMD. Note that this does
      * *not* track the APP_CMD status bit!
      */
     bool expecting_acmd;
-    int blk_written;
+    uint32_t blk_written;
     uint64_t data_start;
     uint32_t data_offset;
     uint8_t data[512];
@@ -421,8 +426,9 @@ static void sd_reset(SDState *sd, BlockDriverState *bdrv)
     if (sd->wp_groups)
         g_free(sd->wp_groups);
     sd->wp_switch = bdrv ? bdrv_is_read_only(bdrv) : false;
-    sd->wp_groups = bitmap_new(sect);
-    memset(sd->function_group, 0, sizeof(int) * 6);
+    sd->wpgrps_size = sect;
+    sd->wp_groups = bitmap_new(sd->wpgrps_size);
+    memset(sd->function_group, 0, sizeof(sd->function_group));
     sd->erase_start = 0;
     sd->erase_end = 0;
     sd->size = size;
@@ -446,6 +452,38 @@ static const BlockDevOps sd_block_ops = {
     .change_media_cb = sd_cardchange,
 };
 
+static const VMStateDescription sd_vmstate = {
+    .name = "sd-card",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(mode, SDState),
+        VMSTATE_INT32(state, SDState),
+        VMSTATE_UINT8_ARRAY(cid, SDState, 16),
+        VMSTATE_UINT8_ARRAY(csd, SDState, 16),
+        VMSTATE_UINT16(rca, SDState),
+        VMSTATE_UINT32(card_status, SDState),
+        VMSTATE_PARTIAL_BUFFER(sd_status, SDState, 1),
+        VMSTATE_UINT32(vhs, SDState),
+        VMSTATE_BITMAP(wp_groups, SDState, 0, wpgrps_size),
+        VMSTATE_UINT32(blk_len, SDState),
+        VMSTATE_UINT32(erase_start, SDState),
+        VMSTATE_UINT32(erase_end, SDState),
+        VMSTATE_UINT8_ARRAY(pwd, SDState, 16),
+        VMSTATE_UINT32(pwd_len, SDState),
+        VMSTATE_UINT8_ARRAY(function_group, SDState, 6),
+        VMSTATE_UINT8(current_cmd, SDState),
+        VMSTATE_BOOL(expecting_acmd, SDState),
+        VMSTATE_UINT32(blk_written, SDState),
+        VMSTATE_UINT64(data_start, SDState),
+        VMSTATE_UINT32(data_offset, SDState),
+        VMSTATE_UINT8_ARRAY(data, SDState, 512),
+        VMSTATE_BUFFER_UNSAFE(buf, SDState, 1, 512),
+        VMSTATE_BOOL(enable, SDState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 /* We do not model the chip select pin, so allow the board to select
    whether card should be in SSI or MMC/SD mode.  It is also up to the
    board to ensure that ssi transfers only occur when the chip select
@@ -463,6 +501,7 @@ SDState *sd_init(BlockDriverState *bs, bool is_spi)
         bdrv_attach_dev_nofail(sd->bdrv, sd);
         bdrv_set_dev_ops(sd->bdrv, &sd_block_ops, sd);
     }
+    vmstate_register(NULL, -1, &sd_vmstate, sd);
     return sd;
 }
 
@@ -476,19 +515,28 @@ void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert)
 
 static void sd_erase(SDState *sd)
 {
-    int i, start, end;
+    int i;
+    uint64_t erase_start = sd->erase_start;
+    uint64_t erase_end = sd->erase_end;
+
     if (!sd->erase_start || !sd->erase_end) {
         sd->card_status |= ERASE_SEQ_ERROR;
         return;
     }
 
-    start = sd_addr_to_wpnum(sd->erase_start);
-    end = sd_addr_to_wpnum(sd->erase_end);
+    if (extract32(sd->ocr, OCR_CCS_BITN, 1)) {
+        /* High capacity memory card: erase units are 512 byte blocks */
+        erase_start *= 512;
+        erase_end *= 512;
+    }
+
+    erase_start = sd_addr_to_wpnum(erase_start);
+    erase_end = sd_addr_to_wpnum(erase_end);
     sd->erase_start = 0;
     sd->erase_end = 0;
     sd->csd[14] |= 0x40;
 
-    for (i = start; i <= end; i++) {
+    for (i = erase_start; i <= erase_end; i++) {
         if (test_bit(i, sd->wp_groups)) {
             sd->card_status |= WP_ERASE_SKIP;
         }
@@ -567,7 +615,7 @@ static void sd_lock_command(SDState *sd)
             sd->card_status |= LOCK_UNLOCK_FAILED;
             return;
         }
-        bitmap_zero(sd->wp_groups, sd_addr_to_wpnum(sd->size) + 1);
+        bitmap_zero(sd->wp_groups, sd->wpgrps_size);
         sd->csd[14] &= ~0x10;
         sd->card_status &= ~CARD_IS_LOCKED;
         sd->pwd_len = 0;
@@ -1391,8 +1439,8 @@ send_response:
         int i;
         DPRINTF("Response:");
         for (i = 0; i < rsplen; i++)
-            printf(" %02x", response[i]);
-        printf(" state %d\n", sd->state);
+            fprintf(stderr, " %02x", response[i]);
+        fprintf(stderr, " state %d\n", sd->state);
     } else {
         DPRINTF("No response %d\n", sd->state);
     }
@@ -1407,7 +1455,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
 
     DPRINTF("sd_blk_read: addr = 0x%08llx, len = %d\n",
             (unsigned long long) addr, len);
-    if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+    if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
         fprintf(stderr, "sd_blk_read: read error on host side\n");
         return;
     }
@@ -1415,7 +1463,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
     if (end > (addr & ~511) + 512) {
         memcpy(sd->data, sd->buf + (addr & 511), 512 - (addr & 511));
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) == -1) {
+        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_read: read error on host side\n");
             return;
         }
@@ -1429,29 +1477,31 @@ static void sd_blk_write(SDState *sd, uint64_t addr, uint32_t len)
     uint64_t end = addr + len;
 
     if ((addr & 511) || len < 512)
-        if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+        if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
 
     if (end > (addr & ~511) + 512) {
         memcpy(sd->buf + (addr & 511), sd->data, 512 - (addr & 511));
-        if (bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+        if (bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
             return;
         }
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) == -1) {
+        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
         memcpy(sd->buf, sd->data + 512 - (addr & 511), end & 511);
-        if (bdrv_write(sd->bdrv, end >> 9, sd->buf, 1) == -1)
+        if (bdrv_write(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
+        }
     } else {
         memcpy(sd->buf + (addr & 511), sd->data, len);
-        if (!sd->bdrv || bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) == -1)
+        if (!sd->bdrv || bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
+        }
     }
 }
 
diff --git a/hw/sd.h b/hw/sd.h
index 4eb9679acd8..d9b97e44664 100644
--- a/hw/sd.h
+++ b/hw/sd.h
@@ -50,6 +50,7 @@
 #define READY_FOR_DATA		(1 << 8)
 #define APP_CMD			(1 << 5)
 #define AKE_SEQ_ERROR		(1 << 3)
+#define OCR_CCS_BITN        30
 
 typedef enum {
     sd_none = -1,
diff --git a/hw/serial-isa.c b/hw/serial-isa.c
new file mode 100644
index 00000000000..5a6f51f856d
--- /dev/null
+++ b/hw/serial-isa.c
@@ -0,0 +1,130 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "serial.h"
+#include "isa.h"
+
+typedef struct ISASerialState {
+    ISADevice dev;
+    uint32_t index;
+    uint32_t iobase;
+    uint32_t isairq;
+    SerialState state;
+} ISASerialState;
+
+static const int isa_serial_io[MAX_SERIAL_PORTS] = {
+    0x3f8, 0x2f8, 0x3e8, 0x2e8
+};
+static const int isa_serial_irq[MAX_SERIAL_PORTS] = {
+    4, 3, 4, 3
+};
+
+static int serial_isa_initfn(ISADevice *dev)
+{
+    static int index;
+    ISASerialState *isa = DO_UPCAST(ISASerialState, dev, dev);
+    SerialState *s = &isa->state;
+
+    if (isa->index == -1) {
+        isa->index = index;
+    }
+    if (isa->index >= MAX_SERIAL_PORTS) {
+        return -1;
+    }
+    if (isa->iobase == -1) {
+        isa->iobase = isa_serial_io[isa->index];
+    }
+    if (isa->isairq == -1) {
+        isa->isairq = isa_serial_irq[isa->index];
+    }
+    index++;
+
+    s->baudbase = 115200;
+    isa_init_irq(dev, &s->irq, isa->isairq);
+    serial_init_core(s);
+    qdev_set_legacy_instance_id(&dev->qdev, isa->iobase, 3);
+
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    isa_register_ioport(dev, &s->io, isa->iobase);
+    return 0;
+}
+
+static const VMStateDescription vmstate_isa_serial = {
+    .name = "serial",
+    .version_id = 3,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(state, ISASerialState, 0, vmstate_serial, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property serial_isa_properties[] = {
+    DEFINE_PROP_UINT32("index",  ISASerialState, index,   -1),
+    DEFINE_PROP_HEX32("iobase",  ISASerialState, iobase,  -1),
+    DEFINE_PROP_UINT32("irq",    ISASerialState, isairq,  -1),
+    DEFINE_PROP_CHR("chardev",   ISASerialState, state.chr),
+    DEFINE_PROP_UINT32("wakeup", ISASerialState, state.wakeup, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void serial_isa_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+    ic->init = serial_isa_initfn;
+    dc->vmsd = &vmstate_isa_serial;
+    dc->props = serial_isa_properties;
+}
+
+static const TypeInfo serial_isa_info = {
+    .name          = "isa-serial",
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISASerialState),
+    .class_init    = serial_isa_class_initfn,
+};
+
+static void serial_register_types(void)
+{
+    type_register_static(&serial_isa_info);
+}
+
+type_init(serial_register_types)
+
+bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr)
+{
+    ISADevice *dev;
+
+    dev = isa_try_create(bus, "isa-serial");
+    if (!dev) {
+        return false;
+    }
+    qdev_prop_set_uint32(&dev->qdev, "index", index);
+    qdev_prop_set_chr(&dev->qdev, "chardev", chr);
+    if (qdev_init(&dev->qdev) < 0) {
+        return false;
+    }
+    return true;
+}
diff --git a/hw/serial-pci.c b/hw/serial-pci.c
new file mode 100644
index 00000000000..1c31353f6df
--- /dev/null
+++ b/hw/serial-pci.c
@@ -0,0 +1,252 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/* see docs/specs/pci-serial.txt */
+
+#include "serial.h"
+#include "pci/pci.h"
+
+#define PCI_SERIAL_MAX_PORTS 4
+
+typedef struct PCISerialState {
+    PCIDevice dev;
+    SerialState state;
+} PCISerialState;
+
+typedef struct PCIMultiSerialState {
+    PCIDevice    dev;
+    MemoryRegion iobar;
+    uint32_t     ports;
+    char         *name[PCI_SERIAL_MAX_PORTS];
+    SerialState  state[PCI_SERIAL_MAX_PORTS];
+    uint32_t     level[PCI_SERIAL_MAX_PORTS];
+    qemu_irq     *irqs;
+} PCIMultiSerialState;
+
+static int serial_pci_init(PCIDevice *dev)
+{
+    PCISerialState *pci = DO_UPCAST(PCISerialState, dev, dev);
+    SerialState *s = &pci->state;
+
+    s->baudbase = 115200;
+    serial_init_core(s);
+
+    pci->dev.config[PCI_INTERRUPT_PIN] = 0x01;
+    s->irq = pci->dev.irq[0];
+
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io);
+    return 0;
+}
+
+static void multi_serial_irq_mux(void *opaque, int n, int level)
+{
+    PCIMultiSerialState *pci = opaque;
+    int i, pending = 0;
+
+    pci->level[n] = level;
+    for (i = 0; i < pci->ports; i++) {
+        if (pci->level[i]) {
+            pending = 1;
+        }
+    }
+    qemu_set_irq(pci->dev.irq[0], pending);
+}
+
+static int multi_serial_pci_init(PCIDevice *dev)
+{
+    PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
+    PCIMultiSerialState *pci = DO_UPCAST(PCIMultiSerialState, dev, dev);
+    SerialState *s;
+    int i;
+
+    switch (pc->device_id) {
+    case 0x0003:
+        pci->ports = 2;
+        break;
+    case 0x0004:
+        pci->ports = 4;
+        break;
+    }
+    assert(pci->ports > 0);
+    assert(pci->ports <= PCI_SERIAL_MAX_PORTS);
+
+    pci->dev.config[PCI_INTERRUPT_PIN] = 0x01;
+    memory_region_init(&pci->iobar, "multiserial", 8 * pci->ports);
+    pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &pci->iobar);
+    pci->irqs = qemu_allocate_irqs(multi_serial_irq_mux, pci,
+                                   pci->ports);
+
+    for (i = 0; i < pci->ports; i++) {
+        s = pci->state + i;
+        s->baudbase = 115200;
+        serial_init_core(s);
+        s->irq = pci->irqs[i];
+        pci->name[i] = g_strdup_printf("uart #%d", i+1);
+        memory_region_init_io(&s->io, &serial_io_ops, s, pci->name[i], 8);
+        memory_region_add_subregion(&pci->iobar, 8 * i, &s->io);
+    }
+    return 0;
+}
+
+static void serial_pci_exit(PCIDevice *dev)
+{
+    PCISerialState *pci = DO_UPCAST(PCISerialState, dev, dev);
+    SerialState *s = &pci->state;
+
+    serial_exit_core(s);
+    memory_region_destroy(&s->io);
+}
+
+static void multi_serial_pci_exit(PCIDevice *dev)
+{
+    PCIMultiSerialState *pci = DO_UPCAST(PCIMultiSerialState, dev, dev);
+    SerialState *s;
+    int i;
+
+    for (i = 0; i < pci->ports; i++) {
+        s = pci->state + i;
+        serial_exit_core(s);
+        memory_region_destroy(&s->io);
+        g_free(pci->name[i]);
+    }
+    memory_region_destroy(&pci->iobar);
+    qemu_free_irqs(pci->irqs);
+}
+
+static const VMStateDescription vmstate_pci_serial = {
+    .name = "pci-serial",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, PCISerialState),
+        VMSTATE_STRUCT(state, PCISerialState, 0, vmstate_serial, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pci_multi_serial = {
+    .name = "pci-serial-multi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, PCIMultiSerialState),
+        VMSTATE_STRUCT_ARRAY(state, PCIMultiSerialState, PCI_SERIAL_MAX_PORTS,
+                             0, vmstate_serial, SerialState),
+        VMSTATE_UINT32_ARRAY(level, PCIMultiSerialState, PCI_SERIAL_MAX_PORTS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev",  PCISerialState, state.chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static Property multi_2x_serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev1",  PCIMultiSerialState, state[0].chr),
+    DEFINE_PROP_CHR("chardev2",  PCIMultiSerialState, state[1].chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static Property multi_4x_serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev1",  PCIMultiSerialState, state[0].chr),
+    DEFINE_PROP_CHR("chardev2",  PCIMultiSerialState, state[1].chr),
+    DEFINE_PROP_CHR("chardev3",  PCIMultiSerialState, state[2].chr),
+    DEFINE_PROP_CHR("chardev4",  PCIMultiSerialState, state[3].chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = serial_pci_init;
+    pc->exit = serial_pci_exit;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_serial;
+    dc->props = serial_pci_properties;
+}
+
+static void multi_2x_serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = multi_serial_pci_init;
+    pc->exit = multi_serial_pci_exit;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL2;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_multi_serial;
+    dc->props = multi_2x_serial_pci_properties;
+}
+
+static void multi_4x_serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = multi_serial_pci_init;
+    pc->exit = multi_serial_pci_exit;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL4;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_multi_serial;
+    dc->props = multi_4x_serial_pci_properties;
+}
+
+static const TypeInfo serial_pci_info = {
+    .name          = "pci-serial",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCISerialState),
+    .class_init    = serial_pci_class_initfn,
+};
+
+static const TypeInfo multi_2x_serial_pci_info = {
+    .name          = "pci-serial-2x",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIMultiSerialState),
+    .class_init    = multi_2x_serial_pci_class_initfn,
+};
+
+static const TypeInfo multi_4x_serial_pci_info = {
+    .name          = "pci-serial-4x",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIMultiSerialState),
+    .class_init    = multi_4x_serial_pci_class_initfn,
+};
+
+static void serial_pci_register_types(void)
+{
+    type_register_static(&serial_pci_info);
+    type_register_static(&multi_2x_serial_pci_info);
+    type_register_static(&multi_4x_serial_pci_info);
+}
+
+type_init(serial_pci_register_types)
diff --git a/hw/serial.c b/hw/serial.c
index a421d1e7bc7..f0ce9b0c150 100644
--- a/hw/serial.c
+++ b/hw/serial.c
@@ -22,12 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "qemu-char.h"
-#include "isa.h"
-#include "pc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+
+#include "serial.h"
+#include "char/char.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_SERIAL
 
@@ -93,8 +92,6 @@
 #define UART_FCR_RFR        0x02    /* RCVR Fifo Reset */
 #define UART_FCR_FE         0x01    /* FIFO Enable */
 
-#define UART_FIFO_LENGTH    16      /* 16550A Fifo Length */
-
 #define XMIT_FIFO           0
 #define RECV_FIFO           1
 #define MAX_XMIT_RETRY      4
@@ -107,64 +104,6 @@ do { fprintf(stderr, "serial: " fmt , ## __VA_ARGS__); } while (0)
 do {} while (0)
 #endif
 
-typedef struct SerialFIFO {
-    uint8_t data[UART_FIFO_LENGTH];
-    uint8_t count;
-    uint8_t itl;                        /* Interrupt Trigger Level */
-    uint8_t tail;
-    uint8_t head;
-} SerialFIFO;
-
-struct SerialState {
-    uint16_t divider;
-    uint8_t rbr; /* receive register */
-    uint8_t thr; /* transmit holding register */
-    uint8_t tsr; /* transmit shift register */
-    uint8_t ier;
-    uint8_t iir; /* read only */
-    uint8_t lcr;
-    uint8_t mcr;
-    uint8_t lsr; /* read only */
-    uint8_t msr; /* read only */
-    uint8_t scr;
-    uint8_t fcr;
-    uint8_t fcr_vmstate; /* we can't write directly this value
-                            it has side effects */
-    /* NOTE: this hidden state is necessary for tx irq generation as
-       it can be reset while reading iir */
-    int thr_ipending;
-    qemu_irq irq;
-    CharDriverState *chr;
-    int last_break_enable;
-    int it_shift;
-    int baudbase;
-    int tsr_retry;
-    uint32_t wakeup;
-
-    uint64_t last_xmit_ts;              /* Time when the last byte was successfully sent out of the tsr */
-    SerialFIFO recv_fifo;
-    SerialFIFO xmit_fifo;
-
-    struct QEMUTimer *fifo_timeout_timer;
-    int timeout_ipending;                   /* timeout interrupt pending state */
-    struct QEMUTimer *transmit_timer;
-
-
-    uint64_t char_transmit_time;               /* time to transmit a char in ticks*/
-    int poll_msl;
-
-    struct QEMUTimer *modem_status_poll;
-    MemoryRegion io;
-};
-
-typedef struct ISASerialState {
-    ISADevice dev;
-    uint32_t index;
-    uint32_t iobase;
-    uint32_t isairq;
-    SerialState state;
-} ISASerialState;
-
 static void serial_receive1(void *opaque, const uint8_t *buf, int size);
 
 static void fifo_clear(SerialState *s, int fifo)
@@ -327,8 +266,6 @@ static void serial_xmit(void *opaque)
             s->tsr = fifo_get(s,XMIT_FIFO);
             if (!s->xmit_fifo.count)
                 s->lsr |= UART_LSR_THRE;
-        } else if ((s->lsr & UART_LSR_THRE)) {
-            return;
         } else {
             s->tsr = s->thr;
             s->lsr |= UART_LSR_THRE;
@@ -340,7 +277,7 @@ static void serial_xmit(void *opaque)
         /* in loopback mode, say that we just received a char */
         serial_receive1(s, &s->tsr, 1);
     } else if (qemu_chr_fe_write(s->chr, &s->tsr, 1) != 1) {
-        if ((s->tsr_retry >= 0) && (s->tsr_retry <= MAX_XMIT_RETRY)) {
+        if ((s->tsr_retry > 0) && (s->tsr_retry <= MAX_XMIT_RETRY)) {
             s->tsr_retry++;
             qemu_mod_timer(s->transmit_timer,  new_xmit_ts + s->char_transmit_time);
             return;
@@ -367,7 +304,8 @@ static void serial_xmit(void *opaque)
 }
 
 
-static void serial_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                unsigned size)
 {
     SerialState *s = opaque;
 
@@ -513,7 +451,7 @@ static void serial_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t serial_ioport_read(void *opaque, uint32_t addr)
+static uint64_t serial_ioport_read(void *opaque, hwaddr addr, unsigned size)
 {
     SerialState *s = opaque;
     uint32_t ret;
@@ -682,12 +620,12 @@ static int serial_post_load(void *opaque, int version_id)
         s->fcr_vmstate = 0;
     }
     /* Initialize fcr via setter to perform essential side-effects */
-    serial_ioport_write(s, 0x02, s->fcr_vmstate);
+    serial_ioport_write(s, 0x02, s->fcr_vmstate, 1);
     serial_update_parameters(s);
     return 0;
 }
 
-static const VMStateDescription vmstate_serial = {
+const VMStateDescription vmstate_serial = {
     .name = "serial",
     .version_id = 3,
     .minimum_version_id = 2,
@@ -736,7 +674,7 @@ static void serial_reset(void *opaque)
     qemu_irq_lower(s->irq);
 }
 
-static void serial_init_core(SerialState *s)
+void serial_init_core(SerialState *s)
 {
     if (!s->chr) {
         fprintf(stderr, "Can't create serial device, empty char device\n");
@@ -754,6 +692,12 @@ static void serial_init_core(SerialState *s)
                           serial_event, s);
 }
 
+void serial_exit_core(SerialState *s)
+{
+    qemu_chr_add_handlers(s->chr, NULL, NULL, NULL, NULL);
+    qemu_unregister_reset(serial_reset, s);
+}
+
 /* Change the main reference oscillator frequency. */
 void serial_set_frequency(SerialState *s, uint32_t frequency)
 {
@@ -761,56 +705,18 @@ void serial_set_frequency(SerialState *s, uint32_t frequency)
     serial_update_parameters(s);
 }
 
-static const int isa_serial_io[MAX_SERIAL_PORTS] = { 0x3f8, 0x2f8, 0x3e8, 0x2e8 };
-static const int isa_serial_irq[MAX_SERIAL_PORTS] = { 4, 3, 4, 3 };
-
-static const MemoryRegionPortio serial_portio[] = {
-    { 0, 8, 1, .read = serial_ioport_read, .write = serial_ioport_write },
-    PORTIO_END_OF_LIST()
-};
-
-static const MemoryRegionOps serial_io_ops = {
-    .old_portio = serial_portio
-};
-
-static int serial_isa_initfn(ISADevice *dev)
-{
-    static int index;
-    ISASerialState *isa = DO_UPCAST(ISASerialState, dev, dev);
-    SerialState *s = &isa->state;
-
-    if (isa->index == -1)
-        isa->index = index;
-    if (isa->index >= MAX_SERIAL_PORTS)
-        return -1;
-    if (isa->iobase == -1)
-        isa->iobase = isa_serial_io[isa->index];
-    if (isa->isairq == -1)
-        isa->isairq = isa_serial_irq[isa->index];
-    index++;
-
-    s->baudbase = 115200;
-    isa_init_irq(dev, &s->irq, isa->isairq);
-    serial_init_core(s);
-    qdev_set_legacy_instance_id(&dev->qdev, isa->iobase, 3);
-
-    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
-    isa_register_ioport(dev, &s->io, isa->iobase);
-    return 0;
-}
-
-static const VMStateDescription vmstate_isa_serial = {
-    .name = "serial",
-    .version_id = 3,
-    .minimum_version_id = 2,
-    .fields      = (VMStateField []) {
-        VMSTATE_STRUCT(state, ISASerialState, 0, vmstate_serial, SerialState),
-        VMSTATE_END_OF_LIST()
-    }
+const MemoryRegionOps serial_io_ops = {
+    .read = serial_ioport_read,
+    .write = serial_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,
-                         CharDriverState *chr)
+                         CharDriverState *chr, MemoryRegion *system_io)
 {
     SerialState *s;
 
@@ -823,25 +729,26 @@ SerialState *serial_init(int base, qemu_irq irq, int baudbase,
 
     vmstate_register(NULL, base, &vmstate_serial, s);
 
-    register_ioport_write(base, 8, 1, serial_ioport_write, s);
-    register_ioport_read(base, 8, 1, serial_ioport_read, s);
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    memory_region_add_subregion(system_io, base, &s->io);
+
     return s;
 }
 
 /* Memory mapped interface */
-static uint64_t serial_mm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t serial_mm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     SerialState *s = opaque;
-    return serial_ioport_read(s, addr >> s->it_shift);
+    return serial_ioport_read(s, addr >> s->it_shift, 1);
 }
 
-static void serial_mm_write(void *opaque, target_phys_addr_t addr,
+static void serial_mm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     SerialState *s = opaque;
     value &= ~0u >> (32 - (size * 8));
-    serial_ioport_write(s, addr >> s->it_shift, value);
+    serial_ioport_write(s, addr >> s->it_shift, value, 1);
 }
 
 static const MemoryRegionOps serial_mm_ops[3] = {
@@ -863,7 +770,7 @@ static const MemoryRegionOps serial_mm_ops[3] = {
 };
 
 SerialState *serial_mm_init(MemoryRegion *address_space,
-                            target_phys_addr_t base, int it_shift,
+                            hwaddr base, int it_shift,
                             qemu_irq irq, int baudbase,
                             CharDriverState *chr, enum device_endian end)
 {
@@ -886,35 +793,3 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
     serial_update_msl(s);
     return s;
 }
-
-static Property serial_isa_properties[] = {
-    DEFINE_PROP_UINT32("index", ISASerialState, index,   -1),
-    DEFINE_PROP_HEX32("iobase", ISASerialState, iobase,  -1),
-    DEFINE_PROP_UINT32("irq",   ISASerialState, isairq,  -1),
-    DEFINE_PROP_CHR("chardev",  ISASerialState, state.chr),
-    DEFINE_PROP_UINT32("wakeup", ISASerialState, state.wakeup, 0),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void serial_isa_class_initfn(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
-    ic->init = serial_isa_initfn;
-    dc->vmsd = &vmstate_isa_serial;
-    dc->props = serial_isa_properties;
-}
-
-static TypeInfo serial_isa_info = {
-    .name          = "isa-serial",
-    .parent        = TYPE_ISA_DEVICE,
-    .instance_size = sizeof(ISASerialState),
-    .class_init    = serial_isa_class_initfn,
-};
-
-static void serial_register_types(void)
-{
-    type_register_static(&serial_isa_info);
-}
-
-type_init(serial_register_types)
diff --git a/hw/serial.h b/hw/serial.h
new file mode 100644
index 00000000000..98ee4241be0
--- /dev/null
+++ b/hw/serial.h
@@ -0,0 +1,103 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef HW_SERIAL_H
+#define HW_SERIAL_H 1
+
+#include "hw.h"
+#include "sysemu/sysemu.h"
+#include "exec/memory.h"
+
+#define UART_FIFO_LENGTH    16      /* 16550A Fifo Length */
+
+typedef struct SerialFIFO {
+    uint8_t data[UART_FIFO_LENGTH];
+    uint8_t count;
+    uint8_t itl;                        /* Interrupt Trigger Level */
+    uint8_t tail;
+    uint8_t head;
+} SerialFIFO;
+
+struct SerialState {
+    uint16_t divider;
+    uint8_t rbr; /* receive register */
+    uint8_t thr; /* transmit holding register */
+    uint8_t tsr; /* transmit shift register */
+    uint8_t ier;
+    uint8_t iir; /* read only */
+    uint8_t lcr;
+    uint8_t mcr;
+    uint8_t lsr; /* read only */
+    uint8_t msr; /* read only */
+    uint8_t scr;
+    uint8_t fcr;
+    uint8_t fcr_vmstate; /* we can't write directly this value
+                            it has side effects */
+    /* NOTE: this hidden state is necessary for tx irq generation as
+       it can be reset while reading iir */
+    int thr_ipending;
+    qemu_irq irq;
+    CharDriverState *chr;
+    int last_break_enable;
+    int it_shift;
+    int baudbase;
+    int tsr_retry;
+    uint32_t wakeup;
+
+    /* Time when the last byte was successfully sent out of the tsr */
+    uint64_t last_xmit_ts;
+    SerialFIFO recv_fifo;
+    SerialFIFO xmit_fifo;
+
+    struct QEMUTimer *fifo_timeout_timer;
+    int timeout_ipending;           /* timeout interrupt pending state */
+    struct QEMUTimer *transmit_timer;
+
+
+    uint64_t char_transmit_time;    /* time to transmit a char in ticks */
+    int poll_msl;
+
+    struct QEMUTimer *modem_status_poll;
+    MemoryRegion io;
+};
+
+extern const VMStateDescription vmstate_serial;
+extern const MemoryRegionOps serial_io_ops;
+
+void serial_init_core(SerialState *s);
+void serial_exit_core(SerialState *s);
+void serial_set_frequency(SerialState *s, uint32_t frequency);
+
+/* legacy pre qom */
+SerialState *serial_init(int base, qemu_irq irq, int baudbase,
+                         CharDriverState *chr, MemoryRegion *system_io);
+SerialState *serial_mm_init(MemoryRegion *address_space,
+                            hwaddr base, int it_shift,
+                            qemu_irq irq, int baudbase,
+                            CharDriverState *chr, enum device_endian end);
+
+/* serial-isa.c */
+bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr);
+
+#endif
diff --git a/hw/sga.c b/hw/sga.c
index a666349f821..29bc3e0246d 100644
--- a/hw/sga.c
+++ b/hw/sga.c
@@ -24,10 +24,10 @@
  * sgabios code originally available at code.google.com/p/sgabios
  *
  */
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define SGABIOS_FILENAME "sgabios.bin"
 
@@ -48,7 +48,7 @@ static void sga_class_initfn(ObjectClass *klass, void *data)
     dc->desc = "Serial Graphics Adapter";
 }
 
-static TypeInfo sga_info = {
+static const TypeInfo sga_info = {
     .name          = "sga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISASGAState),
diff --git a/hw/sh.h b/hw/sh.h
index 40df18c5dca..77bf8aad2c0 100644
--- a/hw/sh.h
+++ b/hw/sh.h
@@ -31,7 +31,7 @@ int sh7750_register_io_device(struct SH7750State *s,
 #define TMU012_FEAT_TOCR   (1 << 0)
 #define TMU012_FEAT_3CHAN  (1 << 1)
 #define TMU012_FEAT_EXTCLK (1 << 2)
-void tmu012_init(struct MemoryRegion *sysmem, target_phys_addr_t base,
+void tmu012_init(struct MemoryRegion *sysmem, hwaddr base,
                  int feat, uint32_t freq,
 		 qemu_irq ch0_irq, qemu_irq ch1_irq,
 		 qemu_irq ch2_irq0, qemu_irq ch2_irq1);
@@ -40,7 +40,7 @@ void tmu012_init(struct MemoryRegion *sysmem, target_phys_addr_t base,
 /* sh_serial.c */
 #define SH_SERIAL_FEAT_SCIF (1 << 0)
 void sh_serial_init(MemoryRegion *sysmem,
-                    target_phys_addr_t base, int feat,
+                    hwaddr base, int feat,
 		     uint32_t freq, CharDriverState *chr,
 		     qemu_irq eri_source,
 		     qemu_irq rxi_source,
diff --git a/hw/sh7750.c b/hw/sh7750.c
index e7129283d19..666f8655ed6 100644
--- a/hw/sh7750.c
+++ b/hw/sh7750.c
@@ -25,12 +25,12 @@
 #include <stdio.h>
 #include "hw.h"
 #include "sh.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sh7750_regs.h"
 #include "sh7750_regnames.h"
 #include "sh_intc.h"
 #include "cpu.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define NB_DEVICES 4
 
@@ -197,19 +197,19 @@ static void portb_changed(SH7750State * s, uint16_t prev)
  Memory
 **********************************************************************/
 
-static void error_access(const char *kind, target_phys_addr_t addr)
+static void error_access(const char *kind, hwaddr addr)
 {
     fprintf(stderr, "%s to %s (0x" TARGET_FMT_plx ") not supported\n",
 	    kind, regname(addr), addr);
 }
 
-static void ignore_access(const char *kind, target_phys_addr_t addr)
+static void ignore_access(const char *kind, hwaddr addr)
 {
     fprintf(stderr, "%s to %s (0x" TARGET_FMT_plx ") ignored\n",
 	    kind, regname(addr), addr);
 }
 
-static uint32_t sh7750_mem_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readb(void *opaque, hwaddr addr)
 {
     switch (addr) {
     default:
@@ -218,7 +218,7 @@ static uint32_t sh7750_mem_readb(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static uint32_t sh7750_mem_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readw(void *opaque, hwaddr addr)
 {
     SH7750State *s = opaque;
 
@@ -252,7 +252,7 @@ static uint32_t sh7750_mem_readw(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static uint32_t sh7750_mem_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readl(void *opaque, hwaddr addr)
 {
     SH7750State *s = opaque;
 
@@ -301,7 +301,7 @@ static uint32_t sh7750_mem_readl(void *opaque, target_phys_addr_t addr)
 
 #define is_in_sdrmx(a, x) (a >= SH7750_SDMR ## x ## _A7 \
 			&& a <= (SH7750_SDMR ## x ## _A7 + SH7750_SDMR ## x ## _REGNB))
-static void sh7750_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writeb(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
 
@@ -314,7 +314,7 @@ static void sh7750_mem_writeb(void *opaque, target_phys_addr_t addr,
     abort();
 }
 
-static void sh7750_mem_writew(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writew(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
     SH7750State *s = opaque;
@@ -366,7 +366,7 @@ static void sh7750_mem_writew(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void sh7750_mem_writel(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writel(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
     SH7750State *s = opaque;
@@ -624,14 +624,14 @@ static struct intc_group groups_irl[] = {
 #define MM_UTLB_DATA     (7)
 #define MM_REGION_TYPE(addr)  ((addr & MM_REGION_MASK) >> 24)
 
-static uint64_t invalid_read(void *opaque, target_phys_addr_t addr)
+static uint64_t invalid_read(void *opaque, hwaddr addr)
 {
     abort();
 
     return 0;
 }
 
-static uint64_t sh7750_mmct_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sh7750_mmct_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     SH7750State *s = opaque;
@@ -669,13 +669,13 @@ static uint64_t sh7750_mmct_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void invalid_write(void *opaque, target_phys_addr_t addr,
+static void invalid_write(void *opaque, hwaddr addr,
                           uint64_t mem_value)
 {
     abort();
 }
 
-static void sh7750_mmct_write(void *opaque, target_phys_addr_t addr,
+static void sh7750_mmct_write(void *opaque, hwaddr addr,
                               uint64_t mem_value, unsigned size)
 {
     SH7750State *s = opaque;
diff --git a/hw/sh_intc.c b/hw/sh_intc.c
index 7d31ced858f..c3f77d5092c 100644
--- a/hw/sh_intc.c
+++ b/hw/sh_intc.c
@@ -219,7 +219,7 @@ static void sh_intc_toggle_mask(struct intc_desc *desc, intc_enum id,
 #endif
 }
 
-static uint64_t sh_intc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t sh_intc_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     struct intc_desc *desc = opaque;
@@ -238,7 +238,7 @@ static uint64_t sh_intc_read(void *opaque, target_phys_addr_t offset,
     return *valuep;
 }
 
-static void sh_intc_write(void *opaque, target_phys_addr_t offset,
+static void sh_intc_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     struct intc_desc *desc = opaque;
diff --git a/hw/sh_intc.h b/hw/sh_intc.h
index 80c94305778..6f11beedddb 100644
--- a/hw/sh_intc.h
+++ b/hw/sh_intc.h
@@ -3,7 +3,7 @@
 
 #include "qemu-common.h"
 #include "irq.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef unsigned char intc_enum;
 
diff --git a/hw/sh_pci.c b/hw/sh_pci.c
index 0cfac46f7fa..077d9570034 100644
--- a/hw/sh_pci.c
+++ b/hw/sh_pci.c
@@ -23,10 +23,10 @@
  */
 #include "sysbus.h"
 #include "sh.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "bswap.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "qemu/bswap.h"
+#include "exec/address-spaces.h"
 
 typedef struct SHPCIState {
     SysBusDevice busdev;
@@ -41,7 +41,7 @@ typedef struct SHPCIState {
     uint32_t iobr;
 } SHPCIState;
 
-static void sh_pci_reg_write (void *p, target_phys_addr_t addr, uint64_t val,
+static void sh_pci_reg_write (void *p, hwaddr addr, uint64_t val,
                               unsigned size)
 {
     SHPCIState *pcic = p;
@@ -69,7 +69,7 @@ static void sh_pci_reg_write (void *p, target_phys_addr_t addr, uint64_t val,
     }
 }
 
-static uint64_t sh_pci_reg_read (void *p, target_phys_addr_t addr,
+static uint64_t sh_pci_reg_read (void *p, hwaddr addr,
                                  unsigned size)
 {
     SHPCIState *pcic = p;
@@ -156,7 +156,7 @@ static void sh_pci_host_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_HITACHI_SH7751R;
 }
 
-static TypeInfo sh_pci_host_info = {
+static const TypeInfo sh_pci_host_info = {
     .name          = "sh_pci_host",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -170,7 +170,7 @@ static void sh_pci_device_class_init(ObjectClass *klass, void *data)
     sdc->init = sh_pci_device_init;
 }
 
-static TypeInfo sh_pci_device_info = {
+static const TypeInfo sh_pci_device_info = {
     .name          = "sh_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SHPCIState),
diff --git a/hw/sh_serial.c b/hw/sh_serial.c
index 1d1883dd206..21c5b1362d1 100644
--- a/hw/sh_serial.c
+++ b/hw/sh_serial.c
@@ -26,8 +26,8 @@
  */
 #include "hw.h"
 #include "sh.h"
-#include "qemu-char.h"
-#include "exec-memory.h"
+#include "char/char.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_SERIAL
 
@@ -78,7 +78,7 @@ static void sh_serial_clear_fifo(sh_serial_state * s)
     s->rx_tail = 0;
 }
 
-static void sh_serial_write(void *opaque, target_phys_addr_t offs,
+static void sh_serial_write(void *opaque, hwaddr offs,
                             uint64_t val, unsigned size)
 {
     sh_serial_state *s = opaque;
@@ -187,11 +187,11 @@ static void sh_serial_write(void *opaque, target_phys_addr_t offs,
     }
 
     fprintf(stderr, "sh_serial: unsupported write to 0x%02"
-            TARGET_PRIxPHYS "\n", offs);
+            HWADDR_PRIx "\n", offs);
     abort();
 }
 
-static uint64_t sh_serial_read(void *opaque, target_phys_addr_t offs,
+static uint64_t sh_serial_read(void *opaque, hwaddr offs,
                                unsigned size)
 {
     sh_serial_state *s = opaque;
@@ -289,7 +289,7 @@ static uint64_t sh_serial_read(void *opaque, target_phys_addr_t offs,
 
     if (ret & ~((1 << 16) - 1)) {
         fprintf(stderr, "sh_serial: unsupported read from 0x%02"
-                TARGET_PRIxPHYS "\n", offs);
+                HWADDR_PRIx "\n", offs);
         abort();
     }
 
@@ -353,7 +353,7 @@ static const MemoryRegionOps sh_serial_ops = {
 };
 
 void sh_serial_init(MemoryRegion *sysmem,
-                    target_phys_addr_t base, int feat,
+                    hwaddr base, int feat,
                     uint32_t freq, CharDriverState *chr,
                     qemu_irq eri_source,
                     qemu_irq rxi_source,
diff --git a/hw/sh_timer.c b/hw/sh_timer.c
index 64bf604ba49..64ea23fce65 100644
--- a/hw/sh_timer.c
+++ b/hw/sh_timer.c
@@ -10,8 +10,8 @@
 
 #include "hw.h"
 #include "sh.h"
-#include "qemu-timer.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 #include "ptimer.h"
 
 //#define DEBUG_TIMER
@@ -59,7 +59,7 @@ static void sh_timer_update(sh_timer_state *s)
     s->int_level = new_level;
 }
 
-static uint32_t sh_timer_read(void *opaque, target_phys_addr_t offset)
+static uint32_t sh_timer_read(void *opaque, hwaddr offset)
 {
     sh_timer_state *s = (sh_timer_state *)opaque;
 
@@ -79,7 +79,7 @@ static uint32_t sh_timer_read(void *opaque, target_phys_addr_t offset)
     }
 }
 
-static void sh_timer_write(void *opaque, target_phys_addr_t offset,
+static void sh_timer_write(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     sh_timer_state *s = (sh_timer_state *)opaque;
@@ -222,7 +222,7 @@ typedef struct {
     int feat;
 } tmu012_state;
 
-static uint64_t tmu012_read(void *opaque, target_phys_addr_t offset,
+static uint64_t tmu012_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     tmu012_state *s = (tmu012_state *)opaque;
@@ -253,7 +253,7 @@ static uint64_t tmu012_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void tmu012_write(void *opaque, target_phys_addr_t offset,
+static void tmu012_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     tmu012_state *s = (tmu012_state *)opaque;
@@ -303,7 +303,7 @@ static const MemoryRegionOps tmu012_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void tmu012_init(MemoryRegion *sysmem, target_phys_addr_t base,
+void tmu012_init(MemoryRegion *sysmem, hwaddr base,
                  int feat, uint32_t freq,
 		 qemu_irq ch0_irq, qemu_irq ch1_irq,
 		 qemu_irq ch2_irq0, qemu_irq ch2_irq1)
diff --git a/hw/sharpsl.h b/hw/sharpsl.h
index 0b3a774f2f8..13981a6d034 100644
--- a/hw/sharpsl.h
+++ b/hw/sharpsl.h
@@ -12,6 +12,6 @@
 /* zaurus.c */
 
 #define SL_PXA_PARAM_BASE	0xa0000a00
-void sl_bootparam_write(target_phys_addr_t ptr);
+void sl_bootparam_write(hwaddr ptr);
 
 #endif
diff --git a/hw/shix.c b/hw/shix.c
index dd9ce174f92..6f2d55a155c 100644
--- a/hw/shix.c
+++ b/hw/shix.c
@@ -29,19 +29,17 @@
 */
 #include "hw.h"
 #include "sh.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME "shix_bios.bin"
 #define BIOS_ADDRESS 0xA0000000
 
-static void shix_init(ram_addr_t ram_size,
-               const char *boot_device,
-	       const char *kernel_filename, const char *kernel_cmdline,
-	       const char *initrd_filename, const char *cpu_model)
+static void shix_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
     int ret;
     CPUSH4State *env;
     struct SH7750State *s;
@@ -94,6 +92,7 @@ static QEMUMachine shix_machine = {
     .desc = "shix card",
     .init = shix_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void shix_machine_init(void)
diff --git a/hw/slavio_intctl.c b/hw/slavio_intctl.c
index 7fdc3be0869..136ceebc803 100644
--- a/hw/slavio_intctl.c
+++ b/hw/slavio_intctl.c
@@ -23,7 +23,7 @@
  */
 
 #include "sun4m.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 #include "trace.h"
 
@@ -78,7 +78,7 @@ typedef struct SLAVIO_INTCTLState {
 static void slavio_check_interrupts(SLAVIO_INTCTLState *s, int set_irqs);
 
 // per-cpu interrupt controller
-static uint64_t slavio_intctl_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_intctl_mem_readl(void *opaque, hwaddr addr,
                                         unsigned size)
 {
     SLAVIO_CPUINTCTLState *s = opaque;
@@ -98,7 +98,7 @@ static uint64_t slavio_intctl_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_intctl_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_intctl_mem_writel(void *opaque, hwaddr addr,
                                      uint64_t val, unsigned size)
 {
     SLAVIO_CPUINTCTLState *s = opaque;
@@ -135,7 +135,7 @@ static const MemoryRegionOps slavio_intctl_mem_ops = {
 };
 
 // master system interrupt controller
-static uint64_t slavio_intctlm_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_intctlm_mem_readl(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     SLAVIO_INTCTLState *s = opaque;
@@ -161,7 +161,7 @@ static uint64_t slavio_intctlm_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_intctlm_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_intctlm_mem_writel(void *opaque, hwaddr addr,
                                       uint64_t val, unsigned size)
 {
     SLAVIO_INTCTLState *s = opaque;
@@ -210,7 +210,7 @@ void slavio_pic_info(Monitor *mon, DeviceState *dev)
     SLAVIO_INTCTLState *s;
     int i;
 
-    sd = sysbus_from_qdev(dev);
+    sd = SYS_BUS_DEVICE(dev);
     s = FROM_SYSBUS(SLAVIO_INTCTLState, sd);
     for (i = 0; i < MAX_CPUS; i++) {
         monitor_printf(mon, "per-cpu %d: pending 0x%08x\n", i,
@@ -230,7 +230,7 @@ void slavio_irq_info(Monitor *mon, DeviceState *dev)
     int i;
     int64_t count;
 
-    sd = sysbus_from_qdev(dev);
+    sd = SYS_BUS_DEVICE(dev);
     s = FROM_SYSBUS(SLAVIO_INTCTLState, sd);
     monitor_printf(mon, "IRQ statistics:\n");
     for (i = 0; i < 32; i++) {
@@ -456,7 +456,7 @@ static void slavio_intctl_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_intctl;
 }
 
-static TypeInfo slavio_intctl_info = {
+static const TypeInfo slavio_intctl_info = {
     .name          = "slavio_intctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLAVIO_INTCTLState),
diff --git a/hw/slavio_misc.c b/hw/slavio_misc.c
index 944835e8800..af24cc1ae82 100644
--- a/hw/slavio_misc.c
+++ b/hw/slavio_misc.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "trace.h"
 
@@ -107,7 +107,7 @@ static void slavio_set_power_fail(void *opaque, int irq, int power_failing)
     slavio_misc_update_irq(s);
 }
 
-static void slavio_cfg_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_cfg_mem_writeb(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -117,7 +117,7 @@ static void slavio_cfg_mem_writeb(void *opaque, target_phys_addr_t addr,
     slavio_misc_update_irq(s);
 }
 
-static uint64_t slavio_cfg_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_cfg_mem_readb(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -138,7 +138,7 @@ static const MemoryRegionOps slavio_cfg_mem_ops = {
     },
 };
 
-static void slavio_diag_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_diag_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -147,7 +147,7 @@ static void slavio_diag_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->diag = val & 0xff;
 }
 
-static uint64_t slavio_diag_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_diag_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -168,7 +168,7 @@ static const MemoryRegionOps slavio_diag_mem_ops = {
     },
 };
 
-static void slavio_mdm_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_mdm_mem_writeb(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -177,7 +177,7 @@ static void slavio_mdm_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->mctrl = val & 0xff;
 }
 
-static uint64_t slavio_mdm_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_mdm_mem_readb(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -198,7 +198,7 @@ static const MemoryRegionOps slavio_mdm_mem_ops = {
     },
 };
 
-static void slavio_aux1_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_aux1_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -215,7 +215,7 @@ static void slavio_aux1_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->aux1 = val & 0xff;
 }
 
-static uint64_t slavio_aux1_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_aux1_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -236,7 +236,7 @@ static const MemoryRegionOps slavio_aux1_mem_ops = {
     },
 };
 
-static void slavio_aux2_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_aux2_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -252,7 +252,7 @@ static void slavio_aux2_mem_writeb(void *opaque, target_phys_addr_t addr,
     slavio_misc_update_irq(s);
 }
 
-static uint64_t slavio_aux2_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_aux2_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -273,7 +273,7 @@ static const MemoryRegionOps slavio_aux2_mem_ops = {
     },
 };
 
-static void apc_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void apc_mem_writeb(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     APCState *s = opaque;
@@ -282,7 +282,7 @@ static void apc_mem_writeb(void *opaque, target_phys_addr_t addr,
     qemu_irq_raise(s->cpu_halt);
 }
 
-static uint64_t apc_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t apc_mem_readb(void *opaque, hwaddr addr,
                               unsigned size)
 {
     uint32_t ret = 0;
@@ -301,7 +301,7 @@ static const MemoryRegionOps apc_mem_ops = {
     }
 };
 
-static uint64_t slavio_sysctrl_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_sysctrl_mem_readl(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     MiscState *s = opaque;
@@ -318,7 +318,7 @@ static uint64_t slavio_sysctrl_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_sysctrl_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_sysctrl_mem_writel(void *opaque, hwaddr addr,
                                       uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -346,7 +346,7 @@ static const MemoryRegionOps slavio_sysctrl_mem_ops = {
     },
 };
 
-static uint64_t slavio_led_mem_readw(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_led_mem_readw(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -363,7 +363,7 @@ static uint64_t slavio_led_mem_readw(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_led_mem_writew(void *opaque, target_phys_addr_t addr,
+static void slavio_led_mem_writew(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -478,7 +478,7 @@ static void slavio_misc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_misc;
 }
 
-static TypeInfo slavio_misc_info = {
+static const TypeInfo slavio_misc_info = {
     .name          = "slavio_misc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MiscState),
@@ -492,7 +492,7 @@ static void apc_class_init(ObjectClass *klass, void *data)
     k->init = apc_init1;
 }
 
-static TypeInfo apc_info = {
+static const TypeInfo apc_info = {
     .name          = "apc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MiscState),
diff --git a/hw/slavio_timer.c b/hw/slavio_timer.c
index 97edebb3baf..68a4c0cca46 100644
--- a/hw/slavio_timer.c
+++ b/hw/slavio_timer.c
@@ -23,7 +23,7 @@
  */
 
 #include "sun4m.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 #include "sysbus.h"
 #include "trace.h"
@@ -130,7 +130,7 @@ static void slavio_timer_irq(void *opaque)
     }
 }
 
-static uint64_t slavio_timer_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_timer_mem_readl(void *opaque, hwaddr addr,
                                        unsigned size)
 {
     TimerContext *tc = opaque;
@@ -190,7 +190,7 @@ static uint64_t slavio_timer_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_timer_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_timer_mem_writel(void *opaque, hwaddr addr,
                                     uint64_t val, unsigned size)
 {
     TimerContext *tc = opaque;
@@ -420,7 +420,7 @@ static void slavio_timer_class_init(ObjectClass *klass, void *data)
     dc->props = slavio_timer_properties;
 }
 
-static TypeInfo slavio_timer_info = {
+static const TypeInfo slavio_timer_info = {
     .name          = "slavio_timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLAVIO_TIMERState),
diff --git a/hw/sm501.c b/hw/sm501.c
index 786e07629c9..b7ac7f9bffc 100644
--- a/hw/sm501.c
+++ b/hw/sm501.c
@@ -24,12 +24,13 @@
 
 #include <stdio.h>
 #include "hw.h"
-#include "pc.h"
-#include "console.h"
+#include "serial.h"
+#include "ui/console.h"
 #include "devices.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
-#include "range.h"
+#include "qemu/range.h"
+#include "ui/pixel_ops.h"
 
 /*
  * Status: 2010/05/07
@@ -456,7 +457,7 @@ typedef struct SM501State {
     DisplayState *ds;
 
     /* status & internal resources */
-    target_phys_addr_t base;
+    hwaddr base;
     uint32_t local_mem_size_index;
     uint8_t * local_mem;
     MemoryRegion local_mem_region;
@@ -726,7 +727,7 @@ static void sm501_2d_operation(SM501State * s)
     }
 }
 
-static uint64_t sm501_system_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_system_config_read(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -779,7 +780,7 @@ static uint64_t sm501_system_config_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_system_config_write(void *opaque, target_phys_addr_t addr,
+static void sm501_system_config_write(void *opaque, hwaddr addr,
                                       uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -837,7 +838,7 @@ static const MemoryRegionOps sm501_system_config_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint32_t sm501_palette_read(void *opaque, target_phys_addr_t addr)
+static uint32_t sm501_palette_read(void *opaque, hwaddr addr)
 {
     SM501State * s = (SM501State *)opaque;
     SM501_DPRINTF("sm501 palette read addr=%x\n", (int)addr);
@@ -850,7 +851,7 @@ static uint32_t sm501_palette_read(void *opaque, target_phys_addr_t addr)
 }
 
 static void sm501_palette_write(void *opaque,
-				target_phys_addr_t addr, uint32_t value)
+				hwaddr addr, uint32_t value)
 {
     SM501State * s = (SM501State *)opaque;
     SM501_DPRINTF("sm501 palette write addr=%x, val=%x\n",
@@ -863,7 +864,7 @@ static void sm501_palette_write(void *opaque,
     *(uint32_t*)&s->dc_palette[addr] = value;
 }
 
-static uint64_t sm501_disp_ctrl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_disp_ctrl_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -958,7 +959,7 @@ static uint64_t sm501_disp_ctrl_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_disp_ctrl_write(void *opaque, target_phys_addr_t addr,
+static void sm501_disp_ctrl_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1073,7 +1074,7 @@ static const MemoryRegionOps sm501_disp_ctrl_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t sm501_2d_engine_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_2d_engine_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1093,7 +1094,7 @@ static uint64_t sm501_2d_engine_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_2d_engine_write(void *opaque, target_phys_addr_t addr,
+static void sm501_2d_engine_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1163,8 +1164,6 @@ static const MemoryRegionOps sm501_2d_engine_ops = {
 
 /* draw line functions for all console modes */
 
-#include "pixel_ops.h"
-
 typedef void draw_line_func(uint8_t *d, const uint8_t *s,
 			    int width, const uint32_t *pal);
 
@@ -1351,7 +1350,7 @@ static void sm501_draw_crt(SM501State * s)
 	} else {
 	    if (y_start >= 0) {
 		/* flush to display */
-		dpy_update(s->ds, 0, y_start, width, y - y_start);
+                dpy_gfx_update(s->ds, 0, y_start, width, y - y_start);
 		y_start = -1;
 	    }
 	}
@@ -1362,7 +1361,7 @@ static void sm501_draw_crt(SM501State * s)
 
     /* complete flush to display */
     if (y_start >= 0)
-	dpy_update(s->ds, 0, y_start, width, y - y_start);
+        dpy_gfx_update(s->ds, 0, y_start, width, y - y_start);
 
     /* clear dirty flags */
     if (page_min != ~0l) {
@@ -1429,9 +1428,9 @@ void sm501_init(MemoryRegion *address_space_mem, uint32_t base,
     qdev_prop_set_uint32(dev, "num-ports", 2);
     qdev_prop_set_taddr(dev, "dma-offset", base);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0,
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0,
                     base + MMIO_BASE_OFFSET + SM501_USB_HOST);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     /* bridge to serial emulation module */
     if (chr) {
diff --git a/hw/smbios.c b/hw/smbios.c
index c57237d2792..a7b8bfc383d 100644
--- a/hw/smbios.c
+++ b/hw/smbios.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "smbios.h"
 #include "loader.h"
 
diff --git a/hw/smbus.c b/hw/smbus.c
index e3cf6a2cc8e..a9085915906 100644
--- a/hw/smbus.c
+++ b/hw/smbus.c
@@ -318,7 +318,7 @@ static void smbus_device_class_init(ObjectClass *klass, void *data)
     sc->send = smbus_i2c_send;
 }
 
-static TypeInfo smbus_device_type_info = {
+static const TypeInfo smbus_device_type_info = {
     .name = TYPE_SMBUS_DEVICE,
     .parent = TYPE_I2C_SLAVE,
     .instance_size = sizeof(SMBusDevice),
diff --git a/hw/smbus_eeprom.c b/hw/smbus_eeprom.c
index 11adab01b86..d36dc7bbe3e 100644
--- a/hw/smbus_eeprom.c
+++ b/hw/smbus_eeprom.c
@@ -123,7 +123,7 @@ static void smbus_eeprom_class_initfn(ObjectClass *klass, void *data)
     dc->props = smbus_eeprom_properties;
 }
 
-static TypeInfo smbus_eeprom_info = {
+static const TypeInfo smbus_eeprom_info = {
     .name          = "smbus-eeprom",
     .parent        = TYPE_SMBUS_DEVICE,
     .instance_size = sizeof(SMBusEEPROMDevice),
diff --git a/hw/smbus_ich9.c b/hw/smbus_ich9.c
new file mode 100644
index 00000000000..16db3a743cf
--- /dev/null
+++ b/hw/smbus_ich9.c
@@ -0,0 +1,127 @@
+/*
+ * ACPI implementation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on acpi.c, but heavily rewritten.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ */
+#include "hw.h"
+#include "pc.h"
+#include "pm_smbus.h"
+#include "pci/pci.h"
+#include "sysemu/sysemu.h"
+#include "i2c.h"
+#include "smbus.h"
+
+#include "ich9.h"
+
+#define TYPE_ICH9_SMB_DEVICE "ICH9 SMB"
+#define ICH9_SMB_DEVICE(obj) \
+     OBJECT_CHECK(ICH9SMBState, (obj), TYPE_ICH9_SMB_DEVICE)
+
+typedef struct ICH9SMBState {
+    PCIDevice dev;
+
+    PMSMBus smb;
+} ICH9SMBState;
+
+static const VMStateDescription vmstate_ich9_smbus = {
+    .name = "ich9_smb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, struct ICH9SMBState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ich9_smbus_write_config(PCIDevice *d, uint32_t address,
+                                    uint32_t val, int len)
+{
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
+
+    pci_default_write_config(d, address, val, len);
+    if (range_covers_byte(address, len, ICH9_SMB_HOSTC)) {
+        uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
+        if ((hostc & ICH9_SMB_HOSTC_HST_EN) &&
+            !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
+            memory_region_set_enabled(&s->smb.io, true);
+        } else {
+            memory_region_set_enabled(&s->smb.io, false);
+        }
+    }
+}
+
+static int ich9_smbus_initfn(PCIDevice *d)
+{
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
+
+    /* TODO? D31IP.SMIP in chipset configuration space */
+    pci_config_set_interrupt_pin(d->config, 0x01); /* interrupt pin 1 */
+
+    pci_set_byte(d->config + ICH9_SMB_HOSTC, 0);
+    /* TODO bar0, bar1: 64bit BAR support*/
+
+    pm_smbus_init(&d->qdev, &s->smb);
+    pci_register_bar(d, ICH9_SMB_SMB_BASE_BAR, PCI_BASE_ADDRESS_SPACE_IO,
+                     &s->smb.io);
+    return 0;
+}
+
+static void ich9_smb_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_ICH9_6;
+    k->revision = ICH9_A2_SMB_REVISION;
+    k->class_id = PCI_CLASS_SERIAL_SMBUS;
+    dc->no_user = 1;
+    dc->vmsd = &vmstate_ich9_smbus;
+    dc->desc = "ICH9 SMBUS Bridge";
+    k->init = ich9_smbus_initfn;
+    k->config_write = ich9_smbus_write_config;
+}
+
+i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base)
+{
+    PCIDevice *d =
+        pci_create_simple_multifunction(bus, devfn, true, TYPE_ICH9_SMB_DEVICE);
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
+    return s->smb.smbus;
+}
+
+static const TypeInfo ich9_smb_info = {
+    .name   = TYPE_ICH9_SMB_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(ICH9SMBState),
+    .class_init = ich9_smb_class_init,
+};
+
+static void ich9_smb_register(void)
+{
+    type_register_static(&ich9_smb_info);
+}
+
+type_init(ich9_smb_register);
diff --git a/hw/smc91c111.c b/hw/smc91c111.c
index d6ef302c6d5..fe2389bf253 100644
--- a/hw/smc91c111.c
+++ b/hw/smc91c111.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 /* For crc32 */
 #include <zlib.h>
@@ -254,7 +254,7 @@ static void smc91c111_queue_tx(smc91c111_state *s, int packet)
 
 static void smc91c111_reset(DeviceState *dev)
 {
-    smc91c111_state *s = FROM_SYSBUS(smc91c111_state, sysbus_from_qdev(dev));
+    smc91c111_state *s = FROM_SYSBUS(smc91c111_state, SYS_BUS_DEVICE(dev));
     s->bank = 0;
     s->tx_fifo_len = 0;
     s->tx_fifo_done_len = 0;
@@ -276,7 +276,7 @@ static void smc91c111_reset(DeviceState *dev)
 #define SET_LOW(name, val) s->name = (s->name & 0xff00) | val
 #define SET_HIGH(name, val) s->name = (s->name & 0xff) | (val << 8)
 
-static void smc91c111_writeb(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writeb(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     smc91c111_state *s = (smc91c111_state *)opaque;
@@ -442,6 +442,7 @@ static void smc91c111_writeb(void *opaque, target_phys_addr_t offset,
             return;
         case 12: /* Early receive.  */
             s->ercv = value & 0x1f;
+            return;
         case 13:
             /* Ignore.  */
             return;
@@ -451,7 +452,7 @@ static void smc91c111_writeb(void *opaque, target_phys_addr_t offset,
     hw_error("smc91c111_write: Bad reg %d:%x\n", s->bank, (int)offset);
 }
 
-static uint32_t smc91c111_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readb(void *opaque, hwaddr offset)
 {
     smc91c111_state *s = (smc91c111_state *)opaque;
 
@@ -595,14 +596,14 @@ static uint32_t smc91c111_readb(void *opaque, target_phys_addr_t offset)
     return 0;
 }
 
-static void smc91c111_writew(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writew(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     smc91c111_writeb(opaque, offset, value & 0xff);
     smc91c111_writeb(opaque, offset + 1, value >> 8);
 }
 
-static void smc91c111_writel(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writel(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     /* 32-bit writes to offset 0xc only actually write to the bank select
@@ -612,7 +613,7 @@ static void smc91c111_writel(void *opaque, target_phys_addr_t offset,
     smc91c111_writew(opaque, offset + 2, value >> 16);
 }
 
-static uint32_t smc91c111_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readw(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = smc91c111_readb(opaque, offset);
@@ -620,7 +621,7 @@ static uint32_t smc91c111_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint32_t smc91c111_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readl(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = smc91c111_readw(opaque, offset);
@@ -774,7 +775,7 @@ static void smc91c111_class_init(ObjectClass *klass, void *data)
     dc->props = smc91c111_properties;
 }
 
-static TypeInfo smc91c111_info = {
+static const TypeInfo smc91c111_info = {
     .name          = "smc91c111",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(smc91c111_state),
@@ -797,7 +798,7 @@ void smc91c111_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "smc91c111");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
diff --git a/hw/soc_dma.c b/hw/soc_dma.c
index 03bc8468dd2..64e8ee1d135 100644
--- a/hw/soc_dma.c
+++ b/hw/soc_dma.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "soc_dma.h"
 
 static void transfer_mem2mem(struct soc_dma_ch_s *ch)
@@ -64,7 +64,7 @@ struct dma_s {
 
     struct memmap_entry_s {
         enum soc_dma_port_type type;
-        target_phys_addr_t addr;
+        hwaddr addr;
         union {
            struct {
                void *opaque;
@@ -105,7 +105,7 @@ static void soc_dma_ch_run(void *opaque)
 }
 
 static inline struct memmap_entry_s *soc_dma_lookup(struct dma_s *dma,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     struct memmap_entry_s *lo;
     int hi;
@@ -255,7 +255,7 @@ struct soc_dma_s *soc_dma_init(int n)
     return &s->soc;
 }
 
-void soc_dma_port_add_fifo(struct soc_dma_s *soc, target_phys_addr_t virt_base,
+void soc_dma_port_add_fifo(struct soc_dma_s *soc, hwaddr virt_base,
                 soc_dma_io_t fn, void *opaque, int out)
 {
     struct memmap_entry_s *entry;
@@ -308,7 +308,7 @@ void soc_dma_port_add_fifo(struct soc_dma_s *soc, target_phys_addr_t virt_base,
 }
 
 void soc_dma_port_add_mem(struct soc_dma_s *soc, uint8_t *phys_base,
-                target_phys_addr_t virt_base, size_t size)
+                hwaddr virt_base, size_t size)
 {
     struct memmap_entry_s *entry;
     struct dma_s *dma = (struct dma_s *) soc;
diff --git a/hw/soc_dma.h b/hw/soc_dma.h
index 904b26c5a80..7379731afd7 100644
--- a/hw/soc_dma.h
+++ b/hw/soc_dma.h
@@ -18,7 +18,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "memory.h"
+#ifndef HW_SOC_DMA_H
+#define HW_SOC_DMA_H 1
+
+
+#include "exec/memory.h"
+#include "hw/irq.h"
 
 struct soc_dma_s;
 struct soc_dma_ch_s;
@@ -51,7 +56,7 @@ struct soc_dma_ch_s {
     int bytes;
     /* Initialised by the DMA module, call soc_dma_ch_update after writing.  */
     enum soc_dma_access_type type[2];
-    target_phys_addr_t vaddr[2];	/* Updated by .transfer_fn().  */
+    hwaddr vaddr[2];	/* Updated by .transfer_fn().  */
     /* Private */
     void *paddr[2];
     soc_dma_io_t io_fn[2];
@@ -91,19 +96,21 @@ void soc_dma_ch_update(struct soc_dma_ch_s *ch);
 void soc_dma_reset(struct soc_dma_s *s);
 struct soc_dma_s *soc_dma_init(int n);
 
-void soc_dma_port_add_fifo(struct soc_dma_s *dma, target_phys_addr_t virt_base,
+void soc_dma_port_add_fifo(struct soc_dma_s *dma, hwaddr virt_base,
                 soc_dma_io_t fn, void *opaque, int out);
 void soc_dma_port_add_mem(struct soc_dma_s *dma, uint8_t *phys_base,
-                target_phys_addr_t virt_base, size_t size);
+                hwaddr virt_base, size_t size);
 
 static inline void soc_dma_port_add_fifo_in(struct soc_dma_s *dma,
-                target_phys_addr_t virt_base, soc_dma_io_t fn, void *opaque)
+                hwaddr virt_base, soc_dma_io_t fn, void *opaque)
 {
     return soc_dma_port_add_fifo(dma, virt_base, fn, opaque, 0);
 }
 
 static inline void soc_dma_port_add_fifo_out(struct soc_dma_s *dma,
-                target_phys_addr_t virt_base, soc_dma_io_t fn, void *opaque)
+                hwaddr virt_base, soc_dma_io_t fn, void *opaque)
 {
     return soc_dma_port_add_fifo(dma, virt_base, fn, opaque, 1);
 }
+
+#endif
diff --git a/hw/spapr.c b/hw/spapr.c
index c34b767c6e9..e88a27aa71f 100644
--- a/hw/spapr.c
+++ b/hw/spapr.c
@@ -24,13 +24,13 @@
  * THE SOFTWARE.
  *
  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "hw.h"
 #include "elf.h"
-#include "net.h"
-#include "blockdev.h"
-#include "cpus.h"
-#include "kvm.h"
+#include "net/net.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/cpus.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 #include "hw/boards.h"
@@ -41,15 +41,15 @@
 #include "hw/spapr_vio.h"
 #include "hw/spapr_pci.h"
 #include "hw/xics.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "pci.h"
-#include "vga-pci.h"
+#include "pci/pci.h"
 
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw/usb.h"
+#include "qemu/config-file.h"
 
 #include <libfdt.h>
 
@@ -77,17 +77,13 @@
 #define MAX_CPUS                256
 #define XICS_IRQS               1024
 
-#define SPAPR_PCI_BUID          0x800000020000001ULL
-#define SPAPR_PCI_MEM_WIN_ADDR  (0x10000000000ULL + 0xA0000000)
-#define SPAPR_PCI_MEM_WIN_SIZE  0x20000000
-#define SPAPR_PCI_IO_WIN_ADDR   (0x10000000000ULL + 0x80000000)
-#define SPAPR_PCI_MSI_WIN_ADDR  (0x10000000000ULL + 0x90000000)
-
 #define PHANDLE_XICP            0x00001111
 
+#define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
+
 sPAPREnvironment *spapr;
 
-int spapr_allocate_irq(int hint, enum xics_irq_type type)
+int spapr_allocate_irq(int hint, bool lsi)
 {
     int irq;
 
@@ -103,13 +99,13 @@ int spapr_allocate_irq(int hint, enum xics_irq_type type)
         return 0;
     }
 
-    xics_set_irq_type(spapr->icp, irq, type);
+    xics_set_irq_type(spapr->icp, irq, lsi);
 
     return irq;
 }
 
 /* Allocate block of consequtive IRQs, returns a number of the first */
-int spapr_allocate_irq_block(int num, enum xics_irq_type type)
+int spapr_allocate_irq_block(int num, bool lsi)
 {
     int first = -1;
     int i;
@@ -117,7 +113,7 @@ int spapr_allocate_irq_block(int num, enum xics_irq_type type)
     for (i = 0; i < num; ++i) {
         int irq;
 
-        irq = spapr_allocate_irq(0, type);
+        irq = spapr_allocate_irq(0, lsi);
         if (!irq) {
             return -1;
         }
@@ -134,37 +130,48 @@ int spapr_allocate_irq_block(int num, enum xics_irq_type type)
     return first;
 }
 
-static int spapr_set_associativity(void *fdt, sPAPREnvironment *spapr)
+static int spapr_fixup_cpu_dt(void *fdt, sPAPREnvironment *spapr)
 {
     int ret = 0, offset;
     CPUPPCState *env;
+    CPUState *cpu;
     char cpu_model[32];
     int smt = kvmppc_smt_threads();
+    uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
 
     assert(spapr->cpu_model);
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = CPU(ppc_env_get_cpu(env));
         uint32_t associativity[] = {cpu_to_be32(0x5),
                                     cpu_to_be32(0x0),
                                     cpu_to_be32(0x0),
                                     cpu_to_be32(0x0),
-                                    cpu_to_be32(env->numa_node),
-                                    cpu_to_be32(env->cpu_index)};
+                                    cpu_to_be32(cpu->numa_node),
+                                    cpu_to_be32(cpu->cpu_index)};
 
-        if ((env->cpu_index % smt) != 0) {
+        if ((cpu->cpu_index % smt) != 0) {
             continue;
         }
 
         snprintf(cpu_model, 32, "/cpus/%s@%x", spapr->cpu_model,
-                 env->cpu_index);
+                 cpu->cpu_index);
 
         offset = fdt_path_offset(fdt, cpu_model);
         if (offset < 0) {
             return offset;
         }
 
-        ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
-                          sizeof(associativity));
+        if (nb_numa_nodes > 1) {
+            ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
+                              sizeof(associativity));
+            if (ret < 0) {
+                return ret;
+            }
+        }
+
+        ret = fdt_setprop(fdt, offset, "ibm,pft-size",
+                          pft_size_prop, sizeof(pft_size_prop));
         if (ret < 0) {
             return ret;
         }
@@ -206,45 +213,37 @@ static size_t create_page_sizes_prop(CPUPPCState *env, uint32_t *prop,
     return (p - prop) * sizeof(uint32_t);
 }
 
+#define _FDT(exp) \
+    do { \
+        int ret = (exp);                                           \
+        if (ret < 0) {                                             \
+            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
+                    #exp, fdt_strerror(ret));                      \
+            exit(1);                                               \
+        }                                                          \
+    } while (0)
+
+
 static void *spapr_create_fdt_skel(const char *cpu_model,
-                                   target_phys_addr_t rma_size,
-                                   target_phys_addr_t initrd_base,
-                                   target_phys_addr_t initrd_size,
-                                   target_phys_addr_t kernel_size,
+                                   hwaddr initrd_base,
+                                   hwaddr initrd_size,
+                                   hwaddr kernel_size,
                                    const char *boot_device,
                                    const char *kernel_cmdline,
-                                   long hash_shift)
+                                   uint32_t epow_irq)
 {
     void *fdt;
     CPUPPCState *env;
-    uint64_t mem_reg_property[2];
     uint32_t start_prop = cpu_to_be32(initrd_base);
     uint32_t end_prop = cpu_to_be32(initrd_base + initrd_size);
-    uint32_t pft_size_prop[] = {0, cpu_to_be32(hash_shift)};
     char hypertas_prop[] = "hcall-pft\0hcall-term\0hcall-dabr\0hcall-interrupt"
         "\0hcall-tce\0hcall-vio\0hcall-splpar\0hcall-bulk";
     char qemu_hypertas_prop[] = "hcall-memop1";
+    uint32_t refpoints[] = {cpu_to_be32(0x4), cpu_to_be32(0x4)};
     uint32_t interrupt_server_ranges_prop[] = {0, cpu_to_be32(smp_cpus)};
-    int i;
     char *modelname;
-    int smt = kvmppc_smt_threads();
+    int i, smt = kvmppc_smt_threads();
     unsigned char vec5[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x80};
-    uint32_t refpoints[] = {cpu_to_be32(0x4), cpu_to_be32(0x4)};
-    uint32_t associativity[] = {cpu_to_be32(0x4), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0)};
-    char mem_name[32];
-    target_phys_addr_t node0_size, mem_start;
-
-#define _FDT(exp) \
-    do { \
-        int ret = (exp);                                           \
-        if (ret < 0) {                                             \
-            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
-                    #exp, fdt_strerror(ret));                      \
-            exit(1);                                               \
-        }                                                          \
-    } while (0)
 
     fdt = g_malloc0(FDT_MAX_SIZE);
     _FDT((fdt_create(fdt, FDT_MAX_SIZE)));
@@ -282,62 +281,15 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
 
         _FDT((fdt_property(fdt, "qemu,boot-kernel", &kprop, sizeof(kprop))));
     }
-    _FDT((fdt_property_string(fdt, "qemu,boot-device", boot_device)));
+    if (boot_device) {
+        _FDT((fdt_property_string(fdt, "qemu,boot-device", boot_device)));
+    }
     _FDT((fdt_property_cell(fdt, "qemu,graphic-width", graphic_width)));
     _FDT((fdt_property_cell(fdt, "qemu,graphic-height", graphic_height)));
     _FDT((fdt_property_cell(fdt, "qemu,graphic-depth", graphic_depth)));
 
     _FDT((fdt_end_node(fdt)));
 
-    /* memory node(s) */
-    node0_size = (nb_numa_nodes > 1) ? node_mem[0] : ram_size;
-    if (rma_size > node0_size) {
-        rma_size = node0_size;
-    }
-
-    /* RMA */
-    mem_reg_property[0] = 0;
-    mem_reg_property[1] = cpu_to_be64(rma_size);
-    _FDT((fdt_begin_node(fdt, "memory@0")));
-    _FDT((fdt_property_string(fdt, "device_type", "memory")));
-    _FDT((fdt_property(fdt, "reg", mem_reg_property,
-        sizeof(mem_reg_property))));
-    _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-        sizeof(associativity))));
-    _FDT((fdt_end_node(fdt)));
-
-    /* RAM: Node 0 */
-    if (node0_size > rma_size) {
-        mem_reg_property[0] = cpu_to_be64(rma_size);
-        mem_reg_property[1] = cpu_to_be64(node0_size - rma_size);
-
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, rma_size);
-        _FDT((fdt_begin_node(fdt, mem_name)));
-        _FDT((fdt_property_string(fdt, "device_type", "memory")));
-        _FDT((fdt_property(fdt, "reg", mem_reg_property,
-                           sizeof(mem_reg_property))));
-        _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-                           sizeof(associativity))));
-        _FDT((fdt_end_node(fdt)));
-    }
-
-    /* RAM: Node 1 and beyond */
-    mem_start = node0_size;
-    for (i = 1; i < nb_numa_nodes; i++) {
-        mem_reg_property[0] = cpu_to_be64(mem_start);
-        mem_reg_property[1] = cpu_to_be64(node_mem[i]);
-        associativity[3] = associativity[4] = cpu_to_be32(i);
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, mem_start);
-        _FDT((fdt_begin_node(fdt, mem_name)));
-        _FDT((fdt_property_string(fdt, "device_type", "memory")));
-        _FDT((fdt_property(fdt, "reg", mem_reg_property,
-            sizeof(mem_reg_property))));
-        _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-            sizeof(associativity))));
-        _FDT((fdt_end_node(fdt)));
-        mem_start += node_mem[i];
-    }
-
     /* cpus */
     _FDT((fdt_begin_node(fdt, "cpus")));
 
@@ -354,7 +306,8 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
     spapr->cpu_model = g_strdup(modelname);
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        int index = env->cpu_index;
+        CPUState *cpu = CPU(ppc_env_get_cpu(env));
+        int index = cpu->cpu_index;
         uint32_t servers_prop[smp_threads];
         uint32_t gservers_prop[smp_threads * 2];
         char *nodename;
@@ -369,14 +322,11 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
             continue;
         }
 
-        if (asprintf(&nodename, "%s@%x", modelname, index) < 0) {
-            fprintf(stderr, "Allocation failure\n");
-            exit(1);
-        }
+        nodename = g_strdup_printf("%s@%x", modelname, index);
 
         _FDT((fdt_begin_node(fdt, nodename)));
 
-        free(nodename);
+        g_free(nodename);
 
         _FDT((fdt_property_cell(fdt, "reg", index)));
         _FDT((fdt_property_string(fdt, "device_type", "cpu")));
@@ -389,8 +339,6 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
         _FDT((fdt_property_cell(fdt, "timebase-frequency", tbfreq)));
         _FDT((fdt_property_cell(fdt, "clock-frequency", cpufreq)));
         _FDT((fdt_property_cell(fdt, "ibm,slb-size", env->slb_nr)));
-        _FDT((fdt_property(fdt, "ibm,pft-size",
-                           pft_size_prop, sizeof(pft_size_prop))));
         _FDT((fdt_property_string(fdt, "status", "okay")));
         _FDT((fdt_property(fdt, "64-bit", NULL, 0)));
 
@@ -453,6 +401,8 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
     _FDT((fdt_property(fdt, "ibm,associativity-reference-points",
         refpoints, sizeof(refpoints))));
 
+    _FDT((fdt_property_cell(fdt, "rtas-error-log-max", RTAS_ERROR_LOG_MAX)));
+
     _FDT((fdt_end_node(fdt)));
 
     /* interrupt controller */
@@ -483,16 +433,81 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
 
     _FDT((fdt_end_node(fdt)));
 
+    /* event-sources */
+    spapr_events_fdt_skel(fdt, epow_irq);
+
     _FDT((fdt_end_node(fdt))); /* close root node */
     _FDT((fdt_finish(fdt)));
 
     return fdt;
 }
 
+static int spapr_populate_memory(sPAPREnvironment *spapr, void *fdt)
+{
+    uint32_t associativity[] = {cpu_to_be32(0x4), cpu_to_be32(0x0),
+                                cpu_to_be32(0x0), cpu_to_be32(0x0),
+                                cpu_to_be32(0x0)};
+    char mem_name[32];
+    hwaddr node0_size, mem_start;
+    uint64_t mem_reg_property[2];
+    int i, off;
+
+    /* memory node(s) */
+    node0_size = (nb_numa_nodes > 1) ? node_mem[0] : ram_size;
+    if (spapr->rma_size > node0_size) {
+        spapr->rma_size = node0_size;
+    }
+
+    /* RMA */
+    mem_reg_property[0] = 0;
+    mem_reg_property[1] = cpu_to_be64(spapr->rma_size);
+    off = fdt_add_subnode(fdt, 0, "memory@0");
+    _FDT(off);
+    _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+    _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                      sizeof(mem_reg_property))));
+    _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                      sizeof(associativity))));
+
+    /* RAM: Node 0 */
+    if (node0_size > spapr->rma_size) {
+        mem_reg_property[0] = cpu_to_be64(spapr->rma_size);
+        mem_reg_property[1] = cpu_to_be64(node0_size - spapr->rma_size);
+
+        sprintf(mem_name, "memory@" TARGET_FMT_lx, spapr->rma_size);
+        off = fdt_add_subnode(fdt, 0, mem_name);
+        _FDT(off);
+        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                          sizeof(mem_reg_property))));
+        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                          sizeof(associativity))));
+    }
+
+    /* RAM: Node 1 and beyond */
+    mem_start = node0_size;
+    for (i = 1; i < nb_numa_nodes; i++) {
+        mem_reg_property[0] = cpu_to_be64(mem_start);
+        mem_reg_property[1] = cpu_to_be64(node_mem[i]);
+        associativity[3] = associativity[4] = cpu_to_be32(i);
+        sprintf(mem_name, "memory@" TARGET_FMT_lx, mem_start);
+        off = fdt_add_subnode(fdt, 0, mem_name);
+        _FDT(off);
+        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                          sizeof(mem_reg_property))));
+        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                          sizeof(associativity))));
+        mem_start += node_mem[i];
+    }
+
+    return 0;
+}
+
 static void spapr_finalize_fdt(sPAPREnvironment *spapr,
-                               target_phys_addr_t fdt_addr,
-                               target_phys_addr_t rtas_addr,
-                               target_phys_addr_t rtas_size)
+                               hwaddr fdt_addr,
+                               hwaddr rtas_addr,
+                               hwaddr rtas_size)
 {
     int ret;
     void *fdt;
@@ -503,6 +518,12 @@ static void spapr_finalize_fdt(sPAPREnvironment *spapr,
     /* open out the base tree into a temp buffer for the final tweaks */
     _FDT((fdt_open_into(spapr->fdt_skel, fdt, FDT_MAX_SIZE)));
 
+    ret = spapr_populate_memory(spapr, fdt);
+    if (ret < 0) {
+        fprintf(stderr, "couldn't setup memory nodes in fdt\n");
+        exit(1);
+    }
+
     ret = spapr_populate_vdevice(spapr->vio_bus, fdt);
     if (ret < 0) {
         fprintf(stderr, "couldn't setup vio devices in fdt\n");
@@ -525,11 +546,9 @@ static void spapr_finalize_fdt(sPAPREnvironment *spapr,
     }
 
     /* Advertise NUMA via ibm,associativity */
-    if (nb_numa_nodes > 1) {
-        ret = spapr_set_associativity(fdt, spapr);
-        if (ret < 0) {
-            fprintf(stderr, "Couldn't set up NUMA device tree properties\n");
-        }
+    ret = spapr_fixup_cpu_dt(fdt, spapr);
+    if (ret < 0) {
+        fprintf(stderr, "Couldn't finalize CPU device tree properties\n");
     }
 
     if (!spapr->has_graphics) {
@@ -554,17 +573,53 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static void emulate_spapr_hypercall(CPUPPCState *env)
+static void emulate_spapr_hypercall(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+
+    if (msr_pr) {
+        hcall_dprintf("Hypercall made with MSR[PR]=1\n");
+        env->gpr[3] = H_PRIVILEGE;
+    } else {
+        env->gpr[3] = spapr_hypercall(cpu, env->gpr[3], &env->gpr[4]);
+    }
+}
+
+static void spapr_reset_htab(sPAPREnvironment *spapr)
 {
-    env->gpr[3] = spapr_hypercall(env, env->gpr[3], &env->gpr[4]);
+    long shift;
+
+    /* allocate hash page table.  For now we always make this 16mb,
+     * later we should probably make it scale to the size of guest
+     * RAM */
+
+    shift = kvmppc_reset_htab(spapr->htab_shift);
+
+    if (shift > 0) {
+        /* Kernel handles htab, we don't need to allocate one */
+        spapr->htab_shift = shift;
+    } else {
+        if (!spapr->htab) {
+            /* Allocate an htab if we don't yet have one */
+            spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
+        }
+
+        /* And clear it */
+        memset(spapr->htab, 0, HTAB_SIZE(spapr));
+    }
+
+    /* Update the RMA size if necessary */
+    if (spapr->vrma_adjust) {
+        spapr->rma_size = kvmppc_rma_size(ram_size, spapr->htab_shift);
+    }
 }
 
-static void spapr_reset(void *opaque)
+static void ppc_spapr_reset(void)
 {
-    sPAPREnvironment *spapr = (sPAPREnvironment *)opaque;
+    /* Reset the hash table & recalc the RMA */
+    spapr_reset_htab(spapr);
 
-    /* flush out the hash table */
-    memset(spapr->htab, 0, spapr->htab_size);
+    qemu_devices_reset();
 
     /* Load the fdt */
     spapr_finalize_fdt(spapr, spapr->fdt_addr, spapr->rtas_addr,
@@ -581,19 +636,61 @@ static void spapr_reset(void *opaque)
 static void spapr_cpu_reset(void *opaque)
 {
     PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     cpu_reset(CPU(cpu));
+
+    /* All CPUs start halted.  CPU0 is unhalted from the machine level
+     * reset code and the rest are explicitly started up by the guest
+     * using an RTAS call */
+    env->halted = 1;
+
+    env->spr[SPR_HIOR] = 0;
+
+    env->external_htab = spapr->htab;
+    env->htab_base = -1;
+    env->htab_mask = HTAB_SIZE(spapr) - 1;
+    env->spr[SPR_SDR1] = (unsigned long)spapr->htab |
+        (spapr->htab_shift - 18);
+}
+
+static void spapr_create_nvram(sPAPREnvironment *spapr)
+{
+    QemuOpts *machine_opts;
+    DeviceState *dev;
+
+    dev = qdev_create(&spapr->vio_bus->bus, "spapr-nvram");
+
+    machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (machine_opts) {
+        const char *drivename;
+
+        drivename = qemu_opt_get(machine_opts, "nvram");
+        if (drivename) {
+            BlockDriverState *bs;
+
+            bs = bdrv_find(drivename);
+            if (!bs) {
+                fprintf(stderr, "No such block device \"%s\" for nvram\n",
+                        drivename);
+                exit(1);
+            }
+            qdev_prop_set_drive_nofail(dev, "drive", bs);
+        }
+    }
+
+    qdev_init_nofail(dev);
+
+    spapr->nvram = (struct sPAPRNVRAM *)dev;
 }
 
 /* Returns whether we want to use VGA or not */
 static int spapr_vga_init(PCIBus *pci_bus)
 {
     switch (vga_interface_type) {
-    case VGA_STD:
-        pci_vga_init(pci_bus);
-        return 1;
     case VGA_NONE:
-        return 0;
+    case VGA_STD:
+        return pci_vga_init(pci_bus) != NULL;
     default:
         fprintf(stderr, "This vga model is not supported,"
                 "currently it only supports -vga std\n");
@@ -603,24 +700,24 @@ static int spapr_vga_init(PCIBus *pci_bus)
 }
 
 /* pSeries LPAR / sPAPR hardware init */
-static void ppc_spapr_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ppc_spapr_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     PowerPCCPU *cpu;
     CPUPPCState *env;
     PCIHostState *phb;
     int i;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    target_phys_addr_t rma_alloc_size, rma_size;
+    hwaddr rma_alloc_size;
     uint32_t initrd_base = 0;
     long kernel_size = 0, initrd_size = 0;
     long load_limit, rtas_limit, fw_size;
-    long pteg_shift = 17;
     char *filename;
 
     msi_supported = true;
@@ -637,20 +734,46 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         hw_error("qemu: Unable to create RMA\n");
         exit(1);
     }
+
     if (rma_alloc_size && (rma_alloc_size < ram_size)) {
-        rma_size = rma_alloc_size;
+        spapr->rma_size = rma_alloc_size;
     } else {
-        rma_size = ram_size;
+        spapr->rma_size = ram_size;
+
+        /* With KVM, we don't actually know whether KVM supports an
+         * unbounded RMA (PR KVM) or is limited by the hash table size
+         * (HV KVM using VRMA), so we always assume the latter
+         *
+         * In that case, we also limit the initial allocations for RTAS
+         * etc... to 256M since we have no way to know what the VRMA size
+         * is going to be as it depends on the size of the hash table
+         * isn't determined yet.
+         */
+        if (kvm_enabled()) {
+            spapr->vrma_adjust = 1;
+            spapr->rma_size = MIN(spapr->rma_size, 0x10000000);
+        }
     }
 
     /* We place the device tree and RTAS just below either the top of the RMA,
      * or just below 2GB, whichever is lowere, so that it can be
      * processed with 32-bit real mode code if necessary */
-    rtas_limit = MIN(rma_size, 0x80000000);
+    rtas_limit = MIN(spapr->rma_size, 0x80000000);
     spapr->rtas_addr = rtas_limit - RTAS_MAX_SIZE;
     spapr->fdt_addr = spapr->rtas_addr - FDT_MAX_SIZE;
     load_limit = spapr->fdt_addr - FW_OVERHEAD;
 
+    /* We aim for a hash table of size 1/128 the size of RAM.  The
+     * normal rule of thumb is 1/64 the size of RAM, but that's much
+     * more than needed for the Linux guests we support. */
+    spapr->htab_shift = 18; /* Minimum architected size */
+    while (spapr->htab_shift <= 46) {
+        if ((1ULL << (spapr->htab_shift + 7)) >= ram_size) {
+            break;
+        }
+        spapr->htab_shift++;
+    }
+
     /* init CPUs */
     if (cpu_model == NULL) {
         cpu_model = kvm_enabled() ? "host" : "POWER7";
@@ -665,11 +788,16 @@ static void ppc_spapr_init(ram_addr_t ram_size,
 
         /* Set time-base frequency to 512 MHz */
         cpu_ppc_tb_init(env, TIMEBASE_FREQ);
-        qemu_register_reset(spapr_cpu_reset, cpu);
 
-        env->hreset_vector = 0x60;
+        /* PAPR always has exception vectors in RAM not ROM */
         env->hreset_excp_prefix = 0;
-        env->gpr[3] = env->cpu_index;
+
+        /* Tell KVM that we're in PAPR mode */
+        if (kvm_enabled()) {
+            kvmppc_set_papr(cpu);
+        }
+
+        qemu_register_reset(spapr_cpu_reset, cpu);
     }
 
     /* allocate RAM */
@@ -683,27 +811,6 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         memory_region_add_subregion(sysmem, nonrma_base, ram);
     }
 
-    /* allocate hash page table.  For now we always make this 16mb,
-     * later we should probably make it scale to the size of guest
-     * RAM */
-    spapr->htab_size = 1ULL << (pteg_shift + 7);
-    spapr->htab = qemu_memalign(spapr->htab_size, spapr->htab_size);
-
-    for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->external_htab = spapr->htab;
-        env->htab_base = -1;
-        env->htab_mask = spapr->htab_size - 1;
-
-        /* Tell KVM that we're in PAPR mode */
-        env->spr[SPR_SDR1] = (unsigned long)spapr->htab |
-                             ((pteg_shift + 7) - 18);
-        env->spr[SPR_HIOR] = 0;
-
-        if (kvm_enabled()) {
-            kvmppc_set_papr(env);
-        }
-    }
-
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
     spapr->rtas_size = load_image_targphys(filename, spapr->rtas_addr,
                                            rtas_limit - spapr->rtas_addr);
@@ -721,7 +828,10 @@ static void ppc_spapr_init(ram_addr_t ram_size,
 
     /* Set up Interrupt Controller */
     spapr->icp = xics_system_init(XICS_IRQS);
-    spapr->next_irq = 16;
+    spapr->next_irq = XICS_IRQ_BASE;
+
+    /* Set up EPOW events infrastructure */
+    spapr_events_init(spapr);
 
     /* Set up IOMMU */
     spapr_iommu_init();
@@ -735,15 +845,13 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         }
     }
 
+    /* We always have at least the nvram device on VIO */
+    spapr_create_nvram(spapr);
+
     /* Set up PCI */
     spapr_pci_rtas_init();
 
-    spapr_create_phb(spapr, "pci", SPAPR_PCI_BUID,
-                     SPAPR_PCI_MEM_WIN_ADDR,
-                     SPAPR_PCI_MEM_WIN_SIZE,
-                     SPAPR_PCI_IO_WIN_ADDR,
-                     SPAPR_PCI_MSI_WIN_ADDR);
-    phb = PCI_HOST_BRIDGE(QLIST_FIRST(&spapr->phbs));
+    phb = spapr_create_phb(spapr, 0, "pci");
 
     for (i = 0; i < nb_nics; i++) {
         NICInfo *nd = &nd_table[i];
@@ -768,7 +876,7 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         spapr->has_graphics = true;
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(spapr->has_graphics)) {
         pci_create_simple(phb->bus, -1, "pci-ohci");
         if (spapr->has_graphics) {
             usbdevice_create("keyboard");
@@ -776,7 +884,7 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         }
     }
 
-    if (rma_size < (MIN_RMA_SLOF << 20)) {
+    if (spapr->rma_size < (MIN_RMA_SLOF << 20)) {
         fprintf(stderr, "qemu: pSeries SLOF firmware requires >= "
                 "%ldM guest RMA (Real Mode Area memory)\n", MIN_RMA_SLOF);
         exit(1);
@@ -827,29 +935,24 @@ static void ppc_spapr_init(ram_addr_t ram_size,
 
     spapr->entry_point = 0x100;
 
-    /* SLOF will startup the secondary CPUs using RTAS */
-    for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->halted = 1;
-    }
-
     /* Prepare the device tree */
-    spapr->fdt_skel = spapr_create_fdt_skel(cpu_model, rma_size,
+    spapr->fdt_skel = spapr_create_fdt_skel(cpu_model,
                                             initrd_base, initrd_size,
                                             kernel_size,
                                             boot_device, kernel_cmdline,
-                                            pteg_shift + 7);
+                                            spapr->epow_irq);
     assert(spapr->fdt_skel != NULL);
-
-    qemu_register_reset(spapr_reset, spapr);
 }
 
 static QEMUMachine spapr_machine = {
     .name = "pseries",
     .desc = "pSeries Logical Partition (PAPR compliant)",
     .init = ppc_spapr_init,
+    .reset = ppc_spapr_reset,
+    .block_default_type = IF_SCSI,
     .max_cpus = MAX_CPUS,
     .no_parallel = 1,
-    .use_scsi = 1,
+    .boot_order = NULL,
 };
 
 static void spapr_machine_init(void)
diff --git a/hw/spapr.h b/hw/spapr.h
index ac34a171e35..3a1f69f2a9f 100644
--- a/hw/spapr.h
+++ b/hw/spapr.h
@@ -1,22 +1,26 @@
 #if !defined(__HW_SPAPR_H__)
 #define __HW_SPAPR_H__
 
-#include "dma.h"
+#include "sysemu/dma.h"
 #include "hw/xics.h"
 
 struct VIOsPAPRBus;
 struct sPAPRPHBState;
+struct sPAPRNVRAM;
 struct icp_state;
 
 typedef struct sPAPREnvironment {
     struct VIOsPAPRBus *vio_bus;
     QLIST_HEAD(, sPAPRPHBState) phbs;
+    struct sPAPRNVRAM *nvram;
     struct icp_state *icp;
 
-    target_phys_addr_t ram_limit;
+    hwaddr ram_limit;
     void *htab;
-    long htab_size;
-    target_phys_addr_t fdt_addr, rtas_addr;
+    long htab_shift;
+    hwaddr rma_size;
+    int vrma_adjust;
+    hwaddr fdt_addr, rtas_addr;
     long rtas_size;
     void *fdt_skel;
     target_ulong entry_point;
@@ -24,6 +28,9 @@ typedef struct sPAPREnvironment {
     int rtc_offset;
     char *cpu_model;
     bool has_graphics;
+
+    uint32_t epow_irq;
+    Notifier epow_notifier;
 } sPAPREnvironment;
 
 #define H_SUCCESS         0
@@ -281,25 +288,25 @@ extern sPAPREnvironment *spapr;
     do { } while (0)
 #endif
 
-typedef target_ulong (*spapr_hcall_fn)(CPUPPCState *env, sPAPREnvironment *spapr,
+typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode,
                                        target_ulong *args);
 
 void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn);
-target_ulong spapr_hypercall(CPUPPCState *env, target_ulong opcode,
+target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args);
 
-int spapr_allocate_irq(int hint, enum xics_irq_type type);
-int spapr_allocate_irq_block(int num, enum xics_irq_type type);
+int spapr_allocate_irq(int hint, bool lsi);
+int spapr_allocate_irq_block(int num, bool lsi);
 
 static inline int spapr_allocate_msi(int hint)
 {
-    return spapr_allocate_irq(hint, XICS_MSI);
+    return spapr_allocate_irq(hint, false);
 }
 
 static inline int spapr_allocate_lsi(int hint)
 {
-    return spapr_allocate_irq(hint, XICS_LSI);
+    return spapr_allocate_irq(hint, true);
 }
 
 static inline uint32_t rtas_ld(target_ulong phys, int n)
@@ -315,12 +322,12 @@ static inline void rtas_st(target_ulong phys, int n, uint32_t val)
 typedef void (*spapr_rtas_fn)(sPAPREnvironment *spapr, uint32_t token,
                               uint32_t nargs, target_ulong args,
                               uint32_t nret, target_ulong rets);
-void spapr_rtas_register(const char *name, spapr_rtas_fn fn);
+int spapr_rtas_register(const char *name, spapr_rtas_fn fn);
 target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
                              uint32_t token, uint32_t nargs, target_ulong args,
                              uint32_t nret, target_ulong rets);
-int spapr_rtas_device_tree_setup(void *fdt, target_phys_addr_t rtas_addr,
-                                 target_phys_addr_t rtas_size);
+int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
+                                 hwaddr rtas_size);
 
 #define SPAPR_TCE_PAGE_SHIFT   12
 #define SPAPR_TCE_PAGE_SIZE    (1ULL << SPAPR_TCE_PAGE_SHIFT)
@@ -333,9 +340,16 @@ typedef struct sPAPRTCE {
 #define SPAPR_VIO_BASE_LIOBN    0x00000000
 #define SPAPR_PCI_BASE_LIOBN    0x80000000
 
+#define RTAS_ERROR_LOG_MAX      2048
+
+
 void spapr_iommu_init(void);
+void spapr_events_init(sPAPREnvironment *spapr);
+void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
 DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size);
 void spapr_tce_free(DMAContext *dma);
+void spapr_tce_reset(DMAContext *dma);
+void spapr_tce_set_bypass(DMAContext *dma, bool bypass);
 int spapr_dma_dt(void *fdt, int node_off, const char *propname,
                  uint32_t liobn, uint64_t window, uint32_t size);
 int spapr_tcet_dma_dt(void *fdt, int node_off, const char *propname,
diff --git a/hw/spapr_events.c b/hw/spapr_events.c
new file mode 100644
index 00000000000..ce78f0922e7
--- /dev/null
+++ b/hw/spapr_events.c
@@ -0,0 +1,321 @@
+/*
+ * QEMU PowerPC pSeries Logical Partition (aka sPAPR) hardware System Emulator
+ *
+ * RTAS events handling
+ *
+ * Copyright (c) 2012 David Gibson, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+#include "cpu.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
+#include "hw/qdev.h"
+#include "sysemu/device_tree.h"
+
+#include "hw/spapr.h"
+#include "hw/spapr_vio.h"
+
+#include <libfdt.h>
+
+struct rtas_error_log {
+    uint32_t summary;
+#define RTAS_LOG_VERSION_MASK                   0xff000000
+#define   RTAS_LOG_VERSION_6                    0x06000000
+#define RTAS_LOG_SEVERITY_MASK                  0x00e00000
+#define   RTAS_LOG_SEVERITY_ALREADY_REPORTED    0x00c00000
+#define   RTAS_LOG_SEVERITY_FATAL               0x00a00000
+#define   RTAS_LOG_SEVERITY_ERROR               0x00800000
+#define   RTAS_LOG_SEVERITY_ERROR_SYNC          0x00600000
+#define   RTAS_LOG_SEVERITY_WARNING             0x00400000
+#define   RTAS_LOG_SEVERITY_EVENT               0x00200000
+#define   RTAS_LOG_SEVERITY_NO_ERROR            0x00000000
+#define RTAS_LOG_DISPOSITION_MASK               0x00180000
+#define   RTAS_LOG_DISPOSITION_FULLY_RECOVERED  0x00000000
+#define   RTAS_LOG_DISPOSITION_LIMITED_RECOVERY 0x00080000
+#define   RTAS_LOG_DISPOSITION_NOT_RECOVERED    0x00100000
+#define RTAS_LOG_OPTIONAL_PART_PRESENT          0x00040000
+#define RTAS_LOG_INITIATOR_MASK                 0x0000f000
+#define   RTAS_LOG_INITIATOR_UNKNOWN            0x00000000
+#define   RTAS_LOG_INITIATOR_CPU                0x00001000
+#define   RTAS_LOG_INITIATOR_PCI                0x00002000
+#define   RTAS_LOG_INITIATOR_MEMORY             0x00004000
+#define   RTAS_LOG_INITIATOR_HOTPLUG            0x00006000
+#define RTAS_LOG_TARGET_MASK                    0x00000f00
+#define   RTAS_LOG_TARGET_UNKNOWN               0x00000000
+#define   RTAS_LOG_TARGET_CPU                   0x00000100
+#define   RTAS_LOG_TARGET_PCI                   0x00000200
+#define   RTAS_LOG_TARGET_MEMORY                0x00000400
+#define   RTAS_LOG_TARGET_HOTPLUG               0x00000600
+#define RTAS_LOG_TYPE_MASK                      0x000000ff
+#define   RTAS_LOG_TYPE_OTHER                   0x00000000
+#define   RTAS_LOG_TYPE_RETRY                   0x00000001
+#define   RTAS_LOG_TYPE_TCE_ERR                 0x00000002
+#define   RTAS_LOG_TYPE_INTERN_DEV_FAIL         0x00000003
+#define   RTAS_LOG_TYPE_TIMEOUT                 0x00000004
+#define   RTAS_LOG_TYPE_DATA_PARITY             0x00000005
+#define   RTAS_LOG_TYPE_ADDR_PARITY             0x00000006
+#define   RTAS_LOG_TYPE_CACHE_PARITY            0x00000007
+#define   RTAS_LOG_TYPE_ADDR_INVALID            0x00000008
+#define   RTAS_LOG_TYPE_ECC_UNCORR              0x00000009
+#define   RTAS_LOG_TYPE_ECC_CORR                0x0000000a
+#define   RTAS_LOG_TYPE_EPOW                    0x00000040
+    uint32_t extended_length;
+} QEMU_PACKED;
+
+struct rtas_event_log_v6 {
+    uint8_t b0;
+#define RTAS_LOG_V6_B0_VALID                          0x80
+#define RTAS_LOG_V6_B0_UNRECOVERABLE_ERROR            0x40
+#define RTAS_LOG_V6_B0_RECOVERABLE_ERROR              0x20
+#define RTAS_LOG_V6_B0_DEGRADED_OPERATION             0x10
+#define RTAS_LOG_V6_B0_PREDICTIVE_ERROR               0x08
+#define RTAS_LOG_V6_B0_NEW_LOG                        0x04
+#define RTAS_LOG_V6_B0_BIGENDIAN                      0x02
+    uint8_t _resv1;
+    uint8_t b2;
+#define RTAS_LOG_V6_B2_POWERPC_FORMAT                 0x80
+#define RTAS_LOG_V6_B2_LOG_FORMAT_MASK                0x0f
+#define   RTAS_LOG_V6_B2_LOG_FORMAT_PLATFORM_EVENT    0x0e
+    uint8_t _resv2[9];
+    uint32_t company;
+#define RTAS_LOG_V6_COMPANY_IBM                 0x49424d00 /* IBM<null> */
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_section_header {
+    uint16_t section_id;
+    uint16_t section_length;
+    uint8_t section_version;
+    uint8_t section_subtype;
+    uint16_t creator_component_id;
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_maina {
+#define RTAS_LOG_V6_SECTION_ID_MAINA                0x5048 /* PH */
+    struct rtas_event_log_v6_section_header hdr;
+    uint32_t creation_date; /* BCD: YYYYMMDD */
+    uint32_t creation_time; /* BCD: HHMMSS00 */
+    uint8_t _platform1[8];
+    char creator_id;
+    uint8_t _resv1[2];
+    uint8_t section_count;
+    uint8_t _resv2[4];
+    uint8_t _platform2[8];
+    uint32_t plid;
+    uint8_t _platform3[4];
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_mainb {
+#define RTAS_LOG_V6_SECTION_ID_MAINB                0x5548 /* UH */
+    struct rtas_event_log_v6_section_header hdr;
+    uint8_t subsystem_id;
+    uint8_t _platform1;
+    uint8_t event_severity;
+    uint8_t event_subtype;
+    uint8_t _platform2[4];
+    uint8_t _resv1[2];
+    uint16_t action_flags;
+    uint8_t _resv2[4];
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_epow {
+#define RTAS_LOG_V6_SECTION_ID_EPOW                 0x4550 /* EP */
+    struct rtas_event_log_v6_section_header hdr;
+    uint8_t sensor_value;
+#define RTAS_LOG_V6_EPOW_ACTION_RESET                    0
+#define RTAS_LOG_V6_EPOW_ACTION_WARN_COOLING             1
+#define RTAS_LOG_V6_EPOW_ACTION_WARN_POWER               2
+#define RTAS_LOG_V6_EPOW_ACTION_SYSTEM_SHUTDOWN          3
+#define RTAS_LOG_V6_EPOW_ACTION_SYSTEM_HALT              4
+#define RTAS_LOG_V6_EPOW_ACTION_MAIN_ENCLOSURE           5
+#define RTAS_LOG_V6_EPOW_ACTION_POWER_OFF                7
+    uint8_t event_modifier;
+#define RTAS_LOG_V6_EPOW_MODIFIER_NORMAL                 1
+#define RTAS_LOG_V6_EPOW_MODIFIER_ON_UPS                 2
+#define RTAS_LOG_V6_EPOW_MODIFIER_CRITICAL               3
+#define RTAS_LOG_V6_EPOW_MODIFIER_TEMPERATURE            4
+    uint8_t extended_modifier;
+#define RTAS_LOG_V6_EPOW_XMODIFIER_SYSTEM_WIDE           0
+#define RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC    1
+    uint8_t _resv;
+    uint64_t reason_code;
+} QEMU_PACKED;
+
+struct epow_log_full {
+    struct rtas_error_log hdr;
+    struct rtas_event_log_v6 v6hdr;
+    struct rtas_event_log_v6_maina maina;
+    struct rtas_event_log_v6_mainb mainb;
+    struct rtas_event_log_v6_epow epow;
+} QEMU_PACKED;
+
+#define EVENT_MASK_INTERNAL_ERRORS           0x80000000
+#define EVENT_MASK_EPOW                      0x40000000
+#define EVENT_MASK_HOTPLUG                   0x10000000
+#define EVENT_MASK_IO                        0x08000000
+
+#define _FDT(exp) \
+    do { \
+        int ret = (exp);                                           \
+        if (ret < 0) {                                             \
+            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
+                    #exp, fdt_strerror(ret));                      \
+            exit(1);                                               \
+        }                                                          \
+    } while (0)
+
+void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq)
+{
+    uint32_t epow_irq_ranges[] = {cpu_to_be32(epow_irq), cpu_to_be32(1)};
+    uint32_t epow_interrupts[] = {cpu_to_be32(epow_irq), 0};
+
+    _FDT((fdt_begin_node(fdt, "event-sources")));
+
+    _FDT((fdt_property(fdt, "interrupt-controller", NULL, 0)));
+    _FDT((fdt_property_cell(fdt, "#interrupt-cells", 2)));
+    _FDT((fdt_property(fdt, "interrupt-ranges",
+                       epow_irq_ranges, sizeof(epow_irq_ranges))));
+
+    _FDT((fdt_begin_node(fdt, "epow-events")));
+    _FDT((fdt_property(fdt, "interrupts",
+                       epow_interrupts, sizeof(epow_interrupts))));
+    _FDT((fdt_end_node(fdt)));
+
+    _FDT((fdt_end_node(fdt)));
+}
+
+static struct epow_log_full *pending_epow;
+static uint32_t next_plid;
+
+static void spapr_powerdown_req(Notifier *n, void *opaque)
+{
+    sPAPREnvironment *spapr = container_of(n, sPAPREnvironment, epow_notifier);
+    struct rtas_error_log *hdr;
+    struct rtas_event_log_v6 *v6hdr;
+    struct rtas_event_log_v6_maina *maina;
+    struct rtas_event_log_v6_mainb *mainb;
+    struct rtas_event_log_v6_epow *epow;
+    struct tm tm;
+    int year;
+
+    if (pending_epow) {
+        /* For now, we just throw away earlier events if two come
+         * along before any are consumed.  This is sufficient for our
+         * powerdown messages, but we'll need more if we do more
+         * general error/event logging */
+        g_free(pending_epow);
+    }
+    pending_epow = g_malloc0(sizeof(*pending_epow));
+    hdr = &pending_epow->hdr;
+    v6hdr = &pending_epow->v6hdr;
+    maina = &pending_epow->maina;
+    mainb = &pending_epow->mainb;
+    epow = &pending_epow->epow;
+
+    hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
+                               | RTAS_LOG_SEVERITY_EVENT
+                               | RTAS_LOG_DISPOSITION_NOT_RECOVERED
+                               | RTAS_LOG_OPTIONAL_PART_PRESENT
+                               | RTAS_LOG_TYPE_EPOW);
+    hdr->extended_length = cpu_to_be32(sizeof(*pending_epow)
+                                       - sizeof(pending_epow->hdr));
+
+    v6hdr->b0 = RTAS_LOG_V6_B0_VALID | RTAS_LOG_V6_B0_NEW_LOG
+        | RTAS_LOG_V6_B0_BIGENDIAN;
+    v6hdr->b2 = RTAS_LOG_V6_B2_POWERPC_FORMAT
+        | RTAS_LOG_V6_B2_LOG_FORMAT_PLATFORM_EVENT;
+    v6hdr->company = cpu_to_be32(RTAS_LOG_V6_COMPANY_IBM);
+
+    maina->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINA);
+    maina->hdr.section_length = cpu_to_be16(sizeof(*maina));
+    /* FIXME: section version, subtype and creator id? */
+    qemu_get_timedate(&tm, spapr->rtc_offset);
+    year = tm.tm_year + 1900;
+    maina->creation_date = cpu_to_be32((to_bcd(year / 100) << 24)
+                                       | (to_bcd(year % 100) << 16)
+                                       | (to_bcd(tm.tm_mon + 1) << 8)
+                                       | to_bcd(tm.tm_mday));
+    maina->creation_time = cpu_to_be32((to_bcd(tm.tm_hour) << 24)
+                                       | (to_bcd(tm.tm_min) << 16)
+                                       | (to_bcd(tm.tm_sec) << 8));
+    maina->creator_id = 'H'; /* Hypervisor */
+    maina->section_count = 3; /* Main-A, Main-B and EPOW */
+    maina->plid = next_plid++;
+
+    mainb->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINB);
+    mainb->hdr.section_length = cpu_to_be16(sizeof(*mainb));
+    /* FIXME: section version, subtype and creator id? */
+    mainb->subsystem_id = 0xa0; /* External environment */
+    mainb->event_severity = 0x00; /* Informational / non-error */
+    mainb->event_subtype = 0xd0; /* Normal shutdown */
+
+    epow->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_EPOW);
+    epow->hdr.section_length = cpu_to_be16(sizeof(*epow));
+    epow->hdr.section_version = 2; /* includes extended modifier */
+    /* FIXME: section subtype and creator id? */
+    epow->sensor_value = RTAS_LOG_V6_EPOW_ACTION_SYSTEM_SHUTDOWN;
+    epow->event_modifier = RTAS_LOG_V6_EPOW_MODIFIER_NORMAL;
+    epow->extended_modifier = RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC;
+
+    qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->epow_irq));
+}
+
+static void check_exception(sPAPREnvironment *spapr,
+                            uint32_t token, uint32_t nargs,
+                            target_ulong args,
+                            uint32_t nret, target_ulong rets)
+{
+    uint32_t mask, buf, len;
+    uint64_t xinfo;
+
+    if ((nargs < 6) || (nargs > 7) || nret != 1) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    xinfo = rtas_ld(args, 1);
+    mask = rtas_ld(args, 2);
+    buf = rtas_ld(args, 4);
+    len = rtas_ld(args, 5);
+    if (nargs == 7) {
+        xinfo |= (uint64_t)rtas_ld(args, 6) << 32;
+    }
+
+    if ((mask & EVENT_MASK_EPOW) && pending_epow) {
+        if (sizeof(*pending_epow) < len) {
+            len = sizeof(*pending_epow);
+        }
+
+        cpu_physical_memory_write(buf, pending_epow, len);
+        g_free(pending_epow);
+        pending_epow = NULL;
+        rtas_st(rets, 0, 0);
+    } else {
+        rtas_st(rets, 0, 1);
+    }
+}
+
+void spapr_events_init(sPAPREnvironment *spapr)
+{
+    spapr->epow_irq = spapr_allocate_msi(0);
+    spapr->epow_notifier.notify = spapr_powerdown_req;
+    qemu_register_powerdown_notifier(&spapr->epow_notifier);
+    spapr_rtas_register("check-exception", check_exception);
+}
diff --git a/hw/spapr_hcall.c b/hw/spapr_hcall.c
index a5990a96178..2889742788c 100644
--- a/hw/spapr_hcall.c
+++ b/hw/spapr_hcall.c
@@ -1,9 +1,6 @@
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "cpu.h"
-#include "dyngen-exec.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
 #include "helper_regs.h"
 #include "hw/spapr.h"
 
@@ -40,22 +37,6 @@
 #define HPTE_V_1TB_SEG          0x4000000000000000ULL
 #define HPTE_V_VRMA_MASK        0x4001ffffff000000ULL
 
-#define HPTE_V_HVLOCK           0x40ULL
-
-static inline int lock_hpte(void *hpte, target_ulong bits)
-{
-    uint64_t pteh;
-
-    pteh = ldq_p(hpte);
-
-    /* We're protected by qemu's global lock here */
-    if (pteh & bits) {
-        return 0;
-    }
-    stq_p(hpte, pteh | HPTE_V_HVLOCK);
-    return 1;
-}
-
 static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
                                      target_ulong pte_index)
 {
@@ -92,9 +73,10 @@ static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
     return rb;
 }
 
-static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_enter(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                             target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong pteh = args[2];
@@ -152,8 +134,7 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
             if (i == 8) {
                 return H_PTEG_FULL;
             }
-            if (((ldq_p(hpte) & HPTE_V_VALID) == 0) &&
-                lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID)) {
+            if ((ldq_p(hpte) & HPTE_V_VALID) == 0) {
                 break;
             }
             hpte += HASH_PTE_SIZE_64;
@@ -161,7 +142,7 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
     } else {
         i = 0;
         hpte = env->external_htab + (pte_index * HASH_PTE_SIZE_64);
-        if (!lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID)) {
+        if (ldq_p(hpte) & HPTE_V_VALID) {
             return H_PTEG_FULL;
         }
     }
@@ -169,7 +150,6 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
     /* eieio();  FIXME: need some sort of barrier for smp? */
     stq_p(hpte, pteh);
 
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
     args[0] = pte_index + i;
     return H_SUCCESS;
 }
@@ -194,11 +174,6 @@ static target_ulong remove_hpte(CPUPPCState *env, target_ulong ptex,
     }
 
     hpte = env->external_htab + (ptex * HASH_PTE_SIZE_64);
-    while (!lock_hpte(hpte, HPTE_V_HVLOCK)) {
-        /* We have no real concurrency in qemu soft-emulation, so we
-         * will never actually have a contested lock */
-        assert(0);
-    }
 
     v = ldq_p(hpte);
     r = ldq_p(hpte + (HASH_PTE_SIZE_64/2));
@@ -206,22 +181,20 @@ static target_ulong remove_hpte(CPUPPCState *env, target_ulong ptex,
     if ((v & HPTE_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
         ((flags & H_ANDCOND) && (v & avpn) != 0)) {
-        stq_p(hpte, v & ~HPTE_V_HVLOCK);
-        assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
         return REMOVE_NOT_FOUND;
     }
-    *vp = v & ~HPTE_V_HVLOCK;
+    *vp = v;
     *rp = r;
     stq_p(hpte, 0);
     rb = compute_tlbie_rb(v, r, ptex);
     ppc_tlb_invalidate_one(env, rb);
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
     return REMOVE_SUCCESS;
 }
 
-static target_ulong h_remove(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                              target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
@@ -265,9 +238,10 @@ static target_ulong h_remove(CPUPPCState *env, sPAPREnvironment *spapr,
 
 #define H_BULK_REMOVE_MAX_BATCH        4
 
-static target_ulong h_bulk_remove(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                   target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     int i;
 
     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
@@ -311,9 +285,10 @@ static target_ulong h_bulk_remove(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_protect(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
@@ -325,19 +300,12 @@ static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
     }
 
     hpte = env->external_htab + (pte_index * HASH_PTE_SIZE_64);
-    while (!lock_hpte(hpte, HPTE_V_HVLOCK)) {
-        /* We have no real concurrency in qemu soft-emulation, so we
-         * will never actually have a contested lock */
-        assert(0);
-    }
 
     v = ldq_p(hpte);
     r = ldq_p(hpte + (HASH_PTE_SIZE_64/2));
 
     if ((v & HPTE_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
-        stq_p(hpte, v & ~HPTE_V_HVLOCK);
-        assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
         return H_NOT_FOUND;
     }
 
@@ -351,12 +319,11 @@ static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
     ppc_tlb_invalidate_one(env, rb);
     stq_p(hpte + (HASH_PTE_SIZE_64/2), r);
     /* Don't need a memory barrier, due to qemu's global lock */
-    stq_p(hpte, v & ~HPTE_V_HVLOCK);
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
+    stq_p(hpte, v);
     return H_SUCCESS;
 }
 
-static target_ulong h_set_dabr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     /* FIXME: actually implement this */
@@ -401,26 +368,26 @@ static target_ulong register_vpa(CPUPPCState *env, target_ulong vpa)
         return H_PARAMETER;
     }
 
-    env->vpa = vpa;
+    env->vpa_addr = vpa;
 
-    tmp = ldub_phys(env->vpa + VPA_SHARED_PROC_OFFSET);
+    tmp = ldub_phys(env->vpa_addr + VPA_SHARED_PROC_OFFSET);
     tmp |= VPA_SHARED_PROC_VAL;
-    stb_phys(env->vpa + VPA_SHARED_PROC_OFFSET, tmp);
+    stb_phys(env->vpa_addr + VPA_SHARED_PROC_OFFSET, tmp);
 
     return H_SUCCESS;
 }
 
 static target_ulong deregister_vpa(CPUPPCState *env, target_ulong vpa)
 {
-    if (env->slb_shadow) {
+    if (env->slb_shadow_addr) {
         return H_RESOURCE;
     }
 
-    if (env->dispatch_trace_log) {
+    if (env->dtl_addr) {
         return H_RESOURCE;
     }
 
-    env->vpa = 0;
+    env->vpa_addr = 0;
     return H_SUCCESS;
 }
 
@@ -442,18 +409,20 @@ static target_ulong register_slb_shadow(CPUPPCState *env, target_ulong addr)
         return H_PARAMETER;
     }
 
-    if (!env->vpa) {
+    if (!env->vpa_addr) {
         return H_RESOURCE;
     }
 
-    env->slb_shadow = addr;
+    env->slb_shadow_addr = addr;
+    env->slb_shadow_size = size;
 
     return H_SUCCESS;
 }
 
 static target_ulong deregister_slb_shadow(CPUPPCState *env, target_ulong addr)
 {
-    env->slb_shadow = 0;
+    env->slb_shadow_addr = 0;
+    env->slb_shadow_size = 0;
     return H_SUCCESS;
 }
 
@@ -472,11 +441,11 @@ static target_ulong register_dtl(CPUPPCState *env, target_ulong addr)
         return H_PARAMETER;
     }
 
-    if (!env->vpa) {
+    if (!env->vpa_addr) {
         return H_RESOURCE;
     }
 
-    env->dispatch_trace_log = addr;
+    env->dtl_addr = addr;
     env->dtl_size = size;
 
     return H_SUCCESS;
@@ -484,13 +453,13 @@ static target_ulong register_dtl(CPUPPCState *env, target_ulong addr)
 
 static target_ulong deregister_dtl(CPUPPCState *env, target_ulong addr)
 {
-    env->dispatch_trace_log = 0;
+    env->dtl_addr = 0;
     env->dtl_size = 0;
 
     return H_SUCCESS;
 }
 
-static target_ulong h_register_vpa(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     target_ulong flags = args[0];
@@ -498,9 +467,11 @@ static target_ulong h_register_vpa(CPUPPCState *env, sPAPREnvironment *spapr,
     target_ulong vpa = args[2];
     target_ulong ret = H_PARAMETER;
     CPUPPCState *tenv;
+    CPUState *tcpu;
 
     for (tenv = first_cpu; tenv; tenv = tenv->next_cpu) {
-        if (tenv->cpu_index == procno) {
+        tcpu = CPU(ppc_env_get_cpu(tenv));
+        if (tcpu->cpu_index == procno) {
             break;
         }
     }
@@ -538,18 +509,22 @@ static target_ulong h_register_vpa(CPUPPCState *env, sPAPREnvironment *spapr,
     return ret;
 }
 
-static target_ulong h_cede(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_cede(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
+
     env->msr |= (1ULL << MSR_EE);
     hreg_compute_hflags(env);
-    if (!cpu_has_work(env)) {
+    if (!cpu_has_work(CPU(cpu))) {
         env->halted = 1;
+        env->exception_index = EXCP_HLT;
+        env->exit_request = 1;
     }
     return H_SUCCESS;
 }
 
-static target_ulong h_rtas(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_rtas(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
     target_ulong rtas_r3 = args[0];
@@ -561,7 +536,7 @@ static target_ulong h_rtas(CPUPPCState *env, sPAPREnvironment *spapr,
                            nret, rtas_r3 + 12 + 4*nargs);
 }
 
-static target_ulong h_logical_load(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_load(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     target_ulong size = args[0];
@@ -584,7 +559,7 @@ static target_ulong h_logical_load(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_store(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong size = args[0];
@@ -608,7 +583,7 @@ static target_ulong h_logical_store(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_memop(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_memop(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong dst   = args[0]; /* Destination address */
@@ -675,14 +650,14 @@ static target_ulong h_logical_memop(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_icbi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_icbi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_dcbf(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
@@ -703,35 +678,29 @@ void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn)
     } else {
         assert((opcode >= KVMPPC_HCALL_BASE) && (opcode <= KVMPPC_HCALL_MAX));
 
-
         slot = &kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
     }
 
-    assert(!(*slot) || (fn == *slot));
+    assert(!(*slot));
     *slot = fn;
 }
 
-target_ulong spapr_hypercall(CPUPPCState *env, target_ulong opcode,
+target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args)
 {
-    if (msr_pr) {
-        hcall_dprintf("Hypercall made with MSR[PR]=1\n");
-        return H_PRIVILEGE;
-    }
-
     if ((opcode <= MAX_HCALL_OPCODE)
         && ((opcode & 0x3) == 0)) {
         spapr_hcall_fn fn = papr_hypercall_table[opcode / 4];
 
         if (fn) {
-            return fn(env, spapr, opcode, args);
+            return fn(cpu, spapr, opcode, args);
         }
     } else if ((opcode >= KVMPPC_HCALL_BASE) &&
                (opcode <= KVMPPC_HCALL_MAX)) {
         spapr_hcall_fn fn = kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
 
         if (fn) {
-            return fn(env, spapr, opcode, args);
+            return fn(cpu, spapr, opcode, args);
         }
     }
 
diff --git a/hw/spapr_iommu.c b/hw/spapr_iommu.c
index 53b731773ab..d8a098cb1b2 100644
--- a/hw/spapr_iommu.c
+++ b/hw/spapr_iommu.c
@@ -17,10 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "qdev.h"
 #include "kvm_ppc.h"
-#include "dma.h"
+#include "sysemu/dma.h"
+#include "exec/address-spaces.h"
 
 #include "hw/spapr.h"
 
@@ -42,6 +43,7 @@ struct sPAPRTCETable {
     uint32_t liobn;
     uint32_t window_size;
     sPAPRTCE *table;
+    bool bypass;
     int fd;
     QLIST_ENTRY(sPAPRTCETable) list;
 };
@@ -64,8 +66,8 @@ static sPAPRTCETable *spapr_tce_find_by_liobn(uint32_t liobn)
 
 static int spapr_tce_translate(DMAContext *dma,
                                dma_addr_t addr,
-                               target_phys_addr_t *paddr,
-                               target_phys_addr_t *len,
+                               hwaddr *paddr,
+                               hwaddr *len,
                                DMADirection dir)
 {
     sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
@@ -78,6 +80,12 @@ static int spapr_tce_translate(DMAContext *dma,
             DMA_ADDR_FMT "\n", tcet->liobn, addr);
 #endif
 
+    if (tcet->bypass) {
+        *paddr = addr;
+        *len = (hwaddr)-1;
+        return 0;
+    }
+
     /* Check if we are in bound */
     if (addr >= tcet->window_size) {
 #ifdef DEBUG_TCE
@@ -112,12 +120,18 @@ DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size)
 {
     sPAPRTCETable *tcet;
 
+    if (spapr_tce_find_by_liobn(liobn)) {
+        fprintf(stderr, "Attempted to create TCE table with duplicate"
+                " LIOBN 0x%x\n", liobn);
+        return NULL;
+    }
+
     if (!window_size) {
         return NULL;
     }
 
     tcet = g_malloc0(sizeof(*tcet));
-    dma_context_init(&tcet->dma, spapr_tce_translate, NULL, NULL);
+    dma_context_init(&tcet->dma, &address_space_memory, spapr_tce_translate, NULL, NULL);
 
     tcet->liobn = liobn;
     tcet->window_size = window_size;
@@ -162,6 +176,23 @@ void spapr_tce_free(DMAContext *dma)
     }
 }
 
+void spapr_tce_set_bypass(DMAContext *dma, bool bypass)
+{
+    sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
+
+    tcet->bypass = bypass;
+}
+
+void spapr_tce_reset(DMAContext *dma)
+{
+    sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
+    size_t table_size = (tcet->window_size >> SPAPR_TCE_PAGE_SHIFT)
+        * sizeof(sPAPRTCE);
+
+    tcet->bypass = false;
+    memset(tcet->table, 0, table_size);
+}
+
 static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
                                 target_ulong tce)
 {
@@ -179,7 +210,7 @@ static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
     return H_SUCCESS;
 }
 
-static target_ulong h_put_tce(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_put_tce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong liobn = args[0];
diff --git a/hw/spapr_llan.c b/hw/spapr_llan.c
index bd3f131d7e9..db34b485aad 100644
--- a/hw/spapr_llan.c
+++ b/hw/spapr_llan.c
@@ -25,7 +25,7 @@
  *
  */
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "hw/qdev.h"
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
@@ -264,7 +264,7 @@ static int check_bd(VIOsPAPRVLANDevice *dev, vlan_bd_t bd,
     return 0;
 }
 
-static target_ulong h_register_logical_lan(CPUPPCState *env,
+static target_ulong h_register_logical_lan(PowerPCCPU *cpu,
                                            sPAPREnvironment *spapr,
                                            target_ulong opcode,
                                            target_ulong *args)
@@ -328,7 +328,7 @@ static target_ulong h_register_logical_lan(CPUPPCState *env,
 }
 
 
-static target_ulong h_free_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_free_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -349,7 +349,7 @@ static target_ulong h_free_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr
     return H_SUCCESS;
 }
 
-static target_ulong h_add_logical_lan_buffer(CPUPPCState *env,
+static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
                                              sPAPREnvironment *spapr,
                                              target_ulong opcode,
                                              target_ulong *args)
@@ -398,7 +398,7 @@ static target_ulong h_add_logical_lan_buffer(CPUPPCState *env,
     return H_SUCCESS;
 }
 
-static target_ulong h_send_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_send_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -467,7 +467,7 @@ static target_ulong h_send_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr
     return H_SUCCESS;
 }
 
-static target_ulong h_multicast_ctrl(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_multicast_ctrl(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                      target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -502,7 +502,7 @@ static void spapr_vlan_class_init(ObjectClass *klass, void *data)
     k->rtce_window_size = 0x10000000;
 }
 
-static TypeInfo spapr_vlan_info = {
+static const TypeInfo spapr_vlan_info = {
     .name          = "spapr-vlan",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VIOsPAPRVLANDevice),
diff --git a/hw/spapr_nvram.c b/hw/spapr_nvram.c
new file mode 100644
index 00000000000..680cdba928f
--- /dev/null
+++ b/hw/spapr_nvram.c
@@ -0,0 +1,196 @@
+/*
+ * QEMU sPAPR NVRAM emulation
+ *
+ * Copyright (C) 2012 David Gibson, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <libfdt.h>
+
+#include "sysemu/device_tree.h"
+#include "hw/sysbus.h"
+#include "hw/spapr.h"
+#include "hw/spapr_vio.h"
+
+typedef struct sPAPRNVRAM {
+    VIOsPAPRDevice sdev;
+    uint32_t size;
+    uint8_t *buf;
+    BlockDriverState *drive;
+} sPAPRNVRAM;
+
+#define MIN_NVRAM_SIZE 8192
+#define DEFAULT_NVRAM_SIZE 65536
+#define MAX_NVRAM_SIZE (UINT16_MAX * 16)
+
+static void rtas_nvram_fetch(sPAPREnvironment *spapr,
+                             uint32_t token, uint32_t nargs,
+                             target_ulong args,
+                             uint32_t nret, target_ulong rets)
+{
+    sPAPRNVRAM *nvram = spapr->nvram;
+    hwaddr offset, buffer, len;
+    int alen;
+    void *membuf;
+
+    if ((nargs != 3) || (nret != 2)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    if (!nvram) {
+        rtas_st(rets, 0, -1);
+        rtas_st(rets, 1, 0);
+        return;
+    }
+
+    offset = rtas_ld(args, 0);
+    buffer = rtas_ld(args, 1);
+    len = rtas_ld(args, 2);
+
+    if (((offset + len) < offset)
+        || ((offset + len) > nvram->size)) {
+        rtas_st(rets, 0, -3);
+        rtas_st(rets, 1, 0);
+        return;
+    }
+
+    membuf = cpu_physical_memory_map(buffer, &len, 1);
+    if (nvram->drive) {
+        alen = bdrv_pread(nvram->drive, offset, membuf, len);
+    } else {
+        assert(nvram->buf);
+
+        memcpy(membuf, nvram->buf + offset, len);
+        alen = len;
+    }
+    cpu_physical_memory_unmap(membuf, len, 1, len);
+
+    rtas_st(rets, 0, (alen < len) ? -1 : 0);
+    rtas_st(rets, 1, (alen < 0) ? 0 : alen);
+}
+
+static void rtas_nvram_store(sPAPREnvironment *spapr,
+                             uint32_t token, uint32_t nargs,
+                             target_ulong args,
+                             uint32_t nret, target_ulong rets)
+{
+    sPAPRNVRAM *nvram = spapr->nvram;
+    hwaddr offset, buffer, len;
+    int alen;
+    void *membuf;
+
+    if ((nargs != 3) || (nret != 2)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    if (!nvram) {
+        rtas_st(rets, 0, -1);
+        return;
+    }
+
+    offset = rtas_ld(args, 0);
+    buffer = rtas_ld(args, 1);
+    len = rtas_ld(args, 2);
+
+    if (((offset + len) < offset)
+        || ((offset + len) > nvram->size)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    membuf = cpu_physical_memory_map(buffer, &len, 0);
+    if (nvram->drive) {
+        alen = bdrv_pwrite(nvram->drive, offset, membuf, len);
+    } else {
+        assert(nvram->buf);
+
+        memcpy(nvram->buf + offset, membuf, len);
+        alen = len;
+    }
+    cpu_physical_memory_unmap(membuf, len, 0, len);
+
+    rtas_st(rets, 0, (alen < len) ? -1 : 0);
+    rtas_st(rets, 1, (alen < 0) ? 0 : alen);
+}
+
+static int spapr_nvram_init(VIOsPAPRDevice *dev)
+{
+    sPAPRNVRAM *nvram = (sPAPRNVRAM *)dev;
+
+    if (nvram->drive) {
+        nvram->size = bdrv_getlength(nvram->drive);
+    } else {
+        nvram->size = DEFAULT_NVRAM_SIZE;
+        nvram->buf = g_malloc0(nvram->size);
+    }
+
+    if ((nvram->size < MIN_NVRAM_SIZE) || (nvram->size > MAX_NVRAM_SIZE)) {
+        fprintf(stderr, "spapr-nvram must be between %d and %d bytes in size\n",
+                MIN_NVRAM_SIZE, MAX_NVRAM_SIZE);
+        return -1;
+    }
+
+    spapr_rtas_register("nvram-fetch", rtas_nvram_fetch);
+    spapr_rtas_register("nvram-store", rtas_nvram_store);
+
+    return 0;
+}
+
+static int spapr_nvram_devnode(VIOsPAPRDevice *dev, void *fdt, int node_off)
+{
+    sPAPRNVRAM *nvram = (sPAPRNVRAM *)dev;
+
+    return fdt_setprop_cell(fdt, node_off, "#bytes", nvram->size);
+}
+
+static Property spapr_nvram_properties[] = {
+    DEFINE_SPAPR_PROPERTIES(sPAPRNVRAM, sdev),
+    DEFINE_PROP_DRIVE("drive", sPAPRNVRAM, drive),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void spapr_nvram_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
+
+    k->init = spapr_nvram_init;
+    k->devnode = spapr_nvram_devnode;
+    k->dt_name = "nvram";
+    k->dt_type = "nvram";
+    k->dt_compatible = "qemu,spapr-nvram";
+    dc->props = spapr_nvram_properties;
+}
+
+static const TypeInfo spapr_nvram_type_info = {
+    .name          = "spapr-nvram",
+    .parent        = TYPE_VIO_SPAPR_DEVICE,
+    .instance_size = sizeof(sPAPRNVRAM),
+    .class_init    = spapr_nvram_class_init,
+};
+
+static void spapr_nvram_register_types(void)
+{
+    type_register_static(&spapr_nvram_type_info);
+}
+
+type_init(spapr_nvram_register_types)
diff --git a/hw/spapr_pci.c b/hw/spapr_pci.c
index 661c05bc308..4eacbcfd581 100644
--- a/hw/spapr_pci.c
+++ b/hw/spapr_pci.c
@@ -23,17 +23,17 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
-#include "msi.h"
-#include "msix.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
+#include "pci/pci_host.h"
 #include "hw/spapr.h"
 #include "hw/spapr_pci.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include <libfdt.h>
 #include "trace.h"
 
-#include "hw/pci_internals.h"
+#include "hw/pci/pci_bus.h"
 
 /* Copied from the kernel arch/powerpc/platforms/pseries/msi.c */
 #define RTAS_QUERY_FN           0
@@ -258,7 +258,7 @@ static int spapr_msicfg_find(sPAPRPHBState *phb, uint32_t config_addr,
  * This is required for msi_notify()/msix_notify() which
  * will write at the addresses via spapr_msi_write().
  */
-static void spapr_msi_setmsg(PCIDevice *pdev, target_phys_addr_t addr,
+static void spapr_msi_setmsg(PCIDevice *pdev, hwaddr addr,
                              bool msix, unsigned req_num)
 {
     unsigned i;
@@ -351,7 +351,7 @@ static void rtas_ibm_change_msi(sPAPREnvironment *spapr,
 
     /* There is no cached config, allocate MSIs */
     if (!phb->msi_table[ndev].nvec) {
-        irq = spapr_allocate_irq_block(req_num, XICS_MSI);
+        irq = spapr_allocate_irq_block(req_num, false);
         if (irq < 0) {
             fprintf(stderr, "Cannot allocate MSIs for device#%d", ndev);
             rtas_st(rets, 0, -1); /* Hardware error */
@@ -435,11 +435,11 @@ static void pci_spapr_set_irq(void *opaque, int irq_num, int level)
      */
     sPAPRPHBState *phb = opaque;
 
-    trace_spapr_pci_lsi_set(phb->busname, irq_num, phb->lsi_table[irq_num].irq);
+    trace_spapr_pci_lsi_set(phb->dtbusname, irq_num, phb->lsi_table[irq_num].irq);
     qemu_set_irq(spapr_phb_lsi_qirq(phb, irq_num), level);
 }
 
-static uint64_t spapr_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t spapr_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     switch (size) {
@@ -453,7 +453,7 @@ static uint64_t spapr_io_read(void *opaque, target_phys_addr_t addr,
     assert(0);
 }
 
-static void spapr_io_write(void *opaque, target_phys_addr_t addr,
+static void spapr_io_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     switch (size) {
@@ -483,7 +483,7 @@ static const MemoryRegionOps spapr_io_ops = {
  * data is set to 0.
  * For MSI, the vector number is encoded in least bits in data.
  */
-static void spapr_msi_write(void *opaque, target_phys_addr_t addr,
+static void spapr_msi_write(void *opaque, hwaddr addr,
                             uint64_t data, unsigned size)
 {
     sPAPRPHBState *phb = opaque;
@@ -522,7 +522,63 @@ static int spapr_phb_init(SysBusDevice *s)
     int i;
     PCIBus *bus;
 
+    if (sphb->index != -1) {
+        hwaddr windows_base;
+
+        if ((sphb->buid != -1) || (sphb->dma_liobn != -1)
+            || (sphb->mem_win_addr != -1)
+            || (sphb->io_win_addr != -1)
+            || (sphb->msi_win_addr != -1)) {
+            fprintf(stderr, "Either \"index\" or other parameters must"
+                    " be specified for PAPR PHB, not both\n");
+            return -1;
+        }
+
+        sphb->buid = SPAPR_PCI_BASE_BUID + sphb->index;
+        sphb->dma_liobn = SPAPR_PCI_BASE_LIOBN + sphb->index;
+
+        windows_base = SPAPR_PCI_WINDOW_BASE
+            + sphb->index * SPAPR_PCI_WINDOW_SPACING;
+        sphb->mem_win_addr = windows_base + SPAPR_PCI_MMIO_WIN_OFF;
+        sphb->io_win_addr = windows_base + SPAPR_PCI_IO_WIN_OFF;
+        sphb->msi_win_addr = windows_base + SPAPR_PCI_MSI_WIN_OFF;
+    }
+
+    if (sphb->buid == -1) {
+        fprintf(stderr, "BUID not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->dma_liobn == -1) {
+        fprintf(stderr, "LIOBN not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->mem_win_addr == -1) {
+        fprintf(stderr, "Memory window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->io_win_addr == -1) {
+        fprintf(stderr, "IO window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->msi_win_addr == -1) {
+        fprintf(stderr, "MSI window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (find_phb(spapr, sphb->buid)) {
+        fprintf(stderr, "PCI host bridges must have unique BUIDs\n");
+        return -1;
+    }
+
     sphb->dtbusname = g_strdup_printf("pci@%" PRIx64, sphb->buid);
+    if (!sphb->busname) {
+        sphb->busname = sphb->dtbusname;
+    }
+
     namebuf = alloca(strlen(sphb->dtbusname) + 32);
 
     /* Initialize memory regions */
@@ -565,17 +621,19 @@ static int spapr_phb_init(SysBusDevice *s)
                                     &sphb->msiwindow);
     }
 
-    bus = pci_register_bus(DEVICE(s),
-                           sphb->busname ? sphb->busname : sphb->dtbusname,
+    bus = pci_register_bus(DEVICE(s), sphb->busname,
                            pci_spapr_set_irq, pci_spapr_map_irq, sphb,
                            &sphb->memspace, &sphb->iospace,
                            PCI_DEVFN(0, 0), PCI_NUM_PINS);
     phb->bus = bus;
 
-    sphb->dma_liobn = SPAPR_PCI_BASE_LIOBN | (pci_find_domain(bus) << 16);
     sphb->dma_window_start = 0;
     sphb->dma_window_size = 0x40000000;
     sphb->dma = spapr_tce_new_dma_context(sphb->dma_liobn, sphb->dma_window_size);
+    if (!sphb->dma) {
+        fprintf(stderr, "Unable to create TCE table for %s\n", sphb->dtbusname);
+        return -1;
+    }
     pci_setup_iommu(bus, spapr_pci_dma_context_fn, sphb);
 
     QLIST_INSERT_HEAD(&spapr->phbs, sphb, list);
@@ -595,14 +653,27 @@ static int spapr_phb_init(SysBusDevice *s)
     return 0;
 }
 
+static void spapr_phb_reset(DeviceState *qdev)
+{
+    SysBusDevice *s = SYS_BUS_DEVICE(qdev);
+    sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
+
+    /* Reset the IOMMU state */
+    spapr_tce_reset(sphb->dma);
+}
+
 static Property spapr_phb_properties[] = {
-    DEFINE_PROP_HEX64("buid", sPAPRPHBState, buid, 0),
     DEFINE_PROP_STRING("busname", sPAPRPHBState, busname),
-    DEFINE_PROP_HEX64("mem_win_addr", sPAPRPHBState, mem_win_addr, 0),
-    DEFINE_PROP_HEX64("mem_win_size", sPAPRPHBState, mem_win_size, 0x20000000),
-    DEFINE_PROP_HEX64("io_win_addr", sPAPRPHBState, io_win_addr, 0),
-    DEFINE_PROP_HEX64("io_win_size", sPAPRPHBState, io_win_size, 0x10000),
-    DEFINE_PROP_HEX64("msi_win_addr", sPAPRPHBState, msi_win_addr, 0),
+    DEFINE_PROP_INT32("index", sPAPRPHBState, index, -1),
+    DEFINE_PROP_HEX64("buid", sPAPRPHBState, buid, -1),
+    DEFINE_PROP_HEX32("liobn", sPAPRPHBState, dma_liobn, -1),
+    DEFINE_PROP_HEX64("mem_win_addr", sPAPRPHBState, mem_win_addr, -1),
+    DEFINE_PROP_HEX64("mem_win_size", sPAPRPHBState, mem_win_size,
+                      SPAPR_PCI_MMIO_WIN_SIZE),
+    DEFINE_PROP_HEX64("io_win_addr", sPAPRPHBState, io_win_addr, -1),
+    DEFINE_PROP_HEX64("io_win_size", sPAPRPHBState, io_win_size,
+                      SPAPR_PCI_IO_WIN_SIZE),
+    DEFINE_PROP_HEX64("msi_win_addr", sPAPRPHBState, msi_win_addr, -1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -613,6 +684,7 @@ static void spapr_phb_class_init(ObjectClass *klass, void *data)
 
     sdc->init = spapr_phb_init;
     dc->props = spapr_phb_properties;
+    dc->reset = spapr_phb_reset;
 }
 
 static const TypeInfo spapr_phb_info = {
@@ -622,25 +694,17 @@ static const TypeInfo spapr_phb_info = {
     .class_init    = spapr_phb_class_init,
 };
 
-void spapr_create_phb(sPAPREnvironment *spapr,
-                      const char *busname, uint64_t buid,
-                      uint64_t mem_win_addr, uint64_t mem_win_size,
-                      uint64_t io_win_addr, uint64_t msi_win_addr)
+PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index,
+                               const char *busname)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, TYPE_SPAPR_PCI_HOST_BRIDGE);
-
-    if (busname) {
-        qdev_prop_set_string(dev, "busname", g_strdup(busname));
-    }
-    qdev_prop_set_uint64(dev, "buid", buid);
-    qdev_prop_set_uint64(dev, "mem_win_addr", mem_win_addr);
-    qdev_prop_set_uint64(dev, "mem_win_size", mem_win_size);
-    qdev_prop_set_uint64(dev, "io_win_addr", io_win_addr);
-    qdev_prop_set_uint64(dev, "msi_win_addr", msi_win_addr);
-
+    qdev_prop_set_uint32(dev, "index", index);
+    qdev_prop_set_string(dev, "busname", busname);
     qdev_init_nofail(dev);
+
+    return PCI_HOST_BRIDGE(dev);
 }
 
 /* Macros to operate with address in OF binding to PCI */
diff --git a/hw/spapr_pci.h b/hw/spapr_pci.h
index 670dc62a192..8bb3c62c3d5 100644
--- a/hw/spapr_pci.h
+++ b/hw/spapr_pci.h
@@ -23,8 +23,8 @@
 #if !defined(__HW_SPAPR_PCI_H__)
 #define __HW_SPAPR_PCI_H__
 
-#include "hw/pci.h"
-#include "hw/pci_host.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
 #include "hw/xics.h"
 
 #define SPAPR_MSIX_MAX_DEVS 32
@@ -37,13 +37,14 @@
 typedef struct sPAPRPHBState {
     PCIHostState parent_obj;
 
+    int32_t index;
     uint64_t buid;
     char *busname;
     char *dtbusname;
 
     MemoryRegion memspace, iospace;
-    target_phys_addr_t mem_win_addr, mem_win_size, io_win_addr, io_win_size;
-    target_phys_addr_t msi_win_addr;
+    hwaddr mem_win_addr, mem_win_size, io_win_addr, io_win_size;
+    hwaddr msi_win_addr;
     MemoryRegion memwindow, iowindow, msiwindow;
 
     uint32_t dma_liobn;
@@ -64,18 +65,25 @@ typedef struct sPAPRPHBState {
     QLIST_ENTRY(sPAPRPHBState) list;
 } sPAPRPHBState;
 
+#define SPAPR_PCI_BASE_BUID          0x800000020000000ULL
+
+#define SPAPR_PCI_WINDOW_BASE        0x10000000000ULL
+#define SPAPR_PCI_WINDOW_SPACING     0x1000000000ULL
+#define SPAPR_PCI_MMIO_WIN_OFF       0xA0000000
+#define SPAPR_PCI_MMIO_WIN_SIZE      0x20000000
+#define SPAPR_PCI_IO_WIN_OFF         0x80000000
+#define SPAPR_PCI_IO_WIN_SIZE        0x10000
+#define SPAPR_PCI_MSI_WIN_OFF        0x90000000
+
+#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
+
 static inline qemu_irq spapr_phb_lsi_qirq(struct sPAPRPHBState *phb, int pin)
 {
     return xics_get_qirq(spapr->icp, phb->lsi_table[pin].irq);
 }
 
-#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
-#define SPAPR_PCI_IO_WIN_SIZE        0x10000
-
-void spapr_create_phb(sPAPREnvironment *spapr,
-                      const char *busname, uint64_t buid,
-                      uint64_t mem_win_addr, uint64_t mem_win_size,
-                      uint64_t io_win_addr, uint64_t msi_win_addr);
+PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index,
+                               const char *busname);
 
 int spapr_populate_pci_dt(sPAPRPHBState *phb,
                           uint32_t xics_phandle,
diff --git a/hw/spapr_rtas.c b/hw/spapr_rtas.c
index ae18595150a..5ec787f29dc 100644
--- a/hw/spapr_rtas.c
+++ b/hw/spapr_rtas.c
@@ -25,10 +25,10 @@
  *
  */
 #include "cpu.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "hw/qdev.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
@@ -131,6 +131,7 @@ static void rtas_query_cpu_stopped_state(sPAPREnvironment *spapr,
 {
     target_ulong id;
     CPUPPCState *env;
+    CPUState *cpu;
 
     if (nargs != 1 || nret != 2) {
         rtas_st(rets, 0, -3);
@@ -139,7 +140,8 @@ static void rtas_query_cpu_stopped_state(sPAPREnvironment *spapr,
 
     id = rtas_ld(args, 0);
     for (env = first_cpu; env; env = env->next_cpu) {
-        if (env->cpu_index != id) {
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index != id) {
             continue;
         }
 
@@ -163,6 +165,7 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
                            uint32_t nret, target_ulong rets)
 {
     target_ulong id, start, r3;
+    CPUState *cpu;
     CPUPPCState *env;
 
     if (nargs != 3 || nret != 1) {
@@ -175,7 +178,9 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
     r3 = rtas_ld(args, 2);
 
     for (env = first_cpu; env; env = env->next_cpu) {
-        if (env->cpu_index != id) {
+        cpu = CPU(ppc_env_get_cpu(env));
+
+        if (cpu->cpu_index != id) {
             continue;
         }
 
@@ -184,12 +189,17 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
             return;
         }
 
+        /* This will make sure qemu state is up to date with kvm, and
+         * mark it dirty so our changes get flushed back before the
+         * new cpu enters */
+        kvm_cpu_synchronize_state(env);
+
         env->msr = (1ULL << MSR_SF) | (1ULL << MSR_ME);
         env->nip = start;
         env->gpr[3] = r3;
         env->halted = 0;
 
-        qemu_cpu_kick(env);
+        qemu_cpu_kick(cpu);
 
         rtas_st(rets, 0, 0);
         return;
@@ -234,18 +244,27 @@ target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-void spapr_rtas_register(const char *name, spapr_rtas_fn fn)
+int spapr_rtas_register(const char *name, spapr_rtas_fn fn)
 {
+    int i;
+
+    for (i = 0; i < (rtas_next - rtas_table); i++) {
+        if (strcmp(name, rtas_table[i].name) == 0) {
+            fprintf(stderr, "RTAS call \"%s\" registered twice\n", name);
+            exit(1);
+        }
+    }
+
     assert(rtas_next < (rtas_table + TOKEN_MAX));
 
     rtas_next->name = name;
     rtas_next->fn = fn;
 
-    rtas_next++;
+    return (rtas_next++ - rtas_table) + TOKEN_BASE;
 }
 
-int spapr_rtas_device_tree_setup(void *fdt, target_phys_addr_t rtas_addr,
-                                 target_phys_addr_t rtas_size)
+int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
+                                 hwaddr rtas_size)
 {
     int ret;
     int i;
@@ -284,7 +303,7 @@ int spapr_rtas_device_tree_setup(void *fdt, target_phys_addr_t rtas_addr,
     for (i = 0; i < TOKEN_MAX; i++) {
         struct rtas_call *call = &rtas_table[i];
 
-        if (!call->fn) {
+        if (!call->name) {
             continue;
         }
 
diff --git a/hw/spapr_vio.c b/hw/spapr_vio.c
index 7ca445216d7..34c9ca6b65e 100644
--- a/hw/spapr_vio.c
+++ b/hw/spapr_vio.c
@@ -20,14 +20,14 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "loader.h"
 #include "elf.h"
 #include "hw/sysbus.h"
-#include "kvm.h"
-#include "device_tree.h"
+#include "sysemu/kvm.h"
+#include "sysemu/device_tree.h"
 #include "kvm_ppc.h"
 
 #include "hw/spapr.h"
@@ -80,9 +80,7 @@ static char *vio_format_dev_name(VIOsPAPRDevice *dev)
     char *name;
 
     /* Device tree style name device@reg */
-    if (asprintf(&name, "%s@%x", pc->dt_name, dev->reg) < 0) {
-        return NULL;
-    }
+    name = g_strdup_printf("%s@%x", pc->dt_name, dev->reg);
 
     return name;
 }
@@ -101,12 +99,8 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
     }
 
     dt_name = vio_format_dev_name(dev);
-    if (!dt_name) {
-        return -ENOMEM;
-    }
-
     node_off = fdt_add_subnode(fdt, vdevice_off, dt_name);
-    free(dt_name);
+    g_free(dt_name);
     if (node_off < 0) {
         return node_off;
     }
@@ -161,7 +155,7 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
 /*
  * CRQ handling
  */
-static target_ulong h_reg_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_reg_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -219,7 +213,7 @@ static target_ulong free_crq(VIOsPAPRDevice *dev)
     return H_SUCCESS;
 }
 
-static target_ulong h_free_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_free_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -233,7 +227,7 @@ static target_ulong h_free_crq(CPUPPCState *env, sPAPREnvironment *spapr,
     return free_crq(dev);
 }
 
-static target_ulong h_send_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_send_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -256,7 +250,7 @@ static target_ulong h_send_crq(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_HARDWARE;
 }
 
-static target_ulong h_enable_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_enable_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                  target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -316,17 +310,10 @@ int spapr_vio_send_crq(VIOsPAPRDevice *dev, uint8_t *crq)
 
 static void spapr_vio_quiesce_one(VIOsPAPRDevice *dev)
 {
-    VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-    uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
-
     if (dev->dma) {
-        spapr_tce_free(dev->dma);
+        spapr_tce_reset(dev->dma);
     }
-    dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
-
-    dev->crq.qladdr = 0;
-    dev->crq.qsize = 0;
-    dev->crq.qnext = 0;
+    free_crq(dev);
 }
 
 static void rtas_set_tce_bypass(sPAPREnvironment *spapr, uint32_t token,
@@ -348,16 +335,14 @@ static void rtas_set_tce_bypass(sPAPREnvironment *spapr, uint32_t token,
         rtas_st(rets, 0, -3);
         return;
     }
-    if (enable) {
-        spapr_tce_free(dev->dma);
-        dev->dma = NULL;
-    } else {
-        VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-        uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
 
-        dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    if (!dev->dma) {
+        rtas_st(rets, 0, -3);
+        return;
     }
 
+    spapr_tce_set_bypass(dev->dma, !!enable);
+
     rtas_st(rets, 0, 0);
 }
 
@@ -409,9 +394,10 @@ static void spapr_vio_busdev_reset(DeviceState *qdev)
     VIOsPAPRDevice *dev = DO_UPCAST(VIOsPAPRDevice, qdev, qdev);
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
 
-    if (dev->crq.qsize) {
-        free_crq(dev);
-    }
+    /* Shut down the request queue and TCEs if necessary */
+    spapr_vio_quiesce_one(dev);
+
+    dev->signal_state = 0;
 
     if (pc->reset) {
         pc->reset(dev);
@@ -422,7 +408,6 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
 {
     VIOsPAPRDevice *dev = (VIOsPAPRDevice *)qdev;
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-    uint32_t liobn;
     char *id;
 
     if (dev->reg != -1) {
@@ -453,9 +438,6 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
     /* Don't overwrite ids assigned on the command line */
     if (!dev->qdev.id) {
         id = vio_format_dev_name(dev);
-        if (!id) {
-            return -1;
-        }
         dev->qdev.id = id;
     }
 
@@ -464,13 +446,15 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
         return -1;
     }
 
-    liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
-    dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    if (pc->rtce_window_size) {
+        uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
+        dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    }
 
     return pc->init(dev);
 }
 
-static target_ulong h_vio_signal(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_vio_signal(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                  target_ulong opcode,
                                  target_ulong *args)
 {
@@ -508,7 +492,7 @@ VIOsPAPRBus *spapr_vio_bus_init(void)
 
     qbus = qbus_create(TYPE_SPAPR_VIO_BUS, dev, "spapr-vio");
     bus = DO_UPCAST(VIOsPAPRBus, bus, qbus);
-    bus->next_reg = 0x1000;
+    bus->next_reg = 0x71000000;
 
     /* hcall-vio */
     spapr_register_hypercall(H_VIO_SIGNAL, h_vio_signal);
@@ -543,7 +527,7 @@ static void spapr_vio_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo spapr_vio_bridge_info = {
+static const TypeInfo spapr_vio_bridge_info = {
     .name          = "spapr-vio-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
@@ -559,7 +543,7 @@ static void vio_spapr_device_class_init(ObjectClass *klass, void *data)
     k->props = spapr_vio_props;
 }
 
-static TypeInfo spapr_vio_type_info = {
+static const TypeInfo spapr_vio_type_info = {
     .name = TYPE_VIO_SPAPR_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VIOsPAPRDevice),
@@ -653,20 +637,12 @@ int spapr_populate_chosen_stdout(void *fdt, VIOsPAPRBus *bus)
     }
 
     name = vio_format_dev_name(dev);
-    if (!name) {
-        return -ENOMEM;
-    }
-
-    if (asprintf(&path, "/vdevice/%s", name) < 0) {
-        path = NULL;
-        ret = -ENOMEM;
-        goto out;
-    }
+    path = g_strdup_printf("/vdevice/%s", name);
 
     ret = fdt_setprop_string(fdt, offset, "linux,stdout-path", path);
-out:
-    free(name);
-    free(path);
+
+    g_free(name);
+    g_free(path);
 
     return ret;
 }
diff --git a/hw/spapr_vio.h b/hw/spapr_vio.h
index ea6aa43e260..f98ec0a2e57 100644
--- a/hw/spapr_vio.h
+++ b/hw/spapr_vio.h
@@ -21,7 +21,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dma.h"
+#include "sysemu/dma.h"
 
 #define TYPE_VIO_SPAPR_DEVICE "vio-spapr-device"
 #define VIO_SPAPR_DEVICE(obj) \
@@ -60,7 +60,6 @@ typedef struct VIOsPAPRDeviceClass {
 struct VIOsPAPRDevice {
     DeviceState qdev;
     uint32_t reg;
-    uint32_t flags;
     uint32_t irq;
     target_ulong signal_state;
     VIOsPAPR_CRQ crq;
@@ -132,7 +131,6 @@ void spapr_vscsi_create(VIOsPAPRBus *bus);
 
 VIOsPAPRDevice *spapr_vty_get_default(VIOsPAPRBus *bus);
 
-int spapr_tce_set_bypass(uint32_t unit, uint32_t enable);
 void spapr_vio_quiesce(void);
 
 #endif /* _HW_SPAPR_VIO_H */
diff --git a/hw/spapr_vscsi.c b/hw/spapr_vscsi.c
index 3cf5844e0f4..7fc0e13f9fd 100644
--- a/hw/spapr_vscsi.c
+++ b/hw/spapr_vscsi.c
@@ -34,7 +34,6 @@
 #include "hw.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "net.h" /* Remove that when we can */
 #include "srp.h"
 #include "hw/qdev.h"
 #include "hw/spapr.h"
@@ -737,7 +736,7 @@ static int vscsi_send_adapter_info(VSCSIState *s, vscsi_req *req)
 #endif
     memset(&info, 0, sizeof(info));
     strcpy(info.srp_version, SRP_VERSION);
-    strncpy(info.partition_name, "qemu", sizeof("qemu"));
+    memcpy(info.partition_name, "qemu", sizeof("qemu"));
     info.partition_number = cpu_to_be32(0);
     info.mad_version = cpu_to_be32(1);
     info.os_type = cpu_to_be32(2);
@@ -968,7 +967,7 @@ static void spapr_vscsi_class_init(ObjectClass *klass, void *data)
     k->rtce_window_size = 0x10000000;
 }
 
-static TypeInfo spapr_vscsi_info = {
+static const TypeInfo spapr_vscsi_info = {
     .name          = "spapr-vscsi",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VSCSIState),
diff --git a/hw/spapr_vty.c b/hw/spapr_vty.c
index 5da17a3ff43..5c63eaafa97 100644
--- a/hw/spapr_vty.c
+++ b/hw/spapr_vty.c
@@ -1,5 +1,5 @@
 #include "qdev.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
 
@@ -70,7 +70,7 @@ static int spapr_vty_init(VIOsPAPRDevice *sdev)
 }
 
 /* Forward declaration */
-static target_ulong h_put_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_put_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -97,7 +97,7 @@ static target_ulong h_put_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_get_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_get_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -150,7 +150,7 @@ static void spapr_vty_class_init(ObjectClass *klass, void *data)
     dc->props = spapr_vty_properties;
 }
 
-static TypeInfo spapr_vty_info = {
+static const TypeInfo spapr_vty_info = {
     .name          = "spapr-vty",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VIOsPAPRVTYDevice),
diff --git a/hw/sparc32_dma.c b/hw/sparc32_dma.c
index 1dbf69e8082..6d0df51749c 100644
--- a/hw/sparc32_dma.c
+++ b/hw/sparc32_dma.c
@@ -78,7 +78,7 @@ enum {
 };
 
 /* Note: on sparc, the lance 16 bit bus is swapped */
-void ledma_memory_read(void *opaque, target_phys_addr_t addr,
+void ledma_memory_read(void *opaque, hwaddr addr,
                        uint8_t *buf, int len, int do_bswap)
 {
     DMAState *s = opaque;
@@ -98,7 +98,7 @@ void ledma_memory_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-void ledma_memory_write(void *opaque, target_phys_addr_t addr,
+void ledma_memory_write(void *opaque, hwaddr addr,
                         uint8_t *buf, int len, int do_bswap)
 {
     DMAState *s = opaque;
@@ -165,7 +165,7 @@ void espdma_memory_write(void *opaque, uint8_t *buf, int len)
     s->dmaregs[1] += len;
 }
 
-static uint64_t dma_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dma_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     DMAState *s = opaque;
@@ -182,7 +182,7 @@ static uint64_t dma_mem_read(void *opaque, target_phys_addr_t addr,
     return s->dmaregs[saddr];
 }
 
-static void dma_mem_write(void *opaque, target_phys_addr_t addr,
+static void dma_mem_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     DMAState *s = opaque;
@@ -300,7 +300,7 @@ static void sparc32_dma_class_init(ObjectClass *klass, void *data)
     dc->props = sparc32_dma_properties;
 }
 
-static TypeInfo sparc32_dma_info = {
+static const TypeInfo sparc32_dma_info = {
     .name          = "sparc32_dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(DMAState),
diff --git a/hw/sparc32_dma.h b/hw/sparc32_dma.h
index 8b72c37a982..9497b13d34e 100644
--- a/hw/sparc32_dma.h
+++ b/hw/sparc32_dma.h
@@ -2,9 +2,9 @@
 #define SPARC32_DMA_H
 
 /* sparc32_dma.c */
-void ledma_memory_read(void *opaque, target_phys_addr_t addr,
+void ledma_memory_read(void *opaque, hwaddr addr,
                        uint8_t *buf, int len, int do_bswap);
-void ledma_memory_write(void *opaque, target_phys_addr_t addr,
+void ledma_memory_write(void *opaque, hwaddr addr,
                         uint8_t *buf, int len, int do_bswap);
 void espdma_memory_read(void *opaque, uint8_t *buf, int len);
 void espdma_memory_write(void *opaque, uint8_t *buf, int len);
diff --git a/hw/spitz.c b/hw/spitz.c
index 20e78351916..5bc49fcd273 100644
--- a/hw/spitz.c
+++ b/hw/spitz.c
@@ -13,21 +13,21 @@
 #include "hw.h"
 #include "pxa.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "pcmcia.h"
 #include "i2c.h"
 #include "ssi.h"
 #include "flash.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "devices.h"
 #include "sharpsl.h"
-#include "console.h"
-#include "block.h"
+#include "ui/console.h"
+#include "block/block.h"
 #include "audio/audio.h"
 #include "boards.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #undef REG_FMT
 #define REG_FMT			"0x%02lx"
@@ -60,7 +60,7 @@ typedef struct {
     ECCState ecc;
 } SLNANDState;
 
-static uint64_t sl_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t sl_read(void *opaque, hwaddr addr, unsigned size)
 {
     SLNANDState *s = (SLNANDState *) opaque;
     int ryby;
@@ -102,7 +102,7 @@ static uint64_t sl_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return 0;
 }
 
-static void sl_write(void *opaque, target_phys_addr_t addr,
+static void sl_write(void *opaque, hwaddr addr,
                      uint64_t value, unsigned size)
 {
     SLNANDState *s = (SLNANDState *) opaque;
@@ -156,7 +156,7 @@ static void sl_flash_register(PXA2xxState *cpu, int size)
         qdev_prop_set_uint8(dev, "chip_id", 0xf1);
 
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, FLASH_BASE);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, FLASH_BASE);
 }
 
 static int sl_nand_init(SysBusDevice *dev) {
@@ -459,7 +459,7 @@ static void spitz_keyboard_register(PXA2xxState *cpu)
     SpitzKeyboardState *s;
 
     dev = sysbus_create_simple("spitz-keyboard", -1, NULL);
-    s = FROM_SYSBUS(SpitzKeyboardState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(SpitzKeyboardState, SYS_BUS_DEVICE(dev));
 
     for (i = 0; i < SPITZ_KEY_SENSE_NUM; i ++)
         qdev_connect_gpio_out(dev, i, qdev_get_gpio_in(cpu->gpio, spitz_gpio_key_sense[i]));
@@ -879,15 +879,14 @@ static struct arm_boot_info spitz_binfo = {
     .ram_size = 0x04000000,
 };
 
-static void spitz_common_init(ram_addr_t ram_size,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, enum spitz_model_e model, int arm_id)
+static void spitz_common_init(QEMUMachineInitArgs *args,
+                              enum spitz_model_e model, int arm_id)
 {
     PXA2xxState *mpu;
     DeviceState *scp0, *scp1 = NULL;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
+    const char *cpu_model = args->cpu_model;
 
     if (!cpu_model)
         cpu_model = (model == terrier) ? "pxa270-c5" : "pxa270-c0";
@@ -928,72 +927,60 @@ static void spitz_common_init(ram_addr_t ram_size,
         /* A 4.0 GB microdrive is permanently sitting in CF slot 0.  */
         spitz_microdrive_attach(mpu, 0);
 
-    spitz_binfo.kernel_filename = kernel_filename;
-    spitz_binfo.kernel_cmdline = kernel_cmdline;
-    spitz_binfo.initrd_filename = initrd_filename;
+    spitz_binfo.kernel_filename = args->kernel_filename;
+    spitz_binfo.kernel_cmdline = args->kernel_cmdline;
+    spitz_binfo.initrd_filename = args->initrd_filename;
     spitz_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &spitz_binfo);
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static void spitz_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void spitz_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, spitz, 0x2c9);
+    spitz_common_init(args, spitz, 0x2c9);
 }
 
-static void borzoi_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void borzoi_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, borzoi, 0x33f);
+    spitz_common_init(args, borzoi, 0x33f);
 }
 
-static void akita_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void akita_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, akita, 0x2e8);
+    spitz_common_init(args, akita, 0x2e8);
 }
 
-static void terrier_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void terrier_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, terrier, 0x33f);
+    spitz_common_init(args, terrier, 0x33f);
 }
 
 static QEMUMachine akitapda_machine = {
     .name = "akita",
     .desc = "Akita PDA (PXA270)",
     .init = akita_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine spitzpda_machine = {
     .name = "spitz",
     .desc = "Spitz PDA (PXA270)",
     .init = spitz_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine borzoipda_machine = {
     .name = "borzoi",
     .desc = "Borzoi PDA (PXA270)",
     .init = borzoi_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine terrierpda_machine = {
     .name = "terrier",
     .desc = "Terrier PDA (PXA270)",
     .init = terrier_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void spitz_machine_init(void)
@@ -1039,7 +1026,7 @@ static void sl_nand_class_init(ObjectClass *klass, void *data)
     dc->props = sl_nand_properties;
 }
 
-static TypeInfo sl_nand_info = {
+static const TypeInfo sl_nand_info = {
     .name          = "sl-nand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLNANDState),
@@ -1074,7 +1061,7 @@ static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
     dc->props = spitz_keyboard_properties;
 }
 
-static TypeInfo spitz_keyboard_info = {
+static const TypeInfo spitz_keyboard_info = {
     .name          = "spitz-keyboard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SpitzKeyboardState),
@@ -1083,10 +1070,11 @@ static TypeInfo spitz_keyboard_info = {
 
 static const VMStateDescription vmstate_corgi_ssp_regs = {
     .name = "corgi-ssp",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
     .fields = (VMStateField []) {
+        VMSTATE_SSI_SLAVE(ssidev, CorgiSSPState),
         VMSTATE_UINT32_ARRAY(enable, CorgiSSPState, 3),
         VMSTATE_END_OF_LIST(),
     }
@@ -1102,7 +1090,7 @@ static void corgi_ssp_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_corgi_ssp_regs;
 }
 
-static TypeInfo corgi_ssp_info = {
+static const TypeInfo corgi_ssp_info = {
     .name          = "corgi-ssp",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(CorgiSSPState),
@@ -1115,6 +1103,7 @@ static const VMStateDescription vmstate_spitz_lcdtg_regs = {
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .fields = (VMStateField []) {
+        VMSTATE_SSI_SLAVE(ssidev, SpitzLCDTG),
         VMSTATE_UINT32(bl_intensity, SpitzLCDTG),
         VMSTATE_UINT32(bl_power, SpitzLCDTG),
         VMSTATE_END_OF_LIST(),
@@ -1131,7 +1120,7 @@ static void spitz_lcdtg_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_spitz_lcdtg_regs;
 }
 
-static TypeInfo spitz_lcdtg_info = {
+static const TypeInfo spitz_lcdtg_info = {
     .name          = "spitz-lcdtg",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(SpitzLCDTG),
diff --git a/hw/srp.h b/hw/srp.h
index 3009bd56ce7..5e0cad5c196 100644
--- a/hw/srp.h
+++ b/hw/srp.h
@@ -177,13 +177,13 @@ struct srp_tsk_mgmt {
     uint8_t    reserved1[6];
     uint64_t   tag;
     uint8_t    reserved2[4];
-    uint64_t   lun QEMU_PACKED;
+    uint64_t   lun;
     uint8_t    reserved3[2];
     uint8_t    tsk_mgmt_func;
     uint8_t    reserved4;
     uint64_t   task_tag;
     uint8_t    reserved5[8];
-};
+} QEMU_PACKED;
 
 /*
  * We need the packed attribute because the SRP spec only aligns the
@@ -198,14 +198,14 @@ struct srp_cmd {
     uint8_t    data_in_desc_cnt;
     uint64_t   tag;
     uint8_t    reserved2[4];
-    uint64_t   lun QEMU_PACKED;
+    uint64_t   lun;
     uint8_t    reserved3;
     uint8_t    task_attr;
     uint8_t    reserved4;
     uint8_t    add_cdb_len;
     uint8_t    cdb[16];
     uint8_t    add_data[0];
-};
+} QEMU_PACKED;
 
 enum {
     SRP_RSP_FLAG_RSPVALID = 1 << 0,
diff --git a/hw/ssd0303.c b/hw/ssd0303.c
index 4e1ee6e12b5..8777b1681ae 100644
--- a/hw/ssd0303.c
+++ b/hw/ssd0303.c
@@ -11,7 +11,7 @@
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
 #include "i2c.h"
-#include "console.h"
+#include "ui/console.h"
 
 //#define DEBUG_SSD0303 1
 
@@ -252,7 +252,7 @@ static void ssd0303_update_display(void *opaque)
         }
     }
     s->redraw = 0;
-    dpy_update(s->ds, 0, 0, 96 * MAGNIFY, 16 * MAGNIFY);
+    dpy_gfx_update(s->ds, 0, 0, 96 * MAGNIFY, 16 * MAGNIFY);
 }
 
 static void ssd0303_invalidate_display(void * opaque)
@@ -306,7 +306,7 @@ static void ssd0303_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_ssd0303;
 }
 
-static TypeInfo ssd0303_info = {
+static const TypeInfo ssd0303_info = {
     .name          = "ssd0303",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(ssd0303_state),
diff --git a/hw/ssd0323.c b/hw/ssd0323.c
index b101c5112ca..84c86a52446 100644
--- a/hw/ssd0323.c
+++ b/hw/ssd0323.c
@@ -11,7 +11,7 @@
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
 #include "ssi.h"
-#include "console.h"
+#include "ui/console.h"
 
 //#define DEBUG_SSD0323 1
 
@@ -260,7 +260,7 @@ static void ssd0323_update_display(void *opaque)
         }
     }
     s->redraw = 0;
-    dpy_update(s->ds, 0, 0, 128 * MAGNIFY, 64 * MAGNIFY);
+    dpy_gfx_update(s->ds, 0, 0, 128 * MAGNIFY, 64 * MAGNIFY);
 }
 
 static void ssd0323_invalidate_display(void * opaque)
@@ -279,6 +279,7 @@ static void ssd0323_cd(void *opaque, int n, int level)
 
 static void ssd0323_save(QEMUFile *f, void *opaque)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssd0323_state *s = (ssd0323_state *)opaque;
     int i;
 
@@ -296,10 +297,13 @@ static void ssd0323_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->remap);
     qemu_put_be32(f, s->mode);
     qemu_put_buffer(f, s->framebuffer, sizeof(s->framebuffer));
+
+    qemu_put_be32(f, ss->cs);
 }
 
 static int ssd0323_load(QEMUFile *f, void *opaque, int version_id)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssd0323_state *s = (ssd0323_state *)opaque;
     int i;
 
@@ -321,6 +325,8 @@ static int ssd0323_load(QEMUFile *f, void *opaque, int version_id)
     s->mode = qemu_get_be32(f);
     qemu_get_buffer(f, s->framebuffer, sizeof(s->framebuffer));
 
+    ss->cs = qemu_get_be32(f);
+
     return 0;
 }
 
@@ -348,9 +354,10 @@ static void ssd0323_class_init(ObjectClass *klass, void *data)
 
     k->init = ssd0323_init;
     k->transfer = ssd0323_transfer;
+    k->cs_polarity = SSI_CS_HIGH;
 }
 
-static TypeInfo ssd0323_info = {
+static const TypeInfo ssd0323_info = {
     .name          = "ssd0323",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ssd0323_state),
diff --git a/hw/ssi-sd.c b/hw/ssi-sd.c
index b519bdb29ac..dca8906e7db 100644
--- a/hw/ssi-sd.c
+++ b/hw/ssi-sd.c
@@ -10,7 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "ssi.h"
 #include "sd.h"
 
@@ -197,6 +197,7 @@ static uint32_t ssi_sd_transfer(SSISlave *dev, uint32_t val)
 
 static void ssi_sd_save(QEMUFile *f, void *opaque)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssi_sd_state *s = (ssi_sd_state *)opaque;
     int i;
 
@@ -209,10 +210,13 @@ static void ssi_sd_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->arglen);
     qemu_put_be32(f, s->response_pos);
     qemu_put_be32(f, s->stopping);
+
+    qemu_put_be32(f, ss->cs);
 }
 
 static int ssi_sd_load(QEMUFile *f, void *opaque, int version_id)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssi_sd_state *s = (ssi_sd_state *)opaque;
     int i;
 
@@ -229,6 +233,8 @@ static int ssi_sd_load(QEMUFile *f, void *opaque, int version_id)
     s->response_pos = qemu_get_be32(f);
     s->stopping = qemu_get_be32(f);
 
+    ss->cs = qemu_get_be32(f);
+
     return 0;
 }
 
@@ -250,9 +256,10 @@ static void ssi_sd_class_init(ObjectClass *klass, void *data)
 
     k->init = ssi_sd_init;
     k->transfer = ssi_sd_transfer;
+    k->cs_polarity = SSI_CS_LOW;
 }
 
-static TypeInfo ssi_sd_info = {
+static const TypeInfo ssi_sd_info = {
     .name          = "ssi-sd",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ssi_sd_state),
diff --git a/hw/ssi.c b/hw/ssi.c
index e5f14a0cefd..0b18176f7a7 100644
--- a/hw/ssi.c
+++ b/hw/ssi.c
@@ -2,6 +2,8 @@
  * QEMU Synchronous Serial Interface support
  *
  * Copyright (c) 2009 CodeSourcery.
+ * Copyright (c) 2012 Peter A.G. Crosthwaite (peter.crosthwaite@petalogix.com)
+ * Copyright (c) 2012 PetaLogix Pty Ltd.
  * Written by Paul Brook
  *
  * This code is licensed under the GNU GPL v2.
@@ -25,17 +27,40 @@ static const TypeInfo ssi_bus_info = {
     .instance_size = sizeof(SSIBus),
 };
 
+static void ssi_cs_default(void *opaque, int n, int level)
+{
+    SSISlave *s = SSI_SLAVE(opaque);
+    bool cs = !!level;
+    assert(n == 0);
+    if (s->cs != cs) {
+        SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(s);
+        if (ssc->set_cs) {
+            ssc->set_cs(s, cs);
+        }
+    }
+    s->cs = cs;
+}
+
+static uint32_t ssi_transfer_raw_default(SSISlave *dev, uint32_t val)
+{
+    SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(dev);
+
+    if ((dev->cs && ssc->cs_polarity == SSI_CS_HIGH) ||
+            (!dev->cs && ssc->cs_polarity == SSI_CS_LOW) ||
+            ssc->cs_polarity == SSI_CS_NONE) {
+        return ssc->transfer(dev, val);
+    }
+    return 0;
+}
+
 static int ssi_slave_init(DeviceState *dev)
 {
     SSISlave *s = SSI_SLAVE(dev);
     SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(s);
-    SSIBus *bus;
-    BusChild *kid;
 
-    bus = FROM_QBUS(SSIBus, qdev_get_parent_bus(dev));
-    kid = QTAILQ_FIRST(&bus->qbus.children);
-    if (kid->child != dev || QTAILQ_NEXT(kid, sibling) != NULL) {
-        hw_error("Too many devices on SSI bus");
+    if (ssc->transfer_raw == ssi_transfer_raw_default &&
+            ssc->cs_polarity != SSI_CS_NONE) {
+        qdev_init_gpio_in(&s->qdev, ssi_cs_default, 1);
     }
 
     return ssc->init(s);
@@ -43,12 +68,17 @@ static int ssi_slave_init(DeviceState *dev)
 
 static void ssi_slave_class_init(ObjectClass *klass, void *data)
 {
+    SSISlaveClass *ssc = SSI_SLAVE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
+
     dc->init = ssi_slave_init;
     dc->bus_type = TYPE_SSI_BUS;
+    if (!ssc->transfer_raw) {
+        ssc->transfer_raw = ssi_transfer_raw_default;
+    }
 }
 
-static TypeInfo ssi_slave_info = {
+static const TypeInfo ssi_slave_info = {
     .name = TYPE_SSI_SLAVE,
     .parent = TYPE_DEVICE,
     .class_init = ssi_slave_class_init,
@@ -56,10 +86,15 @@ static TypeInfo ssi_slave_info = {
     .abstract = true,
 };
 
+DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name)
+{
+    return qdev_create(&bus->qbus, name);
+}
+
 DeviceState *ssi_create_slave(SSIBus *bus, const char *name)
 {
-    DeviceState *dev;
-    dev = qdev_create(&bus->qbus, name);
+    DeviceState *dev = ssi_create_slave_no_init(bus, name);
+
     qdev_init_nofail(dev);
     return dev;
 }
@@ -74,18 +109,29 @@ SSIBus *ssi_create_bus(DeviceState *parent, const char *name)
 uint32_t ssi_transfer(SSIBus *bus, uint32_t val)
 {
     BusChild *kid;
-    SSISlave *slave;
     SSISlaveClass *ssc;
+    uint32_t r = 0;
 
-    kid = QTAILQ_FIRST(&bus->qbus.children);
-    if (!kid) {
-        return 0;
+    QTAILQ_FOREACH(kid, &bus->qbus.children, sibling) {
+        SSISlave *slave = SSI_SLAVE(kid->child);
+        ssc = SSI_SLAVE_GET_CLASS(slave);
+        r |= ssc->transfer_raw(slave, val);
     }
-    slave = SSI_SLAVE(kid->child);
-    ssc = SSI_SLAVE_GET_CLASS(slave);
-    return ssc->transfer(slave, val);
+
+    return r;
 }
 
+const VMStateDescription vmstate_ssi_slave = {
+    .name = "SSISlave",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(cs, SSISlave),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void ssi_slave_register_types(void)
 {
     type_register_static(&ssi_bus_info);
@@ -93,3 +139,36 @@ static void ssi_slave_register_types(void)
 }
 
 type_init(ssi_slave_register_types)
+
+typedef struct SSIAutoConnectArg {
+    qemu_irq **cs_linep;
+    SSIBus *bus;
+} SSIAutoConnectArg;
+
+static int ssi_auto_connect_slave(Object *child, void *opaque)
+{
+    SSIAutoConnectArg *arg = opaque;
+    SSISlave *dev = (SSISlave *)object_dynamic_cast(child, TYPE_SSI_SLAVE);
+    qemu_irq cs_line;
+
+    if (!dev) {
+        return 0;
+    }
+
+    cs_line = qdev_get_gpio_in(DEVICE(dev), 0);
+    qdev_set_parent_bus(DEVICE(dev), &arg->bus->qbus);
+    **arg->cs_linep = cs_line;
+    (*arg->cs_linep)++;
+    return 0;
+}
+
+void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_line,
+                             SSIBus *bus)
+{
+    SSIAutoConnectArg arg = {
+        .cs_linep = &cs_line,
+        .bus = bus
+    };
+
+    object_child_foreach(OBJECT(parent), ssi_auto_connect_slave, &arg);
+}
diff --git a/hw/ssi.h b/hw/ssi.h
index 06657d7c16c..a05d60beb4a 100644
--- a/hw/ssi.h
+++ b/hw/ssi.h
@@ -23,28 +23,70 @@ typedef struct SSISlave SSISlave;
 #define SSI_SLAVE_GET_CLASS(obj) \
      OBJECT_GET_CLASS(SSISlaveClass, (obj), TYPE_SSI_SLAVE)
 
+typedef enum {
+    SSI_CS_NONE = 0,
+    SSI_CS_LOW,
+    SSI_CS_HIGH,
+} SSICSMode;
+
 /* Slave devices.  */
 typedef struct SSISlaveClass {
     DeviceClass parent_class;
 
     int (*init)(SSISlave *dev);
+
+    /* if you have standard or no CS behaviour, just override transfer.
+     * This is called when the device cs is active (true by default).
+     */
     uint32_t (*transfer)(SSISlave *dev, uint32_t val);
+    /* called when the CS line changes. Optional, devices only need to implement
+     * this if they have side effects associated with the cs line (beyond
+     * tristating the txrx lines).
+     */
+    int (*set_cs)(SSISlave *dev, bool select);
+    /* define whether or not CS exists and is active low/high */
+    SSICSMode cs_polarity;
+
+    /* if you have non-standard CS behaviour override this to take control
+     * of the CS behaviour at the device level. transfer, set_cs, and
+     * cs_polarity are unused if this is overwritten. Transfer_raw will
+     * always be called for the device for every txrx access to the parent bus
+     */
+    uint32_t (*transfer_raw)(SSISlave *dev, uint32_t val);
 } SSISlaveClass;
 
 struct SSISlave {
     DeviceState qdev;
+
+    /* Chip select state */
+    bool cs;
 };
 
 #define SSI_SLAVE_FROM_QDEV(dev) DO_UPCAST(SSISlave, qdev, dev)
 #define FROM_SSI_SLAVE(type, dev) DO_UPCAST(type, ssidev, dev)
 
+extern const VMStateDescription vmstate_ssi_slave;
+
+#define VMSTATE_SSI_SLAVE(_field, _state) {                          \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(SSISlave),                                  \
+    .vmsd       = &vmstate_ssi_slave,                                \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, SSISlave),    \
+}
+
 DeviceState *ssi_create_slave(SSIBus *bus, const char *name);
+DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name);
 
 /* Master interface.  */
 SSIBus *ssi_create_bus(DeviceState *parent, const char *name);
 
 uint32_t ssi_transfer(SSIBus *bus, uint32_t val);
 
+/* Automatically connect all children nodes a spi controller as slaves */
+void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_lines,
+                             SSIBus *bus);
+
 /* max111x.c */
 void max111x_set_input(DeviceState *dev, int line, uint8_t value);
 
diff --git a/hw/stellaris.c b/hw/stellaris.c
index 562fbbf4943..9b8f2034f11 100644
--- a/hw/stellaris.c
+++ b/hw/stellaris.c
@@ -11,11 +11,11 @@
 #include "ssi.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i2c.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define GPIO_A 0
 #define GPIO_B 1
@@ -141,7 +141,7 @@ static void gptm_tick(void *opaque)
     gptm_update_irq(s);
 }
 
-static uint64_t gptm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t gptm_read(void *opaque, hwaddr offset,
                           unsigned size)
 {
     gptm_state *s = (gptm_state *)opaque;
@@ -190,7 +190,7 @@ static uint64_t gptm_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void gptm_write(void *opaque, target_phys_addr_t offset,
+static void gptm_write(void *opaque, hwaddr offset,
                        uint64_t value, unsigned size)
 {
     gptm_state *s = (gptm_state *)opaque;
@@ -410,7 +410,7 @@ static int ssys_board_class(const ssys_state *s)
     }
 }
 
-static uint64_t ssys_read(void *opaque, target_phys_addr_t offset,
+static uint64_t ssys_read(void *opaque, hwaddr offset,
                           unsigned size)
 {
     ssys_state *s = (ssys_state *)opaque;
@@ -515,7 +515,7 @@ static void ssys_calculate_system_clock(ssys_state *s)
     }
 }
 
-static void ssys_write(void *opaque, target_phys_addr_t offset,
+static void ssys_write(void *opaque, hwaddr offset,
                        uint64_t value, unsigned size)
 {
     ssys_state *s = (ssys_state *)opaque;
@@ -701,7 +701,7 @@ typedef struct {
 #define STELLARIS_I2C_MCS_IDLE    0x20
 #define STELLARIS_I2C_MCS_BUSBSY  0x40
 
-static uint64_t stellaris_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_i2c_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     stellaris_i2c_state *s = (stellaris_i2c_state *)opaque;
@@ -738,7 +738,7 @@ static void stellaris_i2c_update(stellaris_i2c_state *s)
     qemu_set_irq(s->irq, level);
 }
 
-static void stellaris_i2c_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_i2c_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     stellaris_i2c_state *s = (stellaris_i2c_state *)opaque;
@@ -989,7 +989,7 @@ static void stellaris_adc_reset(stellaris_adc_state *s)
     }
 }
 
-static uint64_t stellaris_adc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_adc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     stellaris_adc_state *s = (stellaris_adc_state *)opaque;
@@ -1037,7 +1037,7 @@ static uint64_t stellaris_adc_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void stellaris_adc_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_adc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     stellaris_adc_state *s = (stellaris_adc_state *)opaque;
@@ -1154,57 +1154,6 @@ static int stellaris_adc_init(SysBusDevice *dev)
     return 0;
 }
 
-/* Some boards have both an OLED controller and SD card connected to
-   the same SSI port, with the SD card chip select connected to a
-   GPIO pin.  Technically the OLED chip select is connected to the SSI
-   Fss pin.  We do not bother emulating that as both devices should
-   never be selected simultaneously, and our OLED controller ignores stray
-   0xff commands that occur when deselecting the SD card.  */
-
-typedef struct {
-    SSISlave ssidev;
-    qemu_irq irq;
-    int current_dev;
-    SSIBus *bus[2];
-} stellaris_ssi_bus_state;
-
-static void stellaris_ssi_bus_select(void *opaque, int irq, int level)
-{
-    stellaris_ssi_bus_state *s = (stellaris_ssi_bus_state *)opaque;
-
-    s->current_dev = level;
-}
-
-static uint32_t stellaris_ssi_bus_transfer(SSISlave *dev, uint32_t val)
-{
-    stellaris_ssi_bus_state *s = FROM_SSI_SLAVE(stellaris_ssi_bus_state, dev);
-
-    return ssi_transfer(s->bus[s->current_dev], val);
-}
-
-static const VMStateDescription vmstate_stellaris_ssi_bus = {
-    .name = "stellaris_ssi_bus",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
-    .fields      = (VMStateField[]) {
-        VMSTATE_INT32(current_dev, stellaris_ssi_bus_state),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
-static int stellaris_ssi_bus_init(SSISlave *dev)
-{
-    stellaris_ssi_bus_state *s = FROM_SSI_SLAVE(stellaris_ssi_bus_state, dev);
-
-    s->bus[0] = ssi_create_bus(&dev->qdev, "ssi0");
-    s->bus[1] = ssi_create_bus(&dev->qdev, "ssi1");
-    qdev_init_gpio_in(&dev->qdev, stellaris_ssi_bus_select, 1);
-
-    vmstate_register(&dev->qdev, -1, &vmstate_stellaris_ssi_bus, s);
-    return 0;
-}
-
 /* Board init.  */
 static stellaris_board_info stellaris_boards[] = {
   { "LM3S811EVB",
@@ -1305,19 +1254,25 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     if (board->dc2 & (1 << 4)) {
         dev = sysbus_create_simple("pl022", 0x40008000, pic[7]);
         if (board->peripherals & BP_OLED_SSI) {
-            DeviceState *mux;
             void *bus;
-
+            DeviceState *sddev;
+            DeviceState *ssddev;
+
+            /* Some boards have both an OLED controller and SD card connected to
+             * the same SSI port, with the SD card chip select connected to a
+             * GPIO pin.  Technically the OLED chip select is connected to the
+             * SSI Fss pin.  We do not bother emulating that as both devices
+             * should never be selected simultaneously, and our OLED controller
+             * ignores stray 0xff commands that occur when deselecting the SD
+             * card.
+             */
             bus = qdev_get_child_bus(dev, "ssi");
-            mux = ssi_create_slave(bus, "evb6965-ssi");
-            gpio_out[GPIO_D][0] = qdev_get_gpio_in(mux, 0);
-
-            bus = qdev_get_child_bus(mux, "ssi0");
-            ssi_create_slave(bus, "ssi-sd");
 
-            bus = qdev_get_child_bus(mux, "ssi1");
-            dev = ssi_create_slave(bus, "ssd0323");
-            gpio_out[GPIO_C][7] = qdev_get_gpio_in(dev, 0);
+            sddev = ssi_create_slave(bus, "ssi-sd");
+            ssddev = ssi_create_slave(bus, "ssd0323");
+            gpio_out[GPIO_D][0] = qemu_irq_split(qdev_get_gpio_in(sddev, 0),
+                                                 qdev_get_gpio_in(ssddev, 0));
+            gpio_out[GPIO_C][7] = qdev_get_gpio_in(ssddev, 1);
 
             /* Make sure the select pin is high.  */
             qemu_irq_raise(gpio_out[GPIO_D][0]);
@@ -1331,8 +1286,8 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         enet = qdev_create(NULL, "stellaris_enet");
         qdev_set_nic_properties(enet, &nd_table[0]);
         qdev_init_nofail(enet);
-        sysbus_mmio_map(sysbus_from_qdev(enet), 0, 0x40048000);
-        sysbus_connect_irq(sysbus_from_qdev(enet), 0, pic[42]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(enet), 0, 0x40048000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, pic[42]);
     }
     if (board->peripherals & BP_GAMEPAD) {
         qemu_irq gpad_irq[5];
@@ -1358,19 +1313,17 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
 }
 
 /* FIXME: Figure out how to generate these from stellaris_boards.  */
-static void lm3s811evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void lm3s811evb_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[0]);
 }
 
-static void lm3s6965evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void lm3s6965evb_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
@@ -1378,12 +1331,14 @@ static QEMUMachine lm3s811evb_machine = {
     .name = "lm3s811evb",
     .desc = "Stellaris LM3S811EVB",
     .init = lm3s811evb_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine lm3s6965evb_machine = {
     .name = "lm3s6965evb",
     .desc = "Stellaris LM3S6965EVB",
     .init = lm3s6965evb_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void stellaris_machine_init(void)
@@ -1394,21 +1349,6 @@ static void stellaris_machine_init(void)
 
 machine_init(stellaris_machine_init);
 
-static void stellaris_ssi_bus_class_init(ObjectClass *klass, void *data)
-{
-    SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
-
-    k->init = stellaris_ssi_bus_init;
-    k->transfer = stellaris_ssi_bus_transfer;
-}
-
-static TypeInfo stellaris_ssi_bus_info = {
-    .name          = "evb6965-ssi",
-    .parent        = TYPE_SSI_SLAVE,
-    .instance_size = sizeof(stellaris_ssi_bus_state),
-    .class_init    = stellaris_ssi_bus_class_init,
-};
-
 static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
@@ -1416,7 +1356,7 @@ static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_i2c_init;
 }
 
-static TypeInfo stellaris_i2c_info = {
+static const TypeInfo stellaris_i2c_info = {
     .name          = "stellaris-i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_i2c_state),
@@ -1430,7 +1370,7 @@ static void stellaris_gptm_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_gptm_init;
 }
 
-static TypeInfo stellaris_gptm_info = {
+static const TypeInfo stellaris_gptm_info = {
     .name          = "stellaris-gptm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(gptm_state),
@@ -1444,7 +1384,7 @@ static void stellaris_adc_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_adc_init;
 }
 
-static TypeInfo stellaris_adc_info = {
+static const TypeInfo stellaris_adc_info = {
     .name          = "stellaris-adc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_adc_state),
@@ -1456,7 +1396,6 @@ static void stellaris_register_types(void)
     type_register_static(&stellaris_i2c_info);
     type_register_static(&stellaris_gptm_info);
     type_register_static(&stellaris_adc_info);
-    type_register_static(&stellaris_ssi_bus_info);
 }
 
 type_init(stellaris_register_types)
diff --git a/hw/stellaris_enet.c b/hw/stellaris_enet.c
index bc97280cca1..5e9053fa262 100644
--- a/hw/stellaris_enet.c
+++ b/hw/stellaris_enet.c
@@ -7,7 +7,7 @@
  * This code is licensed under the GPL.
  */
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include <zlib.h>
 
 //#define DEBUG_STELLARIS_ENET 1
@@ -130,7 +130,7 @@ static int stellaris_enet_can_receive(NetClientState *nc)
     return (s->np < 31);
 }
 
-static uint64_t stellaris_enet_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_enet_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     stellaris_enet_state *s = (stellaris_enet_state *)opaque;
@@ -198,7 +198,7 @@ static uint64_t stellaris_enet_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void stellaris_enet_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_enet_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     stellaris_enet_state *s = (stellaris_enet_state *)opaque;
@@ -434,7 +434,7 @@ static void stellaris_enet_class_init(ObjectClass *klass, void *data)
     dc->props = stellaris_enet_properties;
 }
 
-static TypeInfo stellaris_enet_info = {
+static const TypeInfo stellaris_enet_info = {
     .name          = "stellaris_enet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_enet_state),
diff --git a/hw/stellaris_input.c b/hw/stellaris_input.c
index 68c600c04c5..7a95c3fc885 100644
--- a/hw/stellaris_input.c
+++ b/hw/stellaris_input.c
@@ -8,7 +8,7 @@
  */
 #include "hw.h"
 #include "devices.h"
-#include "console.h"
+#include "ui/console.h"
 
 typedef struct {
     qemu_irq irq;
diff --git a/hw/stream.c b/hw/stream.c
index be57e8b247d..d4cf84d4c03 100644
--- a/hw/stream.c
+++ b/hw/stream.c
@@ -8,7 +8,7 @@ stream_push(StreamSlave *sink, uint8_t *buf, size_t len, uint32_t *app)
     k->push(sink, buf, len, app);
 }
 
-static TypeInfo stream_slave_info = {
+static const TypeInfo stream_slave_info = {
     .name          = TYPE_STREAM_SLAVE,
     .parent        = TYPE_INTERFACE,
     .class_size = sizeof(StreamSlaveClass),
diff --git a/hw/stream.h b/hw/stream.h
index 21123a90894..f6137d6e258 100644
--- a/hw/stream.h
+++ b/hw/stream.h
@@ -2,7 +2,7 @@
 #define STREAM_H 1
 
 #include "qemu-common.h"
-#include "qemu/object.h"
+#include "qom/object.h"
 
 /* stream slave. Used until qdev provides a generic way.  */
 #define TYPE_STREAM_SLAVE "stream-slave"
diff --git a/hw/strongarm.c b/hw/strongarm.c
index 7150eeb2db7..ab736e300e4 100644
--- a/hw/strongarm.c
+++ b/hw/strongarm.c
@@ -28,9 +28,10 @@
  */
 #include "sysbus.h"
 #include "strongarm.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "ssi.h"
 
 //#define DEBUG
@@ -59,7 +60,7 @@
 #endif
 
 static struct {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irq;
 } sa_serial[] = {
     { 0x80010000, SA_PIC_UART1 },
@@ -113,7 +114,7 @@ static void strongarm_pic_set_irq(void *opaque, int irq, int level)
     strongarm_pic_update(s);
 }
 
-static uint64_t strongarm_pic_mem_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_pic_mem_read(void *opaque, hwaddr offset,
                                        unsigned size)
 {
     StrongARMPICState *s = opaque;
@@ -138,7 +139,7 @@ static uint64_t strongarm_pic_mem_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void strongarm_pic_mem_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_pic_mem_write(void *opaque, hwaddr offset,
                                     uint64_t value, unsigned size)
 {
     StrongARMPICState *s = opaque;
@@ -211,7 +212,7 @@ static void strongarm_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_pic_regs;
 }
 
-static TypeInfo strongarm_pic_info = {
+static const TypeInfo strongarm_pic_info = {
     .name          = "strongarm_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPICState),
@@ -294,7 +295,7 @@ static inline void strongarm_rtc_hz_tick(void *opaque)
     strongarm_rtc_int_update(s);
 }
 
-static uint64_t strongarm_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_rtc_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     StrongARMRTCState *s = opaque;
@@ -316,7 +317,7 @@ static uint64_t strongarm_rtc_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void strongarm_rtc_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_rtc_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     StrongARMRTCState *s = opaque;
@@ -432,7 +433,7 @@ static void strongarm_rtc_sysbus_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_rtc_regs;
 }
 
-static TypeInfo strongarm_rtc_sysbus_info = {
+static const TypeInfo strongarm_rtc_sysbus_info = {
     .name          = "strongarm-rtc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMRTCState),
@@ -517,7 +518,7 @@ static void strongarm_gpio_handler_update(StrongARMGPIOInfo *s)
     s->prev_level = level;
 }
 
-static uint64_t strongarm_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_gpio_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     StrongARMGPIOInfo *s = opaque;
@@ -559,7 +560,7 @@ static uint64_t strongarm_gpio_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void strongarm_gpio_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_gpio_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     StrongARMGPIOInfo *s = opaque;
@@ -609,7 +610,7 @@ static const MemoryRegionOps strongarm_gpio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static DeviceState *strongarm_gpio_init(target_phys_addr_t base,
+static DeviceState *strongarm_gpio_init(hwaddr base,
                 DeviceState *pic)
 {
     DeviceState *dev;
@@ -618,9 +619,9 @@ static DeviceState *strongarm_gpio_init(target_phys_addr_t base,
     dev = qdev_create(NULL, "strongarm-gpio");
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     for (i = 0; i < 12; i++)
-        sysbus_connect_irq(sysbus_from_qdev(dev), i,
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i,
                     qdev_get_gpio_in(pic, SA_PIC_GPIO0_EDGE + i));
 
     return dev;
@@ -673,7 +674,7 @@ static void strongarm_gpio_class_init(ObjectClass *klass, void *data)
     dc->desc = "StrongARM GPIO controller";
 }
 
-static TypeInfo strongarm_gpio_info = {
+static const TypeInfo strongarm_gpio_info = {
     .name          = "strongarm-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMGPIOInfo),
@@ -729,7 +730,7 @@ static void strongarm_ppc_handler_update(StrongARMPPCInfo *s)
     s->prev_level = level;
 }
 
-static uint64_t strongarm_ppc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_ppc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     StrongARMPPCInfo *s = opaque;
@@ -759,7 +760,7 @@ static uint64_t strongarm_ppc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void strongarm_ppc_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_ppc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     StrongARMPPCInfo *s = opaque;
@@ -839,7 +840,7 @@ static void strongarm_ppc_class_init(ObjectClass *klass, void *data)
     dc->desc = "StrongARM PPC controller";
 }
 
-static TypeInfo strongarm_ppc_info = {
+static const TypeInfo strongarm_ppc_info = {
     .name          = "strongarm-ppc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPPCInfo),
@@ -1095,7 +1096,7 @@ static void strongarm_uart_tx(void *opaque)
     strongarm_uart_update_int_status(s);
 }
 
-static uint64_t strongarm_uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_uart_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     StrongARMUARTState *s = opaque;
@@ -1137,7 +1138,7 @@ static uint64_t strongarm_uart_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void strongarm_uart_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_uart_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     StrongARMUARTState *s = opaque;
@@ -1298,7 +1299,7 @@ static void strongarm_uart_class_init(ObjectClass *klass, void *data)
     dc->props = strongarm_uart_properties;
 }
 
-static TypeInfo strongarm_uart_info = {
+static const TypeInfo strongarm_uart_info = {
     .name          = "strongarm-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMUARTState),
@@ -1376,7 +1377,7 @@ static void strongarm_ssp_fifo_update(StrongARMSSPState *s)
     strongarm_ssp_int_update(s);
 }
 
-static uint64_t strongarm_ssp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_ssp_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     StrongARMSSPState *s = opaque;
@@ -1409,7 +1410,7 @@ static uint64_t strongarm_ssp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void strongarm_ssp_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_ssp_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     StrongARMSSPState *s = opaque;
@@ -1537,7 +1538,7 @@ static void strongarm_ssp_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_ssp_regs;
 }
 
-static TypeInfo strongarm_ssp_info = {
+static const TypeInfo strongarm_ssp_info = {
     .name          = "strongarm-ssp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMSSPState),
@@ -1596,9 +1597,9 @@ StrongARMState *sa1110_init(MemoryRegion *sysmem,
         DeviceState *dev = qdev_create(NULL, "strongarm-uart");
         qdev_prop_set_chr(dev, "chardev", serial_hds[i]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0,
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0,
                 sa_serial[i].io_base);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0,
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0,
                 qdev_get_gpio_in(s->pic, sa_serial[i].irq));
     }
 
diff --git a/hw/strongarm.h b/hw/strongarm.h
index d30dd6ac5e4..2893f94445d 100644
--- a/hw/strongarm.h
+++ b/hw/strongarm.h
@@ -1,7 +1,7 @@
 #ifndef _STRONGARM_H
 #define _STRONGARM_H
 
-#include "memory.h"
+#include "exec/memory.h"
 
 #define SA_CS0          0x00000000
 #define SA_CS1          0x08000000
diff --git a/hw/sun4c_intctl.c b/hw/sun4c_intctl.c
index 8dfa5ecab6a..f8f4d023a35 100644
--- a/hw/sun4c_intctl.c
+++ b/hw/sun4c_intctl.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "sun4m.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 
 //#define DEBUG_IRQ_COUNT
@@ -61,7 +61,7 @@ typedef struct Sun4c_INTCTLState {
 
 static void sun4c_check_interrupts(void *opaque);
 
-static uint64_t sun4c_intctl_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sun4c_intctl_mem_read(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     Sun4c_INTCTLState *s = opaque;
@@ -73,7 +73,7 @@ static uint64_t sun4c_intctl_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sun4c_intctl_mem_write(void *opaque, target_phys_addr_t addr,
+static void sun4c_intctl_mem_write(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     Sun4c_INTCTLState *s = opaque;
@@ -94,29 +94,6 @@ static const MemoryRegionOps sun4c_intctl_mem_ops = {
     },
 };
 
-void sun4c_pic_info(Monitor *mon, void *opaque)
-{
-    Sun4c_INTCTLState *s = opaque;
-
-    monitor_printf(mon, "master: pending 0x%2.2x, enabled 0x%2.2x\n",
-                   s->pending, s->reg);
-}
-
-void sun4c_irq_info(Monitor *mon, void *opaque)
-{
-#ifndef DEBUG_IRQ_COUNT
-    monitor_printf(mon, "irq statistic code not compiled.\n");
-#else
-    Sun4c_INTCTLState *s = opaque;
-    int64_t count;
-
-    monitor_printf(mon, "IRQ statistics:\n");
-    count = s->irq_count;
-    if (count > 0)
-        monitor_printf(mon, " %" PRId64 "\n", count);
-#endif
-}
-
 static const uint32_t intbit_to_level[] = { 0, 1, 4, 6, 8, 10, 0, 14, };
 
 static void sun4c_check_interrupts(void *opaque)
@@ -216,7 +193,7 @@ static void sun4c_intctl_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_sun4c_intctl;
 }
 
-static TypeInfo sun4c_intctl_info = {
+static const TypeInfo sun4c_intctl_info = {
     .name          = "sun4c_intctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Sun4c_INTCTLState),
diff --git a/hw/sun4m.c b/hw/sun4m.c
index 0f909b5f867..9903f443cba 100644
--- a/hw/sun4m.c
+++ b/hw/sun4m.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sun4m.h"
 #include "nvram.h"
 #include "sparc32_dma.h"
 #include "fdc.h"
-#include "sysemu.h"
-#include "net.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 #include "boards.h"
 #include "firmware_abi.h"
 #include "esp.h"
@@ -40,7 +40,7 @@
 #include "qdev-addr.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "trace.h"
 
 /*
@@ -87,16 +87,16 @@
 #define ESCC_CLOCK 4915200
 
 struct sun4m_hwdef {
-    target_phys_addr_t iommu_base, iommu_pad_base, iommu_pad_len, slavio_base;
-    target_phys_addr_t intctl_base, counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base, fd_base;
-    target_phys_addr_t afx_base, idreg_base, dma_base, esp_base, le_base;
-    target_phys_addr_t tcx_base, cs_base, apc_base, aux1_base, aux2_base;
-    target_phys_addr_t bpp_base, dbri_base, sx_base;
+    hwaddr iommu_base, iommu_pad_base, iommu_pad_len, slavio_base;
+    hwaddr intctl_base, counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base, fd_base;
+    hwaddr afx_base, idreg_base, dma_base, esp_base, le_base;
+    hwaddr tcx_base, cs_base, apc_base, aux1_base, aux2_base;
+    hwaddr bpp_base, dbri_base, sx_base;
     struct {
-        target_phys_addr_t reg_base, vram_base;
+        hwaddr reg_base, vram_base;
     } vsimm[MAX_VSIMMS];
-    target_phys_addr_t ecc_base;
+    hwaddr ecc_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t ecc_version;
@@ -108,13 +108,13 @@ struct sun4m_hwdef {
 #define MAX_IOUNITS 5
 
 struct sun4d_hwdef {
-    target_phys_addr_t iounit_bases[MAX_IOUNITS], slavio_base;
-    target_phys_addr_t counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base;
-    target_phys_addr_t espdma_base, esp_base;
-    target_phys_addr_t ledma_base, le_base;
-    target_phys_addr_t tcx_base;
-    target_phys_addr_t sbi_base;
+    hwaddr iounit_bases[MAX_IOUNITS], slavio_base;
+    hwaddr counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base;
+    hwaddr espdma_base, esp_base;
+    hwaddr ledma_base, le_base;
+    hwaddr tcx_base;
+    hwaddr sbi_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t iounit_version;
@@ -123,11 +123,11 @@ struct sun4d_hwdef {
 };
 
 struct sun4c_hwdef {
-    target_phys_addr_t iommu_base, slavio_base;
-    target_phys_addr_t intctl_base, counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base, fd_base;
-    target_phys_addr_t idreg_base, dma_base, esp_base, le_base;
-    target_phys_addr_t tcx_base, aux1_base;
+    hwaddr iommu_base, slavio_base;
+    hwaddr intctl_base, counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base, fd_base;
+    hwaddr idreg_base, dma_base, esp_base, le_base;
+    hwaddr tcx_base, aux1_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t iommu_version;
@@ -216,13 +216,13 @@ static void nvram_init(M48t59State *nvram, uint8_t *macaddr,
 
 static DeviceState *slavio_intctl;
 
-void sun4m_pic_info(Monitor *mon)
+void sun4m_pic_info(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_pic_info(mon, slavio_intctl);
 }
 
-void sun4m_irq_info(Monitor *mon)
+void sun4m_irq_info(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_irq_info(mon, slavio_intctl);
@@ -253,21 +253,24 @@ void cpu_check_irqs(CPUSPARCState *env)
     }
 }
 
-static void cpu_kick_irq(CPUSPARCState *env)
+static void cpu_kick_irq(SPARCCPU *cpu)
 {
+    CPUSPARCState *env = &cpu->env;
+
     env->halted = 0;
     cpu_check_irqs(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
 }
 
 static void cpu_set_irq(void *opaque, int irq, int level)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     if (level) {
         trace_sun4m_cpu_set_irq_raise(irq);
         env->pil_in |= 1 << irq;
-        cpu_kick_irq(env);
+        cpu_kick_irq(cpu);
     } else {
         trace_sun4m_cpu_set_irq_lower(irq);
         env->pil_in &= ~(1 << irq);
@@ -370,7 +373,7 @@ static unsigned long sun4m_load_kernel(const char *kernel_filename,
     return kernel_size;
 }
 
-static void *iommu_init(target_phys_addr_t addr, uint32_t version, qemu_irq irq)
+static void *iommu_init(hwaddr addr, uint32_t version, qemu_irq irq)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -378,14 +381,14 @@ static void *iommu_init(target_phys_addr_t addr, uint32_t version, qemu_irq irq)
     dev = qdev_create(NULL, "iommu");
     qdev_prop_set_uint32(dev, "version", version);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, addr);
 
     return s;
 }
 
-static void *sparc32_dma_init(target_phys_addr_t daddr, qemu_irq parent_irq,
+static void *sparc32_dma_init(hwaddr daddr, qemu_irq parent_irq,
                               void *iommu, qemu_irq *dev_irq, int is_ledma)
 {
     DeviceState *dev;
@@ -395,7 +398,7 @@ static void *sparc32_dma_init(target_phys_addr_t daddr, qemu_irq parent_irq,
     qdev_prop_set_ptr(dev, "iommu_opaque", iommu);
     qdev_prop_set_uint32(dev, "is_ledma", is_ledma);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, parent_irq);
     *dev_irq = qdev_get_gpio_in(dev, 0);
     sysbus_mmio_map(s, 0, daddr);
@@ -403,7 +406,7 @@ static void *sparc32_dma_init(target_phys_addr_t daddr, qemu_irq parent_irq,
     return s;
 }
 
-static void lance_init(NICInfo *nd, target_phys_addr_t leaddr,
+static void lance_init(NICInfo *nd, hwaddr leaddr,
                        void *dma_opaque, qemu_irq irq)
 {
     DeviceState *dev;
@@ -416,15 +419,15 @@ static void lance_init(NICInfo *nd, target_phys_addr_t leaddr,
     qdev_set_nic_properties(dev, nd);
     qdev_prop_set_ptr(dev, "dma", dma_opaque);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, leaddr);
     sysbus_connect_irq(s, 0, irq);
     reset = qdev_get_gpio_in(dev, 0);
     qdev_connect_gpio_out(dma_opaque, 0, reset);
 }
 
-static DeviceState *slavio_intctl_init(target_phys_addr_t addr,
-                                       target_phys_addr_t addrg,
+static DeviceState *slavio_intctl_init(hwaddr addr,
+                                       hwaddr addrg,
                                        qemu_irq **parent_irq)
 {
     DeviceState *dev;
@@ -434,7 +437,7 @@ static DeviceState *slavio_intctl_init(target_phys_addr_t addr,
     dev = qdev_create(NULL, "slavio_intctl");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_CPUS; i++) {
         for (j = 0; j < MAX_PILS; j++) {
@@ -452,7 +455,7 @@ static DeviceState *slavio_intctl_init(target_phys_addr_t addr,
 #define SYS_TIMER_OFFSET      0x10000ULL
 #define CPU_TIMER_OFFSET(cpu) (0x1000ULL * cpu)
 
-static void slavio_timer_init_all(target_phys_addr_t addr, qemu_irq master_irq,
+static void slavio_timer_init_all(hwaddr addr, qemu_irq master_irq,
                                   qemu_irq *cpu_irqs, unsigned int num_cpus)
 {
     DeviceState *dev;
@@ -462,25 +465,36 @@ static void slavio_timer_init_all(target_phys_addr_t addr, qemu_irq master_irq,
     dev = qdev_create(NULL, "slavio_timer");
     qdev_prop_set_uint32(dev, "num_cpus", num_cpus);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, master_irq);
     sysbus_mmio_map(s, 0, addr + SYS_TIMER_OFFSET);
 
     for (i = 0; i < MAX_CPUS; i++) {
-        sysbus_mmio_map(s, i + 1, addr + (target_phys_addr_t)CPU_TIMER_OFFSET(i));
+        sysbus_mmio_map(s, i + 1, addr + (hwaddr)CPU_TIMER_OFFSET(i));
         sysbus_connect_irq(s, i + 1, cpu_irqs[i]);
     }
 }
 
+static qemu_irq  slavio_system_powerdown;
+
+static void slavio_powerdown_req(Notifier *n, void *opaque)
+{
+    qemu_irq_raise(slavio_system_powerdown);
+}
+
+static Notifier slavio_system_powerdown_notifier = {
+    .notify = slavio_powerdown_req
+};
+
 #define MISC_LEDS 0x01600000
 #define MISC_CFG  0x01800000
 #define MISC_DIAG 0x01a00000
 #define MISC_MDM  0x01b00000
 #define MISC_SYS  0x01f00000
 
-static void slavio_misc_init(target_phys_addr_t base,
-                             target_phys_addr_t aux1_base,
-                             target_phys_addr_t aux2_base, qemu_irq irq,
+static void slavio_misc_init(hwaddr base,
+                             hwaddr aux1_base,
+                             hwaddr aux2_base, qemu_irq irq,
                              qemu_irq fdc_tc)
 {
     DeviceState *dev;
@@ -488,7 +502,7 @@ static void slavio_misc_init(target_phys_addr_t base,
 
     dev = qdev_create(NULL, "slavio_misc");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     if (base) {
         /* 8 bit registers */
         /* Slavio control */
@@ -514,10 +528,11 @@ static void slavio_misc_init(target_phys_addr_t base,
     }
     sysbus_connect_irq(s, 0, irq);
     sysbus_connect_irq(s, 1, fdc_tc);
-    qemu_system_powerdown = qdev_get_gpio_in(dev, 0);
+    slavio_system_powerdown = qdev_get_gpio_in(dev, 0);
+    qemu_register_powerdown_notifier(&slavio_system_powerdown_notifier);
 }
 
-static void ecc_init(target_phys_addr_t base, qemu_irq irq, uint32_t version)
+static void ecc_init(hwaddr base, qemu_irq irq, uint32_t version)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -525,7 +540,7 @@ static void ecc_init(target_phys_addr_t base, qemu_irq irq, uint32_t version)
     dev = qdev_create(NULL, "eccmemctl");
     qdev_prop_set_uint32(dev, "version", version);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, base);
     if (version == 0) { // SS-600MP only
@@ -533,20 +548,20 @@ static void ecc_init(target_phys_addr_t base, qemu_irq irq, uint32_t version)
     }
 }
 
-static void apc_init(target_phys_addr_t power_base, qemu_irq cpu_halt)
+static void apc_init(hwaddr power_base, qemu_irq cpu_halt)
 {
     DeviceState *dev;
     SysBusDevice *s;
 
     dev = qdev_create(NULL, "apc");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* Power management (APC) XXX: not a Slavio device */
     sysbus_mmio_map(s, 0, power_base);
     sysbus_connect_irq(s, 0, cpu_halt);
 }
 
-static void tcx_init(target_phys_addr_t addr, int vram_size, int width,
+static void tcx_init(hwaddr addr, int vram_size, int width,
                      int height, int depth)
 {
     DeviceState *dev;
@@ -559,7 +574,7 @@ static void tcx_init(target_phys_addr_t addr, int vram_size, int width,
     qdev_prop_set_uint16(dev, "height", height);
     qdev_prop_set_uint16(dev, "depth", depth);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* 8-bit plane */
     sysbus_mmio_map(s, 0, addr + 0x00800000ULL);
     /* DAC */
@@ -582,14 +597,14 @@ static void tcx_init(target_phys_addr_t addr, int vram_size, int width,
 /* NCR89C100/MACIO Internal ID register */
 static const uint8_t idreg_data[] = { 0xfe, 0x81, 0x01, 0x03 };
 
-static void idreg_init(target_phys_addr_t addr)
+static void idreg_init(hwaddr addr)
 {
     DeviceState *dev;
     SysBusDevice *s;
 
     dev = qdev_create(NULL, "macio_idreg");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
     cpu_physical_memory_write_rom(addr, idreg_data, sizeof(idreg_data));
@@ -618,7 +633,7 @@ static void idreg_class_init(ObjectClass *klass, void *data)
     k->init = idreg_init1;
 }
 
-static TypeInfo idreg_info = {
+static const TypeInfo idreg_info = {
     .name          = "macio_idreg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IDRegState),
@@ -631,14 +646,14 @@ typedef struct AFXState {
 } AFXState;
 
 /* SS-5 TCX AFX register */
-static void afx_init(target_phys_addr_t addr)
+static void afx_init(hwaddr addr)
 {
     DeviceState *dev;
     SysBusDevice *s;
 
     dev = qdev_create(NULL, "tcx_afx");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 }
@@ -660,7 +675,7 @@ static void afx_class_init(ObjectClass *klass, void *data)
     k->init = afx_init1;
 }
 
-static TypeInfo afx_info = {
+static const TypeInfo afx_info = {
     .name          = "tcx_afx",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(AFXState),
@@ -675,11 +690,11 @@ typedef struct PROMState {
 /* Boot PROM (OpenBIOS) */
 static uint64_t translate_prom_address(void *opaque, uint64_t addr)
 {
-    target_phys_addr_t *base_addr = (target_phys_addr_t *)opaque;
+    hwaddr *base_addr = (hwaddr *)opaque;
     return addr + *base_addr - PROM_VADDR;
 }
 
-static void prom_init(target_phys_addr_t addr, const char *bios_name)
+static void prom_init(hwaddr addr, const char *bios_name)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -688,7 +703,7 @@ static void prom_init(target_phys_addr_t addr, const char *bios_name)
 
     dev = qdev_create(NULL, "openprom");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 
@@ -737,7 +752,7 @@ static void prom_class_init(ObjectClass *klass, void *data)
     dc->props = prom_properties;
 }
 
-static TypeInfo prom_info = {
+static const TypeInfo prom_info = {
     .name          = "openprom",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PROMState),
@@ -762,7 +777,7 @@ static int ram_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size,
+static void ram_init(hwaddr addr, ram_addr_t RAM_size,
                      uint64_t max_mem)
 {
     DeviceState *dev;
@@ -778,7 +793,7 @@ static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size,
         exit(1);
     }
     dev = qdev_create(NULL, "memory");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     d = FROM_SYSBUS(RamDevice, s);
     d->size = RAM_size;
@@ -801,7 +816,7 @@ static void ram_class_init(ObjectClass *klass, void *data)
     dc->props = ram_properties;
 }
 
-static TypeInfo ram_info = {
+static const TypeInfo ram_info = {
     .name          = "memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RamDevice),
@@ -828,7 +843,7 @@ static void cpu_devinit(const char *cpu_model, unsigned int id,
         qemu_register_reset(secondary_cpu_reset, cpu);
         env->halted = 1;
     }
-    *cpu_irqs = qemu_allocate_irqs(cpu_set_irq, env, MAX_PILS);
+    *cpu_irqs = qemu_allocate_irqs(cpu_set_irq, cpu, MAX_PILS);
     env->prom_addr = prom_addr;
 }
 
@@ -1006,6 +1021,7 @@ static void sun4m_hw_init(const struct sun4m_hwdef *hwdef, ram_addr_t RAM_size,
                  hwdef->ecc_version);
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1015,9 +1031,7 @@ static void sun4m_hw_init(const struct sun4m_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
                        strlen(kernel_cmdline) + 1);
     } else {
@@ -1291,92 +1305,118 @@ static const struct sun4m_hwdef sun4m_hwdefs[] = {
 };
 
 /* SPARCstation 5 hardware initialisation */
-static void ss5_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss5_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 10 hardware initialisation */
-static void ss10_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void ss10_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[1], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCserver 600MP hardware initialisation */
-static void ss600mp_init(ram_addr_t RAM_size,
-                         const char *boot_device,
-                         const char *kernel_filename,
-                         const char *kernel_cmdline,
-                         const char *initrd_filename, const char *cpu_model)
+static void ss600mp_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[2], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 20 hardware initialisation */
-static void ss20_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void ss20_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[3], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation Voyager hardware initialisation */
-static void vger_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void vger_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[4], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation LX hardware initialisation */
-static void ss_lx_init(ram_addr_t RAM_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+static void ss_lx_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[5], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 4 hardware initialisation */
-static void ss4_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss4_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[6], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCClassic hardware initialisation */
-static void scls_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void scls_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[7], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCbook hardware initialisation */
-static void sbook_init(ram_addr_t RAM_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+static void sbook_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[8], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
@@ -1385,67 +1425,76 @@ static QEMUMachine ss5_machine = {
     .name = "SS-5",
     .desc = "Sun4m platform, SPARCstation 5",
     .init = ss5_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss10_machine = {
     .name = "SS-10",
     .desc = "Sun4m platform, SPARCstation 10",
     .init = ss10_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss600mp_machine = {
     .name = "SS-600MP",
     .desc = "Sun4m platform, SPARCserver 600MP",
     .init = ss600mp_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss20_machine = {
     .name = "SS-20",
     .desc = "Sun4m platform, SPARCstation 20",
     .init = ss20_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine voyager_machine = {
     .name = "Voyager",
     .desc = "Sun4m platform, SPARCstation Voyager",
     .init = vger_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss_lx_machine = {
     .name = "LX",
     .desc = "Sun4m platform, SPARCstation LX",
     .init = ss_lx_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss4_machine = {
     .name = "SS-4",
     .desc = "Sun4m platform, SPARCstation 4",
     .init = ss4_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine scls_machine = {
     .name = "SPARCClassic",
     .desc = "Sun4m platform, SPARCClassic",
     .init = scls_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sbook_machine = {
     .name = "SPARCbook",
     .desc = "Sun4m platform, SPARCbook",
     .init = sbook_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static const struct sun4d_hwdef sun4d_hwdefs[] = {
@@ -1503,7 +1552,7 @@ static const struct sun4d_hwdef sun4d_hwdefs[] = {
     },
 };
 
-static DeviceState *sbi_init(target_phys_addr_t addr, qemu_irq **parent_irq)
+static DeviceState *sbi_init(hwaddr addr, qemu_irq **parent_irq)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -1512,7 +1561,7 @@ static DeviceState *sbi_init(target_phys_addr_t addr, qemu_irq **parent_irq)
     dev = qdev_create(NULL, "sbi");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_CPUS; i++) {
         sysbus_connect_irq(s, i, *parent_irq[i]);
@@ -1564,7 +1613,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
     }
 
     for (i = 0; i < MAX_IOUNITS; i++)
-        if (hwdef->iounit_bases[i] != (target_phys_addr_t)-1)
+        if (hwdef->iounit_bases[i] != (hwaddr)-1)
             iounits[i] = iommu_init(hwdef->iounit_bases[i],
                                     hwdef->iounit_version,
                                     sbi_irq[0]);
@@ -1617,6 +1666,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
                "Sun4d");
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1626,9 +1676,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, 0);
     }
@@ -1639,21 +1687,27 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
 }
 
 /* SPARCserver 1000 hardware initialisation */
-static void ss1000_init(ram_addr_t RAM_size,
-                        const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void ss1000_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4d_hw_init(&sun4d_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCcenter 2000 hardware initialisation */
-static void ss2000_init(ram_addr_t RAM_size,
-                        const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void ss2000_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4d_hw_init(&sun4d_hwdefs[1], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
@@ -1662,16 +1716,18 @@ static QEMUMachine ss1000_machine = {
     .name = "SS-1000",
     .desc = "Sun4d platform, SPARCserver 1000",
     .init = ss1000_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 8,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss2000_machine = {
     .name = "SS-2000",
     .desc = "Sun4d platform, SPARCcenter 2000",
     .init = ss2000_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 20,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static const struct sun4c_hwdef sun4c_hwdefs[] = {
@@ -1697,7 +1753,7 @@ static const struct sun4c_hwdef sun4c_hwdefs[] = {
     },
 };
 
-static DeviceState *sun4c_intctl_init(target_phys_addr_t addr,
+static DeviceState *sun4c_intctl_init(hwaddr addr,
                                       qemu_irq *parent_irq)
 {
     DeviceState *dev;
@@ -1707,7 +1763,7 @@ static DeviceState *sun4c_intctl_init(target_phys_addr_t addr,
     dev = qdev_create(NULL, "sun4c_intctl");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_PILS; i++) {
         sysbus_connect_irq(s, i, parent_irq[i]);
@@ -1778,7 +1834,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
               slavio_irq[1], serial_hds[0], serial_hds[1],
               ESCC_CLOCK, 1);
 
-    if (hwdef->fd_base != (target_phys_addr_t)-1) {
+    if (hwdef->fd_base != (hwaddr)-1) {
         /* there is zero or one floppy drive */
         memset(fd, 0, sizeof(fd));
         fd[0] = drive_get(IF_FLOPPY, 0, 0);
@@ -1811,6 +1867,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
                "Sun4c");
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1820,9 +1877,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, 0);
     }
@@ -1833,11 +1888,14 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
 }
 
 /* SPARCstation 2 hardware initialisation */
-static void ss2_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss2_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4c_hw_init(&sun4c_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
@@ -1846,7 +1904,8 @@ static QEMUMachine ss2_machine = {
     .name = "SS-2",
     .desc = "Sun4c platform, SPARCstation 2",
     .init = ss2_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sun4m_register_types(void)
diff --git a/hw/sun4m.h b/hw/sun4m.h
index 504c3af4136..0361eeed41c 100644
--- a/hw/sun4m.h
+++ b/hw/sun4m.h
@@ -6,17 +6,17 @@
 /* Devices used by sparc32 system.  */
 
 /* iommu.c */
-void sparc_iommu_memory_rw(void *opaque, target_phys_addr_t addr,
+void sparc_iommu_memory_rw(void *opaque, hwaddr addr,
                                  uint8_t *buf, int len, int is_write);
 static inline void sparc_iommu_memory_read(void *opaque,
-                                           target_phys_addr_t addr,
+                                           hwaddr addr,
                                            uint8_t *buf, int len)
 {
     sparc_iommu_memory_rw(opaque, addr, buf, len, 0);
 }
 
 static inline void sparc_iommu_memory_write(void *opaque,
-                                            target_phys_addr_t addr,
+                                            hwaddr addr,
                                             uint8_t *buf, int len)
 {
     sparc_iommu_memory_rw(opaque, addr, buf, len, 1);
@@ -26,13 +26,9 @@ static inline void sparc_iommu_memory_write(void *opaque,
 void slavio_pic_info(Monitor *mon, DeviceState *dev);
 void slavio_irq_info(Monitor *mon, DeviceState *dev);
 
-/* sun4c_intctl.c */
-void sun4c_pic_info(Monitor *mon, void *opaque);
-void sun4c_irq_info(Monitor *mon, void *opaque);
-
 /* sun4m.c */
-void sun4m_pic_info(Monitor *mon);
-void sun4m_irq_info(Monitor *mon);
+void sun4m_pic_info(Monitor *mon, const QDict *qdict);
+void sun4m_irq_info(Monitor *mon, const QDict *qdict);
 
 /* sparc32_dma.c */
 #include "sparc32_dma.h"
diff --git a/hw/sun4m_iommu.c b/hw/sun4m_iommu.c
index ebefa91b7aa..8f9635f3430 100644
--- a/hw/sun4m_iommu.c
+++ b/hw/sun4m_iommu.c
@@ -130,16 +130,16 @@ typedef struct IOMMUState {
     SysBusDevice busdev;
     MemoryRegion iomem;
     uint32_t regs[IOMMU_NREGS];
-    target_phys_addr_t iostart;
+    hwaddr iostart;
     qemu_irq irq;
     uint32_t version;
 } IOMMUState;
 
-static uint64_t iommu_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t iommu_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     IOMMUState *s = opaque;
-    target_phys_addr_t saddr;
+    hwaddr saddr;
     uint32_t ret;
 
     saddr = addr >> 2;
@@ -157,11 +157,11 @@ static uint64_t iommu_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void iommu_mem_write(void *opaque, target_phys_addr_t addr,
+static void iommu_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     IOMMUState *s = opaque;
-    target_phys_addr_t saddr;
+    hwaddr saddr;
 
     saddr = addr >> 2;
     trace_sun4m_iommu_mem_writel(saddr, val);
@@ -249,11 +249,11 @@ static const MemoryRegionOps iommu_mem_ops = {
     },
 };
 
-static uint32_t iommu_page_get_flags(IOMMUState *s, target_phys_addr_t addr)
+static uint32_t iommu_page_get_flags(IOMMUState *s, hwaddr addr)
 {
     uint32_t ret;
-    target_phys_addr_t iopte;
-    target_phys_addr_t pa = addr;
+    hwaddr iopte;
+    hwaddr pa = addr;
 
     iopte = s->regs[IOMMU_BASE] << 4;
     addr &= ~s->iostart;
@@ -264,17 +264,17 @@ static uint32_t iommu_page_get_flags(IOMMUState *s, target_phys_addr_t addr)
     return ret;
 }
 
-static target_phys_addr_t iommu_translate_pa(target_phys_addr_t addr,
+static hwaddr iommu_translate_pa(hwaddr addr,
                                              uint32_t pte)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     pa = ((pte & IOPTE_PAGE) << 4) + (addr & ~IOMMU_PAGE_MASK);
     trace_sun4m_iommu_translate_pa(addr, pa, pte);
     return pa;
 }
 
-static void iommu_bad_addr(IOMMUState *s, target_phys_addr_t addr,
+static void iommu_bad_addr(IOMMUState *s, hwaddr addr,
                            int is_write)
 {
     trace_sun4m_iommu_bad_addr(addr);
@@ -286,12 +286,12 @@ static void iommu_bad_addr(IOMMUState *s, target_phys_addr_t addr,
     qemu_irq_raise(s->irq);
 }
 
-void sparc_iommu_memory_rw(void *opaque, target_phys_addr_t addr,
+void sparc_iommu_memory_rw(void *opaque, hwaddr addr,
                            uint8_t *buf, int len, int is_write)
 {
     int l;
     uint32_t flags;
-    target_phys_addr_t page, phys_addr;
+    hwaddr page, phys_addr;
 
     while (len > 0) {
         page = addr & IOMMU_PAGE_MASK;
@@ -373,7 +373,7 @@ static void iommu_class_init(ObjectClass *klass, void *data)
     dc->props = iommu_properties;
 }
 
-static TypeInfo iommu_info = {
+static const TypeInfo iommu_info = {
     .name          = "iommu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IOMMUState),
diff --git a/hw/sun4u.c b/hw/sun4u.c
index 07cd04273a8..9fbda29ac4e 100644
--- a/hw/sun4u.c
+++ b/hw/sun4u.c
@@ -22,14 +22,15 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "apb_pci.h"
 #include "pc.h"
+#include "serial.h"
 #include "nvram.h"
 #include "fdc.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "firmware_abi.h"
 #include "fw_cfg.h"
@@ -37,9 +38,8 @@
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
-#include "exec-memory.h"
-#include "vga-pci.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_IRQ
 //#define DEBUG_EBUS
@@ -311,16 +311,19 @@ void cpu_check_irqs(CPUSPARCState *env)
     }
 }
 
-static void cpu_kick_irq(CPUSPARCState *env)
+static void cpu_kick_irq(SPARCCPU *cpu)
 {
+    CPUSPARCState *env = &cpu->env;
+
     env->halted = 0;
     cpu_check_irqs(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
 }
 
 static void cpu_set_ivec_irq(void *opaque, int irq, int level)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     if (level) {
         if (!(env->ivec_status & 0x20)) {
@@ -367,7 +370,7 @@ void cpu_get_timer(QEMUFile *f, CPUTimer *s)
     qemu_get_timer(f, s->qtimer);
 }
 
-static CPUTimer* cpu_timer_create(const char* name, CPUSPARCState *env,
+static CPUTimer *cpu_timer_create(const char *name, SPARCCPU *cpu,
                                   QEMUBHFunc *cb, uint32_t frequency,
                                   uint64_t disabled_mask)
 {
@@ -380,7 +383,7 @@ static CPUTimer* cpu_timer_create(const char* name, CPUSPARCState *env,
     timer->disabled = 1;
     timer->clock_offset = qemu_get_clock_ns(vm_clock);
 
-    timer->qtimer = qemu_new_timer_ns(vm_clock, cb, env);
+    timer->qtimer = qemu_new_timer_ns(vm_clock, cb, cpu);
 
     return timer;
 }
@@ -419,7 +422,8 @@ static void main_cpu_reset(void *opaque)
 
 static void tick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->tick;
 
@@ -431,12 +435,13 @@ static void tick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_TIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static void stick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->stick;
 
@@ -448,12 +453,13 @@ static void stick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_STIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static void hstick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->hstick;
 
@@ -465,7 +471,7 @@ static void hstick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_STIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static int64_t cpu_to_timer_ticks(int64_t cpu_ticks, uint32_t frequency)
@@ -612,7 +618,7 @@ static void ebus_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_BRIDGE_OTHER;
 }
 
-static TypeInfo ebus_info = {
+static const TypeInfo ebus_info = {
     .name          = "ebus",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(EbusState),
@@ -626,12 +632,12 @@ typedef struct PROMState {
 
 static uint64_t translate_prom_address(void *opaque, uint64_t addr)
 {
-    target_phys_addr_t *base_addr = (target_phys_addr_t *)opaque;
+    hwaddr *base_addr = (hwaddr *)opaque;
     return addr + *base_addr - PROM_VADDR;
 }
 
 /* Boot PROM (OpenBIOS) */
-static void prom_init(target_phys_addr_t addr, const char *bios_name)
+static void prom_init(hwaddr addr, const char *bios_name)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -640,7 +646,7 @@ static void prom_init(target_phys_addr_t addr, const char *bios_name)
 
     dev = qdev_create(NULL, "openprom");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 
@@ -689,7 +695,7 @@ static void prom_class_init(ObjectClass *klass, void *data)
     dc->props = prom_properties;
 }
 
-static TypeInfo prom_info = {
+static const TypeInfo prom_info = {
     .name          = "openprom",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PROMState),
@@ -715,7 +721,7 @@ static int ram_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size)
+static void ram_init(hwaddr addr, ram_addr_t RAM_size)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -723,7 +729,7 @@ static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size)
 
     /* allocate RAM */
     dev = qdev_create(NULL, "memory");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     d = FROM_SYSBUS(RamDevice, s);
     d->size = RAM_size;
@@ -746,7 +752,7 @@ static void ram_class_init(ObjectClass *klass, void *data)
     dc->props = ram_properties;
 }
 
-static TypeInfo ram_info = {
+static const TypeInfo ram_info = {
     .name          = "memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RamDevice),
@@ -773,13 +779,13 @@ static SPARCCPU *cpu_devinit(const char *cpu_model, const struct hwdef *hwdef)
     }
     env = &cpu->env;
 
-    env->tick = cpu_timer_create("tick", env, tick_irq,
+    env->tick = cpu_timer_create("tick", cpu, tick_irq,
                                   tick_frequency, TICK_NPT_MASK);
 
-    env->stick = cpu_timer_create("stick", env, stick_irq,
+    env->stick = cpu_timer_create("stick", cpu, stick_irq,
                                    stick_frequency, TICK_INT_DIS);
 
-    env->hstick = cpu_timer_create("hstick", env, hstick_irq,
+    env->hstick = cpu_timer_create("hstick", cpu, hstick_irq,
                                     hstick_frequency, TICK_INT_DIS);
 
     reset_info = g_malloc0(sizeof(ResetData));
@@ -798,7 +804,6 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
                         const struct hwdef *hwdef)
 {
     SPARCCPU *cpu;
-    CPUSPARCState *env;
     M48t59State *nvram;
     unsigned int i;
     uint64_t initrd_addr, initrd_size, kernel_addr, kernel_size, kernel_entry;
@@ -811,14 +816,13 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
 
     /* init CPUs */
     cpu = cpu_devinit(cpu_model, hwdef);
-    env = &cpu->env;
 
     /* set up devices */
     ram_init(0, RAM_size);
 
     prom_init(hwdef->prom_addr, bios_name);
 
-    ivec_irqs = qemu_allocate_irqs(cpu_set_ivec_irq, env, IVEC_MAX);
+    ivec_irqs = qemu_allocate_irqs(cpu_set_ivec_irq, cpu, IVEC_MAX);
     pci_bus = pci_apb_init(APB_SPECIAL_BASE, APB_MEM_BASE, ivec_irqs, &pci_bus2,
                            &pci_bus3, &pbm_irqs);
     pci_vga_init(pci_bus);
@@ -874,6 +878,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
                            (uint8_t *)&nd_table[0].macaddr);
 
     fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -882,9 +887,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
                        strlen(kernel_cmdline) + 1);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, 0);
     }
@@ -930,31 +933,40 @@ static const struct hwdef hwdefs[] = {
 };
 
 /* Sun4u hardware initialisation */
-static void sun4u_init(ram_addr_t RAM_size,
-                       const char *boot_devices,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
-{
+static void sun4u_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[0]);
 }
 
 /* Sun4v hardware initialisation */
-static void sun4v_init(ram_addr_t RAM_size,
-                       const char *boot_devices,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
-{
+static void sun4v_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[1]);
 }
 
 /* Niagara hardware initialisation */
-static void niagara_init(ram_addr_t RAM_size,
-                         const char *boot_devices,
-                         const char *kernel_filename, const char *kernel_cmdline,
-                         const char *initrd_filename, const char *cpu_model)
-{
+static void niagara_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[2]);
 }
@@ -965,6 +977,7 @@ static QEMUMachine sun4u_machine = {
     .init = sun4u_init,
     .max_cpus = 1, // XXX for now
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sun4v_machine = {
@@ -972,6 +985,7 @@ static QEMUMachine sun4v_machine = {
     .desc = "Sun4v platform",
     .init = sun4v_init,
     .max_cpus = 1, // XXX for now
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine niagara_machine = {
@@ -979,6 +993,7 @@ static QEMUMachine niagara_machine = {
     .desc = "Sun4v platform, Niagara",
     .init = niagara_init,
     .max_cpus = 1, // XXX for now
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sun4u_register_types(void)
diff --git a/hw/sysbus.c b/hw/sysbus.c
index 9d8b1eaf7d8..6d9d1df4192 100644
--- a/hw/sysbus.c
+++ b/hw/sysbus.c
@@ -18,8 +18,8 @@
  */
 
 #include "sysbus.h"
-#include "monitor.h"
-#include "exec-memory.h"
+#include "monitor/monitor.h"
+#include "exec/address-spaces.h"
 
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *sysbus_get_fw_dev_path(DeviceState *dev);
@@ -48,7 +48,7 @@ void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq)
     }
 }
 
-void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr)
+void sysbus_mmio_map(SysBusDevice *dev, int n, hwaddr addr)
 {
     assert(n >= 0 && n < dev->num_mmio);
 
@@ -56,7 +56,7 @@ void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr)
         /* ??? region already mapped here.  */
         return;
     }
-    if (dev->mmio[n].addr != (target_phys_addr_t)-1) {
+    if (dev->mmio[n].addr != (hwaddr)-1) {
         /* Unregister previous mapping.  */
         memory_region_del_subregion(get_system_memory(), dev->mmio[n].memory);
     }
@@ -122,7 +122,7 @@ static int sysbus_device_init(DeviceState *dev)
 }
 
 DeviceState *sysbus_create_varargs(const char *name,
-                                   target_phys_addr_t addr, ...)
+                                   hwaddr addr, ...)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -131,9 +131,9 @@ DeviceState *sysbus_create_varargs(const char *name,
     int n;
 
     dev = qdev_create(NULL, name);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
     }
     va_start(va, addr);
@@ -151,7 +151,7 @@ DeviceState *sysbus_create_varargs(const char *name,
 }
 
 DeviceState *sysbus_try_create_varargs(const char *name,
-                                       target_phys_addr_t addr, ...)
+                                       hwaddr addr, ...)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -163,9 +163,9 @@ DeviceState *sysbus_try_create_varargs(const char *name,
     if (!dev) {
         return NULL;
     }
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
     }
     va_start(va, addr);
@@ -184,8 +184,8 @@ DeviceState *sysbus_try_create_varargs(const char *name,
 
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 {
-    SysBusDevice *s = sysbus_from_qdev(dev);
-    target_phys_addr_t size;
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
+    hwaddr size;
     int i;
 
     monitor_printf(mon, "%*sirq %d\n", indent, "", s->num_irq);
@@ -198,7 +198,7 @@ static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 
 static char *sysbus_get_fw_dev_path(DeviceState *dev)
 {
-    SysBusDevice *s = sysbus_from_qdev(dev);
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
     char path[40];
     int off;
 
@@ -211,16 +211,16 @@ static char *sysbus_get_fw_dev_path(DeviceState *dev)
         snprintf(path + off, sizeof(path) - off, "@i%04x", s->pio[0]);
     }
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
-void sysbus_add_memory(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem)
 {
     memory_region_add_subregion(get_system_memory(), addr, mem);
 }
 
-void sysbus_add_memory_overlap(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory_overlap(SysBusDevice *dev, hwaddr addr,
                                MemoryRegion *mem, unsigned priority)
 {
     memory_region_add_subregion_overlap(get_system_memory(), addr, mem,
@@ -232,7 +232,7 @@ void sysbus_del_memory(SysBusDevice *dev, MemoryRegion *mem)
     memory_region_del_subregion(get_system_memory(), mem);
 }
 
-void sysbus_add_io(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem)
 {
     memory_region_add_subregion(get_system_io(), addr, mem);
@@ -255,7 +255,7 @@ static void sysbus_device_class_init(ObjectClass *klass, void *data)
     k->bus_type = TYPE_SYSTEM_BUS;
 }
 
-static TypeInfo sysbus_device_type_info = {
+static const TypeInfo sysbus_device_type_info = {
     .name = TYPE_SYS_BUS_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(SysBusDevice),
@@ -274,7 +274,7 @@ static void main_system_bus_create(void)
     main_system_bus = g_malloc0(system_bus_info.instance_size);
     qbus_create_inplace(main_system_bus, TYPE_SYSTEM_BUS, NULL,
                         "main-system-bus");
-    main_system_bus->glib_allocated = true;
+    OBJECT(main_system_bus)->free = g_free;
     object_property_add_child(container_get(qdev_get_machine(),
                                             "/unattached"),
                               "sysbus", OBJECT(main_system_bus), NULL);
diff --git a/hw/sysbus.h b/hw/sysbus.h
index acfbcfba529..a7fcded6e76 100644
--- a/hw/sysbus.h
+++ b/hw/sysbus.h
@@ -4,7 +4,7 @@
 /* Devices attached directly to the main system bus.  */
 
 #include "qdev.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 #define QDEV_MAX_MMIO 32
 #define QDEV_MAX_PIO 32
@@ -36,7 +36,7 @@ struct SysBusDevice {
     qemu_irq *irqp[QDEV_MAX_IRQ];
     int num_mmio;
     struct {
-        target_phys_addr_t addr;
+        hwaddr addr;
         MemoryRegion *memory;
     } mmio[QDEV_MAX_MMIO];
     int num_pio;
@@ -44,7 +44,6 @@ struct SysBusDevice {
 };
 
 /* Macros to compensate for lack of type inheritance in C.  */
-#define sysbus_from_qdev(dev) ((SysBusDevice *)(dev))
 #define FROM_SYSBUS(type, dev) DO_UPCAST(type, busdev, dev)
 
 void *sysbus_new(void);
@@ -56,31 +55,31 @@ void sysbus_init_ioports(SysBusDevice *dev, pio_addr_t ioport, pio_addr_t size);
 
 
 void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq);
-void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr);
-void sysbus_add_memory(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_mmio_map(SysBusDevice *dev, int n, hwaddr addr);
+void sysbus_add_memory(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem);
-void sysbus_add_memory_overlap(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory_overlap(SysBusDevice *dev, hwaddr addr,
                                MemoryRegion *mem, unsigned priority);
 void sysbus_del_memory(SysBusDevice *dev, MemoryRegion *mem);
-void sysbus_add_io(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
                    MemoryRegion *mem);
 void sysbus_del_io(SysBusDevice *dev, MemoryRegion *mem);
 MemoryRegion *sysbus_address_space(SysBusDevice *dev);
 
 /* Legacy helper function for creating devices.  */
 DeviceState *sysbus_create_varargs(const char *name,
-                                 target_phys_addr_t addr, ...);
+                                 hwaddr addr, ...);
 DeviceState *sysbus_try_create_varargs(const char *name,
-                                       target_phys_addr_t addr, ...);
+                                       hwaddr addr, ...);
 static inline DeviceState *sysbus_create_simple(const char *name,
-                                              target_phys_addr_t addr,
+                                              hwaddr addr,
                                               qemu_irq irq)
 {
     return sysbus_create_varargs(name, addr, irq, NULL);
 }
 
 static inline DeviceState *sysbus_try_create_simple(const char *name,
-                                                    target_phys_addr_t addr,
+                                                    hwaddr addr,
                                                     qemu_irq irq)
 {
     return sysbus_try_create_varargs(name, addr, irq, NULL);
diff --git a/hw/tc6393xb.c b/hw/tc6393xb.c
index 420925ccb35..e815f83198f 100644
--- a/hw/tc6393xb.c
+++ b/hw/tc6393xb.c
@@ -13,9 +13,9 @@
 #include "hw.h"
 #include "devices.h"
 #include "flash.h"
-#include "console.h"
-#include "pixel_ops.h"
-#include "blockdev.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
+#include "sysemu/blockdev.h"
 
 #define IRQ_TC6393_NAND		0
 #define IRQ_TC6393_MMC		1
@@ -215,7 +215,7 @@ static void tc6393xb_sub_irq(void *opaque, int line, int level) {
     case SCR_ ##N(1): return s->scr.N[1];       \
     case SCR_ ##N(2): return s->scr.N[2]
 
-static uint32_t tc6393xb_scr_readb(TC6393xbState *s, target_phys_addr_t addr)
+static uint32_t tc6393xb_scr_readb(TC6393xbState *s, hwaddr addr)
 {
     switch (addr) {
         case SCR_REVID:
@@ -276,7 +276,7 @@ static uint32_t tc6393xb_scr_readb(TC6393xbState *s, target_phys_addr_t addr)
     case SCR_ ##N(1): s->scr.N[1] = value; return;   \
     case SCR_ ##N(2): s->scr.N[2] = value; return
 
-static void tc6393xb_scr_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value)
+static void tc6393xb_scr_writeb(TC6393xbState *s, hwaddr addr, uint32_t value)
 {
     switch (addr) {
         SCR_REG_B(ISR);
@@ -327,7 +327,7 @@ static void tc6393xb_nand_irq(TC6393xbState *s) {
             (s->nand.imr & 0x80) && (s->nand.imr & s->nand.isr));
 }
 
-static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, target_phys_addr_t addr) {
+static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, hwaddr addr) {
     switch (addr) {
         case NAND_CFG_COMMAND:
             return s->nand_enable ? 2 : 0;
@@ -340,7 +340,7 @@ static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, target_phys_addr_t add
     fprintf(stderr, "tc6393xb_nand_cfg: unhandled read at %08x\n", (uint32_t) addr);
     return 0;
 }
-static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value) {
+static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, hwaddr addr, uint32_t value) {
     switch (addr) {
         case NAND_CFG_COMMAND:
             s->nand_enable = (value & 0x2);
@@ -357,7 +357,7 @@ static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, target_phys_addr_t addr,
 					(uint32_t) addr, value & 0xff);
 }
 
-static uint32_t tc6393xb_nand_readb(TC6393xbState *s, target_phys_addr_t addr) {
+static uint32_t tc6393xb_nand_readb(TC6393xbState *s, hwaddr addr) {
     switch (addr) {
         case NAND_DATA + 0:
         case NAND_DATA + 1:
@@ -376,7 +376,7 @@ static uint32_t tc6393xb_nand_readb(TC6393xbState *s, target_phys_addr_t addr) {
     fprintf(stderr, "tc6393xb_nand: unhandled read at %08x\n", (uint32_t) addr);
     return 0;
 }
-static void tc6393xb_nand_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value) {
+static void tc6393xb_nand_writeb(TC6393xbState *s, hwaddr addr, uint32_t value) {
 //    fprintf(stderr, "tc6393xb_nand: write at %08x: %02x\n",
 //					(uint32_t) addr, value & 0xff);
     switch (addr) {
@@ -454,7 +454,7 @@ static void tc6393xb_draw_graphic(TC6393xbState *s, int full_update)
             return;
     }
 
-    dpy_update(s->ds, 0, 0, s->scr_width, s->scr_height);
+    dpy_gfx_update(s->ds, 0, 0, s->scr_width, s->scr_height);
 }
 
 static void tc6393xb_draw_blank(TC6393xbState *s, int full_update)
@@ -472,7 +472,7 @@ static void tc6393xb_draw_blank(TC6393xbState *s, int full_update)
         d += ds_get_linesize(s->ds);
     }
 
-    dpy_update(s->ds, 0, 0, s->scr_width, s->scr_height);
+    dpy_gfx_update(s->ds, 0, 0, s->scr_width, s->scr_height);
 }
 
 static void tc6393xb_update_display(void *opaque)
@@ -499,7 +499,7 @@ static void tc6393xb_update_display(void *opaque)
 }
 
 
-static uint64_t tc6393xb_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t tc6393xb_readb(void *opaque, hwaddr addr,
                                unsigned size)
 {
     TC6393xbState *s = opaque;
@@ -522,7 +522,7 @@ static uint64_t tc6393xb_readb(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void tc6393xb_writeb(void *opaque, target_phys_addr_t addr,
+static void tc6393xb_writeb(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size) {
     TC6393xbState *s = opaque;
 
diff --git a/hw/tcx.c b/hw/tcx.c
index ac7dcb428ce..0ce2952f739 100644
--- a/hw/tcx.c
+++ b/hw/tcx.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "console.h"
-#include "pixel_ops.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
 
@@ -36,7 +37,7 @@
 
 typedef struct TCXState {
     SysBusDevice busdev;
-    target_phys_addr_t addr;
+    hwaddr addr;
     DisplayState *ds;
     uint8_t *vram;
     uint32_t *vram24, *cplane;
@@ -56,8 +57,10 @@ typedef struct TCXState {
     uint8_t dac_index, dac_state;
 } TCXState;
 
-static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch);
-static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch);
+static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
+static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
 
 static void tcx_set_dirty(TCXState *s)
 {
@@ -266,8 +269,8 @@ static void tcx_update_display(void *opaque)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(ts->ds, 0, y_start,
-                           ts->width, y - y_start);
+                dpy_gfx_update(ts->ds, 0, y_start,
+                               ts->width, y - y_start);
                 y_start = -1;
             }
             d += dd * 4;
@@ -276,8 +279,8 @@ static void tcx_update_display(void *opaque)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(ts->ds, 0, y_start,
-                   ts->width, y - y_start);
+        dpy_gfx_update(ts->ds, 0, y_start,
+                       ts->width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -342,8 +345,8 @@ static void tcx24_update_display(void *opaque)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(ts->ds, 0, y_start,
-                           ts->width, y - y_start);
+                dpy_gfx_update(ts->ds, 0, y_start,
+                               ts->width, y - y_start);
                 y_start = -1;
             }
             d += dd * 4;
@@ -354,8 +357,8 @@ static void tcx24_update_display(void *opaque)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(ts->ds, 0, y_start,
-                   ts->width, y - y_start);
+        dpy_gfx_update(ts->ds, 0, y_start,
+                       ts->width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -430,13 +433,13 @@ static void tcx_reset(DeviceState *d)
     s->dac_state = 0;
 }
 
-static uint64_t tcx_dac_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t tcx_dac_readl(void *opaque, hwaddr addr,
                               unsigned size)
 {
     return 0;
 }
 
-static void tcx_dac_writel(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void tcx_dac_writel(void *opaque, hwaddr addr, uint64_t val,
                            unsigned size)
 {
     TCXState *s = opaque;
@@ -470,7 +473,6 @@ static void tcx_dac_writel(void *opaque, target_phys_addr_t addr, uint64_t val,
     default:
         break;
     }
-    return;
 }
 
 static const MemoryRegionOps tcx_dac_ops = {
@@ -483,13 +485,13 @@ static const MemoryRegionOps tcx_dac_ops = {
     },
 };
 
-static uint64_t dummy_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t dummy_readl(void *opaque, hwaddr addr,
                             unsigned size)
 {
     return 0;
 }
 
-static void dummy_writel(void *opaque, target_phys_addr_t addr,
+static void dummy_writel(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
 }
@@ -574,45 +576,76 @@ static int tcx_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp)
 {
     TCXState *s = opaque;
     FILE *f;
     uint8_t *d, *d1, v;
-    int y, x;
+    int ret, y, x;
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
-    fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = s->vram;
     for(y = 0; y < s->height; y++) {
         d = d1;
         for(x = 0; x < s->width; x++) {
             v = *d;
-            fputc(s->r[v], f);
-            fputc(s->g[v], f);
-            fputc(s->b[v], f);
+            ret = fputc(s->r[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
+            ret = fputc(s->g[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
+            ret = fputc(s->b[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
             d++;
         }
         d1 += MAXX;
     }
+
+out:
     fclose(f);
     return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
-static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch,
+                              Error **errp)
 {
     TCXState *s = opaque;
     FILE *f;
     uint8_t *d, *d1, v;
     uint32_t *s24, *cptr, dval;
-    int y, x;
+    int ret, y, x;
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
-    fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = s->vram;
     s24 = s->vram24;
     cptr = s->cplane;
@@ -621,20 +654,46 @@ static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch)
         for(x = 0; x < s->width; x++, d++, s24++) {
             if ((*cptr++ & 0xff000000) == 0x03000000) { // 24-bit direct
                 dval = *s24 & 0x00ffffff;
-                fputc((dval >> 16) & 0xff, f);
-                fputc((dval >> 8) & 0xff, f);
-                fputc(dval & 0xff, f);
+                ret = fputc((dval >> 16) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((dval >> 8) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(dval & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
             } else {
                 v = *d;
-                fputc(s->r[v], f);
-                fputc(s->g[v], f);
-                fputc(s->b[v], f);
+                ret = fputc(s->r[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->g[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->b[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
             }
         }
         d1 += MAXX;
     }
+
+out:
     fclose(f);
     return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 static Property tcx_properties[] = {
@@ -657,7 +716,7 @@ static void tcx_class_init(ObjectClass *klass, void *data)
     dc->props = tcx_properties;
 }
 
-static TypeInfo tcx_info = {
+static const TypeInfo tcx_info = {
     .name          = "SUNW,tcx",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(TCXState),
diff --git a/hw/tmp105.c b/hw/tmp105.c
index 8e8dbd94eba..3ad2d2f04c9 100644
--- a/hw/tmp105.c
+++ b/hw/tmp105.c
@@ -20,20 +20,8 @@
 
 #include "hw.h"
 #include "i2c.h"
-
-typedef struct {
-    I2CSlave i2c;
-    uint8_t len;
-    uint8_t buf[2];
-    qemu_irq pin;
-
-    uint8_t pointer;
-    uint8_t config;
-    int16_t temperature;
-    int16_t limit[2];
-    int faults;
-    uint8_t alarm;
-} TMP105State;
+#include "tmp105.h"
+#include "qapi/visitor.h"
 
 static void tmp105_interrupt_update(TMP105State *s)
 {
@@ -64,15 +52,30 @@ static void tmp105_alarm_update(TMP105State *s)
     tmp105_interrupt_update(s);
 }
 
+static void tmp105_get_temperature(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
+{
+    TMP105State *s = TMP105(obj);
+    int64_t value = s->temperature;
+
+    visit_type_int(v, &value, name, errp);
+}
+
 /* Units are 0.001 centigrades relative to 0 C.  */
-void tmp105_set(I2CSlave *i2c, int temp)
+static void tmp105_set_temperature(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(obj);
+    int64_t temp;
 
+    visit_type_int(v, &temp, name, errp);
+    if (error_is_set(errp)) {
+        return;
+    }
     if (temp >= 128000 || temp < -128000) {
-        fprintf(stderr, "%s: values is out of range (%i.%03i C)\n",
-                        __FUNCTION__, temp / 1000, temp % 1000);
-        exit(-1);
+        error_setg(errp, "value %" PRId64 ".%03" PRIu64 " °C is out of range",
+                   temp / 1000, temp % 1000);
+        return;
     }
 
     s->temperature = ((int16_t) (temp * 0x800 / 128000)) << 4;
@@ -92,22 +95,22 @@ static void tmp105_read(TMP105State *s)
     }
 
     switch (s->pointer & 3) {
-    case 0:	/* Temperature */
+    case TMP105_REG_TEMPERATURE:
         s->buf[s->len ++] = (((uint16_t) s->temperature) >> 8);
         s->buf[s->len ++] = (((uint16_t) s->temperature) >> 0) &
                 (0xf0 << ((~s->config >> 5) & 3));		/* R */
         break;
 
-    case 1:	/* Configuration */
+    case TMP105_REG_CONFIG:
         s->buf[s->len ++] = s->config;
         break;
 
-    case 2:	/* T_LOW */
+    case TMP105_REG_T_LOW:
         s->buf[s->len ++] = ((uint16_t) s->limit[0]) >> 8;
         s->buf[s->len ++] = ((uint16_t) s->limit[0]) >> 0;
         break;
 
-    case 3:	/* T_HIGH */
+    case TMP105_REG_T_HIGH:
         s->buf[s->len ++] = ((uint16_t) s->limit[1]) >> 8;
         s->buf[s->len ++] = ((uint16_t) s->limit[1]) >> 0;
         break;
@@ -117,10 +120,10 @@ static void tmp105_read(TMP105State *s)
 static void tmp105_write(TMP105State *s)
 {
     switch (s->pointer & 3) {
-    case 0:	/* Temperature */
+    case TMP105_REG_TEMPERATURE:
         break;
 
-    case 1:	/* Configuration */
+    case TMP105_REG_CONFIG:
         if (s->buf[0] & ~s->config & (1 << 0))			/* SD */
             printf("%s: TMP105 shutdown\n", __FUNCTION__);
         s->config = s->buf[0];
@@ -128,8 +131,8 @@ static void tmp105_write(TMP105State *s)
         tmp105_alarm_update(s);
         break;
 
-    case 2:	/* T_LOW */
-    case 3:	/* T_HIGH */
+    case TMP105_REG_T_LOW:
+    case TMP105_REG_T_HIGH:
         if (s->len >= 3)
             s->limit[s->pointer & 1] = (int16_t)
                     ((((uint16_t) s->buf[0]) << 8) | s->buf[1]);
@@ -140,23 +143,27 @@ static void tmp105_write(TMP105State *s)
 
 static int tmp105_rx(I2CSlave *i2c)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (s->len < 2)
+    if (s->len < 2) {
         return s->buf[s->len ++];
-    else
+    } else {
         return 0xff;
+    }
 }
 
 static int tmp105_tx(I2CSlave *i2c, uint8_t data)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (!s->len ++)
+    if (s->len == 0) {
         s->pointer = data;
-    else {
-        if (s->len <= 2)
+        s->len++;
+    } else {
+        if (s->len <= 2) {
             s->buf[s->len - 1] = data;
+        }
+        s->len++;
         tmp105_write(s);
     }
 
@@ -165,10 +172,11 @@ static int tmp105_tx(I2CSlave *i2c, uint8_t data)
 
 static void tmp105_event(I2CSlave *i2c, enum i2c_event event)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (event == I2C_START_RECV)
+    if (event == I2C_START_RECV) {
         tmp105_read(s);
+    }
 
     s->len = 0;
 }
@@ -204,7 +212,7 @@ static const VMStateDescription vmstate_tmp105 = {
 
 static void tmp105_reset(I2CSlave *i2c)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
     s->temperature = 0;
     s->pointer = 0;
@@ -217,7 +225,7 @@ static void tmp105_reset(I2CSlave *i2c)
 
 static int tmp105_init(I2CSlave *i2c)
 {
-    TMP105State *s = FROM_I2C_SLAVE(TMP105State, i2c);
+    TMP105State *s = TMP105(i2c);
 
     qdev_init_gpio_out(&i2c->qdev, &s->pin, 1);
 
@@ -226,6 +234,13 @@ static int tmp105_init(I2CSlave *i2c)
     return 0;
 }
 
+static void tmp105_initfn(Object *obj)
+{
+    object_property_add(obj, "temperature", "int",
+                        tmp105_get_temperature,
+                        tmp105_set_temperature, NULL, NULL, NULL);
+}
+
 static void tmp105_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -238,10 +253,11 @@ static void tmp105_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_tmp105;
 }
 
-static TypeInfo tmp105_info = {
-    .name          = "tmp105",
+static const TypeInfo tmp105_info = {
+    .name          = TYPE_TMP105,
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(TMP105State),
+    .instance_init = tmp105_initfn,
     .class_init    = tmp105_class_init,
 };
 
diff --git a/hw/tmp105.h b/hw/tmp105.h
new file mode 100644
index 00000000000..d2189191e2a
--- /dev/null
+++ b/hw/tmp105.h
@@ -0,0 +1,47 @@
+/*
+ * Texas Instruments TMP105 Temperature Sensor
+ *
+ * Browse the data sheet:
+ *
+ *    http://www.ti.com/lit/gpn/tmp105
+ *
+ * Copyright (C) 2012 Alex Horn <alex.horn@cs.ox.ac.uk>
+ * Copyright (C) 2008-2012 Andrzej Zaborowski <balrogg@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_TMP105_H
+#define QEMU_TMP105_H
+
+#include "i2c.h"
+#include "tmp105_regs.h"
+
+#define TYPE_TMP105 "tmp105"
+#define TMP105(obj) OBJECT_CHECK(TMP105State, (obj), TYPE_TMP105)
+
+/**
+ * TMP105State:
+ * @config: Bits 5 and 6 (value 32 and 64) determine the precision of the
+ * temperature. See Table 8 in the data sheet.
+ *
+ * @see_also: http://www.ti.com/lit/gpn/tmp105
+ */
+typedef struct TMP105State {
+    /*< private >*/
+    I2CSlave i2c;
+    /*< public >*/
+
+    uint8_t len;
+    uint8_t buf[2];
+    qemu_irq pin;
+
+    uint8_t pointer;
+    uint8_t config;
+    int16_t temperature;
+    int16_t limit[2];
+    int faults;
+    uint8_t alarm;
+} TMP105State;
+
+#endif
diff --git a/hw/tmp105_regs.h b/hw/tmp105_regs.h
new file mode 100644
index 00000000000..9b55abaf904
--- /dev/null
+++ b/hw/tmp105_regs.h
@@ -0,0 +1,50 @@
+/*
+ * Texas Instruments TMP105 Temperature Sensor I2C messages
+ *
+ * Browse the data sheet:
+ *
+ *    http://www.ti.com/lit/gpn/tmp105
+ *
+ * Copyright (C) 2012 Alex Horn <alex.horn@cs.ox.ac.uk>
+ * Copyright (C) 2008-2012 Andrzej Zaborowski <balrogg@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_TMP105_MSGS_H
+#define QEMU_TMP105_MSGS_H
+
+/**
+ * TMP105Reg:
+ * @TMP105_REG_TEMPERATURE: Temperature register
+ * @TMP105_REG_CONFIG: Configuration register
+ * @TMP105_REG_T_LOW: Low temperature register (also known as T_hyst)
+ * @TMP105_REG_T_HIGH: High temperature register (also known as T_OS)
+ *
+ * The following temperature sensors are
+ * compatible with the TMP105 registers:
+ * - adt75
+ * - ds1775
+ * - ds75
+ * - lm75
+ * - lm75a
+ * - max6625
+ * - max6626
+ * - mcp980x
+ * - stds75
+ * - tcn75
+ * - tmp100
+ * - tmp101
+ * - tmp105
+ * - tmp175
+ * - tmp275
+ * - tmp75
+ **/
+typedef enum TMP105Reg {
+    TMP105_REG_TEMPERATURE = 0,
+    TMP105_REG_CONFIG,
+    TMP105_REG_T_LOW,
+    TMP105_REG_T_HIGH,
+} TMP105Reg;
+
+#endif
diff --git a/hw/tosa.c b/hw/tosa.c
index 297a8c2ed0e..efea109795d 100644
--- a/hw/tosa.c
+++ b/hw/tosa.c
@@ -17,13 +17,13 @@
 #include "devices.h"
 #include "sharpsl.h"
 #include "pcmcia.h"
-#include "block.h"
+#include "block/block.h"
 #include "boards.h"
 #include "i2c.h"
 #include "ssi.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define TOSA_RAM    0x04000000
 #define TOSA_ROM	0x00800000
@@ -205,11 +205,12 @@ static struct arm_boot_info tosa_binfo = {
     .ram_size = 0x04000000,
 };
 
-static void tosa_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void tosa_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
     PXA2xxState *mpu;
@@ -250,6 +251,7 @@ static QEMUMachine tosapda_machine = {
     .name = "tosa",
     .desc = "Tosa PDA (PXA255)",
     .init = tosa_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void tosapda_machine_init(void)
@@ -269,7 +271,7 @@ static void tosa_dac_class_init(ObjectClass *klass, void *data)
     k->send = tosa_dac_send;
 }
 
-static TypeInfo tosa_dac_info = {
+static const TypeInfo tosa_dac_info = {
     .name          = "tosa_dac",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(TosaDACState),
@@ -284,7 +286,7 @@ static void tosa_ssp_class_init(ObjectClass *klass, void *data)
     k->transfer = tosa_ssp_tansfer;
 }
 
-static TypeInfo tosa_ssp_info = {
+static const TypeInfo tosa_ssp_info = {
     .name          = "tosa-ssp",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(SSISlave),
diff --git a/hw/tpci200.c b/hw/tpci200.c
new file mode 100644
index 00000000000..a4823fb9f2c
--- /dev/null
+++ b/hw/tpci200.c
@@ -0,0 +1,671 @@
+/*
+ * QEMU TEWS TPCI200 IndustryPack carrier emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+#include "pci/pci.h"
+#include "qemu/bitops.h"
+#include <stdio.h>
+
+/* #define DEBUG_TPCI */
+
+#ifdef DEBUG_TPCI
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "TPCI200: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) do { } while (0)
+#endif
+
+#define N_MODULES 4
+
+#define IP_ID_SPACE  2
+#define IP_INT_SPACE 3
+#define IP_IO_SPACE_ADDR_MASK  0x7F
+#define IP_ID_SPACE_ADDR_MASK  0x3F
+#define IP_INT_SPACE_ADDR_MASK 0x3F
+
+#define STATUS_INT(IP, INTNO) BIT((IP) * 2 + (INTNO))
+#define STATUS_TIME(IP)       BIT((IP) + 12)
+#define STATUS_ERR_ANY        0xF00
+
+#define CTRL_CLKRATE          BIT(0)
+#define CTRL_RECOVER          BIT(1)
+#define CTRL_TIME_INT         BIT(2)
+#define CTRL_ERR_INT          BIT(3)
+#define CTRL_INT_EDGE(INTNO)  BIT(4 + (INTNO))
+#define CTRL_INT(INTNO)       BIT(6 + (INTNO))
+
+#define REG_REV_ID    0x00
+#define REG_IP_A_CTRL 0x02
+#define REG_IP_B_CTRL 0x04
+#define REG_IP_C_CTRL 0x06
+#define REG_IP_D_CTRL 0x08
+#define REG_RESET     0x0A
+#define REG_STATUS    0x0C
+#define IP_N_FROM_REG(REG) ((REG) / 2 - 1)
+
+typedef struct {
+    PCIDevice dev;
+    IPackBus bus;
+    MemoryRegion mmio;
+    MemoryRegion io;
+    MemoryRegion las0;
+    MemoryRegion las1;
+    MemoryRegion las2;
+    MemoryRegion las3;
+    bool big_endian[3];
+    uint8_t ctrl[N_MODULES];
+    uint16_t status;
+    uint8_t int_set;
+} TPCI200State;
+
+#define TYPE_TPCI200 "tpci200"
+
+#define TPCI200(obj) \
+    OBJECT_CHECK(TPCI200State, (obj), TYPE_TPCI200)
+
+static const uint8_t local_config_regs[] = {
+    0x00, 0xFF, 0xFF, 0x0F, 0x00, 0xFC, 0xFF, 0x0F, 0x00, 0x00, 0x00,
+    0x0E, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+    0x00, 0x08, 0x01, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00, 0x01,
+    0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0xA0, 0x60, 0x41, 0xD4,
+    0xA2, 0x20, 0x41, 0x14, 0xA2, 0x20, 0x41, 0x14, 0xA2, 0x20, 0x01,
+    0x14, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x08, 0x01, 0x02,
+    0x00, 0x04, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x80, 0x02, 0x41,
+    0x00, 0x00, 0x00, 0x00, 0x40, 0x7A, 0x00, 0x52, 0x92, 0x24, 0x02
+};
+
+static void adjust_addr(bool big_endian, hwaddr *addr, unsigned size)
+{
+    /* During 8 bit access in big endian mode,
+       odd and even addresses are swapped */
+    if (big_endian && size == 1) {
+        *addr ^= 1;
+    }
+}
+
+static uint64_t adjust_value(bool big_endian, uint64_t *val, unsigned size)
+{
+    /* Local spaces only support 8/16 bit access,
+     * so there's no need to care for sizes > 2 */
+    if (big_endian && size == 2) {
+        *val = bswap16(*val);
+    }
+    return *val;
+}
+
+static void tpci200_set_irq(void *opaque, int intno, int level)
+{
+    IPackDevice *ip = opaque;
+    IPackBus *bus = IPACK_BUS(qdev_get_parent_bus(DEVICE(ip)));
+    PCIDevice *pcidev = PCI_DEVICE(BUS(bus)->parent);
+    TPCI200State *dev = TPCI200(pcidev);
+    unsigned ip_n = ip->slot;
+    uint16_t prev_status = dev->status;
+
+    assert(ip->slot >= 0 && ip->slot < N_MODULES);
+
+    /* The requested interrupt must be enabled in the IP CONTROL
+     * register */
+    if (!(dev->ctrl[ip_n] & CTRL_INT(intno))) {
+        return;
+    }
+
+    /* Update the interrupt status in the IP STATUS register */
+    if (level) {
+        dev->status |=  STATUS_INT(ip_n, intno);
+    } else {
+        dev->status &= ~STATUS_INT(ip_n, intno);
+    }
+
+    /* Return if there are no changes */
+    if (dev->status == prev_status) {
+        return;
+    }
+
+    DPRINTF("IP %u INT%u#: %u\n", ip_n, intno, level);
+
+    /* Check if the interrupt is edge sensitive */
+    if (dev->ctrl[ip_n] & CTRL_INT_EDGE(intno)) {
+        if (level) {
+            qemu_set_irq(dev->dev.irq[0], !dev->int_set);
+            qemu_set_irq(dev->dev.irq[0],  dev->int_set);
+        }
+    } else {
+        unsigned i, j;
+        uint16_t level_status = dev->status;
+
+        /* Check if there are any level sensitive interrupts set by
+           removing the ones that are edge sensitive from the status
+           register */
+        for (i = 0; i < N_MODULES; i++) {
+            for (j = 0; j < 2; j++) {
+                if (dev->ctrl[i] & CTRL_INT_EDGE(j)) {
+                    level_status &= ~STATUS_INT(i, j);
+                }
+            }
+        }
+
+        if (level_status && !dev->int_set) {
+            qemu_irq_raise(dev->dev.irq[0]);
+            dev->int_set = 1;
+        } else if (!level_status && dev->int_set) {
+            qemu_irq_lower(dev->dev.irq[0]);
+            dev->int_set = 0;
+        }
+    }
+}
+
+static uint64_t tpci200_read_cfg(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    uint8_t ret = 0;
+    if (addr < ARRAY_SIZE(local_config_regs)) {
+        ret = local_config_regs[addr];
+    }
+    /* Endianness is stored in the first bit of these registers */
+    if ((addr == 0x2b && s->big_endian[0]) ||
+        (addr == 0x2f && s->big_endian[1]) ||
+        (addr == 0x33 && s->big_endian[2])) {
+        ret |= 1;
+    }
+    DPRINTF("Read from LCR 0x%x: 0x%x\n", (unsigned) addr, (unsigned) ret);
+    return ret;
+}
+
+static void tpci200_write_cfg(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
+{
+    TPCI200State *s = opaque;
+    /* Endianness is stored in the first bit of these registers */
+    if (addr == 0x2b || addr == 0x2f || addr == 0x33) {
+        unsigned las = (addr - 0x2b) / 4;
+        s->big_endian[las] = val & 1;
+        DPRINTF("LAS%u big endian mode: %u\n", las, (unsigned) val & 1);
+    } else {
+        DPRINTF("Write to LCR 0x%x: 0x%x\n", (unsigned) addr, (unsigned) val);
+    }
+}
+
+static uint64_t tpci200_read_las0(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    uint64_t ret = 0;
+
+    switch (addr) {
+
+    case REG_REV_ID:
+        DPRINTF("Read REVISION ID\n"); /* Current value is 0x00 */
+        break;
+
+    case REG_IP_A_CTRL:
+    case REG_IP_B_CTRL:
+    case REG_IP_C_CTRL:
+    case REG_IP_D_CTRL:
+        {
+            unsigned ip_n = IP_N_FROM_REG(addr);
+            ret = s->ctrl[ip_n];
+            DPRINTF("Read IP %c CONTROL: 0x%x\n", 'A' + ip_n, (unsigned) ret);
+        }
+        break;
+
+    case REG_RESET:
+        DPRINTF("Read RESET\n"); /* Not implemented */
+        break;
+
+    case REG_STATUS:
+        ret = s->status;
+        DPRINTF("Read STATUS: 0x%x\n", (unsigned) ret);
+        break;
+
+    /* Reserved */
+    default:
+        DPRINTF("Unsupported read from LAS0 0x%x\n", (unsigned) addr);
+        break;
+    }
+
+    return adjust_value(s->big_endian[0], &ret, size);
+}
+
+static void tpci200_write_las0(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+
+    adjust_value(s->big_endian[0], &val, size);
+
+    switch (addr) {
+
+    case REG_REV_ID:
+        DPRINTF("Write Revision ID: 0x%x\n", (unsigned) val); /* No effect */
+        break;
+
+    case REG_IP_A_CTRL:
+    case REG_IP_B_CTRL:
+    case REG_IP_C_CTRL:
+    case REG_IP_D_CTRL:
+        {
+            unsigned ip_n = IP_N_FROM_REG(addr);
+            s->ctrl[ip_n] = val;
+            DPRINTF("Write IP %c CONTROL: 0x%x\n", 'A' + ip_n, (unsigned) val);
+        }
+        break;
+
+    case REG_RESET:
+        DPRINTF("Write RESET: 0x%x\n", (unsigned) val); /* Not implemented */
+        break;
+
+    case REG_STATUS:
+        {
+            unsigned i;
+
+            for (i = 0; i < N_MODULES; i++) {
+                IPackDevice *ip = ipack_device_find(&s->bus, i);
+
+                if (ip != NULL) {
+                    if (val & STATUS_INT(i, 0)) {
+                        DPRINTF("Clear IP %c INT0# status\n", 'A' + i);
+                        qemu_irq_lower(ip->irq[0]);
+                    }
+                    if (val & STATUS_INT(i, 1)) {
+                        DPRINTF("Clear IP %c INT1# status\n", 'A' + i);
+                        qemu_irq_lower(ip->irq[1]);
+                    }
+                }
+
+                if (val & STATUS_TIME(i)) {
+                    DPRINTF("Clear IP %c timeout\n", 'A' + i);
+                    s->status &= ~STATUS_TIME(i);
+                }
+            }
+
+            if (val & STATUS_ERR_ANY) {
+                DPRINTF("Unexpected write to STATUS register: 0x%x\n",
+                        (unsigned) val);
+            }
+        }
+        break;
+
+    /* Reserved */
+    default:
+        DPRINTF("Unsupported write to LAS0 0x%x: 0x%x\n",
+                (unsigned) addr, (unsigned) val);
+        break;
+    }
+}
+
+static uint64_t tpci200_read_las1(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    unsigned ip_n, space;
+    uint8_t offset;
+
+    adjust_addr(s->big_endian[1], &addr, size);
+
+    /*
+     * The address is divided into the IP module number (0-4), the IP
+     * address space (I/O, ID, INT) and the offset within that space.
+     */
+    ip_n = addr >> 8;
+    space = (addr >> 6) & 3;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS1: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        switch (space) {
+
+        case IP_ID_SPACE:
+            offset = addr & IP_ID_SPACE_ADDR_MASK;
+            if (k->id_read) {
+                ret = k->id_read(ip, offset);
+            }
+            break;
+
+        case IP_INT_SPACE:
+            offset = addr & IP_INT_SPACE_ADDR_MASK;
+
+            /* Read address 0 to ACK IP INT0# and address 2 to ACK IP INT1# */
+            if (offset == 0 || offset == 2) {
+                unsigned intno = offset / 2;
+                bool int_set = s->status & STATUS_INT(ip_n, intno);
+                bool int_edge_sensitive = s->ctrl[ip_n] & CTRL_INT_EDGE(intno);
+                if (int_set && !int_edge_sensitive) {
+                    qemu_irq_lower(ip->irq[intno]);
+                }
+            }
+
+            if (k->int_read) {
+                ret = k->int_read(ip, offset);
+            }
+            break;
+
+        default:
+            offset = addr & IP_IO_SPACE_ADDR_MASK;
+            if (k->io_read) {
+                ret = k->io_read(ip, offset);
+            }
+            break;
+        }
+    }
+
+    return adjust_value(s->big_endian[1], &ret, size);
+}
+
+static void tpci200_write_las1(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    unsigned ip_n, space;
+    uint8_t offset;
+
+    adjust_addr(s->big_endian[1], &addr, size);
+    adjust_value(s->big_endian[1], &val, size);
+
+    /*
+     * The address is divided into the IP module number, the IP
+     * address space (I/O, ID, INT) and the offset within that space.
+     */
+    ip_n = addr >> 8;
+    space = (addr >> 6) & 3;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS1: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        switch (space) {
+
+        case IP_ID_SPACE:
+            offset = addr & IP_ID_SPACE_ADDR_MASK;
+            if (k->id_write) {
+                k->id_write(ip, offset, val);
+            }
+            break;
+
+        case IP_INT_SPACE:
+            offset = addr & IP_INT_SPACE_ADDR_MASK;
+            if (k->int_write) {
+                k->int_write(ip, offset, val);
+            }
+            break;
+
+        default:
+            offset = addr & IP_IO_SPACE_ADDR_MASK;
+            if (k->io_write) {
+                k->io_write(ip, offset, val);
+            }
+            break;
+        }
+    }
+}
+
+static uint64_t tpci200_read_las2(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    unsigned ip_n;
+    uint32_t offset;
+
+    adjust_addr(s->big_endian[2], &addr, size);
+
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    ip_n = addr >> 23;
+    offset = addr & 0x7fffff;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS2: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_read16) {
+            ret = k->mem_read16(ip, offset);
+        }
+    }
+
+    return adjust_value(s->big_endian[2], &ret, size);
+}
+
+static void tpci200_write_las2(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    unsigned ip_n;
+    uint32_t offset;
+
+    adjust_addr(s->big_endian[2], &addr, size);
+    adjust_value(s->big_endian[2], &val, size);
+
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    ip_n = addr >> 23;
+    offset = addr & 0x7fffff;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS2: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_write16) {
+            k->mem_write16(ip, offset, val);
+        }
+    }
+}
+
+static uint64_t tpci200_read_las3(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    unsigned ip_n = addr >> 22;
+    uint32_t offset = addr & 0x3fffff;
+
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS3: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_read8) {
+            ret = k->mem_read8(ip, offset);
+        }
+    }
+
+    return ret;
+}
+
+static void tpci200_write_las3(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    unsigned ip_n = addr >> 22;
+    uint32_t offset = addr & 0x3fffff;
+
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS3: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_write8) {
+            k->mem_write8(ip, offset, val);
+        }
+    }
+}
+
+static const MemoryRegionOps tpci200_cfg_ops = {
+    .read = tpci200_read_cfg,
+    .write = tpci200_write_cfg,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 4
+    },
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1
+    }
+};
+
+static const MemoryRegionOps tpci200_las0_ops = {
+    .read = tpci200_read_las0,
+    .write = tpci200_write_las0,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 2,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las1_ops = {
+    .read = tpci200_read_las1,
+    .write = tpci200_write_las1,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las2_ops = {
+    .read = tpci200_read_las2,
+    .write = tpci200_write_las2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las3_ops = {
+    .read = tpci200_read_las3,
+    .write = tpci200_write_las3,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 1
+    }
+};
+
+static int tpci200_initfn(PCIDevice *pci_dev)
+{
+    TPCI200State *s = TPCI200(pci_dev);
+    uint8_t *c = s->dev.config;
+
+    pci_set_word(c + PCI_COMMAND, 0x0003);
+    pci_set_word(c + PCI_STATUS,  0x0280);
+
+    pci_set_byte(c + PCI_INTERRUPT_PIN, 0x01); /* Interrupt pin A */
+
+    pci_set_byte(c + PCI_CAPABILITY_LIST, 0x40);
+    pci_set_long(c + 0x40, 0x48014801);
+    pci_set_long(c + 0x48, 0x00024C06);
+    pci_set_long(c + 0x4C, 0x00000003);
+
+    memory_region_init_io(&s->mmio, &tpci200_cfg_ops,
+                          s, "tpci200_mmio", 128);
+    memory_region_init_io(&s->io,   &tpci200_cfg_ops,
+                          s, "tpci200_io",   128);
+    memory_region_init_io(&s->las0, &tpci200_las0_ops,
+                          s, "tpci200_las0", 256);
+    memory_region_init_io(&s->las1, &tpci200_las1_ops,
+                          s, "tpci200_las1", 1024);
+    memory_region_init_io(&s->las2, &tpci200_las2_ops,
+                          s, "tpci200_las2", 1024*1024*32);
+    memory_region_init_io(&s->las3, &tpci200_las3_ops,
+                          s, "tpci200_las3", 1024*1024*16);
+    pci_register_bar(&s->dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mmio);
+    pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_IO,     &s->io);
+    pci_register_bar(&s->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las0);
+    pci_register_bar(&s->dev, 3, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las1);
+    pci_register_bar(&s->dev, 4, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las2);
+    pci_register_bar(&s->dev, 5, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las3);
+
+    ipack_bus_new_inplace(&s->bus, DEVICE(&s->dev), NULL,
+                          N_MODULES, tpci200_set_irq);
+
+    return 0;
+}
+
+static void tpci200_exitfn(PCIDevice *pci_dev)
+{
+    TPCI200State *s = TPCI200(pci_dev);
+
+    memory_region_destroy(&s->mmio);
+    memory_region_destroy(&s->io);
+    memory_region_destroy(&s->las0);
+    memory_region_destroy(&s->las1);
+    memory_region_destroy(&s->las2);
+    memory_region_destroy(&s->las3);
+}
+
+static const VMStateDescription vmstate_tpci200 = {
+    .name = "tpci200",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, TPCI200State),
+        VMSTATE_BOOL_ARRAY(big_endian, TPCI200State, 3),
+        VMSTATE_UINT8_ARRAY(ctrl, TPCI200State, N_MODULES),
+        VMSTATE_UINT16(status, TPCI200State),
+        VMSTATE_UINT8(int_set, TPCI200State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void tpci200_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = tpci200_initfn;
+    k->exit = tpci200_exitfn;
+    k->vendor_id = PCI_VENDOR_ID_TEWS;
+    k->device_id = PCI_DEVICE_ID_TEWS_TPCI200;
+    k->class_id = PCI_CLASS_BRIDGE_OTHER;
+    k->subsystem_vendor_id = PCI_VENDOR_ID_TEWS;
+    k->subsystem_id = 0x300A;
+    dc->desc = "TEWS TPCI200 IndustryPack carrier";
+    dc->vmsd = &vmstate_tpci200;
+}
+
+static const TypeInfo tpci200_info = {
+    .name          = TYPE_TPCI200,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(TPCI200State),
+    .class_init    = tpci200_class_init,
+};
+
+static void tpci200_register_types(void)
+{
+    type_register_static(&tpci200_info);
+}
+
+type_init(tpci200_register_types)
diff --git a/hw/tsc2005.c b/hw/tsc2005.c
index 9a500ebb3d2..740ff86aa8a 100644
--- a/hw/tsc2005.c
+++ b/hw/tsc2005.c
@@ -19,8 +19,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "devices.h"
 
 #define TSC_CUT_RESOLUTION(value, p)	((value) >> (16 - (p ? 12 : 10)))
diff --git a/hw/tsc210x.c b/hw/tsc210x.c
index 3c448a6f0f5..2076c355d22 100644
--- a/hw/tsc210x.c
+++ b/hw/tsc210x.c
@@ -21,8 +21,8 @@
 
 #include "hw.h"
 #include "audio/audio.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "omap.h"	/* For I2SCodec and uWireSlave */
 #include "devices.h"
 
diff --git a/hw/tusb6010.c b/hw/tusb6010.c
index 5ba8da6d6ad..2c7d0336510 100644
--- a/hw/tusb6010.c
+++ b/hw/tusb6010.c
@@ -19,7 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "usb.h"
 #include "omap.h"
 #include "irq.h"
@@ -281,7 +281,7 @@ static void tusb_gpio_intr_update(TUSBState *s)
 extern CPUReadMemoryFunc * const musb_read[];
 extern CPUWriteMemoryFunc * const musb_write[];
 
-static uint32_t tusb_async_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readb(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
 
@@ -298,7 +298,7 @@ static uint32_t tusb_async_readb(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t tusb_async_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readh(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
 
@@ -315,7 +315,7 @@ static uint32_t tusb_async_readh(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t tusb_async_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readw(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
     int offset = addr & 0xfff;
@@ -438,7 +438,7 @@ static uint32_t tusb_async_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void tusb_async_writeb(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writeb(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
@@ -459,7 +459,7 @@ static void tusb_async_writeb(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void tusb_async_writeh(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
@@ -480,7 +480,7 @@ static void tusb_async_writeh(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void tusb_async_writew(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writew(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
@@ -740,7 +740,7 @@ static void tusb6010_irq(void *opaque, int source, int level)
 
 static void tusb6010_reset(DeviceState *dev)
 {
-    TUSBState *s = FROM_SYSBUS(TUSBState, sysbus_from_qdev(dev));
+    TUSBState *s = FROM_SYSBUS(TUSBState, SYS_BUS_DEVICE(dev));
     int i;
 
     s->test_reset = TUSB_PROD_TEST_RESET_VAL;
@@ -798,7 +798,7 @@ static void tusb6010_class_init(ObjectClass *klass, void *data)
     dc->reset = tusb6010_reset;
 }
 
-static TypeInfo tusb6010_info = {
+static const TypeInfo tusb6010_info = {
     .name          = "tusb6010",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(TUSBState),
diff --git a/hw/twl92230.c b/hw/twl92230.c
index 0d70d8498dd..70d9b03e553 100644
--- a/hw/twl92230.c
+++ b/hw/twl92230.c
@@ -20,10 +20,10 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i2c.h"
-#include "sysemu.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "ui/console.h"
 
 #define VERBOSE 1
 
@@ -867,7 +867,7 @@ static void twl92230_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_menelaus;
 }
 
-static TypeInfo twl92230_info = {
+static const TypeInfo twl92230_info = {
     .name          = "twl92230",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(MenelausState),
diff --git a/uboot_image.h b/hw/uboot_image.h
index 9fc2760b53e..9fc2760b53e 100644
--- a/uboot_image.h
+++ b/hw/uboot_image.h
diff --git a/hw/unin_pci.c b/hw/unin_pci.c
index d1cc68062b7..f1c3c20f375 100644
--- a/hw/unin_pci.c
+++ b/hw/unin_pci.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 
 /* debug UniNorth */
 //#define DEBUG_UNIN
@@ -111,7 +111,7 @@ static uint32_t unin_get_config_reg(uint32_t reg, uint32_t addr)
     return retval;
 }
 
-static void unin_data_write(void *opaque, target_phys_addr_t addr,
+static void unin_data_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned len)
 {
     UNINState *s = opaque;
@@ -123,7 +123,7 @@ static void unin_data_write(void *opaque, target_phys_addr_t addr,
                    val, len);
 }
 
-static uint64_t unin_data_read(void *opaque, target_phys_addr_t addr,
+static uint64_t unin_data_read(void *opaque, hwaddr addr,
                                unsigned len)
 {
     UNINState *s = opaque;
diff --git a/hw/usb.h b/hw/usb.h
index b8fceec8951..bc42639b167 100644
--- a/hw/usb.h
+++ b/hw/usb.h
@@ -26,7 +26,7 @@
  */
 
 #include "qdev.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /* Constants related to the USB / PCI interaction */
 #define USB_SBRN    0x60 /* Serial Bus Release Number Register */
@@ -38,12 +38,15 @@
 #define USB_TOKEN_IN    0x69 /* device -> host */
 #define USB_TOKEN_OUT   0xe1 /* host -> device */
 
-#define USB_RET_NODEV   (-1)
-#define USB_RET_NAK     (-2)
-#define USB_RET_STALL   (-3)
-#define USB_RET_BABBLE  (-4)
-#define USB_RET_IOERROR (-5)
-#define USB_RET_ASYNC   (-6)
+#define USB_RET_SUCCESS           (0)
+#define USB_RET_NODEV             (-1)
+#define USB_RET_NAK               (-2)
+#define USB_RET_STALL             (-3)
+#define USB_RET_BABBLE            (-4)
+#define USB_RET_IOERROR           (-5)
+#define USB_RET_ASYNC             (-6)
+#define USB_RET_ADD_TO_QUEUE      (-7)
+#define USB_RET_REMOVE_FROM_QUEUE (-8)
 
 #define USB_SPEED_LOW   0
 #define USB_SPEED_FULL  1
@@ -135,8 +138,15 @@
 #define USB_DT_OTHER_SPEED_CONFIG       0x07
 #define USB_DT_DEBUG                    0x0A
 #define USB_DT_INTERFACE_ASSOC          0x0B
+#define USB_DT_BOS                      0x0F
+#define USB_DT_DEVICE_CAPABILITY        0x10
 #define USB_DT_CS_INTERFACE             0x24
 #define USB_DT_CS_ENDPOINT              0x25
+#define USB_DT_ENDPOINT_COMPANION       0x30
+
+#define USB_DEV_CAP_WIRELESS            0x01
+#define USB_DEV_CAP_USB2_EXT            0x02
+#define USB_DEV_CAP_SUPERSPEED          0x03
 
 #define USB_ENDPOINT_XFER_CONTROL	0
 #define USB_ENDPOINT_XFER_ISOC		1
@@ -151,6 +161,7 @@ typedef struct USBBusOps USBBusOps;
 typedef struct USBPort USBPort;
 typedef struct USBDevice USBDevice;
 typedef struct USBPacket USBPacket;
+typedef struct USBCombinedPacket USBCombinedPacket;
 typedef struct USBEndpoint USBEndpoint;
 
 typedef struct USBDesc USBDesc;
@@ -186,6 +197,7 @@ struct USBEndpoint {
 
 enum USBDeviceFlags {
     USB_DEV_FLAG_FULL_PATH,
+    USB_DEV_FLAG_IS_HOST,
 };
 
 /* definition of a USB device */
@@ -218,6 +230,7 @@ struct USBDevice {
     USBEndpoint ep_out[USB_MAX_ENDPOINTS];
 
     QLIST_HEAD(, USBDescString) strings;
+    const USBDesc *usb_desc; /* Overrides class usb_desc if not NULL */
     const USBDescDevice *device;
 
     int configuration;
@@ -270,22 +283,36 @@ typedef struct USBDeviceClass {
      * Process control request.
      * Called from handle_packet().
      *
-     * Returns length or one of the USB_RET_ codes.
+     * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
+     * then the number of bytes transferred is stored in p->actual_length
      */
-    int (*handle_control)(USBDevice *dev, USBPacket *p, int request, int value,
-                          int index, int length, uint8_t *data);
+    void (*handle_control)(USBDevice *dev, USBPacket *p, int request, int value,
+                           int index, int length, uint8_t *data);
 
     /*
      * Process data transfers (both BULK and ISOC).
      * Called from handle_packet().
      *
-     * Returns length or one of the USB_RET_ codes.
+     * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
+     * then the number of bytes transferred is stored in p->actual_length
      */
-    int (*handle_data)(USBDevice *dev, USBPacket *p);
+    void (*handle_data)(USBDevice *dev, USBPacket *p);
 
     void (*set_interface)(USBDevice *dev, int interface,
                           int alt_old, int alt_new);
 
+    /*
+     * Called when the hcd is done queuing packets for an endpoint, only
+     * necessary for devices which can return USB_RET_ADD_TO_QUEUE.
+     */
+    void (*flush_ep_queue)(USBDevice *dev, USBEndpoint *ep);
+
+    /*
+     * Called by the hcd to let the device know the queue for an endpoint
+     * has been unlinked / stopped. Optional may be NULL.
+     */
+    void (*ep_stopped)(USBDevice *dev, USBEndpoint *ep);
+
     const char *product_desc;
     const USBDesc *usb_desc;
 } USBDeviceClass;
@@ -336,16 +363,28 @@ struct USBPacket {
     USBEndpoint *ep;
     QEMUIOVector iov;
     uint64_t parameter; /* control transfers */
-    int result; /* transfer length or USB_RET_* status code */
+    bool short_not_ok;
+    bool int_req;
+    int status; /* USB_RET_* status code */
+    int actual_length; /* Number of bytes actually transferred */
     /* Internal use by the USB layer.  */
     USBPacketState state;
+    USBCombinedPacket *combined;
     QTAILQ_ENTRY(USBPacket) queue;
+    QTAILQ_ENTRY(USBPacket) combined_entry;
+};
+
+struct USBCombinedPacket {
+    USBPacket *first;
+    QTAILQ_HEAD(packets_head, USBPacket) packets;
+    QEMUIOVector iov;
 };
 
 void usb_packet_init(USBPacket *p);
 void usb_packet_set_state(USBPacket *p, USBPacketState state);
 void usb_packet_check_state(USBPacket *p, USBPacketState expected);
-void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id);
+void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id,
+                      bool short_not_ok, bool int_req);
 void usb_packet_addbuf(USBPacket *p, void *ptr, size_t len);
 int usb_packet_map(USBPacket *p, QEMUSGList *sgl);
 void usb_packet_unmap(USBPacket *p, QEMUSGList *sgl);
@@ -361,8 +400,9 @@ static inline bool usb_packet_is_inflight(USBPacket *p)
 
 USBDevice *usb_find_device(USBPort *port, uint8_t addr);
 
-int usb_handle_packet(USBDevice *dev, USBPacket *p);
+void usb_handle_packet(USBDevice *dev, USBPacket *p);
 void usb_packet_complete(USBDevice *dev, USBPacket *p);
+void usb_packet_complete_one(USBDevice *dev, USBPacket *p);
 void usb_cancel_packet(USBPacket * p);
 
 void usb_ep_init(USBDevice *dev);
@@ -377,6 +417,12 @@ void usb_ep_set_max_packet_size(USBDevice *dev, int pid, int ep,
                                 uint16_t raw);
 int usb_ep_get_max_packet_size(USBDevice *dev, int pid, int ep);
 void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled);
+USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
+                                    uint64_t id);
+
+void usb_ep_combine_input_packets(USBEndpoint *ep);
+void usb_combined_input_packet_complete(USBDevice *dev, USBPacket *p);
+void usb_combined_packet_cancel(USBDevice *dev, USBPacket *p);
 
 void usb_attach(USBPort *port);
 void usb_detach(USBPort *port);
@@ -389,7 +435,7 @@ int set_usb_string(uint8_t *buf, const char *str);
 /* usb-linux.c */
 USBDevice *usb_host_device_open(USBBus *bus, const char *devname);
 int usb_host_device_close(const char *devname);
-void usb_host_info(Monitor *mon);
+void usb_host_info(Monitor *mon, const QDict *qdict);
 
 /* usb-bt.c */
 USBDevice *usb_bt_init(USBBus *bus, HCIInfo *hci);
@@ -489,17 +535,33 @@ void usb_device_handle_attach(USBDevice *dev);
 
 void usb_device_handle_reset(USBDevice *dev);
 
-int usb_device_handle_control(USBDevice *dev, USBPacket *p, int request, int value,
-                              int index, int length, uint8_t *data);
+void usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
+                               int val, int index, int length, uint8_t *data);
 
-int usb_device_handle_data(USBDevice *dev, USBPacket *p);
+void usb_device_handle_data(USBDevice *dev, USBPacket *p);
 
 void usb_device_set_interface(USBDevice *dev, int interface,
                               int alt_old, int alt_new);
 
+void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep);
+
+void usb_device_ep_stopped(USBDevice *dev, USBEndpoint *ep);
+
 const char *usb_device_get_product_desc(USBDevice *dev);
 
 const USBDesc *usb_device_get_usb_desc(USBDevice *dev);
 
-#endif
+int ehci_create_ich9_with_companions(PCIBus *bus, int slot);
 
+/* quirks.c */
+
+/* In bulk endpoints are streaming data sources (iow behave like isoc eps) */
+#define USB_QUIRK_BUFFER_BULK_IN	0x01
+/* Bulk pkts in FTDI format, need special handling when combining packets */
+#define USB_QUIRK_IS_FTDI		0x02
+
+int usb_get_quirks(uint16_t vendor_id, uint16_t product_id,
+                   uint8_t interface_class, uint8_t interface_subclass,
+                   uint8_t interface_protocol);
+
+#endif
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 4225136d0f7..d1bbbc06e7e 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -1,14 +1,14 @@
-hw-obj-$(CONFIG_USB_UHCI) += hcd-uhci.o
-hw-obj-$(CONFIG_USB_OHCI) += hcd-ohci.o
-hw-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o
-hw-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
-hw-obj-y += libhw.o
+common-obj-$(CONFIG_USB_UHCI) += hcd-uhci.o
+common-obj-$(CONFIG_USB_OHCI) += hcd-ohci.o
+common-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o hcd-ehci-pci.o hcd-ehci-sysbus.o
+common-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
+common-obj-y += libhw.o
 
-hw-obj-$(CONFIG_SMARTCARD) += dev-smartcard-reader.o
-hw-obj-$(CONFIG_USB_REDIR) += redirect.o
+common-obj-$(CONFIG_USB_REDIR) += redirect.o quirks.o
 
-common-obj-y += core.o bus.o desc.o dev-hub.o
+common-obj-y += core.o combined-packet.o bus.o desc.o dev-hub.o
 common-obj-y += host-$(HOST_USB).o dev-bluetooth.o
 common-obj-y += dev-hid.o dev-storage.o dev-wacom.o
 common-obj-y += dev-serial.o dev-network.o dev-audio.o
+common-obj-y += dev-smartcard-reader.o
 common-obj-y += dev-uas.o
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index b649360dd38..e58cd9ade23 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -1,8 +1,8 @@
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/qdev.h"
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 #include "trace.h"
 
 static void usb_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent);
@@ -140,24 +140,21 @@ void usb_device_handle_reset(USBDevice *dev)
     }
 }
 
-int usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
-                              int value, int index, int length, uint8_t *data)
+void usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
+                               int value, int index, int length, uint8_t *data)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
     if (klass->handle_control) {
-        return klass->handle_control(dev, p, request, value, index, length,
-                                         data);
+        klass->handle_control(dev, p, request, value, index, length, data);
     }
-    return -ENOSYS;
 }
 
-int usb_device_handle_data(USBDevice *dev, USBPacket *p)
+void usb_device_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
     if (klass->handle_data) {
-        return klass->handle_data(dev, p);
+        klass->handle_data(dev, p);
     }
-    return -ENOSYS;
 }
 
 const char *usb_device_get_product_desc(USBDevice *dev)
@@ -169,6 +166,9 @@ const char *usb_device_get_product_desc(USBDevice *dev)
 const USBDesc *usb_device_get_usb_desc(USBDevice *dev)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (dev->usb_desc) {
+        return dev->usb_desc;
+    }
     return klass->usb_desc;
 }
 
@@ -181,6 +181,22 @@ void usb_device_set_interface(USBDevice *dev, int interface,
     }
 }
 
+void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
+{
+    USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (klass->flush_ep_queue) {
+        klass->flush_ep_queue(dev, ep);
+    }
+}
+
+void usb_device_ep_stopped(USBDevice *dev, USBEndpoint *ep)
+{
+    USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (klass->ep_stopped) {
+        klass->ep_stopped(dev, ep);
+    }
+}
+
 static int usb_qdev_init(DeviceState *qdev)
 {
     USBDevice *dev = USB_DEVICE(qdev);
@@ -526,7 +542,7 @@ static char *usb_get_fw_dev_path(DeviceState *qdev)
     return fw_path;
 }
 
-void usb_info(Monitor *mon)
+void usb_info(Monitor *mon, const QDict *qdict)
 {
     USBBus *bus;
     USBDevice *dev;
@@ -585,6 +601,13 @@ USBDevice *usbdevice_create(const char *cmdline)
         return NULL;
     }
 
+    if (!bus) {
+        error_report("Error: no usb bus to attach usbdevice %s, "
+                     "please try -machine usb=on and check that "
+                     "the machine model supports USB", driver);
+        return NULL;
+    }
+
     if (!f->usbdevice_init) {
         if (*params) {
             error_report("usbdevice %s accepts no params", driver);
@@ -605,7 +628,7 @@ static void usb_device_class_init(ObjectClass *klass, void *data)
     k->props    = usb_props;
 }
 
-static TypeInfo usb_device_type_info = {
+static const TypeInfo usb_device_type_info = {
     .name = TYPE_USB_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(USBDevice),
diff --git a/hw/usb/combined-packet.c b/hw/usb/combined-packet.c
new file mode 100644
index 00000000000..13f6602ad24
--- /dev/null
+++ b/hw/usb/combined-packet.c
@@ -0,0 +1,186 @@
+/*
+ * QEMU USB packet combining code (for input pipelining)
+ *
+ * Copyright(c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "qemu-common.h"
+#include "hw/usb.h"
+#include "qemu/iov.h"
+#include "trace.h"
+
+static void usb_combined_packet_add(USBCombinedPacket *combined, USBPacket *p)
+{
+    qemu_iovec_concat(&combined->iov, &p->iov, 0, p->iov.size);
+    QTAILQ_INSERT_TAIL(&combined->packets, p, combined_entry);
+    p->combined = combined;
+}
+
+/* Note will free combined when the last packet gets removed */
+static void usb_combined_packet_remove(USBCombinedPacket *combined,
+                                       USBPacket *p)
+{
+    assert(p->combined == combined);
+    p->combined = NULL;
+    QTAILQ_REMOVE(&combined->packets, p, combined_entry);
+    if (QTAILQ_EMPTY(&combined->packets)) {
+        g_free(combined);
+    }
+}
+
+/* Also handles completion of non combined packets for pipelined input eps */
+void usb_combined_input_packet_complete(USBDevice *dev, USBPacket *p)
+{
+    USBCombinedPacket *combined = p->combined;
+    USBEndpoint *ep = p->ep;
+    USBPacket *next;
+    int status, actual_length;
+    bool short_not_ok, done = false;
+
+    if (combined == NULL) {
+        usb_packet_complete_one(dev, p);
+        goto leave;
+    }
+
+    assert(combined->first == p && p == QTAILQ_FIRST(&combined->packets));
+
+    status = combined->first->status;
+    actual_length = combined->first->actual_length;
+    short_not_ok = QTAILQ_LAST(&combined->packets, packets_head)->short_not_ok;
+
+    QTAILQ_FOREACH_SAFE(p, &combined->packets, combined_entry, next) {
+        if (!done) {
+            /* Distribute data over uncombined packets */
+            if (actual_length >= p->iov.size) {
+                p->actual_length = p->iov.size;
+            } else {
+                /* Send short or error packet to complete the transfer */
+                p->actual_length = actual_length;
+                done = true;
+            }
+            /* Report status on the last packet */
+            if (done || next == NULL) {
+                p->status = status;
+            } else {
+                p->status = USB_RET_SUCCESS;
+            }
+            p->short_not_ok = short_not_ok;
+            /* Note will free combined when the last packet gets removed! */
+            usb_combined_packet_remove(combined, p);
+            usb_packet_complete_one(dev, p);
+            actual_length -= p->actual_length;
+        } else {
+            /* Remove any leftover packets from the queue */
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            /* Note will free combined on the last packet! */
+            dev->port->ops->complete(dev->port, p);
+        }
+    }
+    /* Do not use combined here, it has been freed! */
+leave:
+    /* Check if there are packets in the queue waiting for our completion */
+    usb_ep_combine_input_packets(ep);
+}
+
+/* May only be called for combined packets! */
+void usb_combined_packet_cancel(USBDevice *dev, USBPacket *p)
+{
+    USBCombinedPacket *combined = p->combined;
+    assert(combined != NULL);
+    USBPacket *first = p->combined->first;
+
+    /* Note will free combined on the last packet! */
+    usb_combined_packet_remove(combined, p);
+    if (p == first) {
+        usb_device_cancel_packet(dev, p);
+    }
+}
+
+/*
+ * Large input transfers can get split into multiple input packets, this
+ * function recombines them, removing the short_not_ok checks which all but
+ * the last packet of such splits transfers have, thereby allowing input
+ * transfer pipelining (which we cannot do on short_not_ok transfers)
+ */
+void usb_ep_combine_input_packets(USBEndpoint *ep)
+{
+    USBPacket *p, *u, *next, *prev = NULL, *first = NULL;
+    USBPort *port = ep->dev->port;
+    int totalsize;
+
+    assert(ep->pipeline);
+    assert(ep->pid == USB_TOKEN_IN);
+
+    QTAILQ_FOREACH_SAFE(p, &ep->queue, queue, next) {
+        /* Empty the queue on a halt */
+        if (ep->halted) {
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            port->ops->complete(port, p);
+            continue;
+        }
+
+        /* Skip packets already submitted to the device */
+        if (p->state == USB_PACKET_ASYNC) {
+            prev = p;
+            continue;
+        }
+        usb_packet_check_state(p, USB_PACKET_QUEUED);
+
+        /*
+         * If the previous (combined) packet has the short_not_ok flag set
+         * stop, as we must not submit packets to the device after a transfer
+         * ending with short_not_ok packet.
+         */
+        if (prev && prev->short_not_ok) {
+            break;
+        }
+
+        if (first) {
+            if (first->combined == NULL) {
+                USBCombinedPacket *combined = g_new0(USBCombinedPacket, 1);
+
+                combined->first = first;
+                QTAILQ_INIT(&combined->packets);
+                qemu_iovec_init(&combined->iov, 2);
+                usb_combined_packet_add(combined, first);
+            }
+            usb_combined_packet_add(first->combined, p);
+        } else {
+            first = p;
+        }
+
+        /* Is this packet the last one of a (combined) transfer? */
+        totalsize = (p->combined) ? p->combined->iov.size : p->iov.size;
+        if ((p->iov.size % ep->max_packet_size) != 0 || !p->short_not_ok ||
+                next == NULL ||
+                /* Work around for Linux usbfs bulk splitting + migration */
+                (totalsize == 16348 && p->int_req)) {
+            usb_device_handle_data(ep->dev, first);
+            assert(first->status == USB_RET_ASYNC);
+            if (first->combined) {
+                QTAILQ_FOREACH(u, &first->combined->packets, combined_entry) {
+                    usb_packet_set_state(u, USB_PACKET_ASYNC);
+                }
+            } else {
+                usb_packet_set_state(first, USB_PACKET_ASYNC);
+            }
+            first = NULL;
+            prev = p;
+        }
+    }
+}
diff --git a/hw/usb/core.c b/hw/usb/core.c
index 2da38e7fd02..d057aab9000 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -25,7 +25,7 @@
  */
 #include "qemu-common.h"
 #include "hw/usb.h"
-#include "iov.h"
+#include "qemu/iov.h"
 #include "trace.h"
 
 void usb_attach(USBPort *port)
@@ -97,17 +97,17 @@ void usb_wakeup(USBEndpoint *ep)
 #define SETUP_STATE_ACK   3
 #define SETUP_STATE_PARAM 4
 
-static int do_token_setup(USBDevice *s, USBPacket *p)
+static void do_token_setup(USBDevice *s, USBPacket *p)
 {
     int request, value, index;
-    int ret = 0;
 
     if (p->iov.size != 8) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     usb_packet_copy(p, s->setup_buf, p->iov.size);
-    p->result = 0;
+    p->actual_length = 0;
     s->setup_len   = (s->setup_buf[7] << 8) | s->setup_buf[6];
     s->setup_index = 0;
 
@@ -116,24 +116,26 @@ static int do_token_setup(USBDevice *s, USBPacket *p)
     index   = (s->setup_buf[5] << 8) | s->setup_buf[4];
 
     if (s->setup_buf[0] & USB_DIR_IN) {
-        ret = usb_device_handle_control(s, p, request, value, index,
-                                        s->setup_len, s->data_buf);
-        if (ret == USB_RET_ASYNC) {
-             s->setup_state = SETUP_STATE_SETUP;
-             return USB_RET_ASYNC;
+        usb_device_handle_control(s, p, request, value, index,
+                                  s->setup_len, s->data_buf);
+        if (p->status == USB_RET_ASYNC) {
+            s->setup_state = SETUP_STATE_SETUP;
+        }
+        if (p->status != USB_RET_SUCCESS) {
+            return;
         }
-        if (ret < 0)
-            return ret;
 
-        if (ret < s->setup_len)
-            s->setup_len = ret;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
+        }
         s->setup_state = SETUP_STATE_DATA;
     } else {
         if (s->setup_len > sizeof(s->data_buf)) {
             fprintf(stderr,
                 "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
                 s->setup_len, sizeof(s->data_buf));
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
+            return;
         }
         if (s->setup_len == 0)
             s->setup_state = SETUP_STATE_ACK;
@@ -141,13 +143,12 @@ static int do_token_setup(USBDevice *s, USBPacket *p)
             s->setup_state = SETUP_STATE_DATA;
     }
 
-    return ret;
+    p->actual_length = 8;
 }
 
-static int do_token_in(USBDevice *s, USBPacket *p)
+static void do_token_in(USBDevice *s, USBPacket *p)
 {
     int request, value, index;
-    int ret = 0;
 
     assert(p->ep->nr == 0);
 
@@ -158,19 +159,15 @@ static int do_token_in(USBDevice *s, USBPacket *p)
     switch(s->setup_state) {
     case SETUP_STATE_ACK:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
-            ret = usb_device_handle_control(s, p, request, value, index,
-                                            s->setup_len, s->data_buf);
-            if (ret == USB_RET_ASYNC) {
-                return USB_RET_ASYNC;
+            usb_device_handle_control(s, p, request, value, index,
+                                      s->setup_len, s->data_buf);
+            if (p->status == USB_RET_ASYNC) {
+                return;
             }
             s->setup_state = SETUP_STATE_IDLE;
-            if (ret > 0)
-                return 0;
-            return ret;
+            p->actual_length = 0;
         }
-
-        /* return 0 byte */
-        return 0;
+        break;
 
     case SETUP_STATE_DATA:
         if (s->setup_buf[0] & USB_DIR_IN) {
@@ -180,20 +177,21 @@ static int do_token_in(USBDevice *s, USBPacket *p)
             }
             usb_packet_copy(p, s->data_buf + s->setup_index, len);
             s->setup_index += len;
-            if (s->setup_index >= s->setup_len)
+            if (s->setup_index >= s->setup_len) {
                 s->setup_state = SETUP_STATE_ACK;
-            return len;
+            }
+            return;
         }
-
         s->setup_state = SETUP_STATE_IDLE;
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
 
     default:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
     }
 }
 
-static int do_token_out(USBDevice *s, USBPacket *p)
+static void do_token_out(USBDevice *s, USBPacket *p)
 {
     assert(p->ep->nr == 0);
 
@@ -205,7 +203,7 @@ static int do_token_out(USBDevice *s, USBPacket *p)
         } else {
             /* ignore additional output */
         }
-        return 0;
+        break;
 
     case SETUP_STATE_DATA:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
@@ -215,23 +213,23 @@ static int do_token_out(USBDevice *s, USBPacket *p)
             }
             usb_packet_copy(p, s->data_buf + s->setup_index, len);
             s->setup_index += len;
-            if (s->setup_index >= s->setup_len)
+            if (s->setup_index >= s->setup_len) {
                 s->setup_state = SETUP_STATE_ACK;
-            return len;
+            }
+            return;
         }
-
         s->setup_state = SETUP_STATE_IDLE;
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
 
     default:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
     }
 }
 
-static int do_parameter(USBDevice *s, USBPacket *p)
+static void do_parameter(USBDevice *s, USBPacket *p)
 {
-    int request, value, index;
-    int i, ret = 0;
+    int i, request, value, index;
 
     for (i = 0; i < 8; i++) {
         s->setup_buf[i] = p->parameter >> (i*8);
@@ -249,27 +247,27 @@ static int do_parameter(USBDevice *s, USBPacket *p)
         fprintf(stderr,
                 "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
                 s->setup_len, sizeof(s->data_buf));
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     if (p->pid == USB_TOKEN_OUT) {
         usb_packet_copy(p, s->data_buf, s->setup_len);
     }
 
-    ret = usb_device_handle_control(s, p, request, value, index,
-                                    s->setup_len, s->data_buf);
-    if (ret < 0) {
-        return ret;
+    usb_device_handle_control(s, p, request, value, index,
+                              s->setup_len, s->data_buf);
+    if (p->status == USB_RET_ASYNC) {
+        return;
     }
 
-    if (ret < s->setup_len) {
-        s->setup_len = ret;
+    if (p->actual_length < s->setup_len) {
+        s->setup_len = p->actual_length;
     }
     if (p->pid == USB_TOKEN_IN) {
+        p->actual_length = 0;
         usb_packet_copy(p, s->data_buf, s->setup_len);
     }
-
-    return ret;
 }
 
 /* ctrl complete function for devices which use usb_generic_handle_packet and
@@ -278,30 +276,30 @@ static int do_parameter(USBDevice *s, USBPacket *p)
    usb_packet_complete to complete their async control packets. */
 void usb_generic_async_ctrl_complete(USBDevice *s, USBPacket *p)
 {
-    if (p->result < 0) {
+    if (p->status < 0) {
         s->setup_state = SETUP_STATE_IDLE;
     }
 
     switch (s->setup_state) {
     case SETUP_STATE_SETUP:
-        if (p->result < s->setup_len) {
-            s->setup_len = p->result;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
         }
         s->setup_state = SETUP_STATE_DATA;
-        p->result = 8;
+        p->actual_length = 8;
         break;
 
     case SETUP_STATE_ACK:
         s->setup_state = SETUP_STATE_IDLE;
-        p->result = 0;
+        p->actual_length = 0;
         break;
 
     case SETUP_STATE_PARAM:
-        if (p->result < s->setup_len) {
-            s->setup_len = p->result;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
         }
         if (p->pid == USB_TOKEN_IN) {
-            p->result = 0;
+            p->actual_length = 0;
             usb_packet_copy(p, s->data_buf, s->setup_len);
         }
         break;
@@ -342,40 +340,57 @@ USBDevice *usb_find_device(USBPort *port, uint8_t addr)
     return usb_device_find_device(dev, addr);
 }
 
-static int usb_process_one(USBPacket *p)
+static void usb_process_one(USBPacket *p)
 {
     USBDevice *dev = p->ep->dev;
 
+    /*
+     * Handlers expect status to be initialized to USB_RET_SUCCESS, but it
+     * can be USB_RET_NAK here from a previous usb_process_one() call,
+     * or USB_RET_ASYNC from going through usb_queue_one().
+     */
+    p->status = USB_RET_SUCCESS;
+
     if (p->ep->nr == 0) {
         /* control pipe */
         if (p->parameter) {
-            return do_parameter(dev, p);
+            do_parameter(dev, p);
+            return;
         }
         switch (p->pid) {
         case USB_TOKEN_SETUP:
-            return do_token_setup(dev, p);
+            do_token_setup(dev, p);
+            break;
         case USB_TOKEN_IN:
-            return do_token_in(dev, p);
+            do_token_in(dev, p);
+            break;
         case USB_TOKEN_OUT:
-            return do_token_out(dev, p);
+            do_token_out(dev, p);
+            break;
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
     } else {
         /* data pipe */
-        return usb_device_handle_data(dev, p);
+        usb_device_handle_data(dev, p);
     }
 }
 
-/* Hand over a packet to a device for processing.  Return value
+static void usb_queue_one(USBPacket *p)
+{
+    usb_packet_set_state(p, USB_PACKET_QUEUED);
+    QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+    p->status = USB_RET_ASYNC;
+}
+
+/* Hand over a packet to a device for processing.  p->status ==
    USB_RET_ASYNC indicates the processing isn't finished yet, the
    driver will call usb_packet_complete() when done processing it. */
-int usb_handle_packet(USBDevice *dev, USBPacket *p)
+void usb_handle_packet(USBDevice *dev, USBPacket *p)
 {
-    int ret;
-
     if (dev == NULL) {
-        return USB_RET_NODEV;
+        p->status = USB_RET_NODEV;
+        return;
     }
     assert(dev == p->ep->dev);
     assert(dev->state == USB_STATE_DEFAULT);
@@ -389,34 +404,41 @@ int usb_handle_packet(USBDevice *dev, USBPacket *p)
     }
 
     if (QTAILQ_EMPTY(&p->ep->queue) || p->ep->pipeline) {
-        ret = usb_process_one(p);
-        if (ret == USB_RET_ASYNC) {
+        usb_process_one(p);
+        if (p->status == USB_RET_ASYNC) {
+            /* hcd drivers cannot handle async for isoc */
+            assert(p->ep->type != USB_ENDPOINT_XFER_ISOC);
+            /* using async for interrupt packets breaks migration */
+            assert(p->ep->type != USB_ENDPOINT_XFER_INT ||
+                   (dev->flags & USB_DEV_FLAG_IS_HOST));
             usb_packet_set_state(p, USB_PACKET_ASYNC);
             QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+        } else if (p->status == USB_RET_ADD_TO_QUEUE) {
+            usb_queue_one(p);
         } else {
             /*
              * When pipelining is enabled usb-devices must always return async,
              * otherwise packets can complete out of order!
              */
-            assert(!p->ep->pipeline);
-            p->result = ret;
-            usb_packet_set_state(p, USB_PACKET_COMPLETE);
+            assert(!p->ep->pipeline || QTAILQ_EMPTY(&p->ep->queue));
+            if (p->status != USB_RET_NAK) {
+                usb_packet_set_state(p, USB_PACKET_COMPLETE);
+            }
         }
     } else {
-        ret = USB_RET_ASYNC;
-        usb_packet_set_state(p, USB_PACKET_QUEUED);
-        QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+        usb_queue_one(p);
     }
-    return ret;
 }
 
-static void __usb_packet_complete(USBDevice *dev, USBPacket *p)
+void usb_packet_complete_one(USBDevice *dev, USBPacket *p)
 {
     USBEndpoint *ep = p->ep;
 
-    assert(p->result != USB_RET_ASYNC && p->result != USB_RET_NAK);
+    assert(QTAILQ_FIRST(&ep->queue) == p);
+    assert(p->status != USB_RET_ASYNC && p->status != USB_RET_NAK);
 
-    if (p->result < 0) {
+    if (p->status != USB_RET_SUCCESS ||
+            (p->short_not_ok && (p->actual_length < p->iov.size))) {
         ep->halted = true;
     }
     usb_packet_set_state(p, USB_PACKET_COMPLETE);
@@ -430,25 +452,28 @@ static void __usb_packet_complete(USBDevice *dev, USBPacket *p)
 void usb_packet_complete(USBDevice *dev, USBPacket *p)
 {
     USBEndpoint *ep = p->ep;
-    int ret;
 
     usb_packet_check_state(p, USB_PACKET_ASYNC);
-    assert(QTAILQ_FIRST(&ep->queue) == p);
-    __usb_packet_complete(dev, p);
+    usb_packet_complete_one(dev, p);
 
-    while (!ep->halted && !QTAILQ_EMPTY(&ep->queue)) {
+    while (!QTAILQ_EMPTY(&ep->queue)) {
         p = QTAILQ_FIRST(&ep->queue);
+        if (ep->halted) {
+            /* Empty the queue on a halt */
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            dev->port->ops->complete(dev->port, p);
+            continue;
+        }
         if (p->state == USB_PACKET_ASYNC) {
             break;
         }
         usb_packet_check_state(p, USB_PACKET_QUEUED);
-        ret = usb_process_one(p);
-        if (ret == USB_RET_ASYNC) {
+        usb_process_one(p);
+        if (p->status == USB_RET_ASYNC) {
             usb_packet_set_state(p, USB_PACKET_ASYNC);
             break;
         }
-        p->result = ret;
-        __usb_packet_complete(ep->dev, p);
+        usb_packet_complete_one(ep->dev, p);
     }
 }
 
@@ -520,15 +545,20 @@ void usb_packet_set_state(USBPacket *p, USBPacketState state)
     p->state = state;
 }
 
-void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id)
+void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id,
+                      bool short_not_ok, bool int_req)
 {
     assert(!usb_packet_is_inflight(p));
     assert(p->iov.iov != NULL);
     p->id = id;
     p->pid = pid;
     p->ep = ep;
-    p->result = 0;
+    p->status = USB_RET_SUCCESS;
+    p->actual_length = 0;
     p->parameter = 0;
+    p->short_not_ok = short_not_ok;
+    p->int_req = int_req;
+    p->combined = NULL;
     qemu_iovec_reset(&p->iov);
     usb_packet_set_state(p, USB_PACKET_SETUP);
 }
@@ -540,31 +570,31 @@ void usb_packet_addbuf(USBPacket *p, void *ptr, size_t len)
 
 void usb_packet_copy(USBPacket *p, void *ptr, size_t bytes)
 {
-    assert(p->result >= 0);
-    assert(p->result + bytes <= p->iov.size);
+    assert(p->actual_length >= 0);
+    assert(p->actual_length + bytes <= p->iov.size);
     switch (p->pid) {
     case USB_TOKEN_SETUP:
     case USB_TOKEN_OUT:
-        iov_to_buf(p->iov.iov, p->iov.niov, p->result, ptr, bytes);
+        iov_to_buf(p->iov.iov, p->iov.niov, p->actual_length, ptr, bytes);
         break;
     case USB_TOKEN_IN:
-        iov_from_buf(p->iov.iov, p->iov.niov, p->result, ptr, bytes);
+        iov_from_buf(p->iov.iov, p->iov.niov, p->actual_length, ptr, bytes);
         break;
     default:
         fprintf(stderr, "%s: invalid pid: %x\n", __func__, p->pid);
         abort();
     }
-    p->result += bytes;
+    p->actual_length += bytes;
 }
 
 void usb_packet_skip(USBPacket *p, size_t bytes)
 {
-    assert(p->result >= 0);
-    assert(p->result + bytes <= p->iov.size);
+    assert(p->actual_length >= 0);
+    assert(p->actual_length + bytes <= p->iov.size);
     if (p->pid == USB_TOKEN_IN) {
-        iov_memset(p->iov.iov, p->iov.niov, p->result, 0, bytes);
+        iov_memset(p->iov.iov, p->iov.niov, p->actual_length, 0, bytes);
     }
-    p->result += bytes;
+    p->actual_length += bytes;
 }
 
 void usb_packet_cleanup(USBPacket *p)
@@ -724,3 +754,18 @@ void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled)
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
     uep->pipeline = enabled;
 }
+
+USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
+                                    uint64_t id)
+{
+    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
+    USBPacket *p;
+
+    QTAILQ_FOREACH(p, &uep->queue, queue) {
+        if (p->id == id) {
+            return p;
+        }
+    }
+
+    return NULL;
+}
diff --git a/hw/usb/desc.c b/hw/usb/desc.c
index 0a9d3c9f605..b7c32333d70 100644
--- a/hw/usb/desc.c
+++ b/hw/usb/desc.c
@@ -76,7 +76,8 @@ int usb_desc_device_qualifier(const USBDescDevice *dev,
     return bLength;
 }
 
-int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
+int usb_desc_config(const USBDescConfig *conf, int flags,
+                    uint8_t *dest, size_t len)
 {
     uint8_t  bLength = 0x09;
     uint16_t wTotalLength = 0;
@@ -99,7 +100,7 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
 
     /* handle grouped interfaces if any */
     for (i = 0; i < conf->nif_groups; i++) {
-        rc = usb_desc_iface_group(&(conf->if_groups[i]),
+        rc = usb_desc_iface_group(&(conf->if_groups[i]), flags,
                                   dest + wTotalLength,
                                   len - wTotalLength);
         if (rc < 0) {
@@ -110,7 +111,8 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
 
     /* handle normal (ungrouped / no IAD) interfaces if any */
     for (i = 0; i < conf->nif; i++) {
-        rc = usb_desc_iface(conf->ifs + i, dest + wTotalLength, len - wTotalLength);
+        rc = usb_desc_iface(conf->ifs + i, flags,
+                            dest + wTotalLength, len - wTotalLength);
         if (rc < 0) {
             return rc;
         }
@@ -122,8 +124,8 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
     return wTotalLength;
 }
 
-int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
-                         size_t len)
+int usb_desc_iface_group(const USBDescIfaceAssoc *iad, int flags,
+                         uint8_t *dest, size_t len)
 {
     int pos = 0;
     int i = 0;
@@ -147,7 +149,7 @@ int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
 
     /* handle associated interfaces in this group */
     for (i = 0; i < iad->nif; i++) {
-        int rc = usb_desc_iface(&(iad->ifs[i]), dest + pos, len - pos);
+        int rc = usb_desc_iface(&(iad->ifs[i]), flags, dest + pos, len - pos);
         if (rc < 0) {
             return rc;
         }
@@ -157,7 +159,8 @@ int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
     return pos;
 }
 
-int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
+int usb_desc_iface(const USBDescIface *iface, int flags,
+                   uint8_t *dest, size_t len)
 {
     uint8_t bLength = 0x09;
     int i, rc, pos = 0;
@@ -188,7 +191,7 @@ int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
     }
 
     for (i = 0; i < iface->bNumEndpoints; i++) {
-        rc = usb_desc_endpoint(iface->eps + i, dest + pos, len - pos);
+        rc = usb_desc_endpoint(iface->eps + i, flags, dest + pos, len - pos);
         if (rc < 0) {
             return rc;
         }
@@ -198,13 +201,15 @@ int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
     return pos;
 }
 
-int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len)
+int usb_desc_endpoint(const USBDescEndpoint *ep, int flags,
+                      uint8_t *dest, size_t len)
 {
     uint8_t bLength = ep->is_audio ? 0x09 : 0x07;
     uint8_t extralen = ep->extra ? ep->extra[0] : 0;
+    uint8_t superlen = (flags & USB_DESC_FLAG_SUPER) ? 0x06 : 0;
     USBDescriptor *d = (void *)dest;
 
-    if (len < bLength + extralen) {
+    if (len < bLength + extralen + superlen) {
         return -1;
     }
 
@@ -224,7 +229,21 @@ int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len)
         memcpy(dest + bLength, ep->extra, extralen);
     }
 
-    return bLength + extralen;
+    if (superlen) {
+        USBDescriptor *d = (void *)(dest + bLength + extralen);
+
+        d->bLength                       = 0x06;
+        d->bDescriptorType               = USB_DT_ENDPOINT_COMPANION;
+
+        d->u.super_endpoint.bMaxBurst    = ep->bMaxBurst;
+        d->u.super_endpoint.bmAttributes = ep->bmAttributes_super;
+        d->u.super_endpoint.wBytesPerInterval_lo =
+            usb_lo(ep->wBytesPerInterval);
+        d->u.super_endpoint.wBytesPerInterval_hi =
+            usb_hi(ep->wBytesPerInterval);
+    }
+
+    return bLength + extralen + superlen;
 }
 
 int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len)
@@ -239,6 +258,111 @@ int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len)
     return bLength;
 }
 
+static int usb_desc_cap_usb2_ext(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x07;
+    USBDescriptor *d = (void *)dest;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                          = bLength;
+    d->bDescriptorType                  = USB_DT_DEVICE_CAPABILITY;
+    d->u.cap.bDevCapabilityType         = USB_DEV_CAP_USB2_EXT;
+
+    d->u.cap.u.usb2_ext.bmAttributes_1  = (1 << 1);  /* LPM */
+    d->u.cap.u.usb2_ext.bmAttributes_2  = 0;
+    d->u.cap.u.usb2_ext.bmAttributes_3  = 0;
+    d->u.cap.u.usb2_ext.bmAttributes_4  = 0;
+
+    return bLength;
+}
+
+static int usb_desc_cap_super(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x0a;
+    USBDescriptor *d = (void *)dest;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                           = bLength;
+    d->bDescriptorType                   = USB_DT_DEVICE_CAPABILITY;
+    d->u.cap.bDevCapabilityType          = USB_DEV_CAP_SUPERSPEED;
+
+    d->u.cap.u.super.bmAttributes        = 0;
+    d->u.cap.u.super.wSpeedsSupported_lo = 0;
+    d->u.cap.u.super.wSpeedsSupported_hi = 0;
+    d->u.cap.u.super.bFunctionalitySupport = 0;
+    d->u.cap.u.super.bU1DevExitLat       = 0x0a;
+    d->u.cap.u.super.wU2DevExitLat_lo    = 0x20;
+    d->u.cap.u.super.wU2DevExitLat_hi    = 0;
+
+    if (desc->full) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 1);
+        d->u.cap.u.super.bFunctionalitySupport = 1;
+    }
+    if (desc->high) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 2);
+        if (!d->u.cap.u.super.bFunctionalitySupport) {
+            d->u.cap.u.super.bFunctionalitySupport = 2;
+        }
+    }
+    if (desc->super) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 3);
+        if (!d->u.cap.u.super.bFunctionalitySupport) {
+            d->u.cap.u.super.bFunctionalitySupport = 3;
+        }
+    }
+
+    return bLength;
+}
+
+static int usb_desc_bos(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x05;
+    uint16_t wTotalLength = 0;
+    uint8_t  bNumDeviceCaps = 0;
+    USBDescriptor *d = (void *)dest;
+    int rc;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                      = bLength;
+    d->bDescriptorType              = USB_DT_BOS;
+
+    wTotalLength += bLength;
+
+    if (desc->high != NULL) {
+        rc = usb_desc_cap_usb2_ext(desc, dest + wTotalLength,
+                                   len - wTotalLength);
+        if (rc < 0) {
+            return rc;
+        }
+        wTotalLength += rc;
+        bNumDeviceCaps++;
+    }
+
+    if (desc->super != NULL) {
+        rc = usb_desc_cap_super(desc, dest + wTotalLength,
+                                len - wTotalLength);
+        if (rc < 0) {
+            return rc;
+        }
+        wTotalLength += rc;
+        bNumDeviceCaps++;
+    }
+
+    d->u.bos.wTotalLength_lo = usb_lo(wTotalLength);
+    d->u.bos.wTotalLength_hi = usb_hi(wTotalLength);
+    d->u.bos.bNumDeviceCaps  = bNumDeviceCaps;
+    return wTotalLength;
+}
+
 /* ------------------------------------------------------------------ */
 
 static void usb_desc_ep_init(USBDevice *dev)
@@ -359,6 +483,9 @@ static void usb_desc_setdefaults(USBDevice *dev)
     case USB_SPEED_HIGH:
         dev->device = desc->high;
         break;
+    case USB_SPEED_SUPER:
+        dev->device = desc->super;
+        break;
     }
     usb_desc_set_config(dev, 0);
 }
@@ -376,6 +503,9 @@ void usb_desc_init(USBDevice *dev)
     if (desc->high) {
         dev->speedmask |= USB_SPEED_MASK_HIGH;
     }
+    if (desc->super) {
+        dev->speedmask |= USB_SPEED_MASK_SUPER;
+    }
     usb_desc_setdefaults(dev);
 }
 
@@ -384,7 +514,9 @@ void usb_desc_attach(USBDevice *dev)
     const USBDesc *desc = usb_device_get_usb_desc(dev);
 
     assert(desc != NULL);
-    if (desc->high && (dev->port->speedmask & USB_SPEED_MASK_HIGH)) {
+    if (desc->super && (dev->port->speedmask & USB_SPEED_MASK_SUPER)) {
+        dev->speed = USB_SPEED_SUPER;
+    } else if (desc->high && (dev->port->speedmask & USB_SPEED_MASK_HIGH)) {
         dev->speed = USB_SPEED_HIGH;
     } else if (desc->full && (dev->port->speedmask & USB_SPEED_MASK_FULL)) {
         dev->speed = USB_SPEED_FULL;
@@ -494,14 +626,15 @@ int usb_desc_string(USBDevice *dev, int index, uint8_t *dest, size_t len)
     return pos;
 }
 
-int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len)
+int usb_desc_get_descriptor(USBDevice *dev, USBPacket *p,
+                            int value, uint8_t *dest, size_t len)
 {
     const USBDesc *desc = usb_device_get_usb_desc(dev);
     const USBDescDevice *other_dev;
     uint8_t buf[256];
     uint8_t type = value >> 8;
     uint8_t index = value & 0xff;
-    int ret = -1;
+    int flags, ret = -1;
 
     if (dev->speed == USB_SPEED_HIGH) {
         other_dev = usb_device_get_usb_desc(dev)->full;
@@ -509,6 +642,11 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         other_dev = usb_device_get_usb_desc(dev)->high;
     }
 
+    flags = 0;
+    if (dev->device->bcdUSB >= 0x0300) {
+        flags |= USB_DESC_FLAG_SUPER;
+    }
+
     switch(type) {
     case USB_DT_DEVICE:
         ret = usb_desc_device(&desc->id, dev->device, buf, sizeof(buf));
@@ -516,7 +654,8 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         break;
     case USB_DT_CONFIG:
         if (index < dev->device->bNumConfigurations) {
-            ret = usb_desc_config(dev->device->confs + index, buf, sizeof(buf));
+            ret = usb_desc_config(dev->device->confs + index, flags,
+                                  buf, sizeof(buf));
         }
         trace_usb_desc_config(dev->addr, index, len, ret);
         break;
@@ -524,7 +663,6 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         ret = usb_desc_string(dev, index, buf, sizeof(buf));
         trace_usb_desc_string(dev->addr, index, len, ret);
         break;
-
     case USB_DT_DEVICE_QUALIFIER:
         if (other_dev != NULL) {
             ret = usb_desc_device_qualifier(other_dev, buf, sizeof(buf));
@@ -533,11 +671,16 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         break;
     case USB_DT_OTHER_SPEED_CONFIG:
         if (other_dev != NULL && index < other_dev->bNumConfigurations) {
-            ret = usb_desc_config(other_dev->confs + index, buf, sizeof(buf));
+            ret = usb_desc_config(other_dev->confs + index, flags,
+                                  buf, sizeof(buf));
             buf[0x01] = USB_DT_OTHER_SPEED_CONFIG;
         }
         trace_usb_desc_other_speed_config(dev->addr, index, len, ret);
         break;
+    case USB_DT_BOS:
+        ret = usb_desc_bos(desc, buf, sizeof(buf));
+        trace_usb_desc_bos(dev->addr, len, ret);
+        break;
 
     case USB_DT_DEBUG:
         /* ignore silently */
@@ -554,6 +697,8 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
             ret = len;
         }
         memcpy(dest, buf, ret);
+        p->actual_length = ret;
+        ret = 0;
     }
     return ret;
 }
@@ -573,7 +718,7 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
         break;
 
     case DeviceRequest | USB_REQ_GET_DESCRIPTOR:
-        ret = usb_desc_get_descriptor(dev, value, data, length);
+        ret = usb_desc_get_descriptor(dev, p, value, data, length);
         break;
 
     case DeviceRequest | USB_REQ_GET_CONFIGURATION:
@@ -582,7 +727,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
          * the non zero value of bConfigurationValue.
          */
         data[0] = dev->config ? dev->config->bConfigurationValue : 0;
-        ret = 1;
+        p->actual_length = 1;
+        ret = 0;
         break;
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
         ret = usb_desc_set_config(dev, value);
@@ -607,7 +753,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
             data[0] |= 1 << USB_DEVICE_REMOTE_WAKEUP;
         }
         data[1] = 0x00;
-        ret = 2;
+        p->actual_length = 2;
+        ret = 0;
         break;
     }
     case DeviceOutRequest | USB_REQ_CLEAR_FEATURE:
@@ -630,7 +777,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
             break;
         }
         data[0] = dev->altsetting[index];
-        ret = 1;
+        p->actual_length = 1;
+        ret = 0;
         break;
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
         ret = usb_desc_set_interface(dev, index, value);
diff --git a/hw/usb/desc.h b/hw/usb/desc.h
index 7cf54429459..ddd3e7485c7 100644
--- a/hw/usb/desc.h
+++ b/hw/usb/desc.h
@@ -63,6 +63,37 @@ typedef struct USBDescriptor {
             uint8_t           bRefresh;        /* only audio ep */
             uint8_t           bSynchAddress;   /* only audio ep */
         } endpoint;
+        struct {
+            uint8_t           bMaxBurst;
+            uint8_t           bmAttributes;
+            uint8_t           wBytesPerInterval_lo;
+            uint8_t           wBytesPerInterval_hi;
+        } super_endpoint;
+        struct {
+            uint8_t           wTotalLength_lo;
+            uint8_t           wTotalLength_hi;
+            uint8_t           bNumDeviceCaps;
+        } bos;
+        struct {
+            uint8_t           bDevCapabilityType;
+            union {
+                struct {
+                    uint8_t   bmAttributes_1;
+                    uint8_t   bmAttributes_2;
+                    uint8_t   bmAttributes_3;
+                    uint8_t   bmAttributes_4;
+                } usb2_ext;
+                struct {
+                    uint8_t   bmAttributes;
+                    uint8_t   wSpeedsSupported_lo;
+                    uint8_t   wSpeedsSupported_hi;
+                    uint8_t   bFunctionalitySupport;
+                    uint8_t   bU1DevExitLat;
+                    uint8_t   wU2DevExitLat_lo;
+                    uint8_t   wU2DevExitLat_hi;
+                } super;
+            } u;
+        } cap;
     } u;
 } QEMU_PACKED USBDescriptor;
 
@@ -139,6 +170,11 @@ struct USBDescEndpoint {
 
     uint8_t                   is_audio; /* has bRefresh + bSynchAddress */
     uint8_t                   *extra;
+
+    /* superspeed endpoint companion */
+    uint8_t                   bMaxBurst;
+    uint8_t                   bmAttributes_super;
+    uint16_t                  wBytesPerInterval;
 };
 
 struct USBDescOther {
@@ -152,19 +188,25 @@ struct USBDesc {
     USBDescID                 id;
     const USBDescDevice       *full;
     const USBDescDevice       *high;
+    const USBDescDevice       *super;
     const char* const         *str;
 };
 
+#define USB_DESC_FLAG_SUPER (1 << 1)
+
 /* generate usb packages from structs */
 int usb_desc_device(const USBDescID *id, const USBDescDevice *dev,
                     uint8_t *dest, size_t len);
 int usb_desc_device_qualifier(const USBDescDevice *dev,
                               uint8_t *dest, size_t len);
-int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len);
-int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
-                         size_t len);
-int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len);
-int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len);
+int usb_desc_config(const USBDescConfig *conf, int flags,
+                    uint8_t *dest, size_t len);
+int usb_desc_iface_group(const USBDescIfaceAssoc *iad, int flags,
+                         uint8_t *dest, size_t len);
+int usb_desc_iface(const USBDescIface *iface, int flags,
+                   uint8_t *dest, size_t len);
+int usb_desc_endpoint(const USBDescEndpoint *ep, int flags,
+                      uint8_t *dest, size_t len);
 int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len);
 
 /* control message emulation helpers */
@@ -174,7 +216,8 @@ void usb_desc_set_string(USBDevice *dev, uint8_t index, const char *str);
 void usb_desc_create_serial(USBDevice *dev);
 const char *usb_desc_get_string(USBDevice *dev, uint8_t index);
 int usb_desc_string(USBDevice *dev, int index, uint8_t *dest, size_t len);
-int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len);
+int usb_desc_get_descriptor(USBDevice *dev, USBPacket *p,
+        int value, uint8_t *dest, size_t len);
 int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
         int request, int value, int index, int length, uint8_t *data);
 
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index 79b75fb628c..b8c79b85e93 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -217,7 +217,7 @@ static const USBDescIface desc_iface[] = {
 };
 
 static const USBDescDevice desc_device = {
-    .bcdUSB                        = 0x0200,
+    .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 64,
     .bNumConfigurations            = 1,
     .confs = (USBDescConfig[]) {
@@ -503,7 +503,7 @@ static int usb_audio_set_control(USBAudioState *s, uint8_t attrib,
     return ret;
 }
 
-static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_audio_handle_control(USBDevice *dev, USBPacket *p,
                                     int request, int value, int index,
                                     int length, uint8_t *data)
 {
@@ -518,7 +518,7 @@ static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch (request) {
@@ -534,6 +534,7 @@ static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
             }
             goto fail;
         }
+        p->actual_length = ret;
         break;
 
     case ClassInterfaceOutRequest | CR_SET_CUR:
@@ -557,10 +558,9 @@ fail:
                     "request 0x%04x value 0x%04x index 0x%04x length 0x%04x\n",
                     request, value, index, length);
         }
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_audio_set_interface(USBDevice *dev, int iface,
@@ -583,50 +583,35 @@ static void usb_audio_handle_reset(USBDevice *dev)
     usb_audio_set_output_altset(s, ALTSET_OFF);
 }
 
-static int usb_audio_handle_dataout(USBAudioState *s, USBPacket *p)
+static void usb_audio_handle_dataout(USBAudioState *s, USBPacket *p)
 {
-    int rc;
-
     if (s->out.altset == ALTSET_OFF) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
-    rc = streambuf_put(&s->out.buf, p);
-    if (rc < p->iov.size && s->debug > 1) {
+    streambuf_put(&s->out.buf, p);
+    if (p->actual_length < p->iov.size && s->debug > 1) {
         fprintf(stderr, "usb-audio: output overrun (%zd bytes)\n",
-                p->iov.size - rc);
+                p->iov.size - p->actual_length);
     }
-
-    return 0;
 }
 
-static int usb_audio_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_audio_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBAudioState *s = (USBAudioState *) dev;
-    int ret = 0;
 
-    switch (p->pid) {
-    case USB_TOKEN_OUT:
-        switch (p->ep->nr) {
-        case 1:
-            ret = usb_audio_handle_dataout(s, p);
-            break;
-        default:
-            goto fail;
-        }
-        break;
-
-    default:
-fail:
-        ret = USB_RET_STALL;
-        break;
+    if (p->pid == USB_TOKEN_OUT && p->ep->nr == 1) {
+        usb_audio_handle_dataout(s, p);
+        return;
     }
-    if (ret == USB_RET_STALL && s->debug) {
+
+    p->status = USB_RET_STALL;
+    if (s->debug) {
         fprintf(stderr, "usb-audio: failed data transaction: "
                         "pid 0x%x ep 0x%x len 0x%zx\n",
                         p->pid, p->ep->nr, p->iov.size);
     }
-    return ret;
 }
 
 static void usb_audio_handle_destroy(USBDevice *dev)
@@ -699,7 +684,7 @@ static void usb_audio_class_init(ObjectClass *klass, void *data)
     k->set_interface  = usb_audio_set_interface;
 }
 
-static TypeInfo usb_audio_info = {
+static const TypeInfo usb_audio_info = {
     .name          = "usb-audio",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBAudioState),
diff --git a/hw/usb/dev-bluetooth.c b/hw/usb/dev-bluetooth.c
index 55bc19184b8..adbf9d46970 100644
--- a/hw/usb/dev-bluetooth.c
+++ b/hw/usb/dev-bluetooth.c
@@ -21,12 +21,13 @@
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "hw/bt.h"
 
 struct USBBtState {
     USBDevice dev;
     struct HCIInfo *hci;
+    USBEndpoint *intr;
 
     int config;
 
@@ -285,13 +286,12 @@ static void usb_bt_fifo_enqueue(struct usb_hci_in_fifo_s *fifo,
     fifo->fifo[off].len = len;
 }
 
-static inline int usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
+static inline void usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
                 USBPacket *p)
 {
     int len;
 
-    if (likely(!fifo->len))
-        return USB_RET_STALL;
+    assert(fifo->len != 0);
 
     len = MIN(p->iov.size, fifo->fifo[fifo->start].len);
     usb_packet_copy(p, fifo->fifo[fifo->start].data, len);
@@ -310,8 +310,6 @@ static inline int usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
         fifo->dstart = 0;
         fifo->dsize = DFIFO_LEN_MASK + 1;
     }
-
-    return len;
 }
 
 static inline void usb_bt_fifo_out_enqueue(struct USBBtState *s,
@@ -363,7 +361,7 @@ static void usb_bt_handle_reset(USBDevice *dev)
     s->outsco.len = 0;
 }
 
-static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_bt_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     struct USBBtState *s = (struct USBBtState *) dev->opaque;
@@ -382,16 +380,15 @@ static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
             usb_bt_fifo_reset(&s->sco);
             break;
         }
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case InterfaceRequest | USB_REQ_GET_STATUS:
     case EndpointRequest | USB_REQ_GET_STATUS:
         data[0] = 0x00;
         data[1] = 0x00;
-        ret = 2;
+        p->actual_length = 2;
         break;
     case InterfaceOutRequest | USB_REQ_CLEAR_FEATURE:
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
@@ -407,16 +404,14 @@ static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
         break;
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_bt_handle_data(USBDevice *dev, USBPacket *p)
 {
     struct USBBtState *s = (struct USBBtState *) dev->opaque;
-    int ret = 0;
 
     if (!s->config)
         goto fail;
@@ -425,15 +420,27 @@ static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case USB_EVT_EP:
-            ret = usb_bt_fifo_dequeue(&s->evt, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_NAK;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->evt, p);
             break;
 
         case USB_ACL_EP:
-            ret = usb_bt_fifo_dequeue(&s->acl, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_STALL;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->acl, p);
             break;
 
         case USB_SCO_EP:
-            ret = usb_bt_fifo_dequeue(&s->sco, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_STALL;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->sco, p);
             break;
 
         default:
@@ -460,11 +467,9 @@ static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
 
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_bt_out_hci_packet_event(void *opaque,
@@ -472,6 +477,9 @@ static void usb_bt_out_hci_packet_event(void *opaque,
 {
     struct USBBtState *s = (struct USBBtState *) opaque;
 
+    if (s->evt.len == 0) {
+        usb_wakeup(s->intr);
+    }
     usb_bt_fifo_enqueue(&s->evt, data, len);
 }
 
@@ -494,8 +502,12 @@ static void usb_bt_handle_destroy(USBDevice *dev)
 
 static int usb_bt_initfn(USBDevice *dev)
 {
+    struct USBBtState *s = DO_UPCAST(struct USBBtState, dev, dev);
+
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, USB_EVT_EP);
+
     return 0;
 }
 
@@ -543,7 +555,7 @@ static void usb_bt_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_bt;
 }
 
-static TypeInfo bt_info = {
+static const TypeInfo bt_info = {
     .name          = "usb-bt-dongle",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(struct USBBtState),
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index b3dcd23109b..29b64819abd 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/hid.h"
 
 /* HID interface requests */
@@ -46,6 +46,7 @@ typedef struct USBHIDState {
     USBDevice dev;
     USBEndpoint *intr;
     HIDState hid;
+    uint32_t usb_version;
 } USBHIDState;
 
 enum {
@@ -131,6 +132,36 @@ static const USBDescIface desc_iface_tablet = {
     },
 };
 
+static const USBDescIface desc_iface_tablet2 = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 1,
+    .bInterfaceClass               = USB_CLASS_HID,
+    .bInterfaceProtocol            = 0x02,
+    .ndesc                         = 1,
+    .descs = (USBDescOther[]) {
+        {
+            /* HID descriptor */
+            .data = (uint8_t[]) {
+                0x09,          /*  u8  bLength */
+                USB_DT_HID,    /*  u8  bDescriptorType */
+                0x01, 0x00,    /*  u16 HID_class */
+                0x00,          /*  u8  country_code */
+                0x01,          /*  u8  num_descriptors */
+                USB_DT_REPORT, /*  u8  type: Report */
+                74, 0,         /*  u16 len */
+            },
+        },
+    },
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_INT,
+            .wMaxPacketSize        = 8,
+            .bInterval             = 4, /* 2 ^ (4-1) * 125 usecs = 1 ms */
+        },
+    },
+};
+
 static const USBDescIface desc_iface_keyboard = {
     .bInterfaceNumber              = 0,
     .bNumEndpoints                 = 1,
@@ -196,6 +227,23 @@ static const USBDescDevice desc_device_tablet = {
     },
 };
 
+static const USBDescDevice desc_device_tablet2 = {
+    .bcdUSB                        = 0x0200,
+    .bMaxPacketSize0               = 64,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_TABLET,
+            .bmAttributes          = 0xa0,
+            .bMaxPower             = 50,
+            .nif = 1,
+            .ifs = &desc_iface_tablet2,
+        },
+    },
+};
+
 static const USBDescDevice desc_device_keyboard = {
     .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 8,
@@ -239,6 +287,20 @@ static const USBDesc desc_tablet = {
     .str  = desc_strings,
 };
 
+static const USBDesc desc_tablet2 = {
+    .id = {
+        .idVendor          = 0x0627,
+        .idProduct         = 0x0001,
+        .bcdDevice         = 0,
+        .iManufacturer     = STR_MANUFACTURER,
+        .iProduct          = STR_PRODUCT_TABLET,
+        .iSerialNumber     = STR_SERIALNUMBER,
+    },
+    .full = &desc_device_tablet,
+    .high = &desc_device_tablet2,
+    .str  = desc_strings,
+};
+
 static const USBDesc desc_keyboard = {
     .id = {
         .idVendor          = 0x0627,
@@ -371,7 +433,7 @@ static void usb_hid_handle_reset(USBDevice *dev)
     hid_reset(&us->hid);
 }
 
-static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_hid_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
@@ -380,10 +442,9 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
         /* hid specific requests */
     case InterfaceRequest | USB_REQ_GET_DESCRIPTOR:
@@ -392,15 +453,15 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
             if (hs->kind == HID_MOUSE) {
 		memcpy(data, qemu_mouse_hid_report_descriptor,
 		       sizeof(qemu_mouse_hid_report_descriptor));
-		ret = sizeof(qemu_mouse_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_mouse_hid_report_descriptor);
             } else if (hs->kind == HID_TABLET) {
                 memcpy(data, qemu_tablet_hid_report_descriptor,
 		       sizeof(qemu_tablet_hid_report_descriptor));
-		ret = sizeof(qemu_tablet_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_tablet_hid_report_descriptor);
             } else if (hs->kind == HID_KEYBOARD) {
                 memcpy(data, qemu_keyboard_hid_report_descriptor,
                        sizeof(qemu_keyboard_hid_report_descriptor));
-                ret = sizeof(qemu_keyboard_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_keyboard_hid_report_descriptor);
             }
             break;
         default:
@@ -409,14 +470,14 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         break;
     case GET_REPORT:
         if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
-            ret = hid_pointer_poll(hs, data, length);
+            p->actual_length = hid_pointer_poll(hs, data, length);
         } else if (hs->kind == HID_KEYBOARD) {
-            ret = hid_keyboard_poll(hs, data, length);
+            p->actual_length = hid_keyboard_poll(hs, data, length);
         }
         break;
     case SET_REPORT:
         if (hs->kind == HID_KEYBOARD) {
-            ret = hid_keyboard_write(hs, data, length);
+            p->actual_length = hid_keyboard_write(hs, data, length);
         } else {
             goto fail;
         }
@@ -425,61 +486,57 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         if (hs->kind != HID_KEYBOARD && hs->kind != HID_MOUSE) {
             goto fail;
         }
-        ret = 1;
         data[0] = hs->protocol;
+        p->actual_length = 1;
         break;
     case SET_PROTOCOL:
         if (hs->kind != HID_KEYBOARD && hs->kind != HID_MOUSE) {
             goto fail;
         }
-        ret = 0;
         hs->protocol = value;
         break;
     case GET_IDLE:
-        ret = 1;
         data[0] = hs->idle;
+        p->actual_length = 1;
         break;
     case SET_IDLE:
         hs->idle = (uint8_t) (value >> 8);
-        hid_set_next_idle(hs, qemu_get_clock_ns(vm_clock));
+        hid_set_next_idle(hs);
         if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
             hid_pointer_activate(hs);
         }
-        ret = 0;
         break;
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_hid_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_hid_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
     HIDState *hs = &us->hid;
     uint8_t buf[p->iov.size];
-    int ret = 0;
+    int len = 0;
 
     switch (p->pid) {
     case USB_TOKEN_IN:
         if (p->ep->nr == 1) {
-            int64_t curtime = qemu_get_clock_ns(vm_clock);
             if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
                 hid_pointer_activate(hs);
             }
-            if (!hid_has_events(hs) &&
-                (!hs->idle || hs->next_idle_clock - curtime > 0)) {
-                return USB_RET_NAK;
+            if (!hid_has_events(hs)) {
+                p->status = USB_RET_NAK;
+                return;
             }
-            hid_set_next_idle(hs, curtime);
+            hid_set_next_idle(hs);
             if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
-                ret = hid_pointer_poll(hs, buf, p->iov.size);
+                len = hid_pointer_poll(hs, buf, p->iov.size);
             } else if (hs->kind == HID_KEYBOARD) {
-                ret = hid_keyboard_poll(hs, buf, p->iov.size);
+                len = hid_keyboard_poll(hs, buf, p->iov.size);
             }
-            usb_packet_copy(p, buf, ret);
+            usb_packet_copy(p, buf, len);
         } else {
             goto fail;
         }
@@ -487,10 +544,9 @@ static int usb_hid_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_hid_handle_destroy(USBDevice *dev)
@@ -512,6 +568,21 @@ static int usb_hid_initfn(USBDevice *dev, int kind)
 
 static int usb_tablet_initfn(USBDevice *dev)
 {
+    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+
+    switch (us->usb_version) {
+    case 1:
+        dev->usb_desc = &desc_tablet;
+        break;
+    case 2:
+        dev->usb_desc = &desc_tablet2;
+        break;
+    default:
+        error_report("Invalid usb version %d for usb-tabler (must be 1 or 2)",
+                     us->usb_version);
+        return -1;
+    }
+
     return usb_hid_initfn(dev, HID_TABLET);
 }
 
@@ -566,8 +637,14 @@ static void usb_hid_class_initfn(ObjectClass *klass, void *data)
     uc->handle_control = usb_hid_handle_control;
     uc->handle_data    = usb_hid_handle_data;
     uc->handle_destroy = usb_hid_handle_destroy;
+    uc->handle_attach  = usb_desc_attach;
 }
 
+static Property usb_tablet_properties[] = {
+        DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
+        DEFINE_PROP_END_OF_LIST(),
+};
+
 static void usb_tablet_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -576,11 +653,11 @@ static void usb_tablet_class_initfn(ObjectClass *klass, void *data)
     usb_hid_class_initfn(klass, data);
     uc->init           = usb_tablet_initfn;
     uc->product_desc   = "QEMU USB Tablet";
-    uc->usb_desc       = &desc_tablet;
     dc->vmsd = &vmstate_usb_ptr;
+    dc->props = usb_tablet_properties;
 }
 
-static TypeInfo usb_tablet_info = {
+static const TypeInfo usb_tablet_info = {
     .name          = "usb-tablet",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
@@ -599,7 +676,7 @@ static void usb_mouse_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_ptr;
 }
 
-static TypeInfo usb_mouse_info = {
+static const TypeInfo usb_mouse_info = {
     .name          = "usb-mouse",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
@@ -618,7 +695,7 @@ static void usb_keyboard_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_kbd;
 }
 
-static TypeInfo usb_keyboard_info = {
+static const TypeInfo usb_keyboard_info = {
     .name          = "usb-kbd",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
diff --git a/hw/usb/dev-hub.c b/hw/usb/dev-hub.c
index 8fd30df0e67..79f2f46d554 100644
--- a/hw/usb/dev-hub.c
+++ b/hw/usb/dev-hub.c
@@ -184,6 +184,7 @@ static void usb_hub_detach(USBPort *port1)
         port->wPortStatus &= ~PORT_STAT_ENABLE;
         port->wPortChange |= PORT_STAT_C_ENABLE;
     }
+    usb_wakeup(s->intr);
 }
 
 static void usb_hub_child_detach(USBPort *port1, USBDevice *child)
@@ -288,7 +289,7 @@ static const char *feature_name(int feature)
     return name[feature] ?: "?";
 }
 
-static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHubState *s = (USBHubState *)dev;
@@ -298,7 +299,7 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch(request) {
@@ -306,7 +307,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
         if (value == 0 && index != 0x81) { /* clear ep halt */
             goto fail;
         }
-        ret = 0;
         break;
         /* usb specific requests */
     case GetHubStatus:
@@ -314,7 +314,7 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
         data[1] = 0;
         data[2] = 0;
         data[3] = 0;
-        ret = 4;
+        p->actual_length = 4;
         break;
     case GetPortStatus:
         {
@@ -331,16 +331,14 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             data[1] = port->wPortStatus >> 8;
             data[2] = port->wPortChange;
             data[3] = port->wPortChange >> 8;
-            ret = 4;
+            p->actual_length = 4;
         }
         break;
     case SetHubFeature:
     case ClearHubFeature:
-        if (value == 0 || value == 1) {
-        } else {
+        if (value != 0 && value != 1) {
             goto fail;
         }
-        ret = 0;
         break;
     case SetPortFeature:
         {
@@ -366,6 +364,7 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                     port->wPortChange |= PORT_STAT_C_RESET;
                     /* set enable bit */
                     port->wPortStatus |= PORT_STAT_ENABLE;
+                    usb_wakeup(s->intr);
                 }
                 break;
             case PORT_POWER:
@@ -373,7 +372,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             default:
                 goto fail;
             }
-            ret = 0;
         }
         break;
     case ClearPortFeature:
@@ -413,7 +411,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             default:
                 goto fail;
             }
-            ret = 0;
         }
         break;
     case GetHubDescriptor:
@@ -437,22 +434,20 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                 var_hub_size++;
             }
 
-            ret = sizeof(qemu_hub_hub_descriptor) + var_hub_size;
-            data[0] = ret;
+            p->actual_length = sizeof(qemu_hub_hub_descriptor) + var_hub_size;
+            data[0] = p->actual_length;
             break;
         }
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_hub_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHubState *s = (USBHubState *)dev;
-    int ret;
 
     switch(p->pid) {
     case USB_TOKEN_IN:
@@ -465,7 +460,8 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
             if (p->iov.size == 1) { /* FreeBSD workaround */
                 n = 1;
             } else if (n > p->iov.size) {
-                return USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
+                return;
             }
             status = 0;
             for(i = 0; i < NUM_PORTS; i++) {
@@ -478,9 +474,8 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
                     buf[i] = status >> (8 * i);
                 }
                 usb_packet_copy(p, buf, n);
-                ret = n;
             } else {
-                ret = USB_RET_NAK; /* usb11 11.13.1 */
+                p->status = USB_RET_NAK; /* usb11 11.13.1 */
             }
         } else {
             goto fail;
@@ -489,10 +484,9 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_hub_handle_destroy(USBDevice *dev)
@@ -574,7 +568,7 @@ static void usb_hub_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_hub;
 }
 
-static TypeInfo hub_info = {
+static const TypeInfo hub_info = {
     .name          = "usb-hub",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHubState),
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index c84892c98d7..9dede4c68dc 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -26,10 +26,11 @@
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "net.h"
-#include "qemu-queue.h"
-#include "sysemu.h"
-#include "iov.h"
+#include "net/net.h"
+#include "qemu/queue.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
 
 /*#define TRAFFIC_DEBUG*/
 /* Thanks to NetChip Technologies for donating this product ID.
@@ -639,6 +640,8 @@ typedef struct USBNetState {
     unsigned int in_ptr, in_len;
     uint8_t in_buf[2048];
 
+    USBEndpoint *intr;
+
     char usbstring_mac[13];
     NICState *nic;
     NICConf conf;
@@ -851,6 +854,10 @@ static void *rndis_queue_response(USBNetState *s, unsigned int length)
     struct rndis_response *r =
             g_malloc0(sizeof(struct rndis_response) + length);
 
+    if (QTAILQ_EMPTY(&s->rndis_resp)) {
+        usb_wakeup(s->intr);
+    }
+
     QTAILQ_INSERT_TAIL(&s->rndis_resp, r, entries);
     r->length = length;
 
@@ -1001,6 +1008,13 @@ static int rndis_keepalive_response(USBNetState *s,
     return 0;
 }
 
+/* Prepare to receive the next packet */
+static void usb_net_reset_in_buf(USBNetState *s)
+{
+    s->in_ptr = s->in_len = 0;
+    qemu_flush_queued_packets(&s->nic->nc);
+}
+
 static int rndis_parse(USBNetState *s, uint8_t *data, int length)
 {
     uint32_t msg_type;
@@ -1025,7 +1039,8 @@ static int rndis_parse(USBNetState *s, uint8_t *data, int length)
 
     case RNDIS_RESET_MSG:
         rndis_clear_responsequeue(s);
-        s->out_ptr = s->in_ptr = s->in_len = 0;
+        s->out_ptr = 0;
+        usb_net_reset_in_buf(s);
         return rndis_reset_response(s, (rndis_reset_msg_type *) data);
 
     case RNDIS_KEEPALIVE_MSG:
@@ -1040,7 +1055,7 @@ static void usb_net_handle_reset(USBDevice *dev)
 {
 }
 
-static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_net_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBNetState *s = (USBNetState *) dev;
@@ -1048,10 +1063,9 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch(request) {
     case ClassInterfaceOutRequest | USB_CDC_SEND_ENCAPSULATED_COMMAND:
         if (!is_rndis(s) || value || index != 0) {
@@ -1070,22 +1084,25 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
         }
 #endif
         ret = rndis_parse(s, data, length);
+        if (ret < 0) {
+            p->status = ret;
+        }
         break;
 
     case ClassInterfaceRequest | USB_CDC_GET_ENCAPSULATED_RESPONSE:
         if (!is_rndis(s) || value || index != 0) {
             goto fail;
         }
-        ret = rndis_get_response(s, data);
-        if (!ret) {
+        p->actual_length = rndis_get_response(s, data);
+        if (p->actual_length == 0) {
             data[0] = 0;
-            ret = 1;
+            p->actual_length = 1;
         }
 #ifdef TRAFFIC_DEBUG
         {
             unsigned int i;
             fprintf(stderr, "GET_ENCAPSULATED_RESPONSE:");
-            for (i = 0; i < ret; i++) {
+            for (i = 0; i < p->actual_length; i++) {
                 if (!(i & 15))
                     fprintf(stderr, "\n%04x:", i);
                 fprintf(stderr, " %02x", data[i]);
@@ -1100,72 +1117,67 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
         fprintf(stderr, "usbnet: failed control transaction: "
                         "request 0x%x value 0x%x index 0x%x length 0x%x\n",
                         request, value, index, length);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_net_handle_statusin(USBNetState *s, USBPacket *p)
+static void usb_net_handle_statusin(USBNetState *s, USBPacket *p)
 {
     le32 buf[2];
-    int ret = 8;
 
     if (p->iov.size < 8) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     buf[0] = cpu_to_le32(1);
     buf[1] = cpu_to_le32(0);
     usb_packet_copy(p, buf, 8);
-    if (!s->rndis_resp.tqh_first)
-        ret = USB_RET_NAK;
+    if (!s->rndis_resp.tqh_first) {
+        p->status = USB_RET_NAK;
+    }
 
 #ifdef TRAFFIC_DEBUG
     fprintf(stderr, "usbnet: interrupt poll len %zu return %d",
-            p->iov.size, ret);
-    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", ret);
+            p->iov.size, p->status);
+    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", p->status);
 #endif
-
-    return ret;
 }
 
-static int usb_net_handle_datain(USBNetState *s, USBPacket *p)
+static void usb_net_handle_datain(USBNetState *s, USBPacket *p)
 {
-    int ret = USB_RET_NAK;
+    int len;
 
     if (s->in_ptr > s->in_len) {
-        s->in_ptr = s->in_len = 0;
-        ret = USB_RET_NAK;
-        return ret;
+        usb_net_reset_in_buf(s);
+        p->status = USB_RET_NAK;
+        return;
     }
     if (!s->in_len) {
-        ret = USB_RET_NAK;
-        return ret;
+        p->status = USB_RET_NAK;
+        return;
     }
-    ret = s->in_len - s->in_ptr;
-    if (ret > p->iov.size) {
-        ret = p->iov.size;
+    len = s->in_len - s->in_ptr;
+    if (len > p->iov.size) {
+        len = p->iov.size;
     }
-    usb_packet_copy(p, &s->in_buf[s->in_ptr], ret);
-    s->in_ptr += ret;
+    usb_packet_copy(p, &s->in_buf[s->in_ptr], len);
+    s->in_ptr += len;
     if (s->in_ptr >= s->in_len &&
-                    (is_rndis(s) || (s->in_len & (64 - 1)) || !ret)) {
+                    (is_rndis(s) || (s->in_len & (64 - 1)) || !len)) {
         /* no short packet necessary */
-        s->in_ptr = s->in_len = 0;
+        usb_net_reset_in_buf(s);
     }
 
 #ifdef TRAFFIC_DEBUG
-    fprintf(stderr, "usbnet: data in len %zu return %d", p->iov.size, ret);
-    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", ret);
+    fprintf(stderr, "usbnet: data in len %zu return %d", p->iov.size, len);
+    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", len);
 #endif
-
-    return ret;
 }
 
-static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
+static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
 {
-    int ret = p->iov.size;
     int sz = sizeof(s->out_buf) - s->out_ptr;
     struct rndis_packet_msg_type *msg =
             (struct rndis_packet_msg_type *) s->out_buf;
@@ -1176,21 +1188,23 @@ static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", p->iov.size);
 #endif
 
-    if (sz > ret)
-        sz = ret;
+    if (sz > p->iov.size) {
+        sz = p->iov.size;
+    }
     usb_packet_copy(p, &s->out_buf[s->out_ptr], sz);
     s->out_ptr += sz;
 
     if (!is_rndis(s)) {
-        if (ret < 64) {
+        if (p->iov.size < 64) {
             qemu_send_packet(&s->nic->nc, s->out_buf, s->out_ptr);
             s->out_ptr = 0;
         }
-        return ret;
+        return;
     }
     len = le32_to_cpu(msg->MessageLength);
-    if (s->out_ptr < 8 || s->out_ptr < len)
-        return ret;
+    if (s->out_ptr < 8 || s->out_ptr < len) {
+        return;
+    }
     if (le32_to_cpu(msg->MessageType) == RNDIS_PACKET_MSG) {
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
@@ -1199,24 +1213,21 @@ static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
-
-    return ret;
 }
 
-static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_net_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBNetState *s = (USBNetState *) dev;
-    int ret = 0;
 
     switch(p->pid) {
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case 1:
-            ret = usb_net_handle_statusin(s, p);
+            usb_net_handle_statusin(s, p);
             break;
 
         case 2:
-            ret = usb_net_handle_datain(s, p);
+            usb_net_handle_datain(s, p);
             break;
 
         default:
@@ -1227,7 +1238,7 @@ static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
         switch (p->ep->nr) {
         case 2:
-            ret = usb_net_handle_dataout(s, p);
+            usb_net_handle_dataout(s, p);
             break;
 
         default:
@@ -1237,33 +1248,46 @@ static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
 
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    if (ret == USB_RET_STALL)
+
+    if (p->status == USB_RET_STALL) {
         fprintf(stderr, "usbnet: failed data transaction: "
                         "pid 0x%x ep 0x%x len 0x%zx\n",
                         p->pid, p->ep->nr, p->iov.size);
-    return ret;
+    }
 }
 
 static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     USBNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
-    struct rndis_packet_msg_type *msg;
+    uint8_t *in_buf = s->in_buf;
+    size_t total_size = size;
 
     if (is_rndis(s)) {
-        msg = (struct rndis_packet_msg_type *) s->in_buf;
         if (s->rndis_state != RNDIS_DATA_INITIALIZED) {
             return -1;
         }
-        if (size + sizeof(struct rndis_packet_msg_type) > sizeof(s->in_buf))
-            return -1;
+        total_size += sizeof(struct rndis_packet_msg_type);
+    }
+    if (total_size > sizeof(s->in_buf)) {
+        return -1;
+    }
+
+    /* Only accept packet if input buffer is empty */
+    if (s->in_len > 0) {
+        return 0;
+    }
 
+    if (is_rndis(s)) {
+        struct rndis_packet_msg_type *msg;
+
+        msg = (struct rndis_packet_msg_type *)in_buf;
         memset(msg, 0, sizeof(struct rndis_packet_msg_type));
         msg->MessageType = cpu_to_le32(RNDIS_PACKET_MSG);
-        msg->MessageLength = cpu_to_le32(size + sizeof(struct rndis_packet_msg_type));
-        msg->DataOffset = cpu_to_le32(sizeof(struct rndis_packet_msg_type) - 8);
+        msg->MessageLength = cpu_to_le32(size + sizeof(*msg));
+        msg->DataOffset = cpu_to_le32(sizeof(*msg) - 8);
         msg->DataLength = cpu_to_le32(size);
         /* msg->OOBDataOffset;
          * msg->OOBDataLength;
@@ -1273,14 +1297,11 @@ static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t siz
          * msg->VcHandle;
          * msg->Reserved;
          */
-        memcpy(msg + 1, buf, size);
-        s->in_len = size + sizeof(struct rndis_packet_msg_type);
-    } else {
-        if (size > sizeof(s->in_buf))
-            return -1;
-        memcpy(s->in_buf, buf, size);
-        s->in_len = size;
+        in_buf += sizeof(*msg);
     }
+
+    memcpy(in_buf, buf, size);
+    s->in_len = total_size;
     s->in_ptr = 0;
     return size;
 }
@@ -1335,6 +1356,7 @@ static int usb_net_initfn(USBDevice *dev)
     s->media_state = 0;	/* NDIS_MEDIA_STATE_CONNECTED */;
     s->filter = 0;
     s->vendorid = 0x1234;
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
 
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_usbnet_info, &s->conf,
@@ -1411,7 +1433,7 @@ static void usb_net_class_initfn(ObjectClass *klass, void *data)
     dc->props = net_properties;
 }
 
-static TypeInfo net_info = {
+static const TypeInfo net_info = {
     .name          = "usb-net",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBNetState),
diff --git a/hw/usb/dev-serial.c b/hw/usb/dev-serial.c
index 8aa655286ba..47ac8c9b69d 100644
--- a/hw/usb/dev-serial.c
+++ b/hw/usb/dev-serial.c
@@ -9,10 +9,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 //#define DEBUG_Serial
 
@@ -113,7 +113,7 @@ enum {
 static const USBDescStrings desc_strings = {
     [STR_MANUFACTURER]    = "QEMU",
     [STR_PRODUCT_SERIAL]  = "QEMU USB SERIAL",
-    [STR_PRODUCT_BRAILLE] = "QEMU USB BRAILLE",
+    [STR_PRODUCT_BRAILLE] = "QEMU USB BAUM BRAILLE",
     [STR_SERIALNUMBER]    = "1",
 };
 
@@ -219,7 +219,7 @@ static uint8_t usb_get_modem_lines(USBSerialState *s)
     return ret;
 }
 
-static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_serial_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBSerialState *s = (USBSerialState *)dev;
@@ -228,13 +228,11 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
     DPRINTF("got control %x, value %x\n",request, value);
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
-        ret = 0;
         break;
 
         /* Class specific requests.  */
@@ -323,7 +321,7 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
     case DeviceInVendor | FTDI_GET_MDM_ST:
         data[0] = usb_get_modem_lines(s) | 1;
         data[1] = 0;
-        ret = 2;
+        p->actual_length = 2;
         break;
     case DeviceOutVendor | FTDI_SET_EVENT_CHR:
         /* TODO: handle it */
@@ -338,25 +336,23 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
         break;
     case DeviceInVendor | FTDI_GET_LATENCY:
         data[0] = s->latency;
-        ret = 1;
+        p->actual_length = 1;
         break;
     default:
     fail:
         DPRINTF("got unsupported/bogus control %x, value %x\n", request, value);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_serial_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBSerialState *s = (USBSerialState *)dev;
-    int i, ret = 0;
     uint8_t devep = p->ep->nr;
     struct iovec *iov;
     uint8_t header[2];
-    int first_len, len;
+    int i, first_len, len;
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
@@ -366,6 +362,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             iov = p->iov.iov + i;
             qemu_chr_fe_write(s->cs, iov->iov_base, iov->iov_len);
         }
+        p->actual_length = p->iov.size;
         break;
 
     case USB_TOKEN_IN:
@@ -374,7 +371,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
         first_len = RECV_BUF - s->recv_ptr;
         len = p->iov.size;
         if (len <= 2) {
-            ret = USB_RET_NAK;
+            p->status = USB_RET_NAK;
             break;
         }
         header[0] = usb_get_modem_lines(s) | 1;
@@ -384,7 +381,6 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             s->event_trigger &= ~FTDI_BI;
             header[1] = FTDI_BI;
             usb_packet_copy(p, header, 2);
-            ret = 2;
             break;
         } else {
             header[1] = 0;
@@ -393,7 +389,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
         if (len > s->recv_used)
             len = s->recv_used;
         if (!len) {
-            ret = USB_RET_NAK;
+            p->status = USB_RET_NAK;
             break;
         }
         if (first_len > len)
@@ -404,29 +400,30 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             usb_packet_copy(p, s->recv_buf, len - first_len);
         s->recv_used -= len;
         s->recv_ptr = (s->recv_ptr + len) % RECV_BUF;
-        ret = len + 2;
         break;
 
     default:
         DPRINTF("Bad token\n");
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_serial_handle_destroy(USBDevice *dev)
 {
     USBSerialState *s = (USBSerialState *)dev;
 
-    qemu_chr_delete(s->cs);
+    qemu_chr_add_handlers(s->cs, NULL, NULL, NULL, NULL);
 }
 
 static int usb_serial_can_read(void *opaque)
 {
     USBSerialState *s = opaque;
+
+    if (!s->dev.attached) {
+        return 0;
+    }
     return RECV_BUF - s->recv_used;
 }
 
@@ -469,8 +466,14 @@ static void usb_serial_event(void *opaque, int event)
         case CHR_EVENT_FOCUS:
             break;
         case CHR_EVENT_OPENED:
-            usb_serial_reset(s);
-            /* TODO: Reset USB port */
+            if (!s->dev.attached) {
+                usb_device_attach(&s->dev);
+            }
+            break;
+        case CHR_EVENT_CLOSED:
+            if (s->dev.attached) {
+                usb_device_detach(&s->dev);
+            }
             break;
     }
 }
@@ -481,6 +484,7 @@ static int usb_serial_initfn(USBDevice *dev)
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    dev->auto_attach = 0;
 
     if (!s->cs) {
         error_report("Property chardev is required");
@@ -490,6 +494,10 @@ static int usb_serial_initfn(USBDevice *dev)
     qemu_chr_add_handlers(s->cs, usb_serial_can_read, usb_serial_read,
                           usb_serial_event, s);
     usb_serial_handle_reset(dev);
+
+    if (s->cs->opened && !dev->attached) {
+        usb_device_attach(dev);
+    }
     return 0;
 }
 
@@ -592,7 +600,7 @@ static void usb_serial_class_initfn(ObjectClass *klass, void *data)
     dc->props = serial_properties;
 }
 
-static TypeInfo serial_info = {
+static const TypeInfo serial_info = {
     .name          = "usb-serial",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBSerialState),
@@ -620,7 +628,7 @@ static void usb_braille_class_initfn(ObjectClass *klass, void *data)
     dc->props = braille_properties;
 }
 
-static TypeInfo braille_info = {
+static const TypeInfo braille_info = {
     .name          = "usb-braille",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBSerialState),
diff --git a/hw/usb/dev-smartcard-reader.c b/hw/usb/dev-smartcard-reader.c
index 1ea079176a2..979a473b378 100644
--- a/hw/usb/dev-smartcard-reader.c
+++ b/hw/usb/dev-smartcard-reader.c
@@ -35,10 +35,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #include "hw/ccid.h"
 
@@ -635,39 +635,38 @@ static void ccid_handle_reset(USBDevice *dev)
     ccid_reset(s);
 }
 
-static int ccid_handle_control(USBDevice *dev, USBPacket *p, int request,
+static void ccid_handle_control(USBDevice *dev, USBPacket *p, int request,
                                int value, int index, int length, uint8_t *data)
 {
     USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
-    int ret = 0;
+    int ret;
 
     DPRINTF(s, 1, "got control %x, value %x\n", request, value);
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch (request) {
         /* Class specific requests.  */
     case InterfaceOutClass | CCID_CONTROL_ABORT:
         DPRINTF(s, 1, "ccid_control abort UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     case InterfaceInClass | CCID_CONTROL_GET_CLOCK_FREQUENCIES:
         DPRINTF(s, 1, "ccid_control get clock frequencies UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     case InterfaceInClass | CCID_CONTROL_GET_DATA_RATES:
         DPRINTF(s, 1, "ccid_control get data rates UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     default:
         DPRINTF(s, 1, "got unsupported/bogus control %x, value %x\n",
                 request, value);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static bool ccid_card_inserted(USBCCIDState *s)
@@ -870,18 +869,13 @@ static void ccid_on_apdu_from_guest(USBCCIDState *s, CCID_XferBlock *recv)
     }
 }
 
-/*
- * Handle a single USB_TOKEN_OUT, return value returned to guest.
- * Return value:
- *  0             - all ok
- *  USB_RET_STALL - failed to handle packet
- */
-static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
+static void ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
 {
     CCID_Header *ccid_header;
 
     if (p->iov.size + s->bulk_out_pos > BULK_OUT_DATA_SIZE) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
     ccid_header = (CCID_Header *)s->bulk_out_data;
     usb_packet_copy(p, s->bulk_out_data + s->bulk_out_pos, p->iov.size);
@@ -890,7 +884,7 @@ static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
         DPRINTF(s, D_VERBOSE,
             "usb-ccid: bulk_in: expecting more packets (%zd/%d)\n",
             p->iov.size, ccid_header->dwLength);
-        return 0;
+        return;
     }
     if (s->bulk_out_pos < 10) {
         DPRINTF(s, 1,
@@ -949,60 +943,52 @@ static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
         }
     }
     s->bulk_out_pos = 0;
-    return 0;
 }
 
-static int ccid_bulk_in_copy_to_guest(USBCCIDState *s, USBPacket *p)
+static void ccid_bulk_in_copy_to_guest(USBCCIDState *s, USBPacket *p)
 {
-    int ret = 0;
+    int len = 0;
 
-    assert(p->iov.size > 0);
     ccid_bulk_in_get(s);
     if (s->current_bulk_in != NULL) {
-        ret = MIN(s->current_bulk_in->len - s->current_bulk_in->pos,
+        len = MIN(s->current_bulk_in->len - s->current_bulk_in->pos,
                   p->iov.size);
         usb_packet_copy(p, s->current_bulk_in->data +
-                        s->current_bulk_in->pos, ret);
-        s->current_bulk_in->pos += ret;
+                        s->current_bulk_in->pos, len);
+        s->current_bulk_in->pos += len;
         if (s->current_bulk_in->pos == s->current_bulk_in->len) {
             ccid_bulk_in_release(s);
         }
     } else {
         /* return when device has no data - usb 2.0 spec Table 8-4 */
-        ret = USB_RET_NAK;
+        p->status = USB_RET_NAK;
     }
-    if (ret > 0) {
+    if (len) {
         DPRINTF(s, D_MORE_INFO,
                 "%s: %zd/%d req/act to guest (BULK_IN)\n",
-                __func__, p->iov.size, ret);
+                __func__, p->iov.size, len);
     }
-    if (ret != USB_RET_NAK && ret < p->iov.size) {
+    if (len < p->iov.size) {
         DPRINTF(s, 1,
                 "%s: returning short (EREMOTEIO) %d < %zd\n",
-                __func__, ret, p->iov.size);
+                __func__, len, p->iov.size);
     }
-    return ret;
 }
 
-static int ccid_handle_data(USBDevice *dev, USBPacket *p)
+static void ccid_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
-    int ret = 0;
     uint8_t buf[2];
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
-        ret = ccid_handle_bulk_out(s, p);
+        ccid_handle_bulk_out(s, p);
         break;
 
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case CCID_BULK_IN_EP:
-            if (!p->iov.size) {
-                ret = USB_RET_NAK;
-            } else {
-                ret = ccid_bulk_in_copy_to_guest(s, p);
-            }
+            ccid_bulk_in_copy_to_guest(s, p);
             break;
         case CCID_INT_IN_EP:
             if (s->notify_slot_change) {
@@ -1010,28 +996,27 @@ static int ccid_handle_data(USBDevice *dev, USBPacket *p)
                 buf[0] = CCID_MESSAGE_TYPE_RDR_to_PC_NotifySlotChange;
                 buf[1] = s->bmSlotICCState;
                 usb_packet_copy(p, buf, 2);
-                ret = 2;
                 s->notify_slot_change = false;
                 s->bmSlotICCState &= ~SLOT_0_CHANGED_MASK;
                 DPRINTF(s, D_INFO,
                         "handle_data: int_in: notify_slot_change %X, "
                         "requested len %zd\n",
                         s->bmSlotICCState, p->iov.size);
+            } else {
+                p->status = USB_RET_NAK;
             }
             break;
         default:
             DPRINTF(s, 1, "Bad endpoint\n");
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         break;
     default:
         DPRINTF(s, 1, "Bad token\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void ccid_handle_destroy(USBDevice *dev)
@@ -1337,7 +1322,7 @@ static void ccid_class_initfn(ObjectClass *klass, void *data)
     dc->props = ccid_properties;
 }
 
-static TypeInfo ccid_info = {
+static const TypeInfo ccid_info = {
     .name          = CCID_DEV_NAME,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBCCIDState),
@@ -1353,7 +1338,7 @@ static void ccid_card_class_init(ObjectClass *klass, void *data)
     k->props = ccid_props;
 }
 
-static TypeInfo ccid_card_type_info = {
+static const TypeInfo ccid_card_type_info = {
     .name = TYPE_CCID_CARD,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(CCIDCardState),
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index ff48d91049a..b89d00f7cf1 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -8,15 +8,15 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 #include "hw/scsi.h"
-#include "console.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "blockdev.h"
+#include "ui/console.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/blockdev.h"
 
 //#define DEBUG_MSD
 
@@ -54,12 +54,12 @@ typedef struct {
     struct usb_msd_csw csw;
     SCSIRequest *req;
     SCSIBus bus;
+    /* For async completion.  */
+    USBPacket *packet;
+    /* usb-storage only */
     BlockConf conf;
     char *serial;
-    SCSIDevice *scsi_dev;
     uint32_t removable;
-    /* For async completion.  */
-    USBPacket *packet;
 } MSDState;
 
 struct usb_msd_cbw {
@@ -78,6 +78,7 @@ enum {
     STR_SERIALNUMBER,
     STR_CONFIG_FULL,
     STR_CONFIG_HIGH,
+    STR_CONFIG_SUPER,
 };
 
 static const USBDescStrings desc_strings = {
@@ -86,6 +87,7 @@ static const USBDescStrings desc_strings = {
     [STR_SERIALNUMBER] = "1",
     [STR_CONFIG_FULL]  = "Full speed config (usb 1.1)",
     [STR_CONFIG_HIGH]  = "High speed config (usb 2.0)",
+    [STR_CONFIG_SUPER] = "Super speed config (usb 3.0)",
 };
 
 static const USBDescIface desc_iface_full = {
@@ -158,6 +160,43 @@ static const USBDescDevice desc_device_high = {
     },
 };
 
+static const USBDescIface desc_iface_super = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 2,
+    .bInterfaceClass               = USB_CLASS_MASS_STORAGE,
+    .bInterfaceSubClass            = 0x06, /* SCSI */
+    .bInterfaceProtocol            = 0x50, /* Bulk */
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_BULK,
+            .wMaxPacketSize        = 1024,
+            .bMaxBurst             = 15,
+        },{
+            .bEndpointAddress      = USB_DIR_OUT | 0x02,
+            .bmAttributes          = USB_ENDPOINT_XFER_BULK,
+            .wMaxPacketSize        = 1024,
+            .bMaxBurst             = 15,
+        },
+    }
+};
+
+static const USBDescDevice desc_device_super = {
+    .bcdUSB                        = 0x0300,
+    .bMaxPacketSize0               = 9,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_SUPER,
+            .bmAttributes          = 0xc0,
+            .nif = 1,
+            .ifs = &desc_iface_super,
+        },
+    },
+};
+
 static const USBDesc desc = {
     .id = {
         .idVendor          = 0x46f4, /* CRC16() of "QEMU" */
@@ -167,15 +206,16 @@ static const USBDesc desc = {
         .iProduct          = STR_PRODUCT,
         .iSerialNumber     = STR_SERIALNUMBER,
     },
-    .full = &desc_device_full,
-    .high = &desc_device_high,
-    .str  = desc_strings,
+    .full  = &desc_device_full,
+    .high  = &desc_device_high,
+    .super = &desc_device_super,
+    .str   = desc_strings,
 };
 
 static void usb_msd_copy_data(MSDState *s, USBPacket *p)
 {
     uint32_t len;
-    len = p->iov.size - p->result;
+    len = p->iov.size - p->actual_length;
     if (len > s->scsi_len)
         len = s->scsi_len;
     usb_packet_copy(p, scsi_req_get_buf(s->req) + s->scsi_off, len);
@@ -223,7 +263,8 @@ static void usb_msd_transfer_data(SCSIRequest *req, uint32_t len)
     if (p) {
         usb_msd_copy_data(s, p);
         p = s->packet;
-        if (p && p->result == p->iov.size) {
+        if (p && p->actual_length == p->iov.size) {
+            p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
             usb_msd_packet_complete(s);
         }
     }
@@ -252,7 +293,7 @@ static void usb_msd_command_complete(SCSIRequest *req, uint32_t status, size_t r
             s->mode = USB_MSDM_CBW;
         } else {
             if (s->data_len) {
-                int len = (p->iov.size - p->result);
+                int len = (p->iov.size - p->actual_length);
                 usb_packet_skip(p, len);
                 s->data_len -= len;
             }
@@ -260,6 +301,7 @@ static void usb_msd_command_complete(SCSIRequest *req, uint32_t status, size_t r
                 s->mode = USB_MSDM_CSW;
             }
         }
+        p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
         usb_msd_packet_complete(s);
     } else if (s->data_len == 0) {
         s->mode = USB_MSDM_CSW;
@@ -290,44 +332,53 @@ static void usb_msd_handle_reset(USBDevice *dev)
     assert(s->req == NULL);
 
     if (s->packet) {
-        s->packet->result = USB_RET_STALL;
+        s->packet->status = USB_RET_STALL;
         usb_msd_packet_complete(s);
     }
 
     s->mode = USB_MSDM_CBW;
 }
 
-static int usb_msd_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_msd_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     MSDState *s = (MSDState *)dev;
-    int ret;
+    SCSIDevice *scsi_dev;
+    int ret, maxlun;
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
-        ret = 0;
         break;
         /* Class specific requests.  */
     case ClassInterfaceOutRequest | MassStorageReset:
         /* Reset state ready for the next CBW.  */
         s->mode = USB_MSDM_CBW;
-        ret = 0;
         break;
     case ClassInterfaceRequest | GetMaxLun:
-        data[0] = 0;
-        ret = 1;
+        maxlun = 0;
+        for (;;) {
+            scsi_dev = scsi_device_find(&s->bus, 0, 0, maxlun+1);
+            if (scsi_dev == NULL) {
+                break;
+            }
+            if (scsi_dev->lun != maxlun+1) {
+                break;
+            }
+            maxlun++;
+        }
+        DPRINTF("MaxLun %d\n", maxlun);
+        data[0] = maxlun;
+        p->actual_length = 1;
         break;
     default:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_msd_cancel_io(USBDevice *dev, USBPacket *p)
@@ -342,13 +393,13 @@ static void usb_msd_cancel_io(USBDevice *dev, USBPacket *p)
     }
 }
 
-static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
 {
     MSDState *s = (MSDState *)dev;
     uint32_t tag;
-    int ret = 0;
     struct usb_msd_cbw cbw;
     uint8_t devep = p->ep->nr;
+    SCSIDevice *scsi_dev;
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
@@ -368,7 +419,8 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 goto fail;
             }
             DPRINTF("Command on LUN %d\n", cbw.lun);
-            if (cbw.lun != 0) {
+            scsi_dev = scsi_device_find(&s->bus, 0, 0, cbw.lun);
+            if (scsi_dev == NULL) {
                 fprintf(stderr, "usb-msd: Bad LUN %d\n", cbw.lun);
                 goto fail;
             }
@@ -385,7 +437,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     tag, cbw.flags, cbw.cmd_len, s->data_len);
             assert(le32_to_cpu(s->csw.residue) == 0);
             s->scsi_len = 0;
-            s->req = scsi_req_new(s->scsi_dev, tag, 0, cbw.cmd, NULL);
+            s->req = scsi_req_new(scsi_dev, tag, cbw.lun, cbw.cmd, NULL);
 #ifdef DEBUG_MSD
             scsi_req_print(s->req);
 #endif
@@ -393,7 +445,6 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
             if (s->req && s->req->cmd.xfer != SCSI_XFER_NONE) {
                 scsi_req_continue(s->req);
             }
-            ret = p->result;
             break;
 
         case USB_MSDM_DATAOUT:
@@ -406,7 +457,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 usb_msd_copy_data(s, p);
             }
             if (le32_to_cpu(s->csw.residue)) {
-                int len = p->iov.size - p->result;
+                int len = p->iov.size - p->actual_length;
                 if (len) {
                     usb_packet_skip(p, len);
                     s->data_len -= len;
@@ -415,12 +466,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     }
                 }
             }
-            if (p->result < p->iov.size) {
+            if (p->actual_length < p->iov.size) {
                 DPRINTF("Deferring packet %p [wait data-out]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
-            } else {
-                ret = p->result;
+                p->status = USB_RET_ASYNC;
             }
             break;
 
@@ -441,7 +490,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
             }
             /* Waiting for SCSI write to complete.  */
             s->packet = p;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
             break;
 
         case USB_MSDM_CSW:
@@ -453,11 +502,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 /* still in flight */
                 DPRINTF("Deferring packet %p [wait status]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
+                p->status = USB_RET_ASYNC;
             } else {
                 usb_msd_send_status(s, p);
                 s->mode = USB_MSDM_CBW;
-                ret = 13;
             }
             break;
 
@@ -468,7 +516,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 usb_msd_copy_data(s, p);
             }
             if (le32_to_cpu(s->csw.residue)) {
-                int len = p->iov.size - p->result;
+                int len = p->iov.size - p->actual_length;
                 if (len) {
                     usb_packet_skip(p, len);
                     s->data_len -= len;
@@ -477,12 +525,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     }
                 }
             }
-            if (p->result < p->iov.size) {
+            if (p->actual_length < p->iov.size) {
                 DPRINTF("Deferring packet %p [wait data-in]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
-            } else {
-                ret = p->result;
+                p->status = USB_RET_ASYNC;
             }
             break;
 
@@ -495,11 +541,9 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
     default:
         DPRINTF("Bad token\n");
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_msd_password_cb(void *opaque, int err)
@@ -524,7 +568,7 @@ static void *usb_msd_load_request(QEMUFile *f, SCSIRequest *req)
     return NULL;
 }
 
-static const struct SCSIBusInfo usb_msd_scsi_info = {
+static const struct SCSIBusInfo usb_msd_scsi_info_storage = {
     .tcq = false,
     .max_target = 0,
     .max_lun = 0,
@@ -535,10 +579,22 @@ static const struct SCSIBusInfo usb_msd_scsi_info = {
     .load_request = usb_msd_load_request,
 };
 
-static int usb_msd_initfn(USBDevice *dev)
+static const struct SCSIBusInfo usb_msd_scsi_info_bot = {
+    .tcq = false,
+    .max_target = 0,
+    .max_lun = 15,
+
+    .transfer_data = usb_msd_transfer_data,
+    .complete = usb_msd_command_complete,
+    .cancel = usb_msd_request_cancelled,
+    .load_request = usb_msd_load_request,
+};
+
+static int usb_msd_initfn_storage(USBDevice *dev)
 {
     MSDState *s = DO_UPCAST(MSDState, dev, dev);
     BlockDriverState *bs = s->conf.bs;
+    SCSIDevice *scsi_dev;
 
     if (!bs) {
         error_report("drive property not set");
@@ -566,10 +622,10 @@ static int usb_msd_initfn(USBDevice *dev)
     }
 
     usb_desc_init(dev);
-    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info);
-    s->scsi_dev = scsi_bus_legacy_add_drive(&s->bus, bs, 0, !!s->removable,
+    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info_storage);
+    scsi_dev = scsi_bus_legacy_add_drive(&s->bus, bs, 0, !!s->removable,
                                             s->conf.bootindex);
-    if (!s->scsi_dev) {
+    if (!scsi_dev) {
         return -1;
     }
     s->bus.qbus.allow_hotplug = 0;
@@ -587,6 +643,19 @@ static int usb_msd_initfn(USBDevice *dev)
     return 0;
 }
 
+static int usb_msd_initfn_bot(USBDevice *dev)
+{
+    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+
+    usb_desc_create_serial(dev);
+    usb_desc_init(dev);
+    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info_bot);
+    s->bus.qbus.allow_hotplug = 0;
+    usb_msd_handle_reset(dev);
+
+    return 0;
+}
+
 static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
 {
     static int nr=0;
@@ -669,12 +738,11 @@ static Property msd_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void usb_msd_class_initfn(ObjectClass *klass, void *data)
+static void usb_msd_class_initfn_common(ObjectClass *klass)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_msd_initfn;
     uc->product_desc   = "QEMU USB MSD";
     uc->usb_desc       = &desc;
     uc->cancel_packet  = usb_msd_cancel_io;
@@ -684,19 +752,44 @@ static void usb_msd_class_initfn(ObjectClass *klass, void *data)
     uc->handle_data    = usb_msd_handle_data;
     dc->fw_name = "storage";
     dc->vmsd = &vmstate_usb_msd;
+}
+
+static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+
+    uc->init = usb_msd_initfn_storage;
     dc->props = msd_properties;
+    usb_msd_class_initfn_common(klass);
 }
 
-static TypeInfo msd_info = {
+static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
+{
+    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+
+    uc->init = usb_msd_initfn_bot;
+    usb_msd_class_initfn_common(klass);
+}
+
+static const TypeInfo msd_info = {
     .name          = "usb-storage",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(MSDState),
-    .class_init    = usb_msd_class_initfn,
+    .class_init    = usb_msd_class_initfn_storage,
+};
+
+static const TypeInfo bot_info = {
+    .name          = "usb-bot",
+    .parent        = TYPE_USB_DEVICE,
+    .instance_size = sizeof(MSDState),
+    .class_init    = usb_msd_class_initfn_bot,
 };
 
 static void usb_msd_register_types(void)
 {
     type_register_static(&msd_info);
+    type_register_static(&bot_info);
     usb_legacy_register("usb-storage", "disk", usb_msd_init);
 }
 
diff --git a/hw/usb/dev-uas.c b/hw/usb/dev-uas.c
index 5a0057a36b3..d904d1a40b9 100644
--- a/hw/usb/dev-uas.c
+++ b/hw/usb/dev-uas.c
@@ -10,8 +10,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "trace.h"
 
 #include "hw/usb.h"
@@ -256,10 +256,10 @@ static void usb_uas_send_status_bh(void *opaque)
 
     uas->status = NULL;
     usb_packet_copy(p, &st->status, st->length);
-    p->result = st->length;
     QTAILQ_REMOVE(&uas->results, st, next);
     g_free(st);
 
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
     usb_packet_complete(&uas->dev, p);
 }
 
@@ -349,6 +349,7 @@ static void usb_uas_complete_data_packet(UASRequest *req)
     p = req->data;
     req->data = NULL;
     req->data_async = false;
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
     usb_packet_complete(&req->uas->dev, p);
 }
 
@@ -357,16 +358,16 @@ static void usb_uas_copy_data(UASRequest *req)
     uint32_t length;
 
     length = MIN(req->buf_size - req->buf_off,
-                 req->data->iov.size - req->data->result);
+                 req->data->iov.size - req->data->actual_length);
     trace_usb_uas_xfer_data(req->uas->dev.addr, req->tag, length,
-                            req->data->result, req->data->iov.size,
+                            req->data->actual_length, req->data->iov.size,
                             req->buf_off, req->buf_size);
     usb_packet_copy(req->data, scsi_req_get_buf(req->req) + req->buf_off,
                     length);
     req->buf_off += length;
     req->data_off += length;
 
-    if (req->data->result == req->data->iov.size) {
+    if (req->data->actual_length == req->data->iov.size) {
         usb_uas_complete_data_packet(req);
     }
     if (req->buf_size && req->buf_off == req->buf_size) {
@@ -504,17 +505,17 @@ static void usb_uas_handle_reset(USBDevice *dev)
     }
 }
 
-static int usb_uas_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_uas_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     int ret;
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
     fprintf(stderr, "%s: unhandled control request\n", __func__);
-    return USB_RET_STALL;
+    p->status = USB_RET_STALL;
 }
 
 static void usb_uas_cancel_io(USBDevice *dev, USBPacket *p)
@@ -576,7 +577,6 @@ bad_target:
      */
     usb_uas_queue_response(uas, req->tag, UAS_RC_INVALID_INFO_UNIT, 0);
     g_free(req);
-    return;
 }
 
 static void usb_uas_task(UASDevice *uas, uas_ui *ui)
@@ -640,16 +640,15 @@ bad_target:
 
 incorrect_lun:
     usb_uas_queue_response(uas, tag, UAS_RC_INCORRECT_LUN, 0);
-    return;
 }
 
-static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
 {
     UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
     uas_ui ui;
     UASStatus *st;
     UASRequest *req;
-    int length, ret = 0;
+    int length;
 
     switch (p->ep->nr) {
     case UAS_PIPE_ID_COMMAND:
@@ -658,16 +657,14 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         switch (ui.hdr.id) {
         case UAS_UI_COMMAND:
             usb_uas_command(uas, &ui);
-            ret = length;
             break;
         case UAS_UI_TASK_MGMT:
             usb_uas_task(uas, &ui);
-            ret = length;
             break;
         default:
             fprintf(stderr, "%s: unknown command ui: id 0x%x\n",
                     __func__, ui.hdr.id);
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         break;
@@ -676,11 +673,10 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         if (st == NULL) {
             assert(uas->status == NULL);
             uas->status = p;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
             break;
         }
         usb_packet_copy(p, &st->status, st->length);
-        ret = st->length;
         QTAILQ_REMOVE(&uas->results, st, next);
         g_free(st);
         break;
@@ -689,28 +685,26 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         req = (p->ep->nr == UAS_PIPE_ID_DATA_IN) ? uas->datain : uas->dataout;
         if (req == NULL) {
             fprintf(stderr, "%s: no inflight request\n", __func__);
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         scsi_req_ref(req->req);
         req->data = p;
         usb_uas_copy_data(req);
-        if (p->result == p->iov.size || req->complete) {
+        if (p->actual_length == p->iov.size || req->complete) {
             req->data = NULL;
-            ret = p->result;
         } else {
             req->data_async = true;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
         }
         scsi_req_unref(req->req);
         usb_uas_start_next_transfer(uas);
         break;
     default:
         fprintf(stderr, "%s: invalid endpoint %d\n", __func__, p->ep->nr);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_uas_handle_destroy(USBDevice *dev)
@@ -763,7 +757,7 @@ static void usb_uas_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_uas;
 }
 
-static TypeInfo uas_info = {
+static const TypeInfo uas_info = {
     .name          = "usb-uas",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(UASDevice),
diff --git a/hw/usb/dev-wacom.c b/hw/usb/dev-wacom.c
index ed9a5ee358a..ab9fa2ef489 100644
--- a/hw/usb/dev-wacom.c
+++ b/hw/usb/dev-wacom.c
@@ -26,7 +26,7 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 
@@ -43,6 +43,7 @@
 
 typedef struct USBWacomState {
     USBDevice dev;
+    USBEndpoint *intr;
     QEMUPutMouseEntry *eh_entry;
     int dx, dy, dz, buttons_state;
     int x, y;
@@ -137,6 +138,7 @@ static void usb_mouse_event(void *opaque,
     s->dz += dz1;
     s->buttons_state = buttons_state;
     s->changed = 1;
+    usb_wakeup(s->intr);
 }
 
 static void usb_wacom_event(void *opaque,
@@ -150,6 +152,7 @@ static void usb_wacom_event(void *opaque,
     s->dz += dz;
     s->buttons_state = buttons_state;
     s->changed = 1;
+    usb_wakeup(s->intr);
 }
 
 static inline int int_clamp(int val, int vmin, int vmax)
@@ -250,7 +253,7 @@ static void usb_wacom_handle_reset(USBDevice *dev)
     s->mode = WACOM_MODE_HID;
 }
 
-static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBWacomState *s = (USBWacomState *) dev;
@@ -258,10 +261,9 @@ static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case WACOM_SET_REPORT:
         if (s->mouse_grabbed) {
@@ -269,61 +271,58 @@ static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
             s->mouse_grabbed = 0;
         }
         s->mode = data[0];
-        ret = 0;
         break;
     case WACOM_GET_REPORT:
         data[0] = 0;
         data[1] = s->mode;
-        ret = 2;
+        p->actual_length = 2;
         break;
     /* USB HID requests */
     case HID_GET_REPORT:
         if (s->mode == WACOM_MODE_HID)
-            ret = usb_mouse_poll(s, data, length);
+            p->actual_length = usb_mouse_poll(s, data, length);
         else if (s->mode == WACOM_MODE_WACOM)
-            ret = usb_wacom_poll(s, data, length);
+            p->actual_length = usb_wacom_poll(s, data, length);
         break;
     case HID_GET_IDLE:
-        ret = 1;
         data[0] = s->idle;
+        p->actual_length = 1;
         break;
     case HID_SET_IDLE:
         s->idle = (uint8_t) (value >> 8);
-        ret = 0;
         break;
     default:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_wacom_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_wacom_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBWacomState *s = (USBWacomState *) dev;
     uint8_t buf[p->iov.size];
-    int ret = 0;
+    int len = 0;
 
     switch (p->pid) {
     case USB_TOKEN_IN:
         if (p->ep->nr == 1) {
-            if (!(s->changed || s->idle))
-                return USB_RET_NAK;
+            if (!(s->changed || s->idle)) {
+                p->status = USB_RET_NAK;
+                return;
+            }
             s->changed = 0;
             if (s->mode == WACOM_MODE_HID)
-                ret = usb_mouse_poll(s, buf, p->iov.size);
+                len = usb_mouse_poll(s, buf, p->iov.size);
             else if (s->mode == WACOM_MODE_WACOM)
-                ret = usb_wacom_poll(s, buf, p->iov.size);
-            usb_packet_copy(p, buf, ret);
+                len = usb_wacom_poll(s, buf, p->iov.size);
+            usb_packet_copy(p, buf, len);
             break;
         }
         /* Fall through.  */
     case USB_TOKEN_OUT:
     default:
-        ret = USB_RET_STALL;
-        break;
+        p->status = USB_RET_STALL;
     }
-    return ret;
 }
 
 static void usb_wacom_handle_destroy(USBDevice *dev)
@@ -341,6 +340,7 @@ static int usb_wacom_initfn(USBDevice *dev)
     USBWacomState *s = DO_UPCAST(USBWacomState, dev, dev);
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
     s->changed = 1;
     return 0;
 }
@@ -366,7 +366,7 @@ static void usb_wacom_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_wacom;
 }
 
-static TypeInfo wacom_info = {
+static const TypeInfo wacom_info = {
     .name          = "usb-wacom-tablet",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBWacomState),
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
new file mode 100644
index 00000000000..0eb78269f77
--- /dev/null
+++ b/hw/usb/hcd-ehci-pci.c
@@ -0,0 +1,228 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/usb/hcd-ehci.h"
+#include "qemu/range.h"
+
+typedef struct EHCIPCIInfo {
+    const char *name;
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint8_t  revision;
+} EHCIPCIInfo;
+
+static int usb_ehci_pci_initfn(PCIDevice *dev)
+{
+    EHCIPCIState *i = PCI_EHCI(dev);
+    EHCIState *s = &i->ehci;
+    uint8_t *pci_conf = dev->config;
+
+    pci_set_byte(&pci_conf[PCI_CLASS_PROG], 0x20);
+
+    /* capabilities pointer */
+    pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x00);
+    /* pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x50); */
+
+    pci_set_byte(&pci_conf[PCI_INTERRUPT_PIN], 4); /* interrupt pin D */
+    pci_set_byte(&pci_conf[PCI_MIN_GNT], 0);
+    pci_set_byte(&pci_conf[PCI_MAX_LAT], 0);
+
+    /* pci_conf[0x50] = 0x01; *//* power management caps */
+
+    pci_set_byte(&pci_conf[USB_SBRN], USB_RELEASE_2); /* release # (2.1.4) */
+    pci_set_byte(&pci_conf[0x61], 0x20);  /* frame length adjustment (2.1.5) */
+    pci_set_word(&pci_conf[0x62], 0x00);  /* port wake up capability (2.1.6) */
+
+    pci_conf[0x64] = 0x00;
+    pci_conf[0x65] = 0x00;
+    pci_conf[0x66] = 0x00;
+    pci_conf[0x67] = 0x00;
+    pci_conf[0x68] = 0x01;
+    pci_conf[0x69] = 0x00;
+    pci_conf[0x6a] = 0x00;
+    pci_conf[0x6b] = 0x00;  /* USBLEGSUP */
+    pci_conf[0x6c] = 0x00;
+    pci_conf[0x6d] = 0x00;
+    pci_conf[0x6e] = 0x00;
+    pci_conf[0x6f] = 0xc0;  /* USBLEFCTLSTS */
+
+    s->caps[0x09] = 0x68;        /* EECP */
+
+    s->irq = dev->irq[3];
+    s->dma = pci_dma_context(dev);
+
+    s->capsbase = 0x00;
+    s->opregbase = 0x20;
+
+    usb_ehci_initfn(s, DEVICE(dev));
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mem);
+
+    return 0;
+}
+
+static void usb_ehci_pci_write_config(PCIDevice *dev, uint32_t addr,
+                                      uint32_t val, int l)
+{
+    EHCIPCIState *i = PCI_EHCI(dev);
+    bool busmaster;
+
+    pci_default_write_config(dev, addr, val, l);
+
+    if (!range_covers_byte(addr, l, PCI_COMMAND)) {
+        return;
+    }
+    busmaster = pci_get_word(dev->config + PCI_COMMAND) & PCI_COMMAND_MASTER;
+    i->ehci.dma = busmaster ? pci_dma_context(dev) : NULL;
+}
+
+static Property ehci_pci_properties[] = {
+    DEFINE_PROP_UINT32("maxframes", EHCIPCIState, ehci.maxframes, 128),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vmstate_ehci_pci = {
+    .name        = "ehci",
+    .version_id  = 2,
+    .minimum_version_id  = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(pcidev, EHCIPCIState),
+        VMSTATE_STRUCT(ehci, EHCIPCIState, 2, vmstate_ehci, EHCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ehci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = usb_ehci_pci_initfn;
+    k->class_id = PCI_CLASS_SERIAL_USB;
+    k->config_write = usb_ehci_pci_write_config;
+    k->no_hotplug = 1;
+    dc->vmsd = &vmstate_ehci_pci;
+    dc->props = ehci_pci_properties;
+}
+
+static const TypeInfo ehci_pci_type_info = {
+    .name = TYPE_PCI_EHCI,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(EHCIPCIState),
+    .abstract = true,
+    .class_init = ehci_class_init,
+};
+
+static void ehci_data_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    EHCIPCIInfo *i = data;
+
+    k->vendor_id = i->vendor_id;
+    k->device_id = i->device_id;
+    k->revision = i->revision;
+}
+
+static struct EHCIPCIInfo ehci_pci_info[] = {
+    {
+        .name      = "usb-ehci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801D, /* ich4 */
+        .revision  = 0x10,
+    },{
+        .name      = "ich9-usb-ehci1", /* 00:1d.7 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI1,
+        .revision  = 0x03,
+    },{
+        .name      = "ich9-usb-ehci2", /* 00:1a.7 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI2,
+        .revision  = 0x03,
+    }
+};
+
+static void ehci_pci_register_types(void)
+{
+    TypeInfo ehci_type_info = {
+        .parent        = TYPE_PCI_EHCI,
+        .class_init    = ehci_data_class_init,
+    };
+    int i;
+
+    type_register_static(&ehci_pci_type_info);
+
+    for (i = 0; i < ARRAY_SIZE(ehci_pci_info); i++) {
+        ehci_type_info.name = ehci_pci_info[i].name;
+        ehci_type_info.class_data = ehci_pci_info + i;
+        type_register(&ehci_type_info);
+    }
+}
+
+type_init(ehci_pci_register_types)
+
+struct ehci_companions {
+    const char *name;
+    int func;
+    int port;
+};
+
+static const struct ehci_companions ich9_1d[] = {
+    { .name = "ich9-usb-uhci1", .func = 0, .port = 0 },
+    { .name = "ich9-usb-uhci2", .func = 1, .port = 2 },
+    { .name = "ich9-usb-uhci3", .func = 2, .port = 4 },
+};
+
+static const struct ehci_companions ich9_1a[] = {
+    { .name = "ich9-usb-uhci4", .func = 0, .port = 0 },
+    { .name = "ich9-usb-uhci5", .func = 1, .port = 2 },
+    { .name = "ich9-usb-uhci6", .func = 2, .port = 4 },
+};
+
+int ehci_create_ich9_with_companions(PCIBus *bus, int slot)
+{
+    const struct ehci_companions *comp;
+    PCIDevice *ehci, *uhci;
+    BusState *usbbus;
+    const char *name;
+    int i;
+
+    switch (slot) {
+    case 0x1d:
+        name = "ich9-usb-ehci1";
+        comp = ich9_1d;
+        break;
+    case 0x1a:
+        name = "ich9-usb-ehci2";
+        comp = ich9_1a;
+        break;
+    default:
+        return -1;
+    }
+
+    ehci = pci_create_multifunction(bus, PCI_DEVFN(slot, 7), true, name);
+    qdev_init_nofail(&ehci->qdev);
+    usbbus = QLIST_FIRST(&ehci->qdev.child_bus);
+
+    for (i = 0; i < 3; i++) {
+        uhci = pci_create_multifunction(bus, PCI_DEVFN(slot, comp[i].func),
+                                        true, comp[i].name);
+        qdev_prop_set_string(&uhci->qdev, "masterbus", usbbus->name);
+        qdev_prop_set_uint32(&uhci->qdev, "firstport", comp[i].port);
+        qdev_init_nofail(&uhci->qdev);
+    }
+    return 0;
+}
diff --git a/hw/usb/hcd-ehci-sysbus.c b/hw/usb/hcd-ehci-sysbus.c
new file mode 100644
index 00000000000..b68a66a63b2
--- /dev/null
+++ b/hw/usb/hcd-ehci-sysbus.c
@@ -0,0 +1,105 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/usb/hcd-ehci.h"
+
+static const VMStateDescription vmstate_ehci_sysbus = {
+    .name        = "ehci-sysbus",
+    .version_id  = 2,
+    .minimum_version_id  = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_STRUCT(ehci, EHCISysBusState, 2, vmstate_ehci, EHCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property ehci_sysbus_properties[] = {
+    DEFINE_PROP_UINT32("maxframes", EHCISysBusState, ehci.maxframes, 128),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static int usb_ehci_sysbus_initfn(SysBusDevice *dev)
+{
+    EHCISysBusState *i = SYS_BUS_EHCI(dev);
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_GET_CLASS(dev);
+    EHCIState *s = &i->ehci;
+
+    s->capsbase = sec->capsbase;
+    s->opregbase = sec->opregbase;
+    s->dma = &dma_context_memory;
+
+    usb_ehci_initfn(s, DEVICE(dev));
+    sysbus_init_irq(dev, &s->irq);
+    sysbus_init_mmio(dev, &s->mem);
+    return 0;
+}
+
+static void ehci_sysbus_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = usb_ehci_sysbus_initfn;
+    dc->vmsd = &vmstate_ehci_sysbus;
+    dc->props = ehci_sysbus_properties;
+}
+
+static const TypeInfo ehci_type_info = {
+    .name          = TYPE_SYS_BUS_EHCI,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(EHCISysBusState),
+    .abstract      = true,
+    .class_init    = ehci_sysbus_class_init,
+    .class_size    = sizeof(SysBusEHCIClass),
+};
+
+static void ehci_xlnx_class_init(ObjectClass *oc, void *data)
+{
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_CLASS(oc);
+
+    sec->capsbase = 0x100;
+    sec->opregbase = 0x140;
+}
+
+static const TypeInfo ehci_xlnx_type_info = {
+    .name          = "xlnx,ps7-usb",
+    .parent        = TYPE_SYS_BUS_EHCI,
+    .class_init    = ehci_xlnx_class_init,
+};
+
+static void ehci_exynos4210_class_init(ObjectClass *oc, void *data)
+{
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_CLASS(oc);
+
+    sec->capsbase = 0x0;
+    sec->opregbase = 0x10;
+}
+
+static const TypeInfo ehci_exynos4210_type_info = {
+    .name          = TYPE_EXYNOS4210_EHCI,
+    .parent        = TYPE_SYS_BUS_EHCI,
+    .class_init    = ehci_exynos4210_class_init,
+};
+
+static void ehci_sysbus_register_types(void)
+{
+    type_register_static(&ehci_type_info);
+    type_register_static(&ehci_xlnx_type_info);
+    type_register_static(&ehci_exynos4210_type_info);
+}
+
+type_init(ehci_sysbus_register_types)
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index 017342b56a2..70406592ef4 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -2,6 +2,11 @@
  * QEMU USB EHCI Emulation
  *
  * Copyright(c) 2008  Emutex Ltd. (address@hidden)
+ * Copyright(c) 2011-2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Gerd Hoffmann <kraxel@redhat.com>
+ * Hans de Goede <hdegoede@redhat.com>
  *
  * EHCI project was started by Mark Burkley, with contributions by
  * Niels de Vos.  David S. Ahern continued working on it.  Kevin Wolf,
@@ -22,40 +27,18 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw/hw.h"
-#include "qemu-timer.h"
-#include "hw/usb.h"
-#include "hw/pci.h"
-#include "monitor.h"
-#include "trace.h"
-#include "dma.h"
-
-#define EHCI_DEBUG   0
-
-#if EHCI_DEBUG
-#define DPRINTF printf
-#else
-#define DPRINTF(...)
-#endif
-
-/* internal processing - reset HC to try and recover */
-#define USB_RET_PROCERR   (-99)
-
-#define MMIO_SIZE        0x1000
+#include "hw/usb/hcd-ehci.h"
 
 /* Capability Registers Base Address - section 2.2 */
-#define CAPREGBASE       0x0000
-#define CAPLENGTH        CAPREGBASE + 0x0000  // 1-byte, 0x0001 reserved
-#define HCIVERSION       CAPREGBASE + 0x0002  // 2-bytes, i/f version #
-#define HCSPARAMS        CAPREGBASE + 0x0004  // 4-bytes, structural params
-#define HCCPARAMS        CAPREGBASE + 0x0008  // 4-bytes, capability params
+#define CAPLENGTH        0x0000  /* 1-byte, 0x0001 reserved */
+#define HCIVERSION       0x0002  /* 2-bytes, i/f version # */
+#define HCSPARAMS        0x0004  /* 4-bytes, structural params */
+#define HCCPARAMS        0x0008  /* 4-bytes, capability params */
 #define EECP             HCCPARAMS + 1
-#define HCSPPORTROUTE1   CAPREGBASE + 0x000c
-#define HCSPPORTROUTE2   CAPREGBASE + 0x0010
+#define HCSPPORTROUTE1   0x000c
+#define HCSPPORTROUTE2   0x0010
 
-#define OPREGBASE        0x0020        // Operational Registers Base Address
-
-#define USBCMD           OPREGBASE + 0x0000
+#define USBCMD           0x0000
 #define USBCMD_RUNSTOP   (1 << 0)      // run / Stop
 #define USBCMD_HCRESET   (1 << 1)      // HC Reset
 #define USBCMD_FLS       (3 << 2)      // Frame List Size
@@ -69,7 +52,7 @@
 #define USBCMD_ITC       (0x7f << 16)  // Int Threshold Control
 #define USBCMD_ITC_SH    16            // Int Threshold Control Shift
 
-#define USBSTS           OPREGBASE + 0x0004
+#define USBSTS           0x0004
 #define USBSTS_RO_MASK   0x0000003f
 #define USBSTS_INT       (1 << 0)      // USB Interrupt
 #define USBSTS_ERRINT    (1 << 1)      // Error Interrupt
@@ -86,20 +69,17 @@
  *  Interrupt enable bits correspond to the interrupt active bits in USBSTS
  *  so no need to redefine here.
  */
-#define USBINTR              OPREGBASE + 0x0008
+#define USBINTR              0x0008
 #define USBINTR_MASK         0x0000003f
 
-#define FRINDEX              OPREGBASE + 0x000c
-#define CTRLDSSEGMENT        OPREGBASE + 0x0010
-#define PERIODICLISTBASE     OPREGBASE + 0x0014
-#define ASYNCLISTADDR        OPREGBASE + 0x0018
+#define FRINDEX              0x000c
+#define CTRLDSSEGMENT        0x0010
+#define PERIODICLISTBASE     0x0014
+#define ASYNCLISTADDR        0x0018
 #define ASYNCLISTADDR_MASK   0xffffffe0
 
-#define CONFIGFLAG           OPREGBASE + 0x0040
+#define CONFIGFLAG           0x0040
 
-#define PORTSC               (OPREGBASE + 0x0044)
-#define PORTSC_BEGIN         PORTSC
-#define PORTSC_END           (PORTSC + 4 * NB_PORTS)
 /*
  * Bits that are reserved or are read-only are masked out of values
  * written to us by software
@@ -129,11 +109,13 @@
 
 #define FRAME_TIMER_FREQ 1000
 #define FRAME_TIMER_NS   (1000000000 / FRAME_TIMER_FREQ)
+#define UFRAME_TIMER_NS  (FRAME_TIMER_NS / 8)
 
 #define NB_MAXINTRATE    8        // Max rate at which controller issues ints
-#define NB_PORTS         6        // Number of downstream ports
 #define BUFF_SIZE        5*4096   // Max bytes to transfer per transaction
 #define MAX_QH           100      // Max allowable queue heads in a chain
+#define MIN_UFR_PER_TICK 24       /* Min frames to process when catching up */
+#define PERIODIC_ACTIVE  512      /* Micro-frames */
 
 /*  Internal periodic / asynchronous schedule state machine states
  */
@@ -167,274 +149,6 @@ typedef enum {
 #define NLPTR_TYPE_STITD         2     // split xaction, isoc xfer descriptor
 #define NLPTR_TYPE_FSTN          3     // frame span traversal node
 
-
-/*  EHCI spec version 1.0 Section 3.3
- */
-typedef struct EHCIitd {
-    uint32_t next;
-
-    uint32_t transact[8];
-#define ITD_XACT_ACTIVE          (1 << 31)
-#define ITD_XACT_DBERROR         (1 << 30)
-#define ITD_XACT_BABBLE          (1 << 29)
-#define ITD_XACT_XACTERR         (1 << 28)
-#define ITD_XACT_LENGTH_MASK     0x0fff0000
-#define ITD_XACT_LENGTH_SH       16
-#define ITD_XACT_IOC             (1 << 15)
-#define ITD_XACT_PGSEL_MASK      0x00007000
-#define ITD_XACT_PGSEL_SH        12
-#define ITD_XACT_OFFSET_MASK     0x00000fff
-
-    uint32_t bufptr[7];
-#define ITD_BUFPTR_MASK          0xfffff000
-#define ITD_BUFPTR_SH            12
-#define ITD_BUFPTR_EP_MASK       0x00000f00
-#define ITD_BUFPTR_EP_SH         8
-#define ITD_BUFPTR_DEVADDR_MASK  0x0000007f
-#define ITD_BUFPTR_DEVADDR_SH    0
-#define ITD_BUFPTR_DIRECTION     (1 << 11)
-#define ITD_BUFPTR_MAXPKT_MASK   0x000007ff
-#define ITD_BUFPTR_MAXPKT_SH     0
-#define ITD_BUFPTR_MULT_MASK     0x00000003
-#define ITD_BUFPTR_MULT_SH       0
-} EHCIitd;
-
-/*  EHCI spec version 1.0 Section 3.4
- */
-typedef struct EHCIsitd {
-    uint32_t next;                  // Standard next link pointer
-    uint32_t epchar;
-#define SITD_EPCHAR_IO              (1 << 31)
-#define SITD_EPCHAR_PORTNUM_MASK    0x7f000000
-#define SITD_EPCHAR_PORTNUM_SH      24
-#define SITD_EPCHAR_HUBADD_MASK     0x007f0000
-#define SITD_EPCHAR_HUBADDR_SH      16
-#define SITD_EPCHAR_EPNUM_MASK      0x00000f00
-#define SITD_EPCHAR_EPNUM_SH        8
-#define SITD_EPCHAR_DEVADDR_MASK    0x0000007f
-
-    uint32_t uframe;
-#define SITD_UFRAME_CMASK_MASK      0x0000ff00
-#define SITD_UFRAME_CMASK_SH        8
-#define SITD_UFRAME_SMASK_MASK      0x000000ff
-
-    uint32_t results;
-#define SITD_RESULTS_IOC              (1 << 31)
-#define SITD_RESULTS_PGSEL            (1 << 30)
-#define SITD_RESULTS_TBYTES_MASK      0x03ff0000
-#define SITD_RESULTS_TYBYTES_SH       16
-#define SITD_RESULTS_CPROGMASK_MASK   0x0000ff00
-#define SITD_RESULTS_CPROGMASK_SH     8
-#define SITD_RESULTS_ACTIVE           (1 << 7)
-#define SITD_RESULTS_ERR              (1 << 6)
-#define SITD_RESULTS_DBERR            (1 << 5)
-#define SITD_RESULTS_BABBLE           (1 << 4)
-#define SITD_RESULTS_XACTERR          (1 << 3)
-#define SITD_RESULTS_MISSEDUF         (1 << 2)
-#define SITD_RESULTS_SPLITXSTATE      (1 << 1)
-
-    uint32_t bufptr[2];
-#define SITD_BUFPTR_MASK              0xfffff000
-#define SITD_BUFPTR_CURROFF_MASK      0x00000fff
-#define SITD_BUFPTR_TPOS_MASK         0x00000018
-#define SITD_BUFPTR_TPOS_SH           3
-#define SITD_BUFPTR_TCNT_MASK         0x00000007
-
-    uint32_t backptr;                 // Standard next link pointer
-} EHCIsitd;
-
-/*  EHCI spec version 1.0 Section 3.5
- */
-typedef struct EHCIqtd {
-    uint32_t next;                    // Standard next link pointer
-    uint32_t altnext;                 // Standard next link pointer
-    uint32_t token;
-#define QTD_TOKEN_DTOGGLE             (1 << 31)
-#define QTD_TOKEN_TBYTES_MASK         0x7fff0000
-#define QTD_TOKEN_TBYTES_SH           16
-#define QTD_TOKEN_IOC                 (1 << 15)
-#define QTD_TOKEN_CPAGE_MASK          0x00007000
-#define QTD_TOKEN_CPAGE_SH            12
-#define QTD_TOKEN_CERR_MASK           0x00000c00
-#define QTD_TOKEN_CERR_SH             10
-#define QTD_TOKEN_PID_MASK            0x00000300
-#define QTD_TOKEN_PID_SH              8
-#define QTD_TOKEN_ACTIVE              (1 << 7)
-#define QTD_TOKEN_HALT                (1 << 6)
-#define QTD_TOKEN_DBERR               (1 << 5)
-#define QTD_TOKEN_BABBLE              (1 << 4)
-#define QTD_TOKEN_XACTERR             (1 << 3)
-#define QTD_TOKEN_MISSEDUF            (1 << 2)
-#define QTD_TOKEN_SPLITXSTATE         (1 << 1)
-#define QTD_TOKEN_PING                (1 << 0)
-
-    uint32_t bufptr[5];               // Standard buffer pointer
-#define QTD_BUFPTR_MASK               0xfffff000
-#define QTD_BUFPTR_SH                 12
-} EHCIqtd;
-
-/*  EHCI spec version 1.0 Section 3.6
- */
-typedef struct EHCIqh {
-    uint32_t next;                    // Standard next link pointer
-
-    /* endpoint characteristics */
-    uint32_t epchar;
-#define QH_EPCHAR_RL_MASK             0xf0000000
-#define QH_EPCHAR_RL_SH               28
-#define QH_EPCHAR_C                   (1 << 27)
-#define QH_EPCHAR_MPLEN_MASK          0x07FF0000
-#define QH_EPCHAR_MPLEN_SH            16
-#define QH_EPCHAR_H                   (1 << 15)
-#define QH_EPCHAR_DTC                 (1 << 14)
-#define QH_EPCHAR_EPS_MASK            0x00003000
-#define QH_EPCHAR_EPS_SH              12
-#define EHCI_QH_EPS_FULL              0
-#define EHCI_QH_EPS_LOW               1
-#define EHCI_QH_EPS_HIGH              2
-#define EHCI_QH_EPS_RESERVED          3
-
-#define QH_EPCHAR_EP_MASK             0x00000f00
-#define QH_EPCHAR_EP_SH               8
-#define QH_EPCHAR_I                   (1 << 7)
-#define QH_EPCHAR_DEVADDR_MASK        0x0000007f
-#define QH_EPCHAR_DEVADDR_SH          0
-
-    /* endpoint capabilities */
-    uint32_t epcap;
-#define QH_EPCAP_MULT_MASK            0xc0000000
-#define QH_EPCAP_MULT_SH              30
-#define QH_EPCAP_PORTNUM_MASK         0x3f800000
-#define QH_EPCAP_PORTNUM_SH           23
-#define QH_EPCAP_HUBADDR_MASK         0x007f0000
-#define QH_EPCAP_HUBADDR_SH           16
-#define QH_EPCAP_CMASK_MASK           0x0000ff00
-#define QH_EPCAP_CMASK_SH             8
-#define QH_EPCAP_SMASK_MASK           0x000000ff
-#define QH_EPCAP_SMASK_SH             0
-
-    uint32_t current_qtd;             // Standard next link pointer
-    uint32_t next_qtd;                // Standard next link pointer
-    uint32_t altnext_qtd;
-#define QH_ALTNEXT_NAKCNT_MASK        0x0000001e
-#define QH_ALTNEXT_NAKCNT_SH          1
-
-    uint32_t token;                   // Same as QTD token
-    uint32_t bufptr[5];               // Standard buffer pointer
-#define BUFPTR_CPROGMASK_MASK         0x000000ff
-#define BUFPTR_FRAMETAG_MASK          0x0000001f
-#define BUFPTR_SBYTES_MASK            0x00000fe0
-#define BUFPTR_SBYTES_SH              5
-} EHCIqh;
-
-/*  EHCI spec version 1.0 Section 3.7
- */
-typedef struct EHCIfstn {
-    uint32_t next;                    // Standard next link pointer
-    uint32_t backptr;                 // Standard next link pointer
-} EHCIfstn;
-
-typedef struct EHCIPacket EHCIPacket;
-typedef struct EHCIQueue EHCIQueue;
-typedef struct EHCIState EHCIState;
-
-enum async_state {
-    EHCI_ASYNC_NONE = 0,
-    EHCI_ASYNC_INFLIGHT,
-    EHCI_ASYNC_FINISHED,
-};
-
-struct EHCIPacket {
-    EHCIQueue *queue;
-    QTAILQ_ENTRY(EHCIPacket) next;
-
-    EHCIqtd qtd;           /* copy of current QTD (being worked on) */
-    uint32_t qtdaddr;      /* address QTD read from                 */
-
-    USBPacket packet;
-    QEMUSGList sgl;
-    int pid;
-    uint32_t tbytes;
-    enum async_state async;
-    int usb_status;
-};
-
-struct EHCIQueue {
-    EHCIState *ehci;
-    QTAILQ_ENTRY(EHCIQueue) next;
-    uint32_t seen;
-    uint64_t ts;
-    int async;
-    int revalidate;
-
-    /* cached data from guest - needs to be flushed
-     * when guest removes an entry (doorbell, handshake sequence)
-     */
-    EHCIqh qh;             /* copy of current QH (being worked on) */
-    uint32_t qhaddr;       /* address QH read from                 */
-    uint32_t qtdaddr;      /* address QTD read from                */
-    USBDevice *dev;
-    QTAILQ_HEAD(, EHCIPacket) packets;
-};
-
-typedef QTAILQ_HEAD(EHCIQueueHead, EHCIQueue) EHCIQueueHead;
-
-struct EHCIState {
-    PCIDevice dev;
-    USBBus bus;
-    qemu_irq irq;
-    MemoryRegion mem;
-    int companion_count;
-
-    /* properties */
-    uint32_t maxframes;
-
-    /*
-     *  EHCI spec version 1.0 Section 2.3
-     *  Host Controller Operational Registers
-     */
-    union {
-        uint8_t mmio[MMIO_SIZE];
-        struct {
-            uint8_t cap[OPREGBASE];
-            uint32_t usbcmd;
-            uint32_t usbsts;
-            uint32_t usbintr;
-            uint32_t frindex;
-            uint32_t ctrldssegment;
-            uint32_t periodiclistbase;
-            uint32_t asynclistaddr;
-            uint32_t notused[9];
-            uint32_t configflag;
-            uint32_t portsc[NB_PORTS];
-        };
-    };
-
-    /*
-     *  Internal states, shadow registers, etc
-     */
-    QEMUTimer *frame_timer;
-    QEMUBH *async_bh;
-    uint32_t astate;         /* Current state in asynchronous schedule */
-    uint32_t pstate;         /* Current state in periodic schedule     */
-    USBPort ports[NB_PORTS];
-    USBPort *companion_ports[NB_PORTS];
-    uint32_t usbsts_pending;
-    uint32_t usbsts_frindex;
-    EHCIQueueHead aqueues;
-    EHCIQueueHead pqueues;
-
-    /* which address to look at next */
-    uint32_t a_fetch_addr;
-    uint32_t p_fetch_addr;
-
-    USBPacket ipacket;
-    QEMUSGList isgl;
-
-    uint64_t last_run_ns;
-    uint32_t async_stepdown;
-};
-
 #define SET_LAST_RUN_CLOCK(s) \
     (s)->last_run_ns = qemu_get_clock_ns(vm_clock);
 
@@ -466,25 +180,21 @@ static const char *ehci_state_names[] = {
 };
 
 static const char *ehci_mmio_names[] = {
-    [CAPLENGTH]         = "CAPLENGTH",
-    [HCIVERSION]        = "HCIVERSION",
-    [HCSPARAMS]         = "HCSPARAMS",
-    [HCCPARAMS]         = "HCCPARAMS",
     [USBCMD]            = "USBCMD",
     [USBSTS]            = "USBSTS",
     [USBINTR]           = "USBINTR",
     [FRINDEX]           = "FRINDEX",
     [PERIODICLISTBASE]  = "P-LIST BASE",
     [ASYNCLISTADDR]     = "A-LIST ADDR",
-    [PORTSC_BEGIN]      = "PORTSC #0",
-    [PORTSC_BEGIN + 4]  = "PORTSC #1",
-    [PORTSC_BEGIN + 8]  = "PORTSC #2",
-    [PORTSC_BEGIN + 12] = "PORTSC #3",
-    [PORTSC_BEGIN + 16] = "PORTSC #4",
-    [PORTSC_BEGIN + 20] = "PORTSC #5",
     [CONFIGFLAG]        = "CONFIGFLAG",
 };
 
+static int ehci_state_executing(EHCIQueue *q);
+static int ehci_state_writeback(EHCIQueue *q);
+static int ehci_state_advqueue(EHCIQueue *q);
+static int ehci_fill_queue(EHCIPacket *p);
+static void ehci_free_packet(EHCIPacket *p);
+
 static const char *nr2str(const char **n, size_t len, uint32_t nr)
 {
     if (nr < len && n[nr] != NULL) {
@@ -499,7 +209,7 @@ static const char *state2str(uint32_t state)
     return nr2str(ehci_state_names, ARRAY_SIZE(ehci_state_names), state);
 }
 
-static const char *addr2str(target_phys_addr_t addr)
+static const char *addr2str(hwaddr addr)
 {
     return nr2str(ehci_mmio_names, ARRAY_SIZE(ehci_mmio_names), addr);
 }
@@ -658,7 +368,7 @@ static int ehci_get_fetch_addr(EHCIState *s, int async)
     return async ? s->a_fetch_addr : s->p_fetch_addr;
 }
 
-static void ehci_trace_qh(EHCIQueue *q, target_phys_addr_t addr, EHCIqh *qh)
+static void ehci_trace_qh(EHCIQueue *q, hwaddr addr, EHCIqh *qh)
 {
     /* need three here due to argument count limits */
     trace_usb_ehci_qh_ptrs(q, addr, qh->next,
@@ -676,7 +386,7 @@ static void ehci_trace_qh(EHCIQueue *q, target_phys_addr_t addr, EHCIqh *qh)
                            (bool)(qh->epchar & QH_EPCHAR_I));
 }
 
-static void ehci_trace_qtd(EHCIQueue *q, target_phys_addr_t addr, EHCIqtd *qtd)
+static void ehci_trace_qtd(EHCIQueue *q, hwaddr addr, EHCIqtd *qtd)
 {
     /* need three here due to argument count limits */
     trace_usb_ehci_qtd_ptrs(q, addr, qtd->next, qtd->altnext);
@@ -693,7 +403,7 @@ static void ehci_trace_qtd(EHCIQueue *q, target_phys_addr_t addr, EHCIqtd *qtd)
                             (bool)(qtd->token & QTD_TOKEN_XACTERR));
 }
 
-static void ehci_trace_itd(EHCIState *s, target_phys_addr_t addr, EHCIitd *itd)
+static void ehci_trace_itd(EHCIState *s, hwaddr addr, EHCIitd *itd)
 {
     trace_usb_ehci_itd(addr, itd->next,
                        get_field(itd->bufptr[1], ITD_BUFPTR_MAXPKT),
@@ -702,13 +412,19 @@ static void ehci_trace_itd(EHCIState *s, target_phys_addr_t addr, EHCIitd *itd)
                        get_field(itd->bufptr[0], ITD_BUFPTR_DEVADDR));
 }
 
-static void ehci_trace_sitd(EHCIState *s, target_phys_addr_t addr,
+static void ehci_trace_sitd(EHCIState *s, hwaddr addr,
                             EHCIsitd *sitd)
 {
     trace_usb_ehci_sitd(addr, sitd->next,
                         (bool)(sitd->results & SITD_RESULTS_ACTIVE));
 }
 
+static void ehci_trace_guest_bug(EHCIState *s, const char *message)
+{
+    trace_usb_ehci_guest_bug(message);
+    fprintf(stderr, "ehci warning: %s\n", message);
+}
+
 static inline bool ehci_enabled(EHCIState *s)
 {
     return s->usbcmd & USBCMD_RUNSTOP;
@@ -724,6 +440,136 @@ static inline bool ehci_periodic_enabled(EHCIState *s)
     return ehci_enabled(s) && (s->usbcmd & USBCMD_PSE);
 }
 
+/* Get an array of dwords from main memory */
+static inline int get_dwords(EHCIState *ehci, uint32_t addr,
+                             uint32_t *buf, int num)
+{
+    int i;
+
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
+    for (i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
+        dma_memory_read(ehci->dma, addr, buf, sizeof(*buf));
+        *buf = le32_to_cpu(*buf);
+    }
+
+    return num;
+}
+
+/* Put an array of dwords in to main memory */
+static inline int put_dwords(EHCIState *ehci, uint32_t addr,
+                             uint32_t *buf, int num)
+{
+    int i;
+
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
+    for (i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
+        uint32_t tmp = cpu_to_le32(*buf);
+        dma_memory_write(ehci->dma, addr, &tmp, sizeof(tmp));
+    }
+
+    return num;
+}
+
+static int ehci_get_pid(EHCIqtd *qtd)
+{
+    switch (get_field(qtd->token, QTD_TOKEN_PID)) {
+    case 0:
+        return USB_TOKEN_OUT;
+    case 1:
+        return USB_TOKEN_IN;
+    case 2:
+        return USB_TOKEN_SETUP;
+    default:
+        fprintf(stderr, "bad token\n");
+        return 0;
+    }
+}
+
+static bool ehci_verify_qh(EHCIQueue *q, EHCIqh *qh)
+{
+    uint32_t devaddr = get_field(qh->epchar, QH_EPCHAR_DEVADDR);
+    uint32_t endp    = get_field(qh->epchar, QH_EPCHAR_EP);
+    if ((devaddr != get_field(q->qh.epchar, QH_EPCHAR_DEVADDR)) ||
+        (endp    != get_field(q->qh.epchar, QH_EPCHAR_EP)) ||
+        (qh->current_qtd != q->qh.current_qtd) ||
+        (q->async && qh->next_qtd != q->qh.next_qtd) ||
+        (memcmp(&qh->altnext_qtd, &q->qh.altnext_qtd,
+                                 7 * sizeof(uint32_t)) != 0) ||
+        (q->dev != NULL && q->dev->addr != devaddr)) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+static bool ehci_verify_qtd(EHCIPacket *p, EHCIqtd *qtd)
+{
+    if (p->qtdaddr != p->queue->qtdaddr ||
+        (p->queue->async && !NLPTR_TBIT(p->qtd.next) &&
+            (p->qtd.next != qtd->next)) ||
+        (!NLPTR_TBIT(p->qtd.altnext) && (p->qtd.altnext != qtd->altnext)) ||
+        p->qtd.token != qtd->token ||
+        p->qtd.bufptr[0] != qtd->bufptr[0]) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+static bool ehci_verify_pid(EHCIQueue *q, EHCIqtd *qtd)
+{
+    int ep  = get_field(q->qh.epchar, QH_EPCHAR_EP);
+    int pid = ehci_get_pid(qtd);
+
+    /* Note the pid changing is normal for ep 0 (the control ep) */
+    if (q->last_pid && ep != 0 && pid != q->last_pid) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+/* Finish executing and writeback a packet outside of the regular
+   fetchqh -> fetchqtd -> execute -> writeback cycle */
+static void ehci_writeback_async_complete_packet(EHCIPacket *p)
+{
+    EHCIQueue *q = p->queue;
+    EHCIqtd qtd;
+    EHCIqh qh;
+    int state;
+
+    /* Verify the qh + qtd, like we do when going through fetchqh & fetchqtd */
+    get_dwords(q->ehci, NLPTR_GET(q->qhaddr),
+               (uint32_t *) &qh, sizeof(EHCIqh) >> 2);
+    get_dwords(q->ehci, NLPTR_GET(q->qtdaddr),
+               (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2);
+    if (!ehci_verify_qh(q, &qh) || !ehci_verify_qtd(p, &qtd)) {
+        p->async = EHCI_ASYNC_INITIALIZED;
+        ehci_free_packet(p);
+        return;
+    }
+
+    state = ehci_get_state(q->ehci, q->async);
+    ehci_state_executing(q);
+    ehci_state_writeback(q); /* Frees the packet! */
+    if (!(q->qh.token & QTD_TOKEN_HALT)) {
+        ehci_state_advqueue(q);
+    }
+    ehci_set_state(q->ehci, q->async, state);
+}
+
 /* packet management */
 
 static EHCIPacket *ehci_alloc_packet(EHCIQueue *q)
@@ -740,9 +586,19 @@ static EHCIPacket *ehci_alloc_packet(EHCIQueue *q)
 
 static void ehci_free_packet(EHCIPacket *p)
 {
+    if (p->async == EHCI_ASYNC_FINISHED) {
+        ehci_writeback_async_complete_packet(p);
+        return;
+    }
     trace_usb_ehci_packet_action(p->queue, p, "free");
+    if (p->async == EHCI_ASYNC_INITIALIZED) {
+        usb_packet_unmap(&p->packet, &p->sgl);
+        qemu_sglist_destroy(&p->sgl);
+    }
     if (p->async == EHCI_ASYNC_INFLIGHT) {
         usb_cancel_packet(&p->packet);
+        usb_packet_unmap(&p->packet, &p->sgl);
+        qemu_sglist_destroy(&p->sgl);
     }
     QTAILQ_REMOVE(&p->queue->packets, p, next);
     usb_packet_cleanup(&p->packet);
@@ -766,27 +622,60 @@ static EHCIQueue *ehci_alloc_queue(EHCIState *ehci, uint32_t addr, int async)
     return q;
 }
 
-static void ehci_cancel_queue(EHCIQueue *q)
+static void ehci_queue_stopped(EHCIQueue *q)
+{
+    int endp  = get_field(q->qh.epchar, QH_EPCHAR_EP);
+
+    if (!q->last_pid || !q->dev) {
+        return;
+    }
+
+    usb_device_ep_stopped(q->dev, usb_ep_get(q->dev, q->last_pid, endp));
+}
+
+static int ehci_cancel_queue(EHCIQueue *q)
 {
     EHCIPacket *p;
+    int packets = 0;
 
     p = QTAILQ_FIRST(&q->packets);
     if (p == NULL) {
-        return;
+        goto leave;
     }
 
     trace_usb_ehci_queue_action(q, "cancel");
     do {
         ehci_free_packet(p);
+        packets++;
     } while ((p = QTAILQ_FIRST(&q->packets)) != NULL);
+
+leave:
+    ehci_queue_stopped(q);
+    return packets;
+}
+
+static int ehci_reset_queue(EHCIQueue *q)
+{
+    int packets;
+
+    trace_usb_ehci_queue_action(q, "reset");
+    packets = ehci_cancel_queue(q);
+    q->dev = NULL;
+    q->qtdaddr = 0;
+    q->last_pid = 0;
+    return packets;
 }
 
-static void ehci_free_queue(EHCIQueue *q)
+static void ehci_free_queue(EHCIQueue *q, const char *warn)
 {
     EHCIQueueHead *head = q->async ? &q->ehci->aqueues : &q->ehci->pqueues;
+    int cancelled;
 
     trace_usb_ehci_queue_action(q, "free");
-    ehci_cancel_queue(q);
+    cancelled = ehci_cancel_queue(q);
+    if (warn && cancelled > 0) {
+        ehci_trace_guest_bug(q->ehci, warn);
+    }
     QTAILQ_REMOVE(head, q, next);
     g_free(q);
 }
@@ -805,20 +694,10 @@ static EHCIQueue *ehci_find_queue_by_qh(EHCIState *ehci, uint32_t addr,
     return NULL;
 }
 
-static void ehci_queues_tag_unused_async(EHCIState *ehci)
-{
-    EHCIQueue *q;
-
-    QTAILQ_FOREACH(q, &ehci->aqueues, next) {
-        if (!q->seen) {
-            q->revalidate = 1;
-        }
-    }
-}
-
 static void ehci_queues_rip_unused(EHCIState *ehci, int async)
 {
     EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    const char *warn = async ? "guest unlinked busy QH" : NULL;
     uint64_t maxage = FRAME_TIMER_NS * ehci->maxframes * 4;
     EHCIQueue *q, *tmp;
 
@@ -831,7 +710,19 @@ static void ehci_queues_rip_unused(EHCIState *ehci, int async)
         if (ehci->last_run_ns < q->ts + maxage) {
             continue;
         }
-        ehci_free_queue(q);
+        ehci_free_queue(q, warn);
+    }
+}
+
+static void ehci_queues_rip_unseen(EHCIState *ehci, int async)
+{
+    EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    EHCIQueue *q, *tmp;
+
+    QTAILQ_FOREACH_SAFE(q, head, next, tmp) {
+        if (!q->seen) {
+            ehci_free_queue(q, NULL);
+        }
     }
 }
 
@@ -844,17 +735,18 @@ static void ehci_queues_rip_device(EHCIState *ehci, USBDevice *dev, int async)
         if (q->dev != dev) {
             continue;
         }
-        ehci_free_queue(q);
+        ehci_free_queue(q, NULL);
     }
 }
 
 static void ehci_queues_rip_all(EHCIState *ehci, int async)
 {
     EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    const char *warn = async ? "guest stopped busy async schedule" : NULL;
     EHCIQueue *q, *tmp;
 
     QTAILQ_FOREACH_SAFE(q, head, next, tmp) {
-        ehci_free_queue(q);
+        ehci_free_queue(q, warn);
     }
 }
 
@@ -879,7 +771,6 @@ static void ehci_attach(USBPort *port)
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
-    ehci_commit_irq(s);
 }
 
 static void ehci_detach(USBPort *port)
@@ -909,7 +800,6 @@ static void ehci_detach(USBPort *port)
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
-    ehci_commit_irq(s);
 }
 
 static void ehci_child_detach(USBPort *port, USBDevice *child)
@@ -979,11 +869,24 @@ static int ehci_register_companion(USBBus *bus, USBPort *ports[],
     }
 
     s->companion_count++;
-    s->mmio[0x05] = (s->companion_count << 4) | portcount;
+    s->caps[0x05] = (s->companion_count << 4) | portcount;
 
     return 0;
 }
 
+static void ehci_wakeup_endpoint(USBBus *bus, USBEndpoint *ep)
+{
+    EHCIState *s = container_of(bus, EHCIState, bus);
+    uint32_t portsc = s->portsc[ep->dev->port->index];
+
+    if (portsc & PORTSC_POWNER) {
+        return;
+    }
+
+    s->periodic_sched_active = PERIODIC_ACTIVE;
+    qemu_bh_schedule(s->async_bh);
+}
+
 static USBDevice *ehci_find_device(EHCIState *ehci, uint8_t addr)
 {
     USBDevice *dev;
@@ -1024,7 +927,8 @@ static void ehci_reset(void *opaque)
         }
     }
 
-    memset(&s->mmio[OPREGBASE], 0x00, MMIO_SIZE - OPREGBASE);
+    memset(&s->opreg, 0x00, sizeof(s->opreg));
+    memset(&s->portsc, 0x00, sizeof(s->portsc));
 
     s->usbcmd = NB_MAXINTRATE << USBCMD_ITC_SH;
     s->usbsts = USBSTS_HALT;
@@ -1051,50 +955,43 @@ static void ehci_reset(void *opaque)
     qemu_bh_cancel(s->async_bh);
 }
 
-static uint32_t ehci_mem_readb(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_caps_read(void *ptr, hwaddr addr,
+                               unsigned size)
 {
     EHCIState *s = ptr;
-    uint32_t val;
-
-    val = s->mmio[addr];
-
-    return val;
+    return s->caps[addr];
 }
 
-static uint32_t ehci_mem_readw(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_opreg_read(void *ptr, hwaddr addr,
+                                unsigned size)
 {
     EHCIState *s = ptr;
     uint32_t val;
 
-    val = s->mmio[addr] | (s->mmio[addr+1] << 8);
+    switch (addr) {
+    case FRINDEX:
+        /* Round down to mult of 8, else it can go backwards on migration */
+        val = s->frindex & ~7;
+        break;
+    default:
+        val = s->opreg[addr >> 2];
+    }
 
+    trace_usb_ehci_opreg_read(addr + s->opregbase, addr2str(addr), val);
     return val;
 }
 
-static uint32_t ehci_mem_readl(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_port_read(void *ptr, hwaddr addr,
+                               unsigned size)
 {
     EHCIState *s = ptr;
     uint32_t val;
 
-    val = s->mmio[addr] | (s->mmio[addr+1] << 8) |
-          (s->mmio[addr+2] << 16) | (s->mmio[addr+3] << 24);
-
-    trace_usb_ehci_mmio_readl(addr, addr2str(addr), val);
+    val = s->portsc[addr >> 2];
+    trace_usb_ehci_portsc_read(addr + PORTSC_BEGIN, addr >> 2, val);
     return val;
 }
 
-static void ehci_mem_writeb(void *ptr, target_phys_addr_t addr, uint32_t val)
-{
-    fprintf(stderr, "EHCI doesn't handle byte writes to MMIO\n");
-    exit(1);
-}
-
-static void ehci_mem_writew(void *ptr, target_phys_addr_t addr, uint32_t val)
-{
-    fprintf(stderr, "EHCI doesn't handle 16-bit writes to MMIO\n");
-    exit(1);
-}
-
 static void handle_port_owner_write(EHCIState *s, int port, uint32_t owner)
 {
     USBDevice *dev = s->ports[port].dev;
@@ -1123,11 +1020,17 @@ static void handle_port_owner_write(EHCIState *s, int port, uint32_t owner)
     }
 }
 
-static void handle_port_status_write(EHCIState *s, int port, uint32_t val)
+static void ehci_port_write(void *ptr, hwaddr addr,
+                            uint64_t val, unsigned size)
 {
+    EHCIState *s = ptr;
+    int port = addr >> 2;
     uint32_t *portsc = &s->portsc[port];
+    uint32_t old = *portsc;
     USBDevice *dev = s->ports[port].dev;
 
+    trace_usb_ehci_portsc_write(addr + PORTSC_BEGIN, addr >> 2, val);
+
     /* Clear rwc bits */
     *portsc &= ~(val & PORTSC_RWC_MASK);
     /* The guest may clear, but not set the PED bit */
@@ -1159,39 +1062,20 @@ static void handle_port_status_write(EHCIState *s, int port, uint32_t val)
 
     *portsc &= ~PORTSC_RO_MASK;
     *portsc |= val;
+    trace_usb_ehci_portsc_change(addr + PORTSC_BEGIN, addr >> 2, *portsc, old);
 }
 
-static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
+static void ehci_opreg_write(void *ptr, hwaddr addr,
+                             uint64_t val, unsigned size)
 {
     EHCIState *s = ptr;
-    uint32_t *mmio = (uint32_t *)(&s->mmio[addr]);
+    uint32_t *mmio = s->opreg + (addr >> 2);
     uint32_t old = *mmio;
     int i;
 
-    trace_usb_ehci_mmio_writel(addr, addr2str(addr), val);
-
-    /* Only aligned reads are allowed on OHCI */
-    if (addr & 3) {
-        fprintf(stderr, "usb-ehci: Mis-aligned write to addr 0x"
-                TARGET_FMT_plx "\n", addr);
-        return;
-    }
-
-    if (addr >= PORTSC && addr < PORTSC + 4 * NB_PORTS) {
-        handle_port_status_write(s, (addr-PORTSC)/4, val);
-        trace_usb_ehci_mmio_change(addr, addr2str(addr), *mmio, old);
-        return;
-    }
+    trace_usb_ehci_opreg_write(addr + s->opregbase, addr2str(addr), val);
 
-    if (addr < OPREGBASE) {
-        fprintf(stderr, "usb-ehci: write attempt to read-only register"
-                TARGET_FMT_plx "\n", addr);
-        return;
-    }
-
-
-    /* Do any register specific pre-write processing here.  */
-    switch(addr) {
+    switch (addr) {
     case USBCMD:
         if (val & USBCMD_HCRESET) {
             ehci_reset(s);
@@ -1202,7 +1086,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
         /* not supporting dynamic frame list size at the moment */
         if ((val & USBCMD_FLS) && !(s->usbcmd & USBCMD_FLS)) {
             fprintf(stderr, "attempt to set frame list size -- value %d\n",
-                    val & USBCMD_FLS);
+                    (int)val & USBCMD_FLS);
             val &= ~USBCMD_FLS;
         }
 
@@ -1213,6 +1097,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
              */
             s->async_stepdown = 0;
             qemu_bh_schedule(s->async_bh);
+            trace_usb_ehci_doorbell_ring();
         }
 
         if (((USBCMD_RUNSTOP | USBCMD_PSE | USBCMD_ASE) & val) !=
@@ -1223,7 +1108,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
             s->usbcmd = val; /* Set usbcmd for ehci_update_halt() */
             ehci_update_halt(s);
             s->async_stepdown = 0;
-            qemu_mod_timer(s->frame_timer, qemu_get_clock_ns(vm_clock));
+            qemu_bh_schedule(s->async_bh);
         }
         break;
 
@@ -1236,10 +1121,14 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
 
     case USBINTR:
         val &= USBINTR_MASK;
+        if (ehci_enabled(s) && (USBSTS_FLR & val)) {
+            qemu_bh_schedule(s->async_bh);
+        }
         break;
 
     case FRINDEX:
-        val &= 0x00003ff8; /* frindex is 14bits and always a multiple of 8 */
+        val &= 0x00003fff; /* frindex is 14bits */
+        s->usbsts_frindex = val;
         break;
 
     case CONFIGFLAG:
@@ -1268,38 +1157,8 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
     }
 
     *mmio = val;
-    trace_usb_ehci_mmio_change(addr, addr2str(addr), *mmio, old);
-}
-
-
-// TODO : Put in common header file, duplication from usb-ohci.c
-
-/* Get an array of dwords from main memory */
-static inline int get_dwords(EHCIState *ehci, uint32_t addr,
-                             uint32_t *buf, int num)
-{
-    int i;
-
-    for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
-        pci_dma_read(&ehci->dev, addr, buf, sizeof(*buf));
-        *buf = le32_to_cpu(*buf);
-    }
-
-    return 1;
-}
-
-/* Put an array of dwords in to main memory */
-static inline int put_dwords(EHCIState *ehci, uint32_t addr,
-                             uint32_t *buf, int num)
-{
-    int i;
-
-    for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
-        uint32_t tmp = cpu_to_le32(*buf);
-        pci_dma_write(&ehci->dev, addr, &tmp, sizeof(tmp));
-    }
-
-    return 1;
+    trace_usb_ehci_opreg_change(addr + s->opregbase, addr2str(addr),
+                                *mmio, old);
 }
 
 /*
@@ -1379,12 +1238,12 @@ static int ehci_init_transfer(EHCIPacket *p)
     cpage  = get_field(p->qtd.token, QTD_TOKEN_CPAGE);
     bytes  = get_field(p->qtd.token, QTD_TOKEN_TBYTES);
     offset = p->qtd.bufptr[0] & ~QTD_BUFPTR_MASK;
-    pci_dma_sglist_init(&p->sgl, &p->queue->ehci->dev, 5);
+    qemu_sglist_init(&p->sgl, 5, p->queue->ehci->dma);
 
     while (bytes > 0) {
         if (cpage > 4) {
             fprintf(stderr, "cpage out of range (%d)\n", cpage);
-            return USB_RET_PROCERR;
+            return -1;
         }
 
         page  = p->qtd.bufptr[cpage] & QTD_BUFPTR_MASK;
@@ -1402,16 +1261,16 @@ static int ehci_init_transfer(EHCIPacket *p)
     return 0;
 }
 
-static void ehci_finish_transfer(EHCIQueue *q, int status)
+static void ehci_finish_transfer(EHCIQueue *q, int len)
 {
     uint32_t cpage, offset;
 
-    if (status > 0) {
+    if (len > 0) {
         /* update cpage & offset */
         cpage  = get_field(q->qh.token, QTD_TOKEN_CPAGE);
         offset = q->qh.bufptr[0] & ~QTD_BUFPTR_MASK;
 
-        offset += status;
+        offset += len;
         cpage  += offset >> QTD_BUFPTR_SH;
         offset &= ~QTD_BUFPTR_MASK;
 
@@ -1434,139 +1293,152 @@ static void ehci_async_complete_packet(USBPort *port, USBPacket *packet)
     }
 
     p = container_of(packet, EHCIPacket, packet);
-    trace_usb_ehci_packet_action(p->queue, p, "wakeup");
     assert(p->async == EHCI_ASYNC_INFLIGHT);
+
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        trace_usb_ehci_packet_action(p->queue, p, "remove");
+        ehci_free_packet(p);
+        return;
+    }
+
+    trace_usb_ehci_packet_action(p->queue, p, "wakeup");
     p->async = EHCI_ASYNC_FINISHED;
-    p->usb_status = packet->result;
 
-    if (p->queue->async) {
-        qemu_bh_schedule(p->queue->ehci->async_bh);
+    if (!p->queue->async) {
+        s->periodic_sched_active = PERIODIC_ACTIVE;
     }
+    qemu_bh_schedule(s->async_bh);
 }
 
 static void ehci_execute_complete(EHCIQueue *q)
 {
     EHCIPacket *p = QTAILQ_FIRST(&q->packets);
+    uint32_t tbytes;
 
     assert(p != NULL);
     assert(p->qtdaddr == q->qtdaddr);
-    assert(p->async != EHCI_ASYNC_INFLIGHT);
-    p->async = EHCI_ASYNC_NONE;
+    assert(p->async == EHCI_ASYNC_INITIALIZED ||
+           p->async == EHCI_ASYNC_FINISHED);
 
-    DPRINTF("execute_complete: qhaddr 0x%x, next %x, qtdaddr 0x%x, status %d\n",
-            q->qhaddr, q->qh.next, q->qtdaddr, q->usb_status);
+    DPRINTF("execute_complete: qhaddr 0x%x, next 0x%x, qtdaddr 0x%x, "
+            "status %d, actual_length %d\n",
+            q->qhaddr, q->qh.next, q->qtdaddr,
+            p->packet.status, p->packet.actual_length);
 
-    if (p->usb_status < 0) {
-        switch (p->usb_status) {
-        case USB_RET_IOERROR:
-        case USB_RET_NODEV:
-            q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_XACTERR);
-            set_field(&q->qh.token, 0, QTD_TOKEN_CERR);
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        case USB_RET_STALL:
-            q->qh.token |= QTD_TOKEN_HALT;
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        case USB_RET_NAK:
-            set_field(&q->qh.altnext_qtd, 0, QH_ALTNEXT_NAKCNT);
-            return; /* We're not done yet with this transaction */
-        case USB_RET_BABBLE:
-            q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_BABBLE);
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        default:
-            /* should not be triggerable */
-            fprintf(stderr, "USB invalid response %d\n", p->usb_status);
-            assert(0);
-            break;
-        }
-    } else if ((p->usb_status > p->tbytes) && (p->pid == USB_TOKEN_IN)) {
-        p->usb_status = USB_RET_BABBLE;
+    switch (p->packet.status) {
+    case USB_RET_SUCCESS:
+        break;
+    case USB_RET_IOERROR:
+    case USB_RET_NODEV:
+        q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_XACTERR);
+        set_field(&q->qh.token, 0, QTD_TOKEN_CERR);
+        ehci_raise_irq(q->ehci, USBSTS_ERRINT);
+        break;
+    case USB_RET_STALL:
+        q->qh.token |= QTD_TOKEN_HALT;
+        ehci_raise_irq(q->ehci, USBSTS_ERRINT);
+        break;
+    case USB_RET_NAK:
+        set_field(&q->qh.altnext_qtd, 0, QH_ALTNEXT_NAKCNT);
+        return; /* We're not done yet with this transaction */
+    case USB_RET_BABBLE:
         q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_BABBLE);
         ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-    } else {
-        // TODO check 4.12 for splits
+        break;
+    default:
+        /* should not be triggerable */
+        fprintf(stderr, "USB invalid response %d\n", p->packet.status);
+        assert(0);
+        break;
+    }
 
-        if (p->tbytes && p->pid == USB_TOKEN_IN) {
-            p->tbytes -= p->usb_status;
-        } else {
-            p->tbytes = 0;
+    /* TODO check 4.12 for splits */
+    tbytes = get_field(q->qh.token, QTD_TOKEN_TBYTES);
+    if (tbytes && p->pid == USB_TOKEN_IN) {
+        tbytes -= p->packet.actual_length;
+        if (tbytes) {
+            /* 4.15.1.2 must raise int on a short input packet */
+            ehci_raise_irq(q->ehci, USBSTS_INT);
+            if (q->async) {
+                q->ehci->int_req_by_async = true;
+            }
         }
-
-        DPRINTF("updating tbytes to %d\n", p->tbytes);
-        set_field(&q->qh.token, p->tbytes, QTD_TOKEN_TBYTES);
+    } else {
+        tbytes = 0;
     }
-    ehci_finish_transfer(q, p->usb_status);
+    DPRINTF("updating tbytes to %d\n", tbytes);
+    set_field(&q->qh.token, tbytes, QTD_TOKEN_TBYTES);
+
+    ehci_finish_transfer(q, p->packet.actual_length);
     usb_packet_unmap(&p->packet, &p->sgl);
     qemu_sglist_destroy(&p->sgl);
+    p->async = EHCI_ASYNC_NONE;
 
     q->qh.token ^= QTD_TOKEN_DTOGGLE;
     q->qh.token &= ~QTD_TOKEN_ACTIVE;
 
     if (q->qh.token & QTD_TOKEN_IOC) {
         ehci_raise_irq(q->ehci, USBSTS_INT);
+        if (q->async) {
+            q->ehci->int_req_by_async = true;
+        }
     }
 }
 
-// 4.10.3
-
+/* 4.10.3 returns "again" */
 static int ehci_execute(EHCIPacket *p, const char *action)
 {
     USBEndpoint *ep;
-    int ret;
     int endp;
+    bool spd;
+
+    assert(p->async == EHCI_ASYNC_NONE ||
+           p->async == EHCI_ASYNC_INITIALIZED);
 
     if (!(p->qtd.token & QTD_TOKEN_ACTIVE)) {
         fprintf(stderr, "Attempting to execute inactive qtd\n");
-        return USB_RET_PROCERR;
-    }
-
-    p->tbytes = (p->qtd.token & QTD_TOKEN_TBYTES_MASK) >> QTD_TOKEN_TBYTES_SH;
-    if (p->tbytes > BUFF_SIZE) {
-        fprintf(stderr, "Request for more bytes than allowed\n");
-        return USB_RET_PROCERR;
+        return -1;
     }
 
-    p->pid = (p->qtd.token & QTD_TOKEN_PID_MASK) >> QTD_TOKEN_PID_SH;
-    switch (p->pid) {
-    case 0:
-        p->pid = USB_TOKEN_OUT;
-        break;
-    case 1:
-        p->pid = USB_TOKEN_IN;
-        break;
-    case 2:
-        p->pid = USB_TOKEN_SETUP;
-        break;
-    default:
-        fprintf(stderr, "bad token\n");
-        break;
+    if (get_field(p->qtd.token, QTD_TOKEN_TBYTES) > BUFF_SIZE) {
+        ehci_trace_guest_bug(p->queue->ehci,
+                             "guest requested more bytes than allowed");
+        return -1;
     }
 
-    if (ehci_init_transfer(p) != 0) {
-        return USB_RET_PROCERR;
+    if (!ehci_verify_pid(p->queue, &p->qtd)) {
+        ehci_queue_stopped(p->queue); /* Mark the ep in the prev dir stopped */
     }
-
+    p->pid = ehci_get_pid(&p->qtd);
+    p->queue->last_pid = p->pid;
     endp = get_field(p->queue->qh.epchar, QH_EPCHAR_EP);
     ep = usb_ep_get(p->queue->dev, p->pid, endp);
 
-    usb_packet_setup(&p->packet, p->pid, ep, p->qtdaddr);
-    usb_packet_map(&p->packet, &p->sgl);
+    if (p->async == EHCI_ASYNC_NONE) {
+        if (ehci_init_transfer(p) != 0) {
+            return -1;
+        }
+
+        spd = (p->pid == USB_TOKEN_IN && NLPTR_TBIT(p->qtd.altnext) == 0);
+        usb_packet_setup(&p->packet, p->pid, ep, p->qtdaddr, spd,
+                         (p->qtd.token & QTD_TOKEN_IOC) != 0);
+        usb_packet_map(&p->packet, &p->sgl);
+        p->async = EHCI_ASYNC_INITIALIZED;
+    }
 
     trace_usb_ehci_packet_action(p->queue, p, action);
-    ret = usb_handle_packet(p->queue->dev, &p->packet);
-    DPRINTF("submit: qh %x next %x qtd %x pid %x len %zd "
-            "(total %d) endp %x ret %d\n",
-            q->qhaddr, q->qh.next, q->qtdaddr, q->pid,
-            q->packet.iov.size, q->tbytes, endp, ret);
+    usb_handle_packet(p->queue->dev, &p->packet);
+    DPRINTF("submit: qh 0x%x next 0x%x qtd 0x%x pid 0x%x len %zd endp 0x%x "
+            "status %d actual_length %d\n", p->queue->qhaddr, p->qtd.next,
+            p->qtdaddr, p->pid, p->packet.iov.size, endp, p->packet.status,
+            p->packet.actual_length);
 
-    if (ret > BUFF_SIZE) {
+    if (p->packet.actual_length > BUFF_SIZE) {
         fprintf(stderr, "ret from usb_handle_packet > BUFF_SIZE\n");
-        return USB_RET_PROCERR;
+        return -1;
     }
 
-    return ret;
+    return 1;
 }
 
 /*  4.7.2
@@ -1578,10 +1450,11 @@ static int ehci_process_itd(EHCIState *ehci,
 {
     USBDevice *dev;
     USBEndpoint *ep;
-    int ret;
     uint32_t i, len, pid, dir, devaddr, endp;
     uint32_t pg, off, ptr1, ptr2, max, mult;
 
+    ehci->periodic_sched_active = PERIODIC_ACTIVE;
+
     dir =(itd->bufptr[1] & ITD_BUFPTR_DIRECTION);
     devaddr = get_field(itd->bufptr[0], ITD_BUFPTR_DEVADDR);
     endp = get_field(itd->bufptr[0], ITD_BUFPTR_EP);
@@ -1601,10 +1474,10 @@ static int ehci_process_itd(EHCIState *ehci,
             }
 
             if (len > BUFF_SIZE) {
-                return USB_RET_PROCERR;
+                return -1;
             }
 
-            pci_dma_sglist_init(&ehci->isgl, &ehci->dev, 2);
+            qemu_sglist_init(&ehci->isgl, 2, ehci->dma);
             if (off + len > 4096) {
                 /* transfer crosses page border */
                 uint32_t len2 = off + len - 4096;
@@ -1620,48 +1493,48 @@ static int ehci_process_itd(EHCIState *ehci,
             dev = ehci_find_device(ehci, devaddr);
             ep = usb_ep_get(dev, pid, endp);
             if (ep && ep->type == USB_ENDPOINT_XFER_ISOC) {
-                usb_packet_setup(&ehci->ipacket, pid, ep, addr);
+                usb_packet_setup(&ehci->ipacket, pid, ep, addr, false,
+                                 (itd->transact[i] & ITD_XACT_IOC) != 0);
                 usb_packet_map(&ehci->ipacket, &ehci->isgl);
-                ret = usb_handle_packet(dev, &ehci->ipacket);
-                assert(ret != USB_RET_ASYNC);
+                usb_handle_packet(dev, &ehci->ipacket);
                 usb_packet_unmap(&ehci->ipacket, &ehci->isgl);
             } else {
                 DPRINTF("ISOCH: attempt to addess non-iso endpoint\n");
-                ret = USB_RET_NAK;
+                ehci->ipacket.status = USB_RET_NAK;
+                ehci->ipacket.actual_length = 0;
             }
             qemu_sglist_destroy(&ehci->isgl);
 
-            if (ret < 0) {
-                switch (ret) {
-                default:
-                    fprintf(stderr, "Unexpected iso usb result: %d\n", ret);
-                    /* Fall through */
-                case USB_RET_IOERROR:
-                case USB_RET_NODEV:
-                    /* 3.3.2: XACTERR is only allowed on IN transactions */
-                    if (dir) {
-                        itd->transact[i] |= ITD_XACT_XACTERR;
-                        ehci_raise_irq(ehci, USBSTS_ERRINT);
-                    }
-                    break;
-                case USB_RET_BABBLE:
-                    itd->transact[i] |= ITD_XACT_BABBLE;
+            switch (ehci->ipacket.status) {
+            case USB_RET_SUCCESS:
+                break;
+            default:
+                fprintf(stderr, "Unexpected iso usb result: %d\n",
+                        ehci->ipacket.status);
+                /* Fall through */
+            case USB_RET_IOERROR:
+            case USB_RET_NODEV:
+                /* 3.3.2: XACTERR is only allowed on IN transactions */
+                if (dir) {
+                    itd->transact[i] |= ITD_XACT_XACTERR;
                     ehci_raise_irq(ehci, USBSTS_ERRINT);
-                    break;
-                case USB_RET_NAK:
-                    /* no data for us, so do a zero-length transfer */
-                    ret = 0;
-                    break;
                 }
+                break;
+            case USB_RET_BABBLE:
+                itd->transact[i] |= ITD_XACT_BABBLE;
+                ehci_raise_irq(ehci, USBSTS_ERRINT);
+                break;
+            case USB_RET_NAK:
+                /* no data for us, so do a zero-length transfer */
+                ehci->ipacket.actual_length = 0;
+                break;
             }
-            if (ret >= 0) {
-                if (!dir) {
-                    /* OUT */
-                    set_field(&itd->transact[i], len - ret, ITD_XACT_LENGTH);
-                } else {
-                    /* IN */
-                    set_field(&itd->transact[i], ret, ITD_XACT_LENGTH);
-                }
+            if (!dir) {
+                set_field(&itd->transact[i], len - ehci->ipacket.actual_length,
+                          ITD_XACT_LENGTH); /* OUT */
+            } else {
+                set_field(&itd->transact[i], ehci->ipacket.actual_length,
+                          ITD_XACT_LENGTH); /* IN */
             }
             if (itd->transact[i] & ITD_XACT_IOC) {
                 ehci_raise_irq(ehci, USBSTS_INT);
@@ -1692,8 +1565,10 @@ static int ehci_state_waitlisthead(EHCIState *ehci,  int async)
 
     /*  Find the head of the list (4.9.1.1) */
     for(i = 0; i < MAX_QH; i++) {
-        get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &qh,
-                   sizeof(EHCIqh) >> 2);
+        if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &qh,
+                       sizeof(EHCIqh) >> 2) < 0) {
+            return 0;
+        }
         ehci_trace_qh(NULL, NLPTR_GET(entry), &qh);
 
         if (qh.epchar & QH_EPCHAR_H) {
@@ -1770,8 +1645,7 @@ out:
 
 static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
 {
-    EHCIPacket *p;
-    uint32_t entry, devaddr;
+    uint32_t entry;
     EHCIQueue *q;
     EHCIqh qh;
 
@@ -1780,7 +1654,6 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
     if (NULL == q) {
         q = ehci_alloc_queue(ehci, entry, async);
     }
-    p = QTAILQ_FIRST(&q->packets);
 
     q->seen++;
     if (q->seen > 1) {
@@ -1790,37 +1663,32 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
         goto out;
     }
 
-    get_dwords(ehci, NLPTR_GET(q->qhaddr),
-               (uint32_t *) &qh, sizeof(EHCIqh) >> 2);
-    if (q->revalidate && (q->qh.epchar      != qh.epchar ||
-                          q->qh.epcap       != qh.epcap  ||
-                          q->qh.current_qtd != qh.current_qtd)) {
-        ehci_free_queue(q);
-        q = ehci_alloc_queue(ehci, entry, async);
-        q->seen++;
-        p = NULL;
+    if (get_dwords(ehci, NLPTR_GET(q->qhaddr),
+                   (uint32_t *) &qh, sizeof(EHCIqh) >> 2) < 0) {
+        q = NULL;
+        goto out;
     }
-    q->qh = qh;
-    q->revalidate = 0;
-    ehci_trace_qh(q, NLPTR_GET(q->qhaddr), &q->qh);
+    ehci_trace_qh(q, NLPTR_GET(q->qhaddr), &qh);
 
-    devaddr = get_field(q->qh.epchar, QH_EPCHAR_DEVADDR);
-    if (q->dev != NULL && q->dev->addr != devaddr) {
-        if (!QTAILQ_EMPTY(&q->packets)) {
-            /* should not happen (guest bug) */
-            ehci_cancel_queue(q);
+    /*
+     * The overlay area of the qh should never be changed by the guest,
+     * except when idle, in which case the reset is a nop.
+     */
+    if (!ehci_verify_qh(q, &qh)) {
+        if (ehci_reset_queue(q) > 0) {
+            ehci_trace_guest_bug(ehci, "guest updated active QH");
         }
-        q->dev = NULL;
     }
-    if (q->dev == NULL) {
-        q->dev = ehci_find_device(q->ehci, devaddr);
+    q->qh = qh;
+
+    q->transact_ctr = get_field(q->qh.epcap, QH_EPCAP_MULT);
+    if (q->transact_ctr == 0) { /* Guest bug in some versions of windows */
+        q->transact_ctr = 4;
     }
 
-    if (p && p->async == EHCI_ASYNC_FINISHED) {
-        /* I/O finished -- continue processing queue */
-        trace_usb_ehci_packet_action(p->queue, p, "complete");
-        ehci_set_state(ehci, async, EST_EXECUTING);
-        goto out;
+    if (q->dev == NULL) {
+        q->dev = ehci_find_device(q->ehci,
+                                  get_field(q->qh.epchar, QH_EPCHAR_DEVADDR));
     }
 
     if (async && (q->qh.epchar & QH_EPCHAR_H)) {
@@ -1873,8 +1741,10 @@ static int ehci_state_fetchitd(EHCIState *ehci, int async)
     assert(!async);
     entry = ehci_get_fetch_addr(ehci, async);
 
-    get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &itd,
-               sizeof(EHCIitd) >> 2);
+    if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &itd,
+                   sizeof(EHCIitd) >> 2) < 0) {
+        return -1;
+    }
     ehci_trace_itd(ehci, entry, &itd);
 
     if (ehci_process_itd(ehci, &itd, entry) != 0) {
@@ -1897,8 +1767,10 @@ static int ehci_state_fetchsitd(EHCIState *ehci, int async)
     assert(!async);
     entry = ehci_get_fetch_addr(ehci, async);
 
-    get_dwords(ehci, NLPTR_GET(entry), (uint32_t *)&sitd,
-               sizeof(EHCIsitd) >> 2);
+    if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *)&sitd,
+                   sizeof(EHCIsitd) >> 2) < 0) {
+        return 0;
+    }
     ehci_trace_sitd(ehci, entry, &sitd);
 
     if (!(sitd.results & SITD_RESULTS_ACTIVE)) {
@@ -1957,20 +1829,21 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
 {
     EHCIqtd qtd;
     EHCIPacket *p;
-    int again = 0;
+    int again = 1;
 
-    get_dwords(q->ehci, NLPTR_GET(q->qtdaddr), (uint32_t *) &qtd,
-               sizeof(EHCIqtd) >> 2);
+    if (get_dwords(q->ehci, NLPTR_GET(q->qtdaddr), (uint32_t *) &qtd,
+                   sizeof(EHCIqtd) >> 2) < 0) {
+        return 0;
+    }
     ehci_trace_qtd(q, NLPTR_GET(q->qtdaddr), &qtd);
 
     p = QTAILQ_FIRST(&q->packets);
     if (p != NULL) {
-        if (p->qtdaddr != q->qtdaddr ||
-            (!NLPTR_TBIT(p->qtd.next) && (p->qtd.next != qtd.next)) ||
-            (!NLPTR_TBIT(p->qtd.altnext) && (p->qtd.altnext != qtd.altnext)) ||
-            p->qtd.bufptr[0] != qtd.bufptr[0]) {
-            /* guest bug: guest updated active QH or qTD underneath us */
+        if (!ehci_verify_qtd(p, &qtd)) {
             ehci_cancel_queue(q);
+            if (qtd.token & QTD_TOKEN_ACTIVE) {
+                ehci_trace_guest_bug(q->ehci, "guest updated active qTD");
+            }
             p = NULL;
         } else {
             p->qtd = qtd;
@@ -1979,35 +1852,30 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
     }
 
     if (!(qtd.token & QTD_TOKEN_ACTIVE)) {
-        if (p != NULL) {
-            /* transfer canceled by guest (clear active) */
-            ehci_cancel_queue(q);
-            p = NULL;
-        }
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-        again = 1;
     } else if (p != NULL) {
         switch (p->async) {
         case EHCI_ASYNC_NONE:
-            /* Previously nacked packet (likely interrupt ep) */
-           ehci_set_state(q->ehci, q->async, EST_EXECUTE);
-           break;
+        case EHCI_ASYNC_INITIALIZED:
+            /* Not yet executed (MULT), or previously nacked (int) packet */
+            ehci_set_state(q->ehci, q->async, EST_EXECUTE);
+            break;
         case EHCI_ASYNC_INFLIGHT:
-            /* Unfinyshed async handled packet, go horizontal */
+            /* Check if the guest has added new tds to the queue */
+            again = ehci_fill_queue(QTAILQ_LAST(&q->packets, pkts_head));
+            /* Unfinished async handled packet, go horizontal */
             ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
             break;
         case EHCI_ASYNC_FINISHED:
-            /* Should never happen, as this case is caught by fetchqh */
+            /* Complete executing of the packet */
             ehci_set_state(q->ehci, q->async, EST_EXECUTING);
             break;
         }
-        again = 1;
     } else {
         p = ehci_alloc_packet(q);
         p->qtdaddr = q->qtdaddr;
         p->qtd = qtd;
         ehci_set_state(q->ehci, q->async, EST_EXECUTE);
-        again = 1;
     }
 
     return again;
@@ -2028,33 +1896,52 @@ static int ehci_state_horizqh(EHCIQueue *q)
     return again;
 }
 
-static void ehci_fill_queue(EHCIPacket *p)
+/* Returns "again" */
+static int ehci_fill_queue(EHCIPacket *p)
 {
+    USBEndpoint *ep = p->packet.ep;
     EHCIQueue *q = p->queue;
     EHCIqtd qtd = p->qtd;
     uint32_t qtdaddr;
 
     for (;;) {
-        if (NLPTR_TBIT(qtd.altnext) == 0) {
-            break;
-        }
         if (NLPTR_TBIT(qtd.next) != 0) {
             break;
         }
         qtdaddr = qtd.next;
-        get_dwords(q->ehci, NLPTR_GET(qtdaddr),
-                   (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2);
+        /*
+         * Detect circular td lists, Windows creates these, counting on the
+         * active bit going low after execution to make the queue stop.
+         */
+        QTAILQ_FOREACH(p, &q->packets, next) {
+            if (p->qtdaddr == qtdaddr) {
+                goto leave;
+            }
+        }
+        if (get_dwords(q->ehci, NLPTR_GET(qtdaddr),
+                       (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2) < 0) {
+            return -1;
+        }
         ehci_trace_qtd(q, NLPTR_GET(qtdaddr), &qtd);
         if (!(qtd.token & QTD_TOKEN_ACTIVE)) {
             break;
         }
+        if (!ehci_verify_pid(q, &qtd)) {
+            ehci_trace_guest_bug(q->ehci, "guest queued token with wrong pid");
+            break;
+        }
         p = ehci_alloc_packet(q);
         p->qtdaddr = qtdaddr;
         p->qtd = qtd;
-        p->usb_status = ehci_execute(p, "queue");
-        assert(p->usb_status == USB_RET_ASYNC);
+        if (ehci_execute(p, "queue") == -1) {
+            return -1;
+        }
+        assert(p->packet.status == USB_RET_ASYNC);
         p->async = EHCI_ASYNC_INFLIGHT;
     }
+leave:
+    usb_device_flush_ep_queue(ep->dev, ep);
+    return 1;
 }
 
 static int ehci_state_execute(EHCIQueue *q)
@@ -2071,33 +1958,32 @@ static int ehci_state_execute(EHCIQueue *q)
 
     // TODO verify enough time remains in the uframe as in 4.4.1.1
     // TODO write back ptr to async list when done or out of time
-    // TODO Windows does not seem to ever set the MULT field
 
-    if (!q->async) {
-        int transactCtr = get_field(q->qh.epcap, QH_EPCAP_MULT);
-        if (!transactCtr) {
-            ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-            again = 1;
-            goto out;
-        }
+    /* 4.10.3, bottom of page 82, go horizontal on transaction counter == 0 */
+    if (!q->async && q->transact_ctr == 0) {
+        ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
+        again = 1;
+        goto out;
     }
 
     if (q->async) {
         ehci_set_usbsts(q->ehci, USBSTS_REC);
     }
 
-    p->usb_status = ehci_execute(p, "process");
-    if (p->usb_status == USB_RET_PROCERR) {
-        again = -1;
+    again = ehci_execute(p, "process");
+    if (again == -1) {
         goto out;
     }
-    if (p->usb_status == USB_RET_ASYNC) {
+    if (p->packet.status == USB_RET_ASYNC) {
         ehci_flush_qh(q);
         trace_usb_ehci_packet_action(p->queue, p, "async");
         p->async = EHCI_ASYNC_INFLIGHT;
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-        again = 1;
-        ehci_fill_queue(p);
+        if (q->async) {
+            again = ehci_fill_queue(p);
+        } else {
+            again = 1;
+        }
         goto out;
     }
 
@@ -2117,17 +2003,13 @@ static int ehci_state_executing(EHCIQueue *q)
 
     ehci_execute_complete(q);
 
-    // 4.10.3
-    if (!q->async) {
-        int transactCtr = get_field(q->qh.epcap, QH_EPCAP_MULT);
-        transactCtr--;
-        set_field(&q->qh.epcap, transactCtr, QH_EPCAP_MULT);
-        // 4.10.3, bottom of page 82, should exit this state when transaction
-        // counter decrements to 0
+    /* 4.10.3 */
+    if (!q->async && q->transact_ctr > 0) {
+        q->transact_ctr--;
     }
 
     /* 4.10.5 */
-    if (p->usb_status == USB_RET_NAK) {
+    if (p->packet.status == USB_RET_NAK) {
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
     } else {
         ehci_set_state(q->ehci, q->async, EST_WRITEBACK);
@@ -2163,19 +2045,6 @@ static int ehci_state_writeback(EHCIQueue *q)
      * bit is clear.
      */
     if (q->qh.token & QTD_TOKEN_HALT) {
-        /*
-         * We should not do any further processing on a halted queue!
-         * This is esp. important for bulk endpoints with pipelining enabled
-         * (redirection to a real USB device), where we must cancel all the
-         * transfers after this one so that:
-         * 1) If they've completed already, they are not processed further
-         *    causing more stalls, originating from the same failed transfer
-         * 2) If still in flight, they are cancelled before the guest does
-         *    a clear stall, otherwise the guest and device can loose sync!
-         */
-        while ((p = QTAILQ_FIRST(&q->packets)) != NULL) {
-            ehci_free_packet(p);
-        }
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
         again = 1;
     } else {
@@ -2223,18 +2092,22 @@ static void ehci_advance_state(EHCIState *ehci, int async)
             break;
 
         case EST_ADVANCEQUEUE:
+            assert(q != NULL);
             again = ehci_state_advqueue(q);
             break;
 
         case EST_FETCHQTD:
+            assert(q != NULL);
             again = ehci_state_fetchqtd(q);
             break;
 
         case EST_HORIZONTALQH:
+            assert(q != NULL);
             again = ehci_state_horizqh(q);
             break;
 
         case EST_EXECUTE:
+            assert(q != NULL);
             again = ehci_state_execute(q);
             if (async) {
                 ehci->async_stepdown = 0;
@@ -2252,6 +2125,9 @@ static void ehci_advance_state(EHCIState *ehci, int async)
         case EST_WRITEBACK:
             assert(q != NULL);
             again = ehci_state_writeback(q);
+            if (!async) {
+                ehci->periodic_sched_active = PERIODIC_ACTIVE;
+            }
             break;
 
         default:
@@ -2310,8 +2186,8 @@ static void ehci_advance_async_state(EHCIState *ehci)
          */
         if (ehci->usbcmd & USBCMD_IAAD) {
             /* Remove all unseen qhs from the async qhs queue */
-            ehci_queues_tag_unused_async(ehci);
-            DPRINTF("ASYNC: doorbell request acknowledged\n");
+            ehci_queues_rip_unseen(ehci, async);
+            trace_usb_ehci_doorbell_ack();
             ehci->usbcmd &= ~USBCMD_IAAD;
             ehci_raise_irq(ehci, USBSTS_IAA);
         }
@@ -2355,8 +2231,9 @@ static void ehci_advance_periodic_state(EHCIState *ehci)
         }
         list |= ((ehci->frindex & 0x1ff8) >> 1);
 
-        pci_dma_read(&ehci->dev, list, &entry, sizeof entry);
-        entry = le32_to_cpu(entry);
+        if (get_dwords(ehci, list, &entry, 1) < 0) {
+            break;
+        }
 
         DPRINTF("PERIODIC state adv fr=%d.  [%08X] -> %08X\n",
                 ehci->frindex / 8, list, entry);
@@ -2374,16 +2251,16 @@ static void ehci_advance_periodic_state(EHCIState *ehci)
     }
 }
 
-static void ehci_update_frindex(EHCIState *ehci, int frames)
+static void ehci_update_frindex(EHCIState *ehci, int uframes)
 {
     int i;
 
-    if (!ehci_enabled(ehci)) {
+    if (!ehci_enabled(ehci) && ehci->pstate == EST_INACTIVE) {
         return;
     }
 
-    for (i = 0; i < frames; i++) {
-        ehci->frindex += 8;
+    for (i = 0; i < uframes; i++) {
+        ehci->frindex++;
 
         if (ehci->frindex == 0x00002000) {
             ehci_raise_irq(ehci, USBSTS_FLR);
@@ -2392,7 +2269,7 @@ static void ehci_update_frindex(EHCIState *ehci, int frames)
         if (ehci->frindex == 0x00004000) {
             ehci_raise_irq(ehci, USBSTS_FLR);
             ehci->frindex = 0;
-            if (ehci->usbsts_frindex > 0x00004000) {
+            if (ehci->usbsts_frindex >= 0x00004000) {
                 ehci->usbsts_frindex -= 0x00004000;
             } else {
                 ehci->usbsts_frindex = 0;
@@ -2407,36 +2284,57 @@ static void ehci_frame_timer(void *opaque)
     int need_timer = 0;
     int64_t expire_time, t_now;
     uint64_t ns_elapsed;
-    int frames, skipped_frames;
+    int uframes, skipped_uframes;
     int i;
 
     t_now = qemu_get_clock_ns(vm_clock);
     ns_elapsed = t_now - ehci->last_run_ns;
-    frames = ns_elapsed / FRAME_TIMER_NS;
+    uframes = ns_elapsed / UFRAME_TIMER_NS;
 
     if (ehci_periodic_enabled(ehci) || ehci->pstate != EST_INACTIVE) {
         need_timer++;
-        ehci->async_stepdown = 0;
 
-        if (frames > ehci->maxframes) {
-            skipped_frames = frames - ehci->maxframes;
-            ehci_update_frindex(ehci, skipped_frames);
-            ehci->last_run_ns += FRAME_TIMER_NS * skipped_frames;
-            frames -= skipped_frames;
-            DPRINTF("WARNING - EHCI skipped %d frames\n", skipped_frames);
+        if (uframes > (ehci->maxframes * 8)) {
+            skipped_uframes = uframes - (ehci->maxframes * 8);
+            ehci_update_frindex(ehci, skipped_uframes);
+            ehci->last_run_ns += UFRAME_TIMER_NS * skipped_uframes;
+            uframes -= skipped_uframes;
+            DPRINTF("WARNING - EHCI skipped %d uframes\n", skipped_uframes);
         }
 
-        for (i = 0; i < frames; i++) {
+        for (i = 0; i < uframes; i++) {
+            /*
+             * If we're running behind schedule, we should not catch up
+             * too fast, as that will make some guests unhappy:
+             * 1) We must process a minimum of MIN_UFR_PER_TICK frames,
+             *    otherwise we will never catch up
+             * 2) Process frames until the guest has requested an irq (IOC)
+             */
+            if (i >= MIN_UFR_PER_TICK) {
+                ehci_commit_irq(ehci);
+                if ((ehci->usbsts & USBINTR_MASK) & ehci->usbintr) {
+                    break;
+                }
+            }
+            if (ehci->periodic_sched_active) {
+                ehci->periodic_sched_active--;
+            }
             ehci_update_frindex(ehci, 1);
-            ehci_advance_periodic_state(ehci);
-            ehci->last_run_ns += FRAME_TIMER_NS;
+            if ((ehci->frindex & 7) == 0) {
+                ehci_advance_periodic_state(ehci);
+            }
+            ehci->last_run_ns += UFRAME_TIMER_NS;
         }
     } else {
-        if (ehci->async_stepdown < ehci->maxframes / 2) {
-            ehci->async_stepdown++;
-        }
-        ehci_update_frindex(ehci, frames);
-        ehci->last_run_ns += FRAME_TIMER_NS * frames;
+        ehci->periodic_sched_active = 0;
+        ehci_update_frindex(ehci, uframes);
+        ehci->last_run_ns += UFRAME_TIMER_NS * uframes;
+    }
+
+    if (ehci->periodic_sched_active) {
+        ehci->async_stepdown = 0;
+    } else if (ehci->async_stepdown < ehci->maxframes / 2) {
+        ehci->async_stepdown++;
     }
 
     /*  Async is not inside loop since it executes everything it can once
@@ -2453,28 +2351,48 @@ static void ehci_frame_timer(void *opaque)
         ehci->async_stepdown = 0;
     }
 
+    if (ehci_enabled(ehci) && (ehci->usbintr & USBSTS_FLR)) {
+        need_timer++;
+    }
+
     if (need_timer) {
-        expire_time = t_now + (get_ticks_per_sec()
+        /* If we've raised int, we speed up the timer, so that we quickly
+         * notice any new packets queued up in response */
+        if (ehci->int_req_by_async && (ehci->usbsts & USBSTS_INT)) {
+            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 4);
+            ehci->int_req_by_async = false;
+        } else {
+            expire_time = t_now + (get_ticks_per_sec()
                                * (ehci->async_stepdown+1) / FRAME_TIMER_FREQ);
+        }
         qemu_mod_timer(ehci->frame_timer, expire_time);
     }
 }
 
-static void ehci_async_bh(void *opaque)
-{
-    EHCIState *ehci = opaque;
-    ehci_advance_async_state(ehci);
-}
+static const MemoryRegionOps ehci_mmio_caps_ops = {
+    .read = ehci_caps_read,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static const MemoryRegionOps ehci_mem_ops = {
-    .old_mmio = {
-        .read = { ehci_mem_readb, ehci_mem_readw, ehci_mem_readl },
-        .write = { ehci_mem_writeb, ehci_mem_writew, ehci_mem_writel },
-    },
+static const MemoryRegionOps ehci_mmio_opreg_ops = {
+    .read = ehci_opreg_read,
+    .write = ehci_opreg_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int usb_ehci_initfn(PCIDevice *dev);
+static const MemoryRegionOps ehci_mmio_port_ops = {
+    .read = ehci_port_read,
+    .write = ehci_port_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
 static USBPortOps ehci_port_ops = {
     .attach = ehci_attach,
@@ -2486,8 +2404,20 @@ static USBPortOps ehci_port_ops = {
 
 static USBBusOps ehci_bus_ops = {
     .register_companion = ehci_register_companion,
+    .wakeup_endpoint = ehci_wakeup_endpoint,
 };
 
+static void usb_ehci_pre_save(void *opaque)
+{
+    EHCIState *ehci = opaque;
+    uint32_t new_frindex;
+
+    /* Round down frindex to a multiple of 8 for migration compatibility */
+    new_frindex = ehci->frindex & ~7;
+    ehci->last_run_ns -= (ehci->frindex - new_frindex) * UFRAME_TIMER_NS;
+    ehci->frindex = new_frindex;
+}
+
 static int usb_ehci_post_load(void *opaque, int version_id)
 {
     EHCIState *s = opaque;
@@ -2508,13 +2438,39 @@ static int usb_ehci_post_load(void *opaque, int version_id)
     return 0;
 }
 
-static const VMStateDescription vmstate_ehci = {
-    .name        = "ehci",
+static void usb_ehci_vm_state_change(void *opaque, int running, RunState state)
+{
+    EHCIState *ehci = opaque;
+
+    /*
+     * We don't migrate the EHCIQueue-s, instead we rebuild them for the
+     * schedule in guest memory. We must do the rebuilt ASAP, so that
+     * USB-devices which have async handled packages have a packet in the
+     * ep queue to match the completion with.
+     */
+    if (state == RUN_STATE_RUNNING) {
+        ehci_advance_async_state(ehci);
+    }
+
+    /*
+     * The schedule rebuilt from guest memory could cause the migration dest
+     * to miss a QH unlink, and fail to cancel packets, since the unlinked QH
+     * will never have existed on the destination. Therefor we must flush the
+     * async schedule on savevm to catch any not yet noticed unlinks.
+     */
+    if (state == RUN_STATE_SAVE_VM) {
+        ehci_advance_async_state(ehci);
+        ehci_queues_rip_unseen(ehci, 1);
+    }
+}
+
+const VMStateDescription vmstate_ehci = {
+    .name        = "ehci-core",
     .version_id  = 2,
     .minimum_version_id  = 1,
+    .pre_save    = usb_ehci_pre_save,
     .post_load   = usb_ehci_post_load,
     .fields      = (VMStateField[]) {
-        VMSTATE_PCI_DEVICE(dev, EHCIState),
         /* mmio registers */
         VMSTATE_UINT32(usbcmd, EHCIState),
         VMSTATE_UINT32(usbsts, EHCIState),
@@ -2545,105 +2501,24 @@ static const VMStateDescription vmstate_ehci = {
     }
 };
 
-static Property ehci_properties[] = {
-    DEFINE_PROP_UINT32("maxframes", EHCIState, maxframes, 128),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void ehci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_ehci_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801D; /* ich4 */
-    k->revision = 0x10;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_ehci;
-    dc->props = ehci_properties;
-}
-
-static TypeInfo ehci_info = {
-    .name          = "usb-ehci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(EHCIState),
-    .class_init    = ehci_class_init,
-};
-
-static void ich9_ehci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_ehci_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI1;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_ehci;
-    dc->props = ehci_properties;
-}
-
-static TypeInfo ich9_ehci_info = {
-    .name          = "ich9-usb-ehci1",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(EHCIState),
-    .class_init    = ich9_ehci_class_init,
-};
-
-static int usb_ehci_initfn(PCIDevice *dev)
+void usb_ehci_initfn(EHCIState *s, DeviceState *dev)
 {
-    EHCIState *s = DO_UPCAST(EHCIState, dev, dev);
-    uint8_t *pci_conf = s->dev.config;
     int i;
 
-    pci_set_byte(&pci_conf[PCI_CLASS_PROG], 0x20);
-
-    /* capabilities pointer */
-    pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x00);
-    //pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x50);
-
-    pci_set_byte(&pci_conf[PCI_INTERRUPT_PIN], 4); /* interrupt pin D */
-    pci_set_byte(&pci_conf[PCI_MIN_GNT], 0);
-    pci_set_byte(&pci_conf[PCI_MAX_LAT], 0);
-
-    // pci_conf[0x50] = 0x01; // power management caps
-
-    pci_set_byte(&pci_conf[USB_SBRN], USB_RELEASE_2); // release number (2.1.4)
-    pci_set_byte(&pci_conf[0x61], 0x20);  // frame length adjustment (2.1.5)
-    pci_set_word(&pci_conf[0x62], 0x00);  // port wake up capability (2.1.6)
-
-    pci_conf[0x64] = 0x00;
-    pci_conf[0x65] = 0x00;
-    pci_conf[0x66] = 0x00;
-    pci_conf[0x67] = 0x00;
-    pci_conf[0x68] = 0x01;
-    pci_conf[0x69] = 0x00;
-    pci_conf[0x6a] = 0x00;
-    pci_conf[0x6b] = 0x00;  // USBLEGSUP
-    pci_conf[0x6c] = 0x00;
-    pci_conf[0x6d] = 0x00;
-    pci_conf[0x6e] = 0x00;
-    pci_conf[0x6f] = 0xc0;  // USBLEFCTLSTS
-
-    // 2.2 host controller interface version
-    s->mmio[0x00] = (uint8_t) OPREGBASE;
-    s->mmio[0x01] = 0x00;
-    s->mmio[0x02] = 0x00;
-    s->mmio[0x03] = 0x01;        // HC version
-    s->mmio[0x04] = NB_PORTS;    // Number of downstream ports
-    s->mmio[0x05] = 0x00;        // No companion ports at present
-    s->mmio[0x06] = 0x00;
-    s->mmio[0x07] = 0x00;
-    s->mmio[0x08] = 0x80;        // We can cache whole frame, not 64-bit capable
-    s->mmio[0x09] = 0x68;        // EECP
-    s->mmio[0x0a] = 0x00;
-    s->mmio[0x0b] = 0x00;
-
-    s->irq = s->dev.irq[3];
-
-    usb_bus_new(&s->bus, &ehci_bus_ops, &s->dev.qdev);
+    /* 2.2 host controller interface version */
+    s->caps[0x00] = (uint8_t)(s->opregbase - s->capsbase);
+    s->caps[0x01] = 0x00;
+    s->caps[0x02] = 0x00;
+    s->caps[0x03] = 0x01;        /* HC version */
+    s->caps[0x04] = NB_PORTS;    /* Number of downstream ports */
+    s->caps[0x05] = 0x00;        /* No companion ports at present */
+    s->caps[0x06] = 0x00;
+    s->caps[0x07] = 0x00;
+    s->caps[0x08] = 0x80;        /* We can cache whole frame, no 64-bit */
+    s->caps[0x0a] = 0x00;
+    s->caps[0x0b] = 0x00;
+
+    usb_bus_new(&s->bus, &ehci_bus_ops, dev);
     for(i = 0; i < NB_PORTS; i++) {
         usb_register_port(&s->bus, &s->ports[i], s, i, &ehci_port_ops,
                           USB_SPEED_MASK_HIGH);
@@ -2651,27 +2526,28 @@ static int usb_ehci_initfn(PCIDevice *dev)
     }
 
     s->frame_timer = qemu_new_timer_ns(vm_clock, ehci_frame_timer, s);
-    s->async_bh = qemu_bh_new(ehci_async_bh, s);
+    s->async_bh = qemu_bh_new(ehci_frame_timer, s);
     QTAILQ_INIT(&s->aqueues);
     QTAILQ_INIT(&s->pqueues);
     usb_packet_init(&s->ipacket);
 
     qemu_register_reset(ehci_reset, s);
+    qemu_add_vm_change_state_handler(usb_ehci_vm_state_change, s);
 
-    memory_region_init_io(&s->mem, &ehci_mem_ops, s, "ehci", MMIO_SIZE);
-    pci_register_bar(&s->dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mem);
-
-    return 0;
-}
+    memory_region_init(&s->mem, "ehci", MMIO_SIZE);
+    memory_region_init_io(&s->mem_caps, &ehci_mmio_caps_ops, s,
+                          "capabilities", CAPA_SIZE);
+    memory_region_init_io(&s->mem_opreg, &ehci_mmio_opreg_ops, s,
+                          "operational", PORTSC_BEGIN);
+    memory_region_init_io(&s->mem_ports, &ehci_mmio_port_ops, s,
+                          "ports", PORTSC_END - PORTSC_BEGIN);
 
-static void ehci_register_types(void)
-{
-    type_register_static(&ehci_info);
-    type_register_static(&ich9_ehci_info);
+    memory_region_add_subregion(&s->mem, s->capsbase, &s->mem_caps);
+    memory_region_add_subregion(&s->mem, s->opregbase, &s->mem_opreg);
+    memory_region_add_subregion(&s->mem, s->opregbase + PORTSC_BEGIN,
+                                &s->mem_ports);
 }
 
-type_init(ehci_register_types)
-
 /*
  * vim: expandtab ts=4
  */
diff --git a/hw/usb/hcd-ehci.h b/hw/usb/hcd-ehci.h
new file mode 100644
index 00000000000..e95bb7ec46e
--- /dev/null
+++ b/hw/usb/hcd-ehci.h
@@ -0,0 +1,366 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef HW_USB_EHCI_H
+#define HW_USB_EHCI_H 1
+
+#include "hw/hw.h"
+#include "qemu/timer.h"
+#include "hw/usb.h"
+#include "monitor/monitor.h"
+#include "trace.h"
+#include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/sysbus.h"
+
+#ifndef EHCI_DEBUG
+#define EHCI_DEBUG   0
+#endif
+
+#if EHCI_DEBUG
+#define DPRINTF printf
+#else
+#define DPRINTF(...)
+#endif
+
+#define MMIO_SIZE        0x1000
+#define CAPA_SIZE        0x10
+
+#define PORTSC               0x0044
+#define PORTSC_BEGIN         PORTSC
+#define PORTSC_END           (PORTSC + 4 * NB_PORTS)
+
+#define NB_PORTS         6        /* Number of downstream ports */
+
+typedef struct EHCIPacket EHCIPacket;
+typedef struct EHCIQueue EHCIQueue;
+typedef struct EHCIState EHCIState;
+
+/*  EHCI spec version 1.0 Section 3.3
+ */
+typedef struct EHCIitd {
+    uint32_t next;
+
+    uint32_t transact[8];
+#define ITD_XACT_ACTIVE          (1 << 31)
+#define ITD_XACT_DBERROR         (1 << 30)
+#define ITD_XACT_BABBLE          (1 << 29)
+#define ITD_XACT_XACTERR         (1 << 28)
+#define ITD_XACT_LENGTH_MASK     0x0fff0000
+#define ITD_XACT_LENGTH_SH       16
+#define ITD_XACT_IOC             (1 << 15)
+#define ITD_XACT_PGSEL_MASK      0x00007000
+#define ITD_XACT_PGSEL_SH        12
+#define ITD_XACT_OFFSET_MASK     0x00000fff
+
+    uint32_t bufptr[7];
+#define ITD_BUFPTR_MASK          0xfffff000
+#define ITD_BUFPTR_SH            12
+#define ITD_BUFPTR_EP_MASK       0x00000f00
+#define ITD_BUFPTR_EP_SH         8
+#define ITD_BUFPTR_DEVADDR_MASK  0x0000007f
+#define ITD_BUFPTR_DEVADDR_SH    0
+#define ITD_BUFPTR_DIRECTION     (1 << 11)
+#define ITD_BUFPTR_MAXPKT_MASK   0x000007ff
+#define ITD_BUFPTR_MAXPKT_SH     0
+#define ITD_BUFPTR_MULT_MASK     0x00000003
+#define ITD_BUFPTR_MULT_SH       0
+} EHCIitd;
+
+/*  EHCI spec version 1.0 Section 3.4
+ */
+typedef struct EHCIsitd {
+    uint32_t next;                  /* Standard next link pointer */
+    uint32_t epchar;
+#define SITD_EPCHAR_IO              (1 << 31)
+#define SITD_EPCHAR_PORTNUM_MASK    0x7f000000
+#define SITD_EPCHAR_PORTNUM_SH      24
+#define SITD_EPCHAR_HUBADD_MASK     0x007f0000
+#define SITD_EPCHAR_HUBADDR_SH      16
+#define SITD_EPCHAR_EPNUM_MASK      0x00000f00
+#define SITD_EPCHAR_EPNUM_SH        8
+#define SITD_EPCHAR_DEVADDR_MASK    0x0000007f
+
+    uint32_t uframe;
+#define SITD_UFRAME_CMASK_MASK      0x0000ff00
+#define SITD_UFRAME_CMASK_SH        8
+#define SITD_UFRAME_SMASK_MASK      0x000000ff
+
+    uint32_t results;
+#define SITD_RESULTS_IOC              (1 << 31)
+#define SITD_RESULTS_PGSEL            (1 << 30)
+#define SITD_RESULTS_TBYTES_MASK      0x03ff0000
+#define SITD_RESULTS_TYBYTES_SH       16
+#define SITD_RESULTS_CPROGMASK_MASK   0x0000ff00
+#define SITD_RESULTS_CPROGMASK_SH     8
+#define SITD_RESULTS_ACTIVE           (1 << 7)
+#define SITD_RESULTS_ERR              (1 << 6)
+#define SITD_RESULTS_DBERR            (1 << 5)
+#define SITD_RESULTS_BABBLE           (1 << 4)
+#define SITD_RESULTS_XACTERR          (1 << 3)
+#define SITD_RESULTS_MISSEDUF         (1 << 2)
+#define SITD_RESULTS_SPLITXSTATE      (1 << 1)
+
+    uint32_t bufptr[2];
+#define SITD_BUFPTR_MASK              0xfffff000
+#define SITD_BUFPTR_CURROFF_MASK      0x00000fff
+#define SITD_BUFPTR_TPOS_MASK         0x00000018
+#define SITD_BUFPTR_TPOS_SH           3
+#define SITD_BUFPTR_TCNT_MASK         0x00000007
+
+    uint32_t backptr;                 /* Standard next link pointer */
+} EHCIsitd;
+
+/*  EHCI spec version 1.0 Section 3.5
+ */
+typedef struct EHCIqtd {
+    uint32_t next;                    /* Standard next link pointer */
+    uint32_t altnext;                 /* Standard next link pointer */
+    uint32_t token;
+#define QTD_TOKEN_DTOGGLE             (1 << 31)
+#define QTD_TOKEN_TBYTES_MASK         0x7fff0000
+#define QTD_TOKEN_TBYTES_SH           16
+#define QTD_TOKEN_IOC                 (1 << 15)
+#define QTD_TOKEN_CPAGE_MASK          0x00007000
+#define QTD_TOKEN_CPAGE_SH            12
+#define QTD_TOKEN_CERR_MASK           0x00000c00
+#define QTD_TOKEN_CERR_SH             10
+#define QTD_TOKEN_PID_MASK            0x00000300
+#define QTD_TOKEN_PID_SH              8
+#define QTD_TOKEN_ACTIVE              (1 << 7)
+#define QTD_TOKEN_HALT                (1 << 6)
+#define QTD_TOKEN_DBERR               (1 << 5)
+#define QTD_TOKEN_BABBLE              (1 << 4)
+#define QTD_TOKEN_XACTERR             (1 << 3)
+#define QTD_TOKEN_MISSEDUF            (1 << 2)
+#define QTD_TOKEN_SPLITXSTATE         (1 << 1)
+#define QTD_TOKEN_PING                (1 << 0)
+
+    uint32_t bufptr[5];               /* Standard buffer pointer */
+#define QTD_BUFPTR_MASK               0xfffff000
+#define QTD_BUFPTR_SH                 12
+} EHCIqtd;
+
+/*  EHCI spec version 1.0 Section 3.6
+ */
+typedef struct EHCIqh {
+    uint32_t next;                    /* Standard next link pointer */
+
+    /* endpoint characteristics */
+    uint32_t epchar;
+#define QH_EPCHAR_RL_MASK             0xf0000000
+#define QH_EPCHAR_RL_SH               28
+#define QH_EPCHAR_C                   (1 << 27)
+#define QH_EPCHAR_MPLEN_MASK          0x07FF0000
+#define QH_EPCHAR_MPLEN_SH            16
+#define QH_EPCHAR_H                   (1 << 15)
+#define QH_EPCHAR_DTC                 (1 << 14)
+#define QH_EPCHAR_EPS_MASK            0x00003000
+#define QH_EPCHAR_EPS_SH              12
+#define EHCI_QH_EPS_FULL              0
+#define EHCI_QH_EPS_LOW               1
+#define EHCI_QH_EPS_HIGH              2
+#define EHCI_QH_EPS_RESERVED          3
+
+#define QH_EPCHAR_EP_MASK             0x00000f00
+#define QH_EPCHAR_EP_SH               8
+#define QH_EPCHAR_I                   (1 << 7)
+#define QH_EPCHAR_DEVADDR_MASK        0x0000007f
+#define QH_EPCHAR_DEVADDR_SH          0
+
+    /* endpoint capabilities */
+    uint32_t epcap;
+#define QH_EPCAP_MULT_MASK            0xc0000000
+#define QH_EPCAP_MULT_SH              30
+#define QH_EPCAP_PORTNUM_MASK         0x3f800000
+#define QH_EPCAP_PORTNUM_SH           23
+#define QH_EPCAP_HUBADDR_MASK         0x007f0000
+#define QH_EPCAP_HUBADDR_SH           16
+#define QH_EPCAP_CMASK_MASK           0x0000ff00
+#define QH_EPCAP_CMASK_SH             8
+#define QH_EPCAP_SMASK_MASK           0x000000ff
+#define QH_EPCAP_SMASK_SH             0
+
+    uint32_t current_qtd;             /* Standard next link pointer */
+    uint32_t next_qtd;                /* Standard next link pointer */
+    uint32_t altnext_qtd;
+#define QH_ALTNEXT_NAKCNT_MASK        0x0000001e
+#define QH_ALTNEXT_NAKCNT_SH          1
+
+    uint32_t token;                   /* Same as QTD token */
+    uint32_t bufptr[5];               /* Standard buffer pointer */
+#define BUFPTR_CPROGMASK_MASK         0x000000ff
+#define BUFPTR_FRAMETAG_MASK          0x0000001f
+#define BUFPTR_SBYTES_MASK            0x00000fe0
+#define BUFPTR_SBYTES_SH              5
+} EHCIqh;
+
+/*  EHCI spec version 1.0 Section 3.7
+ */
+typedef struct EHCIfstn {
+    uint32_t next;                    /* Standard next link pointer */
+    uint32_t backptr;                 /* Standard next link pointer */
+} EHCIfstn;
+
+enum async_state {
+    EHCI_ASYNC_NONE = 0,
+    EHCI_ASYNC_INITIALIZED,
+    EHCI_ASYNC_INFLIGHT,
+    EHCI_ASYNC_FINISHED,
+};
+
+struct EHCIPacket {
+    EHCIQueue *queue;
+    QTAILQ_ENTRY(EHCIPacket) next;
+
+    EHCIqtd qtd;           /* copy of current QTD (being worked on) */
+    uint32_t qtdaddr;      /* address QTD read from                 */
+
+    USBPacket packet;
+    QEMUSGList sgl;
+    int pid;
+    enum async_state async;
+};
+
+struct EHCIQueue {
+    EHCIState *ehci;
+    QTAILQ_ENTRY(EHCIQueue) next;
+    uint32_t seen;
+    uint64_t ts;
+    int async;
+    int transact_ctr;
+
+    /* cached data from guest - needs to be flushed
+     * when guest removes an entry (doorbell, handshake sequence)
+     */
+    EHCIqh qh;             /* copy of current QH (being worked on) */
+    uint32_t qhaddr;       /* address QH read from                 */
+    uint32_t qtdaddr;      /* address QTD read from                */
+    int last_pid;          /* pid of last packet executed          */
+    USBDevice *dev;
+    QTAILQ_HEAD(pkts_head, EHCIPacket) packets;
+};
+
+typedef QTAILQ_HEAD(EHCIQueueHead, EHCIQueue) EHCIQueueHead;
+
+struct EHCIState {
+    USBBus bus;
+    qemu_irq irq;
+    MemoryRegion mem;
+    DMAContext *dma;
+    MemoryRegion mem_caps;
+    MemoryRegion mem_opreg;
+    MemoryRegion mem_ports;
+    int companion_count;
+    uint16_t capsbase;
+    uint16_t opregbase;
+
+    /* properties */
+    uint32_t maxframes;
+
+    /*
+     *  EHCI spec version 1.0 Section 2.3
+     *  Host Controller Operational Registers
+     */
+    uint8_t caps[CAPA_SIZE];
+    union {
+        uint32_t opreg[PORTSC_BEGIN/sizeof(uint32_t)];
+        struct {
+            uint32_t usbcmd;
+            uint32_t usbsts;
+            uint32_t usbintr;
+            uint32_t frindex;
+            uint32_t ctrldssegment;
+            uint32_t periodiclistbase;
+            uint32_t asynclistaddr;
+            uint32_t notused[9];
+            uint32_t configflag;
+        };
+    };
+    uint32_t portsc[NB_PORTS];
+
+    /*
+     *  Internal states, shadow registers, etc
+     */
+    QEMUTimer *frame_timer;
+    QEMUBH *async_bh;
+    uint32_t astate;         /* Current state in asynchronous schedule */
+    uint32_t pstate;         /* Current state in periodic schedule     */
+    USBPort ports[NB_PORTS];
+    USBPort *companion_ports[NB_PORTS];
+    uint32_t usbsts_pending;
+    uint32_t usbsts_frindex;
+    EHCIQueueHead aqueues;
+    EHCIQueueHead pqueues;
+
+    /* which address to look at next */
+    uint32_t a_fetch_addr;
+    uint32_t p_fetch_addr;
+
+    USBPacket ipacket;
+    QEMUSGList isgl;
+
+    uint64_t last_run_ns;
+    uint32_t async_stepdown;
+    uint32_t periodic_sched_active;
+    bool int_req_by_async;
+};
+
+extern const VMStateDescription vmstate_ehci;
+
+void usb_ehci_initfn(EHCIState *s, DeviceState *dev);
+
+#define TYPE_PCI_EHCI "pci-ehci-usb"
+#define PCI_EHCI(obj) OBJECT_CHECK(EHCIPCIState, (obj), TYPE_PCI_EHCI)
+
+typedef struct EHCIPCIState {
+    /*< private >*/
+    PCIDevice pcidev;
+    /*< public >*/
+
+    EHCIState ehci;
+} EHCIPCIState;
+
+
+#define TYPE_SYS_BUS_EHCI "sysbus-ehci-usb"
+#define TYPE_EXYNOS4210_EHCI "exynos4210-ehci-usb"
+
+#define SYS_BUS_EHCI(obj) \
+    OBJECT_CHECK(EHCISysBusState, (obj), TYPE_SYS_BUS_EHCI)
+#define SYS_BUS_EHCI_CLASS(class) \
+    OBJECT_CLASS_CHECK(SysBusEHCIClass, (class), TYPE_SYS_BUS_EHCI)
+#define SYS_BUS_EHCI_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(SysBusEHCIClass, (obj), TYPE_SYS_BUS_EHCI)
+
+typedef struct EHCISysBusState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    EHCIState ehci;
+} EHCISysBusState;
+
+typedef struct SysBusEHCIClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    uint16_t capsbase;
+    uint16_t opregbase;
+} SysBusEHCIClass;
+
+#endif
diff --git a/hw/usb/hcd-musb.c b/hw/usb/hcd-musb.c
index 0bb5c7b19e8..64e9e834bf1 100644
--- a/hw/usb/hcd-musb.c
+++ b/hw/usb/hcd-musb.c
@@ -21,7 +21,7 @@
  * Only host-mode and non-DMA accesses are currently supported.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
 #include "hw/irq.h"
 #include "hw/hw.h"
@@ -607,7 +607,6 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
 {
     USBDevice *dev;
     USBEndpoint *uep;
-    int ret;
     int idx = epnum && dir;
     int ttype;
 
@@ -627,19 +626,24 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
     dev = usb_find_device(&s->port, ep->faddr[idx]);
     uep = usb_ep_get(dev, pid, ep->type[idx] & 0xf);
     usb_packet_setup(&ep->packey[dir].p, pid, uep,
-                     (dev->addr << 16) | (uep->nr << 8) | pid);
+                     (dev->addr << 16) | (uep->nr << 8) | pid, false, true);
     usb_packet_addbuf(&ep->packey[dir].p, ep->buf[idx], len);
     ep->packey[dir].ep = ep;
     ep->packey[dir].dir = dir;
 
-    ret = usb_handle_packet(dev, &ep->packey[dir].p);
+    usb_handle_packet(dev, &ep->packey[dir].p);
 
-    if (ret == USB_RET_ASYNC) {
+    if (ep->packey[dir].p.status == USB_RET_ASYNC) {
+        usb_device_flush_ep_queue(dev, uep);
         ep->status[dir] = len;
         return;
     }
 
-    ep->status[dir] = ret;
+    if (ep->packey[dir].p.status == USB_RET_SUCCESS) {
+        ep->status[dir] = ep->packey[dir].p.actual_length;
+    } else {
+        ep->status[dir] = ep->packey[dir].p.status;
+    }
     musb_schedule_cb(&s->port, &ep->packey[dir].p);
 }
 
@@ -753,7 +757,6 @@ static void musb_rx_packet_complete(USBPacket *packey, void *opaque)
 
     if (ep->status[1] == USB_RET_STALL) {
         ep->status[1] = 0;
-        packey->result = 0;
 
         ep->csr[1] |= MGC_M_RXCSR_H_RXSTALL;
         if (!epnum)
@@ -792,14 +795,12 @@ static void musb_rx_packet_complete(USBPacket *packey, void *opaque)
     /* TODO: check len for over/underruns of an OUT packet?  */
     /* TODO: perhaps make use of e->ext_size[1] here.  */
 
-    packey->result = ep->status[1];
-
     if (!(ep->csr[1] & (MGC_M_RXCSR_H_RXSTALL | MGC_M_RXCSR_DATAERROR))) {
         ep->csr[1] |= MGC_M_RXCSR_FIFOFULL | MGC_M_RXCSR_RXPKTRDY;
         if (!epnum)
             ep->csr[0] |= MGC_M_CSR0_RXPKTRDY;
 
-        ep->rxcount = packey->result; /* XXX: MIN(packey->len, ep->maxp[1]); */
+        ep->rxcount = ep->status[1]; /* XXX: MIN(packey->len, ep->maxp[1]); */
         /* In DMA mode: assert DMA request for this EP */
     }
 
@@ -1236,7 +1237,7 @@ static void musb_ep_writeh(void *opaque, int ep, int addr, uint16_t value)
 }
 
 /* Generic control */
-static uint32_t musb_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readb(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep, i;
@@ -1298,7 +1299,7 @@ static uint32_t musb_readb(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writeb(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writeb(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1385,7 +1386,7 @@ static void musb_writeb(void *opaque, target_phys_addr_t addr, uint32_t value)
     };
 }
 
-static uint32_t musb_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readh(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep, i;
@@ -1439,7 +1440,7 @@ static uint32_t musb_readh(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writeh(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writeh(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1495,7 +1496,7 @@ static void musb_writeh(void *opaque, target_phys_addr_t addr, uint32_t value)
     };
 }
 
-static uint32_t musb_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readw(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1513,7 +1514,7 @@ static uint32_t musb_readw(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writew(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writew(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index c36184ae4fe..dd9967b13d8 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -27,9 +27,9 @@
 */
 
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/sysbus.h"
 #include "hw/qdev-dma.h"
 
@@ -430,6 +430,23 @@ static USBDevice *ohci_find_device(OHCIState *ohci, uint8_t addr)
     return NULL;
 }
 
+static void ohci_stop_endpoints(OHCIState *ohci)
+{
+    USBDevice *dev;
+    int i, j;
+
+    for (i = 0; i < ohci->num_ports; i++) {
+        dev = ohci->rhport[i].port.dev;
+        if (dev && dev->attached) {
+            usb_device_ep_stopped(dev, &dev->ep_ctl);
+            for (j = 0; j < USB_MAX_ENDPOINTS; j++) {
+                usb_device_ep_stopped(dev, &dev->ep_in[j]);
+                usb_device_ep_stopped(dev, &dev->ep_out[j]);
+            }
+        }
+    }
+}
+
 /* Reset the controller */
 static void ohci_reset(void *opaque)
 {
@@ -478,6 +495,7 @@ static void ohci_reset(void *opaque)
         usb_cancel_packet(&ohci->usb_packet);
         ohci->async_td = 0;
     }
+    ohci_stop_endpoints(ohci);
     DPRINTF("usb-ohci: Reset %s\n", ohci->name);
 }
 
@@ -807,18 +825,24 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
                          DMA_DIRECTION_TO_DEVICE);
     }
 
-    if (completion) {
-        ret = ohci->usb_packet.result;
-    } else {
+    if (!completion) {
+        bool int_req = relative_frame_number == frame_count &&
+                       OHCI_BM(iso_td.flags, TD_DI) == 0;
         dev = ohci_find_device(ohci, OHCI_BM(ed->flags, ED_FA));
         ep = usb_ep_get(dev, pid, OHCI_BM(ed->flags, ED_EN));
-        usb_packet_setup(&ohci->usb_packet, pid, ep, addr);
+        usb_packet_setup(&ohci->usb_packet, pid, ep, addr, false, int_req);
         usb_packet_addbuf(&ohci->usb_packet, ohci->usb_buf, len);
-        ret = usb_handle_packet(dev, &ohci->usb_packet);
-        if (ret == USB_RET_ASYNC) {
+        usb_handle_packet(dev, &ohci->usb_packet);
+        if (ohci->usb_packet.status == USB_RET_ASYNC) {
+            usb_device_flush_ep_queue(dev, ep);
             return 1;
         }
     }
+    if (ohci->usb_packet.status == USB_RET_SUCCESS) {
+        ret = ohci->usb_packet.actual_length;
+    } else {
+        ret = ohci->usb_packet.status;
+    }
 
 #ifdef DEBUG_ISOCH
     printf("so 0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d\n",
@@ -994,7 +1018,6 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
     }
 #endif
     if (completion) {
-        ret = ohci->usb_packet.result;
         ohci->async_td = 0;
         ohci->async_complete = 0;
     } else {
@@ -1011,17 +1034,25 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
         }
         dev = ohci_find_device(ohci, OHCI_BM(ed->flags, ED_FA));
         ep = usb_ep_get(dev, pid, OHCI_BM(ed->flags, ED_EN));
-        usb_packet_setup(&ohci->usb_packet, pid, ep, addr);
+        usb_packet_setup(&ohci->usb_packet, pid, ep, addr, !flag_r,
+                         OHCI_BM(td.flags, TD_DI) == 0);
         usb_packet_addbuf(&ohci->usb_packet, ohci->usb_buf, pktlen);
-        ret = usb_handle_packet(dev, &ohci->usb_packet);
+        usb_handle_packet(dev, &ohci->usb_packet);
 #ifdef DEBUG_PACKET
-        DPRINTF("ret=%d\n", ret);
+        DPRINTF("status=%d\n", ohci->usb_packet.status);
 #endif
-        if (ret == USB_RET_ASYNC) {
+        if (ohci->usb_packet.status == USB_RET_ASYNC) {
+            usb_device_flush_ep_queue(dev, ep);
             ohci->async_td = addr;
             return 1;
         }
     }
+    if (ohci->usb_packet.status == USB_RET_SUCCESS) {
+        ret = ohci->usb_packet.actual_length;
+    } else {
+        ret = ohci->usb_packet.status;
+    }
+
     if (ret >= 0) {
         if (dir == OHCI_TD_DIR_IN) {
             ohci_copy_td(ohci, &td, ohci->usb_buf, ret,
@@ -1134,6 +1165,8 @@ static int ohci_service_ed_list(OHCIState *ohci, uint32_t head, int completion)
             if (ohci->async_td && addr == ohci->async_td) {
                 usb_cancel_packet(&ohci->usb_packet);
                 ohci->async_td = 0;
+                usb_device_ep_stopped(ohci->usb_packet.ep->dev,
+                                      ohci->usb_packet.ep);
             }
             continue;
         }
@@ -1214,10 +1247,12 @@ static void ohci_frame_boundary(void *opaque)
     }
 
     /* Cancel all pending packets if either of the lists has been disabled.  */
-    if (ohci->async_td &&
-        ohci->old_ctl & (~ohci->ctl) & (OHCI_CTL_BLE | OHCI_CTL_CLE)) {
-        usb_cancel_packet(&ohci->usb_packet);
-        ohci->async_td = 0;
+    if (ohci->old_ctl & (~ohci->ctl) & (OHCI_CTL_BLE | OHCI_CTL_CLE)) {
+        if (ohci->async_td) {
+            usb_cancel_packet(&ohci->usb_packet);
+            ohci->async_td = 0;
+        }
+        ohci_stop_endpoints(ohci);
     }
     ohci->old_ctl = ohci->ctl;
     ohci_process_lists(ohci, 0);
@@ -1470,12 +1505,10 @@ static void ohci_port_set_status(OHCIState *ohci, int portnum, uint32_t val)
 
     if (old_state != port->ctrl)
         ohci_set_interrupt(ohci, OHCI_INTR_RHSC);
-
-    return;
 }
 
 static uint64_t ohci_mem_read(void *opaque,
-                              target_phys_addr_t addr,
+                              hwaddr addr,
                               unsigned size)
 {
     OHCIState *ohci = opaque;
@@ -1598,7 +1631,7 @@ static uint64_t ohci_mem_read(void *opaque,
 }
 
 static void ohci_mem_write(void *opaque,
-                           target_phys_addr_t addr,
+                           hwaddr addr,
                            uint64_t val,
                            unsigned size)
 {
@@ -1703,6 +1736,7 @@ static void ohci_mem_write(void *opaque,
     /* PXA27x specific registers */
     case 24: /* HcStatus */
         ohci->hstatus &= ~(val & ohci->hmask);
+        break;
 
     case 25: /* HcHReset */
         ohci->hreset = val & ~OHCI_HRESET_FSBIR;
@@ -1848,7 +1882,7 @@ static int ohci_init_pxa(SysBusDevice *dev)
 
     /* Cannot fail as we pass NULL for masterbus */
     usb_ohci_init(&s->ohci, &dev->qdev, s->num_ports, s->dma_offset, NULL, 0,
-                  NULL);
+                  &dma_context_memory);
     sysbus_init_irq(dev, &s->ohci.irq);
     sysbus_init_mmio(dev, &s->ohci.mem);
 
@@ -1871,11 +1905,12 @@ static void ohci_pci_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_IPID_USB;
     k->class_id = PCI_CLASS_SERIAL_USB;
+    k->no_hotplug = 1;
     dc->desc = "Apple USB Controller";
     dc->props = ohci_pci_properties;
 }
 
-static TypeInfo ohci_pci_info = {
+static const TypeInfo ohci_pci_info = {
     .name          = "pci-ohci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(OHCIPCIState),
@@ -1898,7 +1933,7 @@ static void ohci_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = ohci_sysbus_properties;
 }
 
-static TypeInfo ohci_sysbus_info = {
+static const TypeInfo ohci_sysbus_info = {
     .name          = "sysbus-ohci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OHCISysBusState),
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index b0db92145a5..60645aa21ff 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -27,10 +27,10 @@
  */
 #include "hw/hw.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
-#include "qemu-timer.h"
-#include "iov.h"
-#include "dma.h"
+#include "hw/pci/pci.h"
+#include "qemu/timer.h"
+#include "qemu/iov.h"
+#include "sysemu/dma.h"
 #include "trace.h"
 
 //#define DEBUG
@@ -75,6 +75,11 @@
 
 #define FRAME_MAX_LOOPS  256
 
+/* Must be large enough to handle 10 frame delay for initial isoc requests */
+#define QH_VALID         32
+
+#define MAX_FRAMES_PER_TICK    (QH_VALID / 2)
+
 #define NB_PORTS 2
 
 enum {
@@ -88,6 +93,23 @@ enum {
 typedef struct UHCIState UHCIState;
 typedef struct UHCIAsync UHCIAsync;
 typedef struct UHCIQueue UHCIQueue;
+typedef struct UHCIInfo UHCIInfo;
+typedef struct UHCIPCIDeviceClass UHCIPCIDeviceClass;
+
+struct UHCIInfo {
+    const char *name;
+    uint16_t   vendor_id;
+    uint16_t   device_id;
+    uint8_t    revision;
+    uint8_t    irq_pin;
+    int        (*initfn)(PCIDevice *dev);
+    bool       unplug;
+};
+
+struct UHCIPCIDeviceClass {
+    PCIDeviceClass parent_class;
+    UHCIInfo       info;
+};
 
 /* 
  * Pending async transaction.
@@ -100,16 +122,17 @@ struct UHCIAsync {
     QEMUSGList sgl;
     UHCIQueue *queue;
     QTAILQ_ENTRY(UHCIAsync) next;
-    uint32_t  td;
-    uint8_t   isoc;
+    uint32_t  td_addr;
     uint8_t   done;
 };
 
 struct UHCIQueue {
+    uint32_t  qh_addr;
     uint32_t  token;
     UHCIState *uhci;
+    USBEndpoint *ep;
     QTAILQ_ENTRY(UHCIQueue) next;
-    QTAILQ_HEAD(, UHCIAsync) asyncs;
+    QTAILQ_HEAD(asyncs_head, UHCIAsync) asyncs;
     int8_t    valid;
 };
 
@@ -134,6 +157,7 @@ struct UHCIState {
     QEMUBH *bh;
     uint32_t frame_bytes;
     uint32_t frame_bandwidth;
+    bool completions_only;
     UHCIPort ports[NB_PORTS];
 
     /* Interrupts that should be raised at the end of the current frame.  */
@@ -147,6 +171,7 @@ struct UHCIState {
     /* Properties */
     char *masterbus;
     uint32_t firstport;
+    uint32_t maxframes;
 };
 
 typedef struct UHCI_TD {
@@ -161,57 +186,93 @@ typedef struct UHCI_QH {
     uint32_t el_link;
 } UHCI_QH;
 
+static void uhci_async_cancel(UHCIAsync *async);
+static void uhci_queue_fill(UHCIQueue *q, UHCI_TD *td);
+
 static inline int32_t uhci_queue_token(UHCI_TD *td)
 {
-    /* covers ep, dev, pid -> identifies the endpoint */
-    return td->token & 0x7ffff;
+    if ((td->token & (0xf << 15)) == 0) {
+        /* ctrl ep, cover ep and dev, not pid! */
+        return td->token & 0x7ff00;
+    } else {
+        /* covers ep, dev, pid -> identifies the endpoint */
+        return td->token & 0x7ffff;
+    }
 }
 
-static UHCIQueue *uhci_queue_get(UHCIState *s, UHCI_TD *td)
+static UHCIQueue *uhci_queue_new(UHCIState *s, uint32_t qh_addr, UHCI_TD *td,
+                                 USBEndpoint *ep)
 {
-    uint32_t token = uhci_queue_token(td);
     UHCIQueue *queue;
 
-    QTAILQ_FOREACH(queue, &s->queues, next) {
-        if (queue->token == token) {
-            return queue;
-        }
-    }
-
     queue = g_new0(UHCIQueue, 1);
     queue->uhci = s;
-    queue->token = token;
+    queue->qh_addr = qh_addr;
+    queue->token = uhci_queue_token(td);
+    queue->ep = ep;
     QTAILQ_INIT(&queue->asyncs);
     QTAILQ_INSERT_HEAD(&s->queues, queue, next);
+    queue->valid = QH_VALID;
     trace_usb_uhci_queue_add(queue->token);
     return queue;
 }
 
-static void uhci_queue_free(UHCIQueue *queue)
+static void uhci_queue_free(UHCIQueue *queue, const char *reason)
 {
     UHCIState *s = queue->uhci;
+    UHCIAsync *async;
+
+    while (!QTAILQ_EMPTY(&queue->asyncs)) {
+        async = QTAILQ_FIRST(&queue->asyncs);
+        uhci_async_cancel(async);
+    }
+    usb_device_ep_stopped(queue->ep->dev, queue->ep);
 
-    trace_usb_uhci_queue_del(queue->token);
+    trace_usb_uhci_queue_del(queue->token, reason);
     QTAILQ_REMOVE(&s->queues, queue, next);
     g_free(queue);
 }
 
-static UHCIAsync *uhci_async_alloc(UHCIQueue *queue, uint32_t addr)
+static UHCIQueue *uhci_queue_find(UHCIState *s, UHCI_TD *td)
+{
+    uint32_t token = uhci_queue_token(td);
+    UHCIQueue *queue;
+
+    QTAILQ_FOREACH(queue, &s->queues, next) {
+        if (queue->token == token) {
+            return queue;
+        }
+    }
+    return NULL;
+}
+
+static bool uhci_queue_verify(UHCIQueue *queue, uint32_t qh_addr, UHCI_TD *td,
+                              uint32_t td_addr, bool queuing)
+{
+    UHCIAsync *first = QTAILQ_FIRST(&queue->asyncs);
+
+    return queue->qh_addr == qh_addr &&
+           queue->token == uhci_queue_token(td) &&
+           (queuing || !(td->ctrl & TD_CTRL_ACTIVE) || first == NULL ||
+            first->td_addr == td_addr);
+}
+
+static UHCIAsync *uhci_async_alloc(UHCIQueue *queue, uint32_t td_addr)
 {
     UHCIAsync *async = g_new0(UHCIAsync, 1);
 
     async->queue = queue;
-    async->td = addr;
+    async->td_addr = td_addr;
     usb_packet_init(&async->packet);
     pci_dma_sglist_init(&async->sgl, &queue->uhci->dev, 1);
-    trace_usb_uhci_packet_add(async->queue->token, async->td);
+    trace_usb_uhci_packet_add(async->queue->token, async->td_addr);
 
     return async;
 }
 
 static void uhci_async_free(UHCIAsync *async)
 {
-    trace_usb_uhci_packet_del(async->queue->token, async->td);
+    trace_usb_uhci_packet_del(async->queue->token, async->td_addr);
     usb_packet_cleanup(&async->packet);
     qemu_sglist_destroy(&async->sgl);
     g_free(async);
@@ -221,21 +282,24 @@ static void uhci_async_link(UHCIAsync *async)
 {
     UHCIQueue *queue = async->queue;
     QTAILQ_INSERT_TAIL(&queue->asyncs, async, next);
-    trace_usb_uhci_packet_link_async(async->queue->token, async->td);
+    trace_usb_uhci_packet_link_async(async->queue->token, async->td_addr);
 }
 
 static void uhci_async_unlink(UHCIAsync *async)
 {
     UHCIQueue *queue = async->queue;
     QTAILQ_REMOVE(&queue->asyncs, async, next);
-    trace_usb_uhci_packet_unlink_async(async->queue->token, async->td);
+    trace_usb_uhci_packet_unlink_async(async->queue->token, async->td_addr);
 }
 
 static void uhci_async_cancel(UHCIAsync *async)
 {
-    trace_usb_uhci_packet_cancel(async->queue->token, async->td, async->done);
+    uhci_async_unlink(async);
+    trace_usb_uhci_packet_cancel(async->queue->token, async->td_addr,
+                                 async->done);
     if (!async->done)
         usb_cancel_packet(&async->packet);
+    usb_packet_unmap(&async->packet, &async->sgl);
     uhci_async_free(async);
 }
 
@@ -258,34 +322,21 @@ static void uhci_async_validate_begin(UHCIState *s)
 static void uhci_async_validate_end(UHCIState *s)
 {
     UHCIQueue *queue, *n;
-    UHCIAsync *async;
 
     QTAILQ_FOREACH_SAFE(queue, &s->queues, next, n) {
-        if (queue->valid > 0) {
-            continue;
-        }
-        while (!QTAILQ_EMPTY(&queue->asyncs)) {
-            async = QTAILQ_FIRST(&queue->asyncs);
-            uhci_async_unlink(async);
-            uhci_async_cancel(async);
+        if (!queue->valid) {
+            uhci_queue_free(queue, "validate-end");
         }
-        uhci_queue_free(queue);
     }
 }
 
 static void uhci_async_cancel_device(UHCIState *s, USBDevice *dev)
 {
-    UHCIQueue *queue;
-    UHCIAsync *curr, *n;
+    UHCIQueue *queue, *n;
 
-    QTAILQ_FOREACH(queue, &s->queues, next) {
-        QTAILQ_FOREACH_SAFE(curr, &queue->asyncs, next, n) {
-            if (!usb_packet_is_inflight(&curr->packet) ||
-                curr->packet.ep->dev != dev) {
-                continue;
-            }
-            uhci_async_unlink(curr);
-            uhci_async_cancel(curr);
+    QTAILQ_FOREACH_SAFE(queue, &s->queues, next, n) {
+        if (queue->ep->dev == dev) {
+            uhci_queue_free(queue, "cancel-device");
         }
     }
 }
@@ -293,38 +344,24 @@ static void uhci_async_cancel_device(UHCIState *s, USBDevice *dev)
 static void uhci_async_cancel_all(UHCIState *s)
 {
     UHCIQueue *queue, *nq;
-    UHCIAsync *curr, *n;
 
     QTAILQ_FOREACH_SAFE(queue, &s->queues, next, nq) {
-        QTAILQ_FOREACH_SAFE(curr, &queue->asyncs, next, n) {
-            uhci_async_unlink(curr);
-            uhci_async_cancel(curr);
-        }
-        uhci_queue_free(queue);
+        uhci_queue_free(queue, "cancel-all");
     }
 }
 
-static UHCIAsync *uhci_async_find_td(UHCIState *s, uint32_t addr, UHCI_TD *td)
+static UHCIAsync *uhci_async_find_td(UHCIState *s, uint32_t td_addr)
 {
-    uint32_t token = uhci_queue_token(td);
     UHCIQueue *queue;
     UHCIAsync *async;
 
     QTAILQ_FOREACH(queue, &s->queues, next) {
-        if (queue->token == token) {
-            break;
-        }
-    }
-    if (queue == NULL) {
-        return NULL;
-    }
-
-    QTAILQ_FOREACH(async, &queue->asyncs, next) {
-        if (async->td == addr) {
-            return async;
+        QTAILQ_FOREACH(async, &queue->asyncs, next) {
+            if (async->td_addr == td_addr) {
+                return async;
+            }
         }
     }
-
     return NULL;
 }
 
@@ -401,7 +438,7 @@ static int uhci_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_uhci = {
     .name = "uhci",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .post_load = uhci_post_load,
@@ -419,44 +456,16 @@ static const VMStateDescription vmstate_uhci = {
         VMSTATE_UINT8(status2, UHCIState),
         VMSTATE_TIMER(frame_timer, UHCIState),
         VMSTATE_INT64_V(expire_time, UHCIState, 2),
+        VMSTATE_UINT32_V(pending_int_mask, UHCIState, 3),
         VMSTATE_END_OF_LIST()
     }
 };
 
-static void uhci_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
-{
-    UHCIState *s = opaque;
-
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x0c:
-        s->sof_timing = val;
-        break;
-    }
-}
-
-static uint32_t uhci_ioport_readb(void *opaque, uint32_t addr)
+static void uhci_port_write(void *opaque, hwaddr addr,
+                            uint64_t val, unsigned size)
 {
     UHCIState *s = opaque;
-    uint32_t val;
 
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x0c:
-        val = s->sof_timing;
-        break;
-    default:
-        val = 0xff;
-        break;
-    }
-    return val;
-}
-
-static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    UHCIState *s = opaque;
-
-    addr &= 0x1f;
     trace_usb_uhci_mmio_writew(addr, val);
 
     switch(addr) {
@@ -466,7 +475,7 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
             trace_usb_uhci_schedule_start();
             s->expire_time = qemu_get_clock_ns(vm_clock) +
                 (get_ticks_per_sec() / FRAME_TIMER_FREQ);
-            qemu_mod_timer(s->frame_timer, qemu_get_clock_ns(vm_clock));
+            qemu_mod_timer(s->frame_timer, s->expire_time);
             s->status &= ~UHCI_STS_HCHALTED;
         } else if (!(val & UHCI_CMD_RS)) {
             s->status |= UHCI_STS_HCHALTED;
@@ -505,6 +514,17 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
         if (s->status & UHCI_STS_HCHALTED)
             s->frnum = val & 0x7ff;
         break;
+    case 0x08:
+        s->fl_base_addr &= 0xffff0000;
+        s->fl_base_addr |= val & ~0xfff;
+        break;
+    case 0x0a:
+        s->fl_base_addr &= 0x0000ffff;
+        s->fl_base_addr |= (val << 16);
+        break;
+    case 0x0c:
+        s->sof_timing = val & 0xff;
+        break;
     case 0x10 ... 0x1f:
         {
             UHCIPort *port;
@@ -524,6 +544,10 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
                 }
             }
             port->ctrl &= UHCI_PORT_READ_ONLY;
+            /* enabled may only be set if a device is connected */
+            if (!(port->ctrl & UHCI_PORT_CCS)) {
+                val &= ~UHCI_PORT_EN;
+            }
             port->ctrl |= (val & ~UHCI_PORT_READ_ONLY);
             /* some bits are reset when a '1' is written to them */
             port->ctrl &= ~(val & UHCI_PORT_WRITE_CLEAR);
@@ -532,12 +556,11 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
+static uint64_t uhci_port_read(void *opaque, hwaddr addr, unsigned size)
 {
     UHCIState *s = opaque;
     uint32_t val;
 
-    addr &= 0x1f;
     switch(addr) {
     case 0x00:
         val = s->cmd;
@@ -551,6 +574,15 @@ static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
     case 0x06:
         val = s->frnum;
         break;
+    case 0x08:
+        val = s->fl_base_addr & 0xffff;
+        break;
+    case 0x0a:
+        val = (s->fl_base_addr >> 16) & 0xffff;
+        break;
+    case 0x0c:
+        val = s->sof_timing;
+        break;
     case 0x10 ... 0x1f:
         {
             UHCIPort *port;
@@ -573,38 +605,6 @@ static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
     return val;
 }
 
-static void uhci_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    UHCIState *s = opaque;
-
-    addr &= 0x1f;
-    trace_usb_uhci_mmio_writel(addr, val);
-
-    switch(addr) {
-    case 0x08:
-        s->fl_base_addr = val & ~0xfff;
-        break;
-    }
-}
-
-static uint32_t uhci_ioport_readl(void *opaque, uint32_t addr)
-{
-    UHCIState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x08:
-        val = s->fl_base_addr;
-        break;
-    default:
-        val = 0xffffffff;
-        break;
-    }
-    trace_usb_uhci_mmio_readl(addr, val);
-    return val;
-}
-
 /* signal resume if controller suspended */
 static void uhci_resume (void *opaque)
 {
@@ -695,30 +695,75 @@ static USBDevice *uhci_find_device(UHCIState *s, uint8_t addr)
     return NULL;
 }
 
-static void uhci_async_complete(USBPort *port, USBPacket *packet);
-static void uhci_process_frame(UHCIState *s);
+static void uhci_read_td(UHCIState *s, UHCI_TD *td, uint32_t link)
+{
+    pci_dma_read(&s->dev, link & ~0xf, td, sizeof(*td));
+    le32_to_cpus(&td->link);
+    le32_to_cpus(&td->ctrl);
+    le32_to_cpus(&td->token);
+    le32_to_cpus(&td->buffer);
+}
+
+static int uhci_handle_td_error(UHCIState *s, UHCI_TD *td, uint32_t td_addr,
+                                int status, uint32_t *int_mask)
+{
+    uint32_t queue_token = uhci_queue_token(td);
+    int ret;
+
+    switch (status) {
+    case USB_RET_NAK:
+        td->ctrl |= TD_CTRL_NAK;
+        return TD_RESULT_NEXT_QH;
+
+    case USB_RET_STALL:
+        td->ctrl |= TD_CTRL_STALL;
+        trace_usb_uhci_packet_complete_stall(queue_token, td_addr);
+        ret = TD_RESULT_NEXT_QH;
+        break;
+
+    case USB_RET_BABBLE:
+        td->ctrl |= TD_CTRL_BABBLE | TD_CTRL_STALL;
+        /* frame interrupted */
+        trace_usb_uhci_packet_complete_babble(queue_token, td_addr);
+        ret = TD_RESULT_STOP_FRAME;
+        break;
+
+    case USB_RET_IOERROR:
+    case USB_RET_NODEV:
+    default:
+        td->ctrl |= TD_CTRL_TIMEOUT;
+        td->ctrl &= ~(3 << TD_CTRL_ERROR_SHIFT);
+        trace_usb_uhci_packet_complete_error(queue_token, td_addr);
+        ret = TD_RESULT_NEXT_QH;
+        break;
+    }
+
+    td->ctrl &= ~TD_CTRL_ACTIVE;
+    s->status |= UHCI_STS_USBERR;
+    if (td->ctrl & TD_CTRL_IOC) {
+        *int_mask |= 0x01;
+    }
+    uhci_update_irq(s);
+    return ret;
+}
 
-/* return -1 if fatal error (frame must be stopped)
-          0 if TD successful
-          1 if TD unsuccessful or inactive
-*/
 static int uhci_complete_td(UHCIState *s, UHCI_TD *td, UHCIAsync *async, uint32_t *int_mask)
 {
-    int len = 0, max_len, err, ret;
+    int len = 0, max_len;
     uint8_t pid;
 
     max_len = ((td->token >> 21) + 1) & 0x7ff;
     pid = td->token & 0xff;
 
-    ret = async->packet.result;
-
     if (td->ctrl & TD_CTRL_IOS)
         td->ctrl &= ~TD_CTRL_ACTIVE;
 
-    if (ret < 0)
-        goto out;
+    if (async->packet.status != USB_RET_SUCCESS) {
+        return uhci_handle_td_error(s, td, async->td_addr,
+                                    async->packet.status, int_mask);
+    }
 
-    len = async->packet.result;
+    len = async->packet.actual_length;
     td->ctrl = (td->ctrl & ~0x7ff) | ((len - 1) & 0x7ff);
 
     /* The NAK bit may have been set by a previous frame, so clear it
@@ -729,179 +774,151 @@ static int uhci_complete_td(UHCIState *s, UHCI_TD *td, UHCIAsync *async, uint32_
         *int_mask |= 0x01;
 
     if (pid == USB_TOKEN_IN) {
-        if (len > max_len) {
-            ret = USB_RET_BABBLE;
-            goto out;
-        }
-
         if ((td->ctrl & TD_CTRL_SPD) && len < max_len) {
             *int_mask |= 0x02;
             /* short packet: do not update QH */
             trace_usb_uhci_packet_complete_shortxfer(async->queue->token,
-                                                    async->td);
+                                                     async->td_addr);
             return TD_RESULT_NEXT_QH;
         }
     }
 
     /* success */
-    trace_usb_uhci_packet_complete_success(async->queue->token, async->td);
+    trace_usb_uhci_packet_complete_success(async->queue->token,
+                                           async->td_addr);
     return TD_RESULT_COMPLETE;
+}
 
-out:
-    /*
-     * We should not do any further processing on a queue with errors!
-     * This is esp. important for bulk endpoints with pipelining enabled
-     * (redirection to a real USB device), where we must cancel all the
-     * transfers after this one so that:
-     * 1) If they've completed already, they are not processed further
-     *    causing more stalls, originating from the same failed transfer
-     * 2) If still in flight, they are cancelled before the guest does
-     *    a clear stall, otherwise the guest and device can loose sync!
-     */
-    while (!QTAILQ_EMPTY(&async->queue->asyncs)) {
-        UHCIAsync *as = QTAILQ_FIRST(&async->queue->asyncs);
-        uhci_async_unlink(as);
-        uhci_async_cancel(as);
-    }
-
-    switch(ret) {
-    case USB_RET_STALL:
-        td->ctrl |= TD_CTRL_STALL;
-        td->ctrl &= ~TD_CTRL_ACTIVE;
-        s->status |= UHCI_STS_USBERR;
-        if (td->ctrl & TD_CTRL_IOC) {
-            *int_mask |= 0x01;
-        }
-        uhci_update_irq(s);
-        trace_usb_uhci_packet_complete_stall(async->queue->token, async->td);
-        return TD_RESULT_NEXT_QH;
+static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
+                          UHCI_TD *td, uint32_t td_addr, uint32_t *int_mask)
+{
+    int ret, max_len;
+    bool spd;
+    bool queuing = (q != NULL);
+    uint8_t pid = td->token & 0xff;
+    UHCIAsync *async = uhci_async_find_td(s, td_addr);
 
-    case USB_RET_BABBLE:
-        td->ctrl |= TD_CTRL_BABBLE | TD_CTRL_STALL;
-        td->ctrl &= ~TD_CTRL_ACTIVE;
-        s->status |= UHCI_STS_USBERR;
-        if (td->ctrl & TD_CTRL_IOC) {
-            *int_mask |= 0x01;
+    if (async) {
+        if (uhci_queue_verify(async->queue, qh_addr, td, td_addr, queuing)) {
+            assert(q == NULL || q == async->queue);
+            q = async->queue;
+        } else {
+            uhci_queue_free(async->queue, "guest re-used pending td");
+            async = NULL;
         }
-        uhci_update_irq(s);
-        /* frame interrupted */
-        trace_usb_uhci_packet_complete_babble(async->queue->token, async->td);
-        return TD_RESULT_STOP_FRAME;
-
-    case USB_RET_NAK:
-        td->ctrl |= TD_CTRL_NAK;
-        if (pid == USB_TOKEN_SETUP)
-            break;
-        return TD_RESULT_NEXT_QH;
-
-    case USB_RET_IOERROR:
-    case USB_RET_NODEV:
-    default:
-	break;
     }
 
-    /* Retry the TD if error count is not zero */
-
-    td->ctrl |= TD_CTRL_TIMEOUT;
-    err = (td->ctrl >> TD_CTRL_ERROR_SHIFT) & 3;
-    if (err != 0) {
-        err--;
-        if (err == 0) {
-            td->ctrl &= ~TD_CTRL_ACTIVE;
-            s->status |= UHCI_STS_USBERR;
-            if (td->ctrl & TD_CTRL_IOC)
-                *int_mask |= 0x01;
-            uhci_update_irq(s);
-            trace_usb_uhci_packet_complete_error(async->queue->token,
-                                                 async->td);
+    if (q == NULL) {
+        q = uhci_queue_find(s, td);
+        if (q && !uhci_queue_verify(q, qh_addr, td, td_addr, queuing)) {
+            uhci_queue_free(q, "guest re-used qh");
+            q = NULL;
         }
     }
-    td->ctrl = (td->ctrl & ~(3 << TD_CTRL_ERROR_SHIFT)) |
-        (err << TD_CTRL_ERROR_SHIFT);
-    return TD_RESULT_NEXT_QH;
-}
 
-static int uhci_handle_td(UHCIState *s, uint32_t addr, UHCI_TD *td,
-                          uint32_t *int_mask, bool queuing)
-{
-    UHCIAsync *async;
-    int len = 0, max_len;
-    uint8_t pid;
-    USBDevice *dev;
-    USBEndpoint *ep;
+    if (q) {
+        q->valid = QH_VALID;
+    }
 
     /* Is active ? */
-    if (!(td->ctrl & TD_CTRL_ACTIVE))
+    if (!(td->ctrl & TD_CTRL_ACTIVE)) {
+        if (async) {
+            /* Guest marked a pending td non-active, cancel the queue */
+            uhci_queue_free(async->queue, "pending td non-active");
+        }
+        /*
+         * ehci11d spec page 22: "Even if the Active bit in the TD is already
+         * cleared when the TD is fetched ... an IOC interrupt is generated"
+         */
+        if (td->ctrl & TD_CTRL_IOC) {
+                *int_mask |= 0x01;
+        }
         return TD_RESULT_NEXT_QH;
+    }
 
-    async = uhci_async_find_td(s, addr, td);
     if (async) {
-        /* Already submitted */
-        async->queue->valid = 32;
-
-        if (!async->done)
-            return TD_RESULT_ASYNC_CONT;
         if (queuing) {
             /* we are busy filling the queue, we are not prepared
                to consume completed packages then, just leave them
                in async state */
             return TD_RESULT_ASYNC_CONT;
         }
+        if (!async->done) {
+            UHCI_TD last_td;
+            UHCIAsync *last = QTAILQ_LAST(&async->queue->asyncs, asyncs_head);
+            /*
+             * While we are waiting for the current td to complete, the guest
+             * may have added more tds to the queue. Note we re-read the td
+             * rather then caching it, as we want to see guest made changes!
+             */
+            uhci_read_td(s, &last_td, last->td_addr);
+            uhci_queue_fill(async->queue, &last_td);
 
+            return TD_RESULT_ASYNC_CONT;
+        }
         uhci_async_unlink(async);
         goto done;
     }
 
+    if (s->completions_only) {
+        return TD_RESULT_ASYNC_CONT;
+    }
+
     /* Allocate new packet */
-    async = uhci_async_alloc(uhci_queue_get(s, td), addr);
+    if (q == NULL) {
+        USBDevice *dev = uhci_find_device(s, (td->token >> 8) & 0x7f);
+        USBEndpoint *ep = usb_ep_get(dev, pid, (td->token >> 15) & 0xf);
 
-    /* valid needs to be large enough to handle 10 frame delay
-     * for initial isochronous requests
-     */
-    async->queue->valid = 32;
-    async->isoc = td->ctrl & TD_CTRL_IOS;
+        if (ep == NULL) {
+            return uhci_handle_td_error(s, td, td_addr, USB_RET_NODEV,
+                                        int_mask);
+        }
+        q = uhci_queue_new(s, qh_addr, td, ep);
+    }
+    async = uhci_async_alloc(q, td_addr);
 
     max_len = ((td->token >> 21) + 1) & 0x7ff;
-    pid = td->token & 0xff;
-
-    dev = uhci_find_device(s, (td->token >> 8) & 0x7f);
-    ep = usb_ep_get(dev, pid, (td->token >> 15) & 0xf);
-    usb_packet_setup(&async->packet, pid, ep, addr);
+    spd = (pid == USB_TOKEN_IN && (td->ctrl & TD_CTRL_SPD) != 0);
+    usb_packet_setup(&async->packet, pid, q->ep, td_addr, spd,
+                     (td->ctrl & TD_CTRL_IOC) != 0);
     qemu_sglist_add(&async->sgl, td->buffer, max_len);
     usb_packet_map(&async->packet, &async->sgl);
 
     switch(pid) {
     case USB_TOKEN_OUT:
     case USB_TOKEN_SETUP:
-        len = usb_handle_packet(dev, &async->packet);
-        if (len >= 0)
-            len = max_len;
+        usb_handle_packet(q->ep->dev, &async->packet);
+        if (async->packet.status == USB_RET_SUCCESS) {
+            async->packet.actual_length = max_len;
+        }
         break;
 
     case USB_TOKEN_IN:
-        len = usb_handle_packet(dev, &async->packet);
+        usb_handle_packet(q->ep->dev, &async->packet);
         break;
 
     default:
         /* invalid pid : frame interrupted */
+        usb_packet_unmap(&async->packet, &async->sgl);
         uhci_async_free(async);
         s->status |= UHCI_STS_HCPERR;
         uhci_update_irq(s);
         return TD_RESULT_STOP_FRAME;
     }
- 
-    if (len == USB_RET_ASYNC) {
+
+    if (async->packet.status == USB_RET_ASYNC) {
         uhci_async_link(async);
+        if (!queuing) {
+            uhci_queue_fill(q, td);
+        }
         return TD_RESULT_ASYNC_START;
     }
 
-    async->packet.result = len;
-
 done:
-    len = uhci_complete_td(s, td, async, int_mask);
+    ret = uhci_complete_td(s, td, async, int_mask);
     usb_packet_unmap(&async->packet, &async->sgl);
     uhci_async_free(async);
-    return len;
+    return ret;
 }
 
 static void uhci_async_complete(USBPort *port, USBPacket *packet)
@@ -909,31 +926,15 @@ static void uhci_async_complete(USBPort *port, USBPacket *packet)
     UHCIAsync *async = container_of(packet, UHCIAsync, packet);
     UHCIState *s = async->queue->uhci;
 
-    if (async->isoc) {
-        UHCI_TD td;
-        uint32_t link = async->td;
-        uint32_t int_mask = 0, val;
-
-        pci_dma_read(&s->dev, link & ~0xf, &td, sizeof(td));
-        le32_to_cpus(&td.link);
-        le32_to_cpus(&td.ctrl);
-        le32_to_cpus(&td.token);
-        le32_to_cpus(&td.buffer);
-
-        uhci_async_unlink(async);
-        uhci_complete_td(s, &td, async, &int_mask);
-        s->pending_int_mask |= int_mask;
-
-        /* update the status bits of the TD */
-        val = cpu_to_le32(td.ctrl);
-        pci_dma_write(&s->dev, (link & ~0xf) + 4, &val, sizeof(val));
-        uhci_async_free(async);
-    } else {
-        async->done = 1;
-        if (s->frame_bytes < s->frame_bandwidth) {
-            qemu_bh_schedule(s->bh);
-        }
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        uhci_async_cancel(async);
+        return;
     }
+
+    async->done = 1;
+    /* Force processing of this packet *now*, needed for migration */
+    s->completions_only = true;
+    qemu_bh_schedule(s->bh);
 }
 
 static int is_valid(uint32_t link)
@@ -978,28 +979,23 @@ static int qhdb_insert(QhDb *db, uint32_t addr)
     return 0;
 }
 
-static void uhci_fill_queue(UHCIState *s, UHCI_TD *td)
+static void uhci_queue_fill(UHCIQueue *q, UHCI_TD *td)
 {
     uint32_t int_mask = 0;
     uint32_t plink = td->link;
-    uint32_t token = uhci_queue_token(td);
     UHCI_TD ptd;
     int ret;
 
     while (is_valid(plink)) {
-        pci_dma_read(&s->dev, plink & ~0xf, &ptd, sizeof(ptd));
-        le32_to_cpus(&ptd.link);
-        le32_to_cpus(&ptd.ctrl);
-        le32_to_cpus(&ptd.token);
-        le32_to_cpus(&ptd.buffer);
+        uhci_read_td(q->uhci, &ptd, plink);
         if (!(ptd.ctrl & TD_CTRL_ACTIVE)) {
             break;
         }
-        if (uhci_queue_token(&ptd) != token) {
+        if (uhci_queue_token(&ptd) != q->token) {
             break;
         }
         trace_usb_uhci_td_queue(plink & ~0xf, ptd.ctrl, ptd.token);
-        ret = uhci_handle_td(s, plink, &ptd, &int_mask, true);
+        ret = uhci_handle_td(q->uhci, q, q->qh_addr, &ptd, plink, &int_mask);
         if (ret == TD_RESULT_ASYNC_CONT) {
             break;
         }
@@ -1007,6 +1003,7 @@ static void uhci_fill_queue(UHCIState *s, UHCI_TD *td)
         assert(int_mask == 0);
         plink = ptd.link;
     }
+    usb_device_flush_ep_queue(q->ep->dev, q->ep);
 }
 
 static void uhci_process_frame(UHCIState *s)
@@ -1029,7 +1026,7 @@ static void uhci_process_frame(UHCIState *s)
     qhdb_reset(&qhdb);
 
     for (cnt = FRAME_MAX_LOOPS; is_valid(link) && cnt; cnt--) {
-        if (s->frame_bytes >= s->frame_bandwidth) {
+        if (!s->completions_only && s->frame_bytes >= s->frame_bandwidth) {
             /* We've reached the usb 1.1 bandwidth, which is
                1280 bytes/frame, stop processing */
             trace_usb_uhci_frame_stop_bandwidth();
@@ -1075,15 +1072,11 @@ static void uhci_process_frame(UHCIState *s)
         }
 
         /* TD */
-        pci_dma_read(&s->dev, link & ~0xf, &td, sizeof(td));
-        le32_to_cpus(&td.link);
-        le32_to_cpus(&td.ctrl);
-        le32_to_cpus(&td.token);
-        le32_to_cpus(&td.buffer);
+        uhci_read_td(s, &td, link);
         trace_usb_uhci_td_load(curr_qh & ~0xf, link & ~0xf, td.ctrl, td.token);
 
         old_td_ctrl = td.ctrl;
-        ret = uhci_handle_td(s, link, &td, &int_mask, false);
+        ret = uhci_handle_td(s, NULL, curr_qh, &td, link, &int_mask);
         if (old_td_ctrl != td.ctrl) {
             /* update the status bits of the TD */
             val = cpu_to_le32(td.ctrl);
@@ -1102,9 +1095,6 @@ static void uhci_process_frame(UHCIState *s)
 
         case TD_RESULT_ASYNC_START:
             trace_usb_uhci_td_async(curr_qh & ~0xf, link & ~0xf);
-            if (is_valid(td.link)) {
-                uhci_fill_queue(s, &td);
-            }
             link = curr_qh ? qh.link : td.link;
             continue;
 
@@ -1148,10 +1138,11 @@ static void uhci_bh(void *opaque)
 static void uhci_frame_timer(void *opaque)
 {
     UHCIState *s = opaque;
+    uint64_t t_now, t_last_run;
+    int i, frames;
+    const uint64_t frame_t = get_ticks_per_sec() / FRAME_TIMER_FREQ;
 
-    /* prepare the timer for the next frame */
-    s->expire_time += (get_ticks_per_sec() / FRAME_TIMER_FREQ);
-    s->frame_bytes = 0;
+    s->completions_only = false;
     qemu_bh_cancel(s->bh);
 
     if (!(s->cmd & UHCI_CMD_RS)) {
@@ -1164,7 +1155,35 @@ static void uhci_frame_timer(void *opaque)
         return;
     }
 
-    /* Complete the previous frame */
+    /* We still store expire_time in our state, for migration */
+    t_last_run = s->expire_time - frame_t;
+    t_now = qemu_get_clock_ns(vm_clock);
+
+    /* Process up to MAX_FRAMES_PER_TICK frames */
+    frames = (t_now - t_last_run) / frame_t;
+    if (frames > s->maxframes) {
+        int skipped = frames - s->maxframes;
+        s->expire_time += skipped * frame_t;
+        s->frnum = (s->frnum + skipped) & 0x7ff;
+        frames -= skipped;
+    }
+    if (frames > MAX_FRAMES_PER_TICK) {
+        frames = MAX_FRAMES_PER_TICK;
+    }
+
+    for (i = 0; i < frames; i++) {
+        s->frame_bytes = 0;
+        trace_usb_uhci_frame_start(s->frnum);
+        uhci_async_validate_begin(s);
+        uhci_process_frame(s);
+        uhci_async_validate_end(s);
+        /* The spec says frnum is the frame currently being processed, and
+         * the guest must look at frnum - 1 on interrupt, so inc frnum now */
+        s->frnum = (s->frnum + 1) & 0x7ff;
+        s->expire_time += frame_t;
+    }
+
+    /* Complete the previous frame(s) */
     if (s->pending_int_mask) {
         s->status2 |= s->pending_int_mask;
         s->status  |= UHCI_STS_USBINT;
@@ -1172,32 +1191,17 @@ static void uhci_frame_timer(void *opaque)
     }
     s->pending_int_mask = 0;
 
-    /* Start new frame */
-    s->frnum = (s->frnum + 1) & 0x7ff;
-
-    trace_usb_uhci_frame_start(s->frnum);
-
-    uhci_async_validate_begin(s);
-
-    uhci_process_frame(s);
-
-    uhci_async_validate_end(s);
-
-    qemu_mod_timer(s->frame_timer, s->expire_time);
+    qemu_mod_timer(s->frame_timer, t_now + frame_t);
 }
 
-static const MemoryRegionPortio uhci_portio[] = {
-    { 0, 32, 2, .write = uhci_ioport_writew, },
-    { 0, 32, 2, .read = uhci_ioport_readw, },
-    { 0, 32, 4, .write = uhci_ioport_writel, },
-    { 0, 32, 4, .read = uhci_ioport_readl, },
-    { 0, 32, 1, .write = uhci_ioport_writeb, },
-    { 0, 32, 1, .read = uhci_ioport_readb, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps uhci_ioport_ops = {
-    .old_portio = uhci_portio,
+    .read  = uhci_port_read,
+    .write = uhci_port_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static USBPortOps uhci_port_ops = {
@@ -1214,6 +1218,7 @@ static USBBusOps uhci_bus_ops = {
 static int usb_uhci_common_initfn(PCIDevice *dev)
 {
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
+    UHCIPCIDeviceClass *u = container_of(pc, UHCIPCIDeviceClass, parent_class);
     UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
     uint8_t *pci_conf = s->dev.config;
     int i;
@@ -1222,20 +1227,7 @@ static int usb_uhci_common_initfn(PCIDevice *dev)
     /* TODO: reset value should be 0. */
     pci_conf[USB_SBRN] = USB_RELEASE_1; // release number
 
-    switch (pc->device_id) {
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI1:
-        s->irq_pin = 0;  /* A */
-        break;
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI2:
-        s->irq_pin = 1;  /* B */
-        break;
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI3:
-        s->irq_pin = 2;  /* C */
-        break;
-    default:
-        s->irq_pin = 3;  /* D */
-        break;
-    }
+    s->irq_pin = u->info.irq_pin;
     pci_config_set_interrupt_pin(pci_conf, s->irq_pin + 1);
 
     if (s->masterbus) {
@@ -1296,146 +1288,112 @@ static Property uhci_properties[] = {
     DEFINE_PROP_STRING("masterbus", UHCIState, masterbus),
     DEFINE_PROP_UINT32("firstport", UHCIState, firstport, 0),
     DEFINE_PROP_UINT32("bandwidth", UHCIState, frame_bandwidth, 1280),
+    DEFINE_PROP_UINT32("maxframes", UHCIState, maxframes, 128),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void piix3_uhci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371SB_2;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo piix3_uhci_info = {
-    .name          = "piix3-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = piix3_uhci_class_init,
-};
-
-static void piix4_uhci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371AB_2;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo piix4_uhci_info = {
-    .name          = "piix4-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = piix4_uhci_class_init,
-};
-
-static void vt82c686b_uhci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_vt82c686b_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_VIA;
-    k->device_id = PCI_DEVICE_ID_VIA_UHCI;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo vt82c686b_uhci_info = {
-    .name          = "vt82c686b-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = vt82c686b_uhci_class_init,
-};
-
-static void ich9_uhci1_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI1;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo ich9_uhci1_info = {
-    .name          = "ich9-usb-uhci1",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci1_class_init,
-};
-
-static void ich9_uhci2_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI2;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo ich9_uhci2_info = {
-    .name          = "ich9-usb-uhci2",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci2_class_init,
-};
-
-static void ich9_uhci3_class_init(ObjectClass *klass, void *data)
+static void uhci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI3;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
+    UHCIPCIDeviceClass *u = container_of(k, UHCIPCIDeviceClass, parent_class);
+    UHCIInfo *info = data;
+
+    k->init = info->initfn ? info->initfn : usb_uhci_common_initfn;
+    k->exit = info->unplug ? usb_uhci_exit : NULL;
+    k->vendor_id = info->vendor_id;
+    k->device_id = info->device_id;
+    k->revision  = info->revision;
+    k->class_id  = PCI_CLASS_SERIAL_USB;
+    k->no_hotplug = 1;
     dc->vmsd = &vmstate_uhci;
     dc->props = uhci_properties;
+    u->info = *info;
 }
 
-static TypeInfo ich9_uhci3_info = {
-    .name          = "ich9-usb-uhci3",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci3_class_init,
+static UHCIInfo uhci_info[] = {
+    {
+        .name       = "piix3-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82371SB_2,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .unplug    = true,
+    },{
+        .name      = "piix4-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82371AB_2,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .unplug    = true,
+    },{
+        .name      = "vt82c686b-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_VIA,
+        .device_id = PCI_DEVICE_ID_VIA_UHCI,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .initfn    = usb_uhci_vt82c686b_initfn,
+        .unplug    = true,
+    },{
+        .name      = "ich9-usb-uhci1", /* 00:1d.0 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI1,
+        .revision  = 0x03,
+        .irq_pin   = 0,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci2", /* 00:1d.1 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI2,
+        .revision  = 0x03,
+        .irq_pin   = 1,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci3", /* 00:1d.2 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI3,
+        .revision  = 0x03,
+        .irq_pin   = 2,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci4", /* 00:1a.0 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI4,
+        .revision  = 0x03,
+        .irq_pin   = 0,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci5", /* 00:1a.1 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI5,
+        .revision  = 0x03,
+        .irq_pin   = 1,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci6", /* 00:1a.2 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI6,
+        .revision  = 0x03,
+        .irq_pin   = 2,
+        .unplug    = false,
+    }
 };
 
 static void uhci_register_types(void)
 {
-    type_register_static(&piix3_uhci_info);
-    type_register_static(&piix4_uhci_info);
-    type_register_static(&vt82c686b_uhci_info);
-    type_register_static(&ich9_uhci1_info);
-    type_register_static(&ich9_uhci2_info);
-    type_register_static(&ich9_uhci3_info);
+    TypeInfo uhci_type_info = {
+        .parent        = TYPE_PCI_DEVICE,
+        .instance_size = sizeof(UHCIState),
+        .class_size    = sizeof(UHCIPCIDeviceClass),
+        .class_init    = uhci_class_init,
+    };
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(uhci_info); i++) {
+        uhci_type_info.name = uhci_info[i].name;
+        uhci_type_info.class_data = uhci_info + i;
+        type_register(&uhci_type_info);
+    }
 }
 
 type_init(uhci_register_types)
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 3eb27fadb88..5fb0c488e81 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -19,10 +19,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
-#include "hw/msi.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
 #include "trace.h"
 
 //#define DEBUG_XHCI
@@ -36,17 +37,14 @@
 #define FIXME() do { fprintf(stderr, "FIXME %s:%d\n", \
                              __func__, __LINE__); abort(); } while (0)
 
-#define MAXSLOTS 8
-#define MAXINTRS 1
+#define MAXPORTS_2 15
+#define MAXPORTS_3 15
 
-#define USB2_PORTS 4
-#define USB3_PORTS 4
-
-#define MAXPORTS (USB2_PORTS+USB3_PORTS)
+#define MAXPORTS (MAXPORTS_2+MAXPORTS_3)
+#define MAXSLOTS 64
+#define MAXINTRS 16
 
 #define TD_QUEUE 24
-#define BG_XFERS 8
-#define BG_PKTS 8
 
 /* Very pessimistic, let's hope it's enough for all cases */
 #define EV_QUEUE (((3*TD_QUEUE)+16)*MAXSLOTS)
@@ -55,24 +53,28 @@
 #define ER_FULL_HACK
 
 #define LEN_CAP         0x40
-#define OFF_OPER        LEN_CAP
 #define LEN_OPER        (0x400 + 0x10 * MAXPORTS)
-#define OFF_RUNTIME     ((OFF_OPER + LEN_OPER + 0x20) & ~0x1f)
-#define LEN_RUNTIME     (0x20 + MAXINTRS * 0x20)
-#define OFF_DOORBELL    (OFF_RUNTIME + LEN_RUNTIME)
+#define LEN_RUNTIME     ((MAXINTRS + 1) * 0x20)
 #define LEN_DOORBELL    ((MAXSLOTS + 1) * 0x20)
 
+#define OFF_OPER        LEN_CAP
+#define OFF_RUNTIME     0x1000
+#define OFF_DOORBELL    0x2000
+#define OFF_MSIX_TABLE  0x3000
+#define OFF_MSIX_PBA    0x3800
 /* must be power of 2 */
-#define LEN_REGS        0x2000
+#define LEN_REGS        0x4000
 
+#if (OFF_OPER + LEN_OPER) > OFF_RUNTIME
+#error Increase OFF_RUNTIME
+#endif
+#if (OFF_RUNTIME + LEN_RUNTIME) > OFF_DOORBELL
+#error Increase OFF_DOORBELL
+#endif
 #if (OFF_DOORBELL + LEN_DOORBELL) > LEN_REGS
 # error Increase LEN_REGS
 #endif
 
-#if MAXINTRS > 1
-# error TODO: only one interrupter supported
-#endif
-
 /* bit definitions */
 #define USBCMD_RS       (1<<0)
 #define USBCMD_HCRST    (1<<1)
@@ -144,6 +146,21 @@ typedef struct XHCITRB {
     bool ccs;
 } XHCITRB;
 
+enum {
+    PLS_U0              =  0,
+    PLS_U1              =  1,
+    PLS_U2              =  2,
+    PLS_U3              =  3,
+    PLS_DISABLED        =  4,
+    PLS_RX_DETECT       =  5,
+    PLS_INACTIVE        =  6,
+    PLS_POLLING         =  7,
+    PLS_RECOVERY        =  8,
+    PLS_HOT_RESET       =  9,
+    PLS_COMPILANCE_MODE = 10,
+    PLS_TEST_MODE       = 11,
+    PLS_RESUME          = 15,
+};
 
 typedef enum TRBType {
     TRB_RESERVED = 0,
@@ -258,6 +275,10 @@ typedef enum TRBCCode {
 
 #define TRB_LK_TC           (1<<1)
 
+#define TRB_INTR_SHIFT          22
+#define TRB_INTR_MASK       0x3ff
+#define TRB_INTR(t)         (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)
+
 #define EP_TYPE_MASK        0x7
 #define EP_TYPE_SHIFT           3
 
@@ -279,6 +300,18 @@ typedef enum TRBCCode {
 #define SLOT_CONTEXT_ENTRIES_MASK 0x1f
 #define SLOT_CONTEXT_ENTRIES_SHIFT 27
 
+typedef struct XHCIState XHCIState;
+
+#define get_field(data, field)                  \
+    (((data) >> field##_SHIFT) & field##_MASK)
+
+#define set_field(data, newval, field) do {                     \
+        uint32_t val = *data;                                   \
+        val &= ~(field##_MASK << field##_SHIFT);                \
+        val |= ((newval) & field##_MASK) << field##_SHIFT;      \
+        *data = val;                                            \
+    } while (0)
+
 typedef enum EPType {
     ET_INVALID = 0,
     ET_ISO_OUT,
@@ -297,64 +330,68 @@ typedef struct XHCIRing {
 } XHCIRing;
 
 typedef struct XHCIPort {
-    USBPort port;
+    XHCIState *xhci;
     uint32_t portsc;
+    uint32_t portnr;
+    USBPort  *uport;
+    uint32_t speedmask;
+    char name[16];
+    MemoryRegion mem;
 } XHCIPort;
 
-struct XHCIState;
-typedef struct XHCIState XHCIState;
-
 typedef struct XHCITransfer {
     XHCIState *xhci;
     USBPacket packet;
+    QEMUSGList sgl;
     bool running_async;
     bool running_retry;
     bool cancelled;
     bool complete;
-    bool backgrounded;
+    bool int_req;
     unsigned int iso_pkts;
     unsigned int slotid;
     unsigned int epid;
     bool in_xfer;
     bool iso_xfer;
-    bool bg_xfer;
 
     unsigned int trb_count;
     unsigned int trb_alloced;
     XHCITRB *trbs;
 
-    unsigned int data_length;
-    unsigned int data_alloced;
-    uint8_t *data;
-
     TRBCCode status;
 
     unsigned int pkts;
     unsigned int pktsize;
     unsigned int cur_pkt;
+
+    uint64_t mfindex_kick;
 } XHCITransfer;
 
 typedef struct XHCIEPContext {
+    XHCIState *xhci;
+    unsigned int slotid;
+    unsigned int epid;
+
     XHCIRing ring;
     unsigned int next_xfer;
     unsigned int comp_xfer;
     XHCITransfer transfers[TD_QUEUE];
     XHCITransfer *retry;
-    bool bg_running;
-    bool bg_updating;
-    unsigned int next_bg;
-    XHCITransfer bg_transfers[BG_XFERS];
     EPType type;
     dma_addr_t pctx;
     unsigned int max_psize;
-    bool has_bg;
     uint32_t state;
+
+    /* iso xfer scheduling */
+    unsigned int interval;
+    int64_t mfindex_last;
+    QEMUTimer *kick_timer;
 } XHCIEPContext;
 
 typedef struct XHCISlot {
     bool enabled;
     dma_addr_t ctx;
-    unsigned int port;
+    USBPort *uport;
     unsigned int devaddr;
     XHCIEPContext * eps[31];
 } XHCISlot;
@@ -369,15 +406,46 @@ typedef struct XHCIEvent {
     uint8_t epid;
 } XHCIEvent;
 
+typedef struct XHCIInterrupter {
+    uint32_t iman;
+    uint32_t imod;
+    uint32_t erstsz;
+    uint32_t erstba_low;
+    uint32_t erstba_high;
+    uint32_t erdp_low;
+    uint32_t erdp_high;
+
+    bool msix_used, er_pcs, er_full;
+
+    dma_addr_t er_start;
+    uint32_t er_size;
+    unsigned int er_ep_idx;
+
+    XHCIEvent ev_buffer[EV_QUEUE];
+    unsigned int ev_buffer_put;
+    unsigned int ev_buffer_get;
+
+} XHCIInterrupter;
+
 struct XHCIState {
     PCIDevice pci_dev;
     USBBus bus;
     qemu_irq irq;
     MemoryRegion mem;
+    MemoryRegion mem_cap;
+    MemoryRegion mem_oper;
+    MemoryRegion mem_runtime;
+    MemoryRegion mem_doorbell;
     const char *name;
-    uint32_t msi;
     unsigned int devaddr;
 
+    /* properties */
+    uint32_t numports_2;
+    uint32_t numports_3;
+    uint32_t numintrs;
+    uint32_t numslots;
+    uint32_t flags;
+
     /* Operational Registers */
     uint32_t usbcmd;
     uint32_t usbsts;
@@ -388,29 +456,15 @@ struct XHCIState {
     uint32_t dcbaap_high;
     uint32_t config;
 
+    USBPort  uports[MAX(MAXPORTS_2, MAXPORTS_3)];
     XHCIPort ports[MAXPORTS];
     XHCISlot slots[MAXSLOTS];
+    uint32_t numports;
 
     /* Runtime Registers */
-    uint32_t mfindex;
-    /* note: we only support one interrupter */
-    uint32_t iman;
-    uint32_t imod;
-    uint32_t erstsz;
-    uint32_t erstba_low;
-    uint32_t erstba_high;
-    uint32_t erdp_low;
-    uint32_t erdp_high;
-
-    dma_addr_t er_start;
-    uint32_t er_size;
-    bool er_pcs;
-    unsigned int er_ep_idx;
-    bool er_full;
-
-    XHCIEvent ev_buffer[EV_QUEUE];
-    unsigned int ev_buffer_put;
-    unsigned int ev_buffer_get;
+    int64_t mfindex_start;
+    QEMUTimer *mfwrap_timer;
+    XHCIInterrupter intr[MAXINTRS];
 
     XHCIRing cmd_ring;
 };
@@ -422,6 +476,18 @@ typedef struct XHCIEvRingSeg {
     uint32_t rsvd;
 } XHCIEvRingSeg;
 
+enum xhci_flags {
+    XHCI_FLAG_USE_MSI = 1,
+    XHCI_FLAG_USE_MSI_X,
+};
+
+static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
+                         unsigned int epid);
+static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
+                                unsigned int epid);
+static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
+static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);
+
 static const char *TRBType_names[] = {
     [TRB_RESERVED]                     = "TRB_RESERVED",
     [TR_NORMAL]                        = "TR_NORMAL",
@@ -460,6 +526,45 @@ static const char *TRBType_names[] = {
     [CR_VENDOR_NEC_CHALLENGE_RESPONSE] = "CR_VENDOR_NEC_CHALLENGE_RESPONSE",
 };
 
+static const char *TRBCCode_names[] = {
+    [CC_INVALID]                       = "CC_INVALID",
+    [CC_SUCCESS]                       = "CC_SUCCESS",
+    [CC_DATA_BUFFER_ERROR]             = "CC_DATA_BUFFER_ERROR",
+    [CC_BABBLE_DETECTED]               = "CC_BABBLE_DETECTED",
+    [CC_USB_TRANSACTION_ERROR]         = "CC_USB_TRANSACTION_ERROR",
+    [CC_TRB_ERROR]                     = "CC_TRB_ERROR",
+    [CC_STALL_ERROR]                   = "CC_STALL_ERROR",
+    [CC_RESOURCE_ERROR]                = "CC_RESOURCE_ERROR",
+    [CC_BANDWIDTH_ERROR]               = "CC_BANDWIDTH_ERROR",
+    [CC_NO_SLOTS_ERROR]                = "CC_NO_SLOTS_ERROR",
+    [CC_INVALID_STREAM_TYPE_ERROR]     = "CC_INVALID_STREAM_TYPE_ERROR",
+    [CC_SLOT_NOT_ENABLED_ERROR]        = "CC_SLOT_NOT_ENABLED_ERROR",
+    [CC_EP_NOT_ENABLED_ERROR]          = "CC_EP_NOT_ENABLED_ERROR",
+    [CC_SHORT_PACKET]                  = "CC_SHORT_PACKET",
+    [CC_RING_UNDERRUN]                 = "CC_RING_UNDERRUN",
+    [CC_RING_OVERRUN]                  = "CC_RING_OVERRUN",
+    [CC_VF_ER_FULL]                    = "CC_VF_ER_FULL",
+    [CC_PARAMETER_ERROR]               = "CC_PARAMETER_ERROR",
+    [CC_BANDWIDTH_OVERRUN]             = "CC_BANDWIDTH_OVERRUN",
+    [CC_CONTEXT_STATE_ERROR]           = "CC_CONTEXT_STATE_ERROR",
+    [CC_NO_PING_RESPONSE_ERROR]        = "CC_NO_PING_RESPONSE_ERROR",
+    [CC_EVENT_RING_FULL_ERROR]         = "CC_EVENT_RING_FULL_ERROR",
+    [CC_INCOMPATIBLE_DEVICE_ERROR]     = "CC_INCOMPATIBLE_DEVICE_ERROR",
+    [CC_MISSED_SERVICE_ERROR]          = "CC_MISSED_SERVICE_ERROR",
+    [CC_COMMAND_RING_STOPPED]          = "CC_COMMAND_RING_STOPPED",
+    [CC_COMMAND_ABORTED]               = "CC_COMMAND_ABORTED",
+    [CC_STOPPED]                       = "CC_STOPPED",
+    [CC_STOPPED_LENGTH_INVALID]        = "CC_STOPPED_LENGTH_INVALID",
+    [CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR]
+    = "CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR",
+    [CC_ISOCH_BUFFER_OVERRUN]          = "CC_ISOCH_BUFFER_OVERRUN",
+    [CC_EVENT_LOST_ERROR]              = "CC_EVENT_LOST_ERROR",
+    [CC_UNDEFINED_ERROR]               = "CC_UNDEFINED_ERROR",
+    [CC_INVALID_STREAM_ID_ERROR]       = "CC_INVALID_STREAM_ID_ERROR",
+    [CC_SECONDARY_BANDWIDTH_ERROR]     = "CC_SECONDARY_BANDWIDTH_ERROR",
+    [CC_SPLIT_TRANSACTION_ERROR]       = "CC_SPLIT_TRANSACTION_ERROR",
+};
+
 static const char *lookup_name(uint32_t index, const char **list, uint32_t llen)
 {
     if (index >= llen || list[index] == NULL) {
@@ -474,8 +579,42 @@ static const char *trb_name(XHCITRB *trb)
                        ARRAY_SIZE(TRBType_names));
 }
 
-static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
-                         unsigned int epid);
+static const char *event_name(XHCIEvent *event)
+{
+    return lookup_name(event->ccode, TRBCCode_names,
+                       ARRAY_SIZE(TRBCCode_names));
+}
+
+static uint64_t xhci_mfindex_get(XHCIState *xhci)
+{
+    int64_t now = qemu_get_clock_ns(vm_clock);
+    return (now - xhci->mfindex_start) / 125000;
+}
+
+static void xhci_mfwrap_update(XHCIState *xhci)
+{
+    const uint32_t bits = USBCMD_RS | USBCMD_EWE;
+    uint32_t mfindex, left;
+    int64_t now;
+
+    if ((xhci->usbcmd & bits) == bits) {
+        now = qemu_get_clock_ns(vm_clock);
+        mfindex = ((now - xhci->mfindex_start) / 125000) & 0x3fff;
+        left = 0x4000 - mfindex;
+        qemu_mod_timer(xhci->mfwrap_timer, now + left * 125000);
+    } else {
+        qemu_del_timer(xhci->mfwrap_timer);
+    }
+}
+
+static void xhci_mfwrap_timer(void *opaque)
+{
+    XHCIState *xhci = opaque;
+    XHCIEvent wrap = { ER_MFINDEX_WRAP, CC_SUCCESS };
+
+    xhci_event(xhci, &wrap, 0);
+    xhci_mfwrap_update(xhci);
+}
 
 static inline dma_addr_t xhci_addr64(uint32_t low, uint32_t high)
 {
@@ -495,29 +634,134 @@ static inline dma_addr_t xhci_mask64(uint64_t addr)
     }
 }
 
-static void xhci_irq_update(XHCIState *xhci)
+static inline void xhci_dma_read_u32s(XHCIState *xhci, dma_addr_t addr,
+                                      uint32_t *buf, size_t len)
+{
+    int i;
+
+    assert((len % sizeof(uint32_t)) == 0);
+
+    pci_dma_read(&xhci->pci_dev, addr, buf, len);
+
+    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+        buf[i] = le32_to_cpu(buf[i]);
+    }
+}
+
+static inline void xhci_dma_write_u32s(XHCIState *xhci, dma_addr_t addr,
+                                       uint32_t *buf, size_t len)
+{
+    int i;
+    uint32_t tmp[len / sizeof(uint32_t)];
+
+    assert((len % sizeof(uint32_t)) == 0);
+
+    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+        tmp[i] = cpu_to_le32(buf[i]);
+    }
+    pci_dma_write(&xhci->pci_dev, addr, tmp, len);
+}
+
+static XHCIPort *xhci_lookup_port(XHCIState *xhci, struct USBPort *uport)
+{
+    int index;
+
+    if (!uport->dev) {
+        return NULL;
+    }
+    switch (uport->dev->speed) {
+    case USB_SPEED_LOW:
+    case USB_SPEED_FULL:
+    case USB_SPEED_HIGH:
+        index = uport->index;
+        break;
+    case USB_SPEED_SUPER:
+        index = uport->index + xhci->numports_2;
+        break;
+    default:
+        return NULL;
+    }
+    return &xhci->ports[index];
+}
+
+static void xhci_intx_update(XHCIState *xhci)
 {
     int level = 0;
 
-    if (xhci->iman & IMAN_IP && xhci->iman & IMAN_IE &&
+    if (msix_enabled(&xhci->pci_dev) ||
+        msi_enabled(&xhci->pci_dev)) {
+        return;
+    }
+
+    if (xhci->intr[0].iman & IMAN_IP &&
+        xhci->intr[0].iman & IMAN_IE &&
         xhci->usbcmd & USBCMD_INTE) {
         level = 1;
     }
 
-    if (xhci->msi && msi_enabled(&xhci->pci_dev)) {
-        if (level) {
-            trace_usb_xhci_irq_msi(0);
-            msi_notify(&xhci->pci_dev, 0);
-        }
+    trace_usb_xhci_irq_intx(level);
+    qemu_set_irq(xhci->irq, level);
+}
+
+static void xhci_msix_update(XHCIState *xhci, int v)
+{
+    bool enabled;
+
+    if (!msix_enabled(&xhci->pci_dev)) {
+        return;
+    }
+
+    enabled = xhci->intr[v].iman & IMAN_IE;
+    if (enabled == xhci->intr[v].msix_used) {
+        return;
+    }
+
+    if (enabled) {
+        trace_usb_xhci_irq_msix_use(v);
+        msix_vector_use(&xhci->pci_dev, v);
+        xhci->intr[v].msix_used = true;
     } else {
-        trace_usb_xhci_irq_intx(level);
-        qemu_set_irq(xhci->irq, level);
+        trace_usb_xhci_irq_msix_unuse(v);
+        msix_vector_unuse(&xhci->pci_dev, v);
+        xhci->intr[v].msix_used = false;
+    }
+}
+
+static void xhci_intr_raise(XHCIState *xhci, int v)
+{
+    xhci->intr[v].erdp_low |= ERDP_EHB;
+    xhci->intr[v].iman |= IMAN_IP;
+    xhci->usbsts |= USBSTS_EINT;
+
+    if (!(xhci->intr[v].iman & IMAN_IE)) {
+        return;
+    }
+
+    if (!(xhci->usbcmd & USBCMD_INTE)) {
+        return;
+    }
+
+    if (msix_enabled(&xhci->pci_dev)) {
+        trace_usb_xhci_irq_msix(v);
+        msix_notify(&xhci->pci_dev, v);
+        return;
+    }
+
+    if (msi_enabled(&xhci->pci_dev)) {
+        trace_usb_xhci_irq_msi(v);
+        msi_notify(&xhci->pci_dev, v);
+        return;
+    }
+
+    if (v == 0) {
+        trace_usb_xhci_irq_intx(1);
+        qemu_set_irq(xhci->irq, 1);
     }
 }
 
 static inline int xhci_running(XHCIState *xhci)
 {
-    return !(xhci->usbsts & USBSTS_HCH) && !xhci->er_full;
+    return !(xhci->usbsts & USBSTS_HCH) && !xhci->intr[0].er_full;
 }
 
 static void xhci_die(XHCIState *xhci)
@@ -526,8 +770,9 @@ static void xhci_die(XHCIState *xhci)
     fprintf(stderr, "xhci: asserted controller error\n");
 }
 
-static void xhci_write_event(XHCIState *xhci, XHCIEvent *event)
+static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     XHCITRB ev_trb;
     dma_addr_t addr;
 
@@ -535,26 +780,28 @@ static void xhci_write_event(XHCIState *xhci, XHCIEvent *event)
     ev_trb.status = cpu_to_le32(event->length | (event->ccode << 24));
     ev_trb.control = (event->slotid << 24) | (event->epid << 16) |
                      event->flags | (event->type << TRB_TYPE_SHIFT);
-    if (xhci->er_pcs) {
+    if (intr->er_pcs) {
         ev_trb.control |= TRB_C;
     }
     ev_trb.control = cpu_to_le32(ev_trb.control);
 
-    trace_usb_xhci_queue_event(xhci->er_ep_idx, trb_name(&ev_trb),
-                               ev_trb.parameter, ev_trb.status, ev_trb.control);
+    trace_usb_xhci_queue_event(v, intr->er_ep_idx, trb_name(&ev_trb),
+                               event_name(event), ev_trb.parameter,
+                               ev_trb.status, ev_trb.control);
 
-    addr = xhci->er_start + TRB_SIZE*xhci->er_ep_idx;
+    addr = intr->er_start + TRB_SIZE*intr->er_ep_idx;
     pci_dma_write(&xhci->pci_dev, addr, &ev_trb, TRB_SIZE);
 
-    xhci->er_ep_idx++;
-    if (xhci->er_ep_idx >= xhci->er_size) {
-        xhci->er_ep_idx = 0;
-        xhci->er_pcs = !xhci->er_pcs;
+    intr->er_ep_idx++;
+    if (intr->er_ep_idx >= intr->er_size) {
+        intr->er_ep_idx = 0;
+        intr->er_pcs = !intr->er_pcs;
     }
 }
 
-static void xhci_events_update(XHCIState *xhci)
+static void xhci_events_update(XHCIState *xhci, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     dma_addr_t erdp;
     unsigned int dp_idx;
     bool do_irq = 0;
@@ -563,122 +810,121 @@ static void xhci_events_update(XHCIState *xhci)
         return;
     }
 
-    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);
-    if (erdp < xhci->er_start ||
-        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {
+    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
+    if (erdp < intr->er_start ||
+        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
         fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
-        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",
-                xhci->er_start, xhci->er_size);
+        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
+                v, intr->er_start, intr->er_size);
         xhci_die(xhci);
         return;
     }
-    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;
-    assert(dp_idx < xhci->er_size);
+    dp_idx = (erdp - intr->er_start) / TRB_SIZE;
+    assert(dp_idx < intr->er_size);
 
     /* NEC didn't read section 4.9.4 of the spec (v1.0 p139 top Note) and thus
      * deadlocks when the ER is full. Hack it by holding off events until
      * the driver decides to free at least half of the ring */
-    if (xhci->er_full) {
-        int er_free = dp_idx - xhci->er_ep_idx;
+    if (intr->er_full) {
+        int er_free = dp_idx - intr->er_ep_idx;
         if (er_free <= 0) {
-            er_free += xhci->er_size;
+            er_free += intr->er_size;
         }
-        if (er_free < (xhci->er_size/2)) {
+        if (er_free < (intr->er_size/2)) {
             DPRINTF("xhci_events_update(): event ring still "
                     "more than half full (hack)\n");
             return;
         }
     }
 
-    while (xhci->ev_buffer_put != xhci->ev_buffer_get) {
-        assert(xhci->er_full);
-        if (((xhci->er_ep_idx+1) % xhci->er_size) == dp_idx) {
+    while (intr->ev_buffer_put != intr->ev_buffer_get) {
+        assert(intr->er_full);
+        if (((intr->er_ep_idx+1) % intr->er_size) == dp_idx) {
             DPRINTF("xhci_events_update(): event ring full again\n");
 #ifndef ER_FULL_HACK
             XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
-            xhci_write_event(xhci, &full);
+            xhci_write_event(xhci, &full, v);
 #endif
             do_irq = 1;
             break;
         }
-        XHCIEvent *event = &xhci->ev_buffer[xhci->ev_buffer_get];
-        xhci_write_event(xhci, event);
-        xhci->ev_buffer_get++;
+        XHCIEvent *event = &intr->ev_buffer[intr->ev_buffer_get];
+        xhci_write_event(xhci, event, v);
+        intr->ev_buffer_get++;
         do_irq = 1;
-        if (xhci->ev_buffer_get == EV_QUEUE) {
-            xhci->ev_buffer_get = 0;
+        if (intr->ev_buffer_get == EV_QUEUE) {
+            intr->ev_buffer_get = 0;
         }
     }
 
     if (do_irq) {
-        xhci->erdp_low |= ERDP_EHB;
-        xhci->iman |= IMAN_IP;
-        xhci->usbsts |= USBSTS_EINT;
-        xhci_irq_update(xhci);
+        xhci_intr_raise(xhci, v);
     }
 
-    if (xhci->er_full && xhci->ev_buffer_put == xhci->ev_buffer_get) {
+    if (intr->er_full && intr->ev_buffer_put == intr->ev_buffer_get) {
         DPRINTF("xhci_events_update(): event ring no longer full\n");
-        xhci->er_full = 0;
+        intr->er_full = 0;
     }
-    return;
 }
 
-static void xhci_event(XHCIState *xhci, XHCIEvent *event)
+static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
+    XHCIInterrupter *intr;
     dma_addr_t erdp;
     unsigned int dp_idx;
 
-    if (xhci->er_full) {
+    if (v >= xhci->numintrs) {
+        DPRINTF("intr nr out of range (%d >= %d)\n", v, xhci->numintrs);
+        return;
+    }
+    intr = &xhci->intr[v];
+
+    if (intr->er_full) {
         DPRINTF("xhci_event(): ER full, queueing\n");
-        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {
+        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
             fprintf(stderr, "xhci: event queue full, dropping event!\n");
             return;
         }
-        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;
-        if (xhci->ev_buffer_put == EV_QUEUE) {
-            xhci->ev_buffer_put = 0;
+        intr->ev_buffer[intr->ev_buffer_put++] = *event;
+        if (intr->ev_buffer_put == EV_QUEUE) {
+            intr->ev_buffer_put = 0;
         }
         return;
     }
 
-    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);
-    if (erdp < xhci->er_start ||
-        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {
+    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
+    if (erdp < intr->er_start ||
+        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
         fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
-        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",
-                xhci->er_start, xhci->er_size);
+        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
+                v, intr->er_start, intr->er_size);
         xhci_die(xhci);
         return;
     }
 
-    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;
-    assert(dp_idx < xhci->er_size);
+    dp_idx = (erdp - intr->er_start) / TRB_SIZE;
+    assert(dp_idx < intr->er_size);
 
-    if ((xhci->er_ep_idx+1) % xhci->er_size == dp_idx) {
+    if ((intr->er_ep_idx+1) % intr->er_size == dp_idx) {
         DPRINTF("xhci_event(): ER full, queueing\n");
 #ifndef ER_FULL_HACK
         XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
         xhci_write_event(xhci, &full);
 #endif
-        xhci->er_full = 1;
-        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {
+        intr->er_full = 1;
+        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
             fprintf(stderr, "xhci: event queue full, dropping event!\n");
             return;
         }
-        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;
-        if (xhci->ev_buffer_put == EV_QUEUE) {
-            xhci->ev_buffer_put = 0;
+        intr->ev_buffer[intr->ev_buffer_put++] = *event;
+        if (intr->ev_buffer_put == EV_QUEUE) {
+            intr->ev_buffer_put = 0;
         }
     } else {
-        xhci_write_event(xhci, event);
+        xhci_write_event(xhci, event, v);
     }
 
-    xhci->erdp_low |= ERDP_EHB;
-    xhci->iman |= IMAN_IP;
-    xhci->usbsts |= USBSTS_EINT;
-
-    xhci_irq_update(xhci);
+    xhci_intr_raise(xhci, v);
 }
 
 static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,
@@ -770,17 +1016,24 @@ static int xhci_ring_chain_length(XHCIState *xhci, const XHCIRing *ring)
     }
 }
 
-static void xhci_er_reset(XHCIState *xhci)
+static void xhci_er_reset(XHCIState *xhci, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     XHCIEvRingSeg seg;
 
+    if (intr->erstsz == 0) {
+        /* disabled */
+        intr->er_start = 0;
+        intr->er_size = 0;
+        return;
+    }
     /* cache the (sole) event ring segment location */
-    if (xhci->erstsz != 1) {
-        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", xhci->erstsz);
+    if (intr->erstsz != 1) {
+        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", intr->erstsz);
         xhci_die(xhci);
         return;
     }
-    dma_addr_t erstba = xhci_addr64(xhci->erstba_low, xhci->erstba_high);
+    dma_addr_t erstba = xhci_addr64(intr->erstba_low, intr->erstba_high);
     pci_dma_read(&xhci->pci_dev, erstba, &seg, sizeof(seg));
     le32_to_cpus(&seg.addr_low);
     le32_to_cpus(&seg.addr_high);
@@ -790,21 +1043,22 @@ static void xhci_er_reset(XHCIState *xhci)
         xhci_die(xhci);
         return;
     }
-    xhci->er_start = xhci_addr64(seg.addr_low, seg.addr_high);
-    xhci->er_size = seg.size;
+    intr->er_start = xhci_addr64(seg.addr_low, seg.addr_high);
+    intr->er_size = seg.size;
 
-    xhci->er_ep_idx = 0;
-    xhci->er_pcs = 1;
-    xhci->er_full = 0;
+    intr->er_ep_idx = 0;
+    intr->er_pcs = 1;
+    intr->er_full = 0;
 
-    DPRINTF("xhci: event ring:" DMA_ADDR_FMT " [%d]\n",
-            xhci->er_start, xhci->er_size);
+    DPRINTF("xhci: event ring[%d]:" DMA_ADDR_FMT " [%d]\n",
+            v, intr->er_start, intr->er_size);
 }
 
 static void xhci_run(XHCIState *xhci)
 {
     trace_usb_xhci_run();
     xhci->usbsts &= ~USBSTS_HCH;
+    xhci->mfindex_start = qemu_get_clock_ns(vm_clock);
 }
 
 static void xhci_stop(XHCIState *xhci)
@@ -818,21 +1072,24 @@ static void xhci_set_ep_state(XHCIState *xhci, XHCIEPContext *epctx,
                               uint32_t state)
 {
     uint32_t ctx[5];
-    if (epctx->state == state) {
-        return;
-    }
 
-    pci_dma_read(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));
+    xhci_dma_read_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
     ctx[0] &= ~EP_STATE_MASK;
     ctx[0] |= state;
     ctx[2] = epctx->ring.dequeue | epctx->ring.ccs;
     ctx[3] = (epctx->ring.dequeue >> 16) >> 16;
     DPRINTF("xhci: set epctx: " DMA_ADDR_FMT " state=%d dequeue=%08x%08x\n",
             epctx->pctx, state, ctx[3], ctx[2]);
-    pci_dma_write(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));
+    xhci_dma_write_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
     epctx->state = state;
 }
 
+static void xhci_ep_kick_timer(void *opaque)
+{
+    XHCIEPContext *epctx = opaque;
+    xhci_kick_ep(epctx->xhci, epctx->slotid, epctx->epid);
+}
+
 static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
                                unsigned int epid, dma_addr_t pctx,
                                uint32_t *ctx)
@@ -843,17 +1100,19 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     int i;
 
     trace_usb_xhci_ep_enable(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     slot = &xhci->slots[slotid-1];
     if (slot->eps[epid-1]) {
-        fprintf(stderr, "xhci: slot %d ep %d already enabled!\n", slotid, epid);
-        return CC_TRB_ERROR;
+        xhci_disable_ep(xhci, slotid, epid);
     }
 
     epctx = g_malloc(sizeof(XHCIEPContext));
     memset(epctx, 0, sizeof(XHCIEPContext));
+    epctx->xhci = xhci;
+    epctx->slotid = slotid;
+    epctx->epid = epid;
 
     slot->eps[epid-1] = epctx;
 
@@ -866,16 +1125,16 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     epctx->pctx = pctx;
     epctx->max_psize = ctx[1]>>16;
     epctx->max_psize *= 1+((ctx[1]>>8)&0xff);
-    epctx->has_bg = false;
-    if (epctx->type == ET_ISO_IN) {
-        epctx->has_bg = true;
-    }
     DPRINTF("xhci: endpoint %d.%d max transaction (burst) size is %d\n",
             epid/2, epid%2, epctx->max_psize);
     for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
         usb_packet_init(&epctx->transfers[i].packet);
     }
 
+    epctx->interval = 1 << (ctx[0] >> 16) & 0xff;
+    epctx->mfindex_last = 0;
+    epctx->kick_timer = qemu_new_timer_ns(vm_clock, xhci_ep_kick_timer, epctx);
+
     epctx->state = EP_RUNNING;
     ctx[0] &= ~EP_STATE_MASK;
     ctx[0] |= EP_RUNNING;
@@ -883,13 +1142,43 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     return CC_SUCCESS;
 }
 
+static int xhci_ep_nuke_one_xfer(XHCITransfer *t)
+{
+    int killed = 0;
+
+    if (t->running_async) {
+        usb_cancel_packet(&t->packet);
+        t->running_async = 0;
+        t->cancelled = 1;
+        DPRINTF("xhci: cancelling transfer, waiting for it to complete\n");
+        killed = 1;
+    }
+    if (t->running_retry) {
+        XHCIEPContext *epctx = t->xhci->slots[t->slotid-1].eps[t->epid-1];
+        if (epctx) {
+            epctx->retry = NULL;
+            qemu_del_timer(epctx->kick_timer);
+        }
+        t->running_retry = 0;
+    }
+    if (t->trbs) {
+        g_free(t->trbs);
+    }
+
+    t->trbs = NULL;
+    t->trb_count = t->trb_alloced = 0;
+
+    return killed;
+}
+
 static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
                                unsigned int epid)
 {
     XHCISlot *slot;
     XHCIEPContext *epctx;
     int i, xferi, killed = 0;
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    USBEndpoint *ep = NULL;
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     DPRINTF("xhci_ep_nuke_xfers(%d, %d)\n", slotid, epid);
@@ -904,52 +1193,15 @@ static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
 
     xferi = epctx->next_xfer;
     for (i = 0; i < TD_QUEUE; i++) {
-        XHCITransfer *t = &epctx->transfers[xferi];
-        if (t->running_async) {
-            usb_cancel_packet(&t->packet);
-            t->running_async = 0;
-            t->cancelled = 1;
-            DPRINTF("xhci: cancelling transfer %d, waiting for it to complete...\n", i);
-            killed++;
-        }
-        if (t->running_retry) {
-            t->running_retry = 0;
-            epctx->retry = NULL;
-        }
-        if (t->backgrounded) {
-            t->backgrounded = 0;
+        if (epctx->transfers[xferi].packet.ep) {
+            ep = epctx->transfers[xferi].packet.ep;
         }
-        if (t->trbs) {
-            g_free(t->trbs);
-        }
-        if (t->data) {
-            g_free(t->data);
-        }
-
-        t->trbs = NULL;
-        t->data = NULL;
-        t->trb_count = t->trb_alloced = 0;
-        t->data_length = t->data_alloced = 0;
+        killed += xhci_ep_nuke_one_xfer(&epctx->transfers[xferi]);
+        epctx->transfers[xferi].packet.ep = NULL;
         xferi = (xferi + 1) % TD_QUEUE;
     }
-    if (epctx->has_bg) {
-        xferi = epctx->next_bg;
-        for (i = 0; i < BG_XFERS; i++) {
-            XHCITransfer *t = &epctx->bg_transfers[xferi];
-            if (t->running_async) {
-                usb_cancel_packet(&t->packet);
-                t->running_async = 0;
-                t->cancelled = 1;
-                DPRINTF("xhci: cancelling bg transfer %d, waiting for it to complete...\n", i);
-                killed++;
-            }
-            if (t->data) {
-                g_free(t->data);
-            }
-
-            t->data = NULL;
-            xferi = (xferi + 1) % BG_XFERS;
-        }
+    if (ep) {
+        usb_device_ep_stopped(ep->dev, ep);
     }
     return killed;
 }
@@ -961,7 +1213,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
 
     trace_usb_xhci_ep_disable(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     slot = &xhci->slots[slotid-1];
@@ -977,6 +1229,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
 
     xhci_set_ep_state(xhci, epctx, EP_DISABLED);
 
+    qemu_free_timer(epctx->kick_timer);
     g_free(epctx);
     slot->eps[epid-1] = NULL;
 
@@ -990,7 +1243,7 @@ static TRBCCode xhci_stop_ep(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
 
     trace_usb_xhci_ep_stop(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
@@ -1024,7 +1277,7 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
     USBDevice *dev;
 
     trace_usb_xhci_ep_reset(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
@@ -1057,7 +1310,7 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
         ep |= 0x80;
     }
 
-    dev = xhci->ports[xhci->slots[slotid-1].port-1].port.dev;
+    dev = xhci->slots[slotid-1].uport->dev;
     if (!dev) {
         return CC_USB_TRANSACTION_ERROR;
     }
@@ -1074,14 +1327,14 @@ static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
     dma_addr_t dequeue;
 
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
         return CC_TRB_ERROR;
     }
 
-    DPRINTF("xhci_set_ep_dequeue(%d, %d, %016"PRIx64")\n", slotid, epid, pdequeue);
+    trace_usb_xhci_ep_set_dequeue(slotid, epid, pdequeue);
     dequeue = xhci_mask64(pdequeue);
 
     slot = &xhci->slots[slotid-1];
@@ -1107,81 +1360,89 @@ static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,
     return CC_SUCCESS;
 }
 
-static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
-                          unsigned int length, bool in_xfer, bool out_xfer,
-                          bool report)
+static int xhci_xfer_create_sgl(XHCITransfer *xfer, int in_xfer)
 {
-    int i;
-    uint32_t edtla = 0;
-    unsigned int transferred = 0;
-    unsigned int left = length;
-    bool reported = 0;
-    bool shortpkt = 0;
-    XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};
     XHCIState *xhci = xfer->xhci;
+    int i;
 
-    DPRINTF("xhci_xfer_data(len=%d, in_xfer=%d, out_xfer=%d, report=%d)\n",
-            length, in_xfer, out_xfer, report);
-
-    assert(!(in_xfer && out_xfer));
-
+    xfer->int_req = false;
+    pci_dma_sglist_init(&xfer->sgl, &xhci->pci_dev, xfer->trb_count);
     for (i = 0; i < xfer->trb_count; i++) {
         XHCITRB *trb = &xfer->trbs[i];
         dma_addr_t addr;
         unsigned int chunk = 0;
 
+        if (trb->control & TRB_TR_IOC) {
+            xfer->int_req = true;
+        }
+
         switch (TRB_TYPE(*trb)) {
         case TR_DATA:
             if ((!(trb->control & TRB_TR_DIR)) != (!in_xfer)) {
                 fprintf(stderr, "xhci: data direction mismatch for TR_DATA\n");
-                xhci_die(xhci);
-                return transferred;
+                goto err;
             }
             /* fallthrough */
         case TR_NORMAL:
         case TR_ISOCH:
             addr = xhci_mask64(trb->parameter);
             chunk = trb->status & 0x1ffff;
+            if (trb->control & TRB_TR_IDT) {
+                if (chunk > 8 || in_xfer) {
+                    fprintf(stderr, "xhci: invalid immediate data TRB\n");
+                    goto err;
+                }
+                qemu_sglist_add(&xfer->sgl, trb->addr, chunk);
+            } else {
+                qemu_sglist_add(&xfer->sgl, addr, chunk);
+            }
+            break;
+        }
+    }
+
+    return 0;
+
+err:
+    qemu_sglist_destroy(&xfer->sgl);
+    xhci_die(xhci);
+    return -1;
+}
+
+static void xhci_xfer_unmap(XHCITransfer *xfer)
+{
+    usb_packet_unmap(&xfer->packet, &xfer->sgl);
+    qemu_sglist_destroy(&xfer->sgl);
+}
+
+static void xhci_xfer_report(XHCITransfer *xfer)
+{
+    uint32_t edtla = 0;
+    unsigned int left;
+    bool reported = 0;
+    bool shortpkt = 0;
+    XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};
+    XHCIState *xhci = xfer->xhci;
+    int i;
+
+    left = xfer->packet.actual_length;
+
+    for (i = 0; i < xfer->trb_count; i++) {
+        XHCITRB *trb = &xfer->trbs[i];
+        unsigned int chunk = 0;
+
+        switch (TRB_TYPE(*trb)) {
+        case TR_DATA:
+        case TR_NORMAL:
+        case TR_ISOCH:
+            chunk = trb->status & 0x1ffff;
             if (chunk > left) {
                 chunk = left;
-                shortpkt = 1;
-            }
-            if (in_xfer || out_xfer) {
-                if (trb->control & TRB_TR_IDT) {
-                    uint64_t idata;
-                    if (chunk > 8 || in_xfer) {
-                        fprintf(stderr, "xhci: invalid immediate data TRB\n");
-                        xhci_die(xhci);
-                        return transferred;
-                    }
-                    idata = le64_to_cpu(trb->parameter);
-                    memcpy(data, &idata, chunk);
-                } else {
-                    DPRINTF("xhci_xfer_data: r/w(%d) %d bytes at "
-                            DMA_ADDR_FMT "\n", in_xfer, chunk, addr);
-                    if (in_xfer) {
-                        pci_dma_write(&xhci->pci_dev, addr, data, chunk);
-                    } else {
-                        pci_dma_read(&xhci->pci_dev, addr, data, chunk);
-                    }
-#ifdef DEBUG_DATA
-                    unsigned int count = chunk;
-                    int i;
-                    if (count > 16) {
-                        count = 16;
-                    }
-                    DPRINTF(" ::");
-                    for (i = 0; i < count; i++) {
-                        DPRINTF(" %02x", data[i]);
-                    }
-                    DPRINTF("\n");
-#endif
+                if (xfer->status == CC_SUCCESS) {
+                    shortpkt = 1;
                 }
             }
             left -= chunk;
-            data += chunk;
             edtla += chunk;
-            transferred += chunk;
             break;
         case TR_STATUS:
             reported = 0;
@@ -1189,8 +1450,9 @@ static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
             break;
         }
 
-        if (report && !reported && (trb->control & TRB_TR_IOC ||
-            (shortpkt && (trb->control & TRB_TR_ISP)))) {
+        if (!reported && ((trb->control & TRB_TR_IOC) ||
+                          (shortpkt && (trb->control & TRB_TR_ISP)) ||
+                          (xfer->status != CC_SUCCESS && left == 0))) {
             event.slotid = xfer->slotid;
             event.epid = xfer->epid;
             event.length = (trb->status & 0x1ffff) - chunk;
@@ -1208,11 +1470,13 @@ static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
                 DPRINTF("xhci_xfer_data: EDTLA=%d\n", event.length);
                 edtla = 0;
             }
-            xhci_event(xhci, &event);
+            xhci_event(xhci, &event, TRB_INTR(*trb));
             reported = 1;
+            if (xfer->status != CC_SUCCESS) {
+                return;
+            }
         }
     }
-    return transferred;
 }
 
 static void xhci_stall_ep(XHCITransfer *xfer)
@@ -1231,184 +1495,47 @@ static void xhci_stall_ep(XHCITransfer *xfer)
 static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer,
                        XHCIEPContext *epctx);
 
-static void xhci_bg_update(XHCIState *xhci, XHCIEPContext *epctx)
+static int xhci_setup_packet(XHCITransfer *xfer)
 {
-    if (epctx->bg_updating) {
-        return;
-    }
-    DPRINTF("xhci_bg_update(%p, %p)\n", xhci, epctx);
-    assert(epctx->has_bg);
-    DPRINTF("xhci: fg=%d bg=%d\n", epctx->comp_xfer, epctx->next_bg);
-    epctx->bg_updating = 1;
-    while (epctx->transfers[epctx->comp_xfer].backgrounded &&
-           epctx->bg_transfers[epctx->next_bg].complete) {
-        XHCITransfer *fg = &epctx->transfers[epctx->comp_xfer];
-        XHCITransfer *bg = &epctx->bg_transfers[epctx->next_bg];
-#if 0
-        DPRINTF("xhci: completing fg %d from bg %d.%d (stat: %d)\n",
-                epctx->comp_xfer, epctx->next_bg, bg->cur_pkt,
-                bg->usbxfer->iso_packet_desc[bg->cur_pkt].status
-               );
-#endif
-        assert(epctx->type == ET_ISO_IN);
-        assert(bg->iso_xfer);
-        assert(bg->in_xfer);
-        uint8_t *p = bg->data + bg->cur_pkt * bg->pktsize;
-#if 0
-        int len = bg->usbxfer->iso_packet_desc[bg->cur_pkt].actual_length;
-        fg->status = libusb_to_ccode(bg->usbxfer->iso_packet_desc[bg->cur_pkt].status);
-#else
-        int len = 0;
-        FIXME();
-#endif
-        fg->complete = 1;
-        fg->backgrounded = 0;
-
-        if (fg->status == CC_STALL_ERROR) {
-            xhci_stall_ep(fg);
-        }
-
-        xhci_xfer_data(fg, p, len, 1, 0, 1);
-
-        epctx->comp_xfer++;
-        if (epctx->comp_xfer == TD_QUEUE) {
-            epctx->comp_xfer = 0;
-        }
-        DPRINTF("next fg xfer: %d\n", epctx->comp_xfer);
-        bg->cur_pkt++;
-        if (bg->cur_pkt == bg->pkts) {
-            bg->complete = 0;
-            if (xhci_submit(xhci, bg, epctx) < 0) {
-                fprintf(stderr, "xhci: bg resubmit failed\n");
-            }
-            epctx->next_bg++;
-            if (epctx->next_bg == BG_XFERS) {
-                epctx->next_bg = 0;
-            }
-            DPRINTF("next bg xfer: %d\n", epctx->next_bg);
-
-        xhci_kick_ep(xhci, fg->slotid, fg->epid);
-        }
-    }
-    epctx->bg_updating = 0;
-}
-
-#if 0
-static void xhci_xfer_cb(struct libusb_transfer *transfer)
-{
-    XHCIState *xhci;
-    XHCITransfer *xfer;
-
-    xfer = (XHCITransfer *)transfer->user_data;
-    xhci = xfer->xhci;
-
-    DPRINTF("xhci_xfer_cb(slot=%d, ep=%d, status=%d)\n", xfer->slotid,
-            xfer->epid, transfer->status);
-
-    assert(xfer->slotid >= 1 && xfer->slotid <= MAXSLOTS);
-    assert(xfer->epid >= 1 && xfer->epid <= 31);
-
-    if (xfer->cancelled) {
-        DPRINTF("xhci: transfer cancelled, not reporting anything\n");
-        xfer->running = 0;
-        return;
-    }
-
-    XHCIEPContext *epctx;
-    XHCISlot *slot;
-    slot = &xhci->slots[xfer->slotid-1];
-    assert(slot->eps[xfer->epid-1]);
-    epctx = slot->eps[xfer->epid-1];
-
-    if (xfer->bg_xfer) {
-        DPRINTF("xhci: background transfer, updating\n");
-        xfer->complete = 1;
-        xfer->running = 0;
-        xhci_bg_update(xhci, epctx);
-        return;
-    }
-
-    if (xfer->iso_xfer) {
-        transfer->status = transfer->iso_packet_desc[0].status;
-        transfer->actual_length = transfer->iso_packet_desc[0].actual_length;
-    }
-
-    xfer->status = libusb_to_ccode(transfer->status);
-
-    xfer->complete = 1;
-    xfer->running = 0;
-
-    if (transfer->status == LIBUSB_TRANSFER_STALL)
-        xhci_stall_ep(xhci, epctx, xfer);
+    XHCIState *xhci = xfer->xhci;
+    USBDevice *dev;
+    USBEndpoint *ep;
+    int dir;
 
-    DPRINTF("xhci: transfer actual length = %d\n", transfer->actual_length);
+    dir = xfer->in_xfer ? USB_TOKEN_IN : USB_TOKEN_OUT;
 
-    if (xfer->in_xfer) {
-        if (xfer->epid == 1) {
-            xhci_xfer_data(xhci, xfer, xfer->data + 8,
-                           transfer->actual_length, 1, 0, 1);
-        } else {
-            xhci_xfer_data(xhci, xfer, xfer->data,
-                           transfer->actual_length, 1, 0, 1);
-        }
+    if (xfer->packet.ep) {
+        ep = xfer->packet.ep;
+        dev = ep->dev;
     } else {
-        xhci_xfer_data(xhci, xfer, NULL, transfer->actual_length, 0, 0, 1);
-    }
-
-    xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
-}
-
-static int xhci_hle_control(XHCIState *xhci, XHCITransfer *xfer,
-                            uint8_t bmRequestType, uint8_t bRequest,
-                            uint16_t wValue, uint16_t wIndex, uint16_t wLength)
-{
-    uint16_t type_req = (bmRequestType << 8) | bRequest;
-
-    switch (type_req) {
-        case 0x0000 | USB_REQ_SET_CONFIGURATION:
-            DPRINTF("xhci: HLE switch configuration\n");
-            return xhci_switch_config(xhci, xfer->slotid, wValue) == 0;
-        case 0x0100 | USB_REQ_SET_INTERFACE:
-            DPRINTF("xhci: HLE set interface altsetting\n");
-            return xhci_set_iface_alt(xhci, xfer->slotid, wIndex, wValue) == 0;
-        case 0x0200 | USB_REQ_CLEAR_FEATURE:
-            if (wValue == 0) { // endpoint halt
-                DPRINTF("xhci: HLE clear halt\n");
-                return xhci_clear_halt(xhci, xfer->slotid, wIndex);
-            }
-        case 0x0000 | USB_REQ_SET_ADDRESS:
-            fprintf(stderr, "xhci: warn: illegal SET_ADDRESS request\n");
-            return 0;
-        default:
-            return 0;
+        if (!xhci->slots[xfer->slotid-1].uport) {
+            fprintf(stderr, "xhci: slot %d has no device\n",
+                    xfer->slotid);
+            return -1;
+        }
+        dev = xhci->slots[xfer->slotid-1].uport->dev;
+        ep = usb_ep_get(dev, dir, xfer->epid >> 1);
     }
-}
-#endif
-
-static int xhci_setup_packet(XHCITransfer *xfer, USBDevice *dev)
-{
-    USBEndpoint *ep;
-    int dir;
 
-    dir = xfer->in_xfer ? USB_TOKEN_IN : USB_TOKEN_OUT;
-    ep = usb_ep_get(dev, dir, xfer->epid >> 1);
-    usb_packet_setup(&xfer->packet, dir, ep, xfer->trbs[0].addr);
-    usb_packet_addbuf(&xfer->packet, xfer->data, xfer->data_length);
+    xhci_xfer_create_sgl(xfer, dir == USB_TOKEN_IN); /* Also sets int_req */
+    usb_packet_setup(&xfer->packet, dir, ep, xfer->trbs[0].addr, false,
+                     xfer->int_req);
+    usb_packet_map(&xfer->packet, &xfer->sgl);
     DPRINTF("xhci: setup packet pid 0x%x addr %d ep %d\n",
             xfer->packet.pid, dev->addr, ep->nr);
     return 0;
 }
 
-static int xhci_complete_packet(XHCITransfer *xfer, int ret)
+static int xhci_complete_packet(XHCITransfer *xfer)
 {
-    if (ret == USB_RET_ASYNC) {
+    if (xfer->packet.status == USB_RET_ASYNC) {
         trace_usb_xhci_xfer_async(xfer);
         xfer->running_async = 1;
         xfer->running_retry = 0;
         xfer->complete = 0;
         xfer->cancelled = 0;
         return 0;
-    } else if (ret == USB_RET_NAK) {
+    } else if (xfer->packet.status == USB_RET_NAK) {
         trace_usb_xhci_xfer_nak(xfer);
         xfer->running_async = 0;
         xfer->running_retry = 1;
@@ -1419,57 +1546,46 @@ static int xhci_complete_packet(XHCITransfer *xfer, int ret)
         xfer->running_async = 0;
         xfer->running_retry = 0;
         xfer->complete = 1;
+        xhci_xfer_unmap(xfer);
     }
 
-    if (ret >= 0) {
+    if (xfer->packet.status == USB_RET_SUCCESS) {
+        trace_usb_xhci_xfer_success(xfer, xfer->packet.actual_length);
         xfer->status = CC_SUCCESS;
-        xhci_xfer_data(xfer, xfer->data, ret, xfer->in_xfer, 0, 1);
-        trace_usb_xhci_xfer_success(xfer, ret);
+        xhci_xfer_report(xfer);
         return 0;
     }
 
     /* error */
-    trace_usb_xhci_xfer_error(xfer, ret);
-    switch (ret) {
+    trace_usb_xhci_xfer_error(xfer, xfer->packet.status);
+    switch (xfer->packet.status) {
     case USB_RET_NODEV:
         xfer->status = CC_USB_TRANSACTION_ERROR;
-        xhci_xfer_data(xfer, xfer->data, 0, xfer->in_xfer, 0, 1);
+        xhci_xfer_report(xfer);
         xhci_stall_ep(xfer);
         break;
     case USB_RET_STALL:
         xfer->status = CC_STALL_ERROR;
-        xhci_xfer_data(xfer, xfer->data, 0, xfer->in_xfer, 0, 1);
+        xhci_xfer_report(xfer);
         xhci_stall_ep(xfer);
         break;
     default:
-        fprintf(stderr, "%s: FIXME: ret = %d\n", __FUNCTION__, ret);
+        fprintf(stderr, "%s: FIXME: status = %d\n", __func__,
+                xfer->packet.status);
         FIXME();
     }
     return 0;
 }
 
-static USBDevice *xhci_find_device(XHCIPort *port, uint8_t addr)
-{
-    if (!(port->portsc & PORTSC_PED)) {
-        return NULL;
-    }
-    return usb_find_device(&port->port, addr);
-}
-
 static int xhci_fire_ctl_transfer(XHCIState *xhci, XHCITransfer *xfer)
 {
     XHCITRB *trb_setup, *trb_status;
     uint8_t bmRequestType;
-    uint16_t wLength;
-    XHCIPort *port;
-    USBDevice *dev;
-    int ret;
 
     trb_setup = &xfer->trbs[0];
     trb_status = &xfer->trbs[xfer->trb_count-1];
 
-    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid,
-                              trb_setup->parameter >> 48);
+    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid);
 
     /* at most one Event Data TRB allowed after STATUS */
     if (TRB_TYPE(*trb_status) == TR_EVDATA && xfer->trb_count > 2) {
@@ -1498,93 +1614,87 @@ static int xhci_fire_ctl_transfer(XHCIState *xhci, XHCITransfer *xfer)
     }
 
     bmRequestType = trb_setup->parameter;
-    wLength = trb_setup->parameter >> 48;
-
-    if (xfer->data && xfer->data_alloced < wLength) {
-        xfer->data_alloced = 0;
-        g_free(xfer->data);
-        xfer->data = NULL;
-    }
-    if (!xfer->data) {
-        DPRINTF("xhci: alloc %d bytes data\n", wLength);
-        xfer->data = g_malloc(wLength+1);
-        xfer->data_alloced = wLength;
-    }
-    xfer->data_length = wLength;
-
-    port = &xhci->ports[xhci->slots[xfer->slotid-1].port-1];
-    dev = xhci_find_device(port, xhci->slots[xfer->slotid-1].devaddr);
-    if (!dev) {
-        fprintf(stderr, "xhci: slot %d port %d has no device\n", xfer->slotid,
-                xhci->slots[xfer->slotid-1].port);
-        return -1;
-    }
 
     xfer->in_xfer = bmRequestType & USB_DIR_IN;
     xfer->iso_xfer = false;
 
-    xhci_setup_packet(xfer, dev);
-    xfer->packet.parameter = trb_setup->parameter;
-    if (!xfer->in_xfer) {
-        xhci_xfer_data(xfer, xfer->data, wLength, 0, 1, 0);
+    if (xhci_setup_packet(xfer) < 0) {
+        return -1;
     }
+    xfer->packet.parameter = trb_setup->parameter;
 
-    ret = usb_handle_packet(dev, &xfer->packet);
+    usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
 
-    xhci_complete_packet(xfer, ret);
+    xhci_complete_packet(xfer);
     if (!xfer->running_async && !xfer->running_retry) {
         xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
     }
     return 0;
 }
 
-static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
+static void xhci_calc_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
+                               XHCIEPContext *epctx, uint64_t mfindex)
 {
-    XHCIPort *port;
-    USBDevice *dev;
-    int ret;
-
-    DPRINTF("xhci_submit(slotid=%d,epid=%d)\n", xfer->slotid, xfer->epid);
-
-    xfer->in_xfer = epctx->type>>2;
-
-    if (xfer->data && xfer->data_alloced < xfer->data_length) {
-        xfer->data_alloced = 0;
-        g_free(xfer->data);
-        xfer->data = NULL;
-    }
-    if (!xfer->data && xfer->data_length) {
-        DPRINTF("xhci: alloc %d bytes data\n", xfer->data_length);
-        xfer->data = g_malloc(xfer->data_length);
-        xfer->data_alloced = xfer->data_length;
-    }
-    if (epctx->type == ET_ISO_IN || epctx->type == ET_ISO_OUT) {
-        if (!xfer->bg_xfer) {
-            xfer->pkts = 1;
+    if (xfer->trbs[0].control & TRB_TR_SIA) {
+        uint64_t asap = ((mfindex + epctx->interval - 1) &
+                         ~(epctx->interval-1));
+        if (asap >= epctx->mfindex_last &&
+            asap <= epctx->mfindex_last + epctx->interval * 4) {
+            xfer->mfindex_kick = epctx->mfindex_last + epctx->interval;
+        } else {
+            xfer->mfindex_kick = asap;
         }
     } else {
-        xfer->pkts = 0;
+        xfer->mfindex_kick = (xfer->trbs[0].control >> TRB_TR_FRAMEID_SHIFT)
+            & TRB_TR_FRAMEID_MASK;
+        xfer->mfindex_kick |= mfindex & ~0x3fff;
+        if (xfer->mfindex_kick < mfindex) {
+            xfer->mfindex_kick += 0x4000;
+        }
     }
+}
 
-    port = &xhci->ports[xhci->slots[xfer->slotid-1].port-1];
-    dev = xhci_find_device(port, xhci->slots[xfer->slotid-1].devaddr);
-    if (!dev) {
-        fprintf(stderr, "xhci: slot %d port %d has no device\n", xfer->slotid,
-                xhci->slots[xfer->slotid-1].port);
-        return -1;
+static void xhci_check_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
+                                XHCIEPContext *epctx, uint64_t mfindex)
+{
+    if (xfer->mfindex_kick > mfindex) {
+        qemu_mod_timer(epctx->kick_timer, qemu_get_clock_ns(vm_clock) +
+                       (xfer->mfindex_kick - mfindex) * 125000);
+        xfer->running_retry = 1;
+    } else {
+        epctx->mfindex_last = xfer->mfindex_kick;
+        qemu_del_timer(epctx->kick_timer);
+        xfer->running_retry = 0;
     }
+}
 
-    xhci_setup_packet(xfer, dev);
+
+static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
+{
+    uint64_t mfindex;
+
+    DPRINTF("xhci_submit(slotid=%d,epid=%d)\n", xfer->slotid, xfer->epid);
+
+    xfer->in_xfer = epctx->type>>2;
 
     switch(epctx->type) {
     case ET_INTR_OUT:
     case ET_INTR_IN:
     case ET_BULK_OUT:
     case ET_BULK_IN:
+        xfer->pkts = 0;
+        xfer->iso_xfer = false;
         break;
     case ET_ISO_OUT:
     case ET_ISO_IN:
-        FIXME();
+        xfer->pkts = 1;
+        xfer->iso_xfer = true;
+        mfindex = xhci_mfindex_get(xhci);
+        xhci_calc_iso_kick(xhci, xfer, epctx, mfindex);
+        xhci_check_iso_kick(xhci, xfer, epctx, mfindex);
+        if (xfer->running_retry) {
+            return -1;
+        }
         break;
     default:
         fprintf(stderr, "xhci: unknown or unhandled EP "
@@ -1593,12 +1703,12 @@ static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx
         return -1;
     }
 
-    if (!xfer->in_xfer) {
-        xhci_xfer_data(xfer, xfer->data, xfer->data_length, 0, 1, 0);
+    if (xhci_setup_packet(xfer) < 0) {
+        return -1;
     }
-    ret = usb_handle_packet(dev, &xfer->packet);
+    usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
 
-    xhci_complete_packet(xfer, ret);
+    xhci_complete_packet(xfer);
     if (!xfer->running_async && !xfer->running_retry) {
         xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
     }
@@ -1607,55 +1717,20 @@ static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx
 
 static int xhci_fire_transfer(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
 {
-    int i;
-    unsigned int length = 0;
-    XHCITRB *trb;
-
-    for (i = 0; i < xfer->trb_count; i++) {
-        trb = &xfer->trbs[i];
-        if (TRB_TYPE(*trb) == TR_NORMAL || TRB_TYPE(*trb) == TR_ISOCH) {
-            length += trb->status & 0x1ffff;
-        }
-    }
-
-    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid, length);
-
-    if (!epctx->has_bg) {
-        xfer->data_length = length;
-        xfer->backgrounded = 0;
-        return xhci_submit(xhci, xfer, epctx);
-    } else {
-        if (!epctx->bg_running) {
-            for (i = 0; i < BG_XFERS; i++) {
-                XHCITransfer *t = &epctx->bg_transfers[i];
-                t->xhci = xhci;
-                t->epid = xfer->epid;
-                t->slotid = xfer->slotid;
-                t->pkts = BG_PKTS;
-                t->pktsize = epctx->max_psize;
-                t->data_length = t->pkts * t->pktsize;
-                t->bg_xfer = 1;
-                if (xhci_submit(xhci, t, epctx) < 0) {
-                    fprintf(stderr, "xhci: bg submit failed\n");
-                    return -1;
-                }
-            }
-            epctx->bg_running = 1;
-        }
-        xfer->backgrounded = 1;
-        xhci_bg_update(xhci, epctx);
-        return 0;
-    }
+    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid);
+    return xhci_submit(xhci, xfer, epctx);
 }
 
 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid)
 {
     XHCIEPContext *epctx;
+    USBEndpoint *ep = NULL;
+    uint64_t mfindex;
     int length;
     int i;
 
     trace_usb_xhci_ep_kick(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     if (!xhci->slots[slotid-1].enabled) {
@@ -1670,18 +1745,34 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
     }
 
     if (epctx->retry) {
-        /* retry nak'ed transfer */
         XHCITransfer *xfer = epctx->retry;
-        int result;
 
         trace_usb_xhci_xfer_retry(xfer);
         assert(xfer->running_retry);
-        xhci_setup_packet(xfer, xfer->packet.ep->dev);
-        result = usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
-        if (result == USB_RET_NAK) {
-            return;
+        if (xfer->iso_xfer) {
+            /* retry delayed iso transfer */
+            mfindex = xhci_mfindex_get(xhci);
+            xhci_check_iso_kick(xhci, xfer, epctx, mfindex);
+            if (xfer->running_retry) {
+                return;
+            }
+            if (xhci_setup_packet(xfer) < 0) {
+                return;
+            }
+            usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
+            assert(xfer->packet.status != USB_RET_NAK);
+            xhci_complete_packet(xfer);
+        } else {
+            /* retry nak'ed transfer */
+            if (xhci_setup_packet(xfer) < 0) {
+                return;
+            }
+            usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
+            if (xfer->packet.status == USB_RET_NAK) {
+                return;
+            }
+            xhci_complete_packet(xfer);
         }
-        xhci_complete_packet(xfer, result);
         assert(!xfer->running_retry);
         epctx->retry = NULL;
     }
@@ -1695,7 +1786,7 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
 
     while (1) {
         XHCITransfer *xfer = &epctx->transfers[epctx->next_xfer];
-        if (xfer->running_async || xfer->running_retry || xfer->backgrounded) {
+        if (xfer->running_async || xfer->running_retry) {
             break;
         }
         length = xhci_ring_chain_length(xhci, &epctx->ring);
@@ -1726,14 +1817,18 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
         if (epid == 1) {
             if (xhci_fire_ctl_transfer(xhci, xfer) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
+                ep = xfer->packet.ep;
             } else {
                 fprintf(stderr, "xhci: error firing CTL transfer\n");
             }
         } else {
             if (xhci_fire_transfer(xhci, xfer, epctx) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
+                ep = xfer->packet.ep;
             } else {
-                fprintf(stderr, "xhci: error firing data transfer\n");
+                if (!xfer->iso_xfer) {
+                    fprintf(stderr, "xhci: error firing data transfer\n");
+                }
             }
         }
 
@@ -1746,14 +1841,17 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
             break;
         }
     }
+    if (ep) {
+        usb_device_flush_ep_queue(ep->dev, ep);
+    }
 }
 
 static TRBCCode xhci_enable_slot(XHCIState *xhci, unsigned int slotid)
 {
     trace_usb_xhci_slot_enable(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     xhci->slots[slotid-1].enabled = 1;
-    xhci->slots[slotid-1].port = 0;
+    xhci->slots[slotid-1].uport = NULL;
     memset(xhci->slots[slotid-1].eps, 0, sizeof(XHCIEPContext*)*31);
 
     return CC_SUCCESS;
@@ -1764,7 +1862,7 @@ static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
     int i;
 
     trace_usb_xhci_slot_disable(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     for (i = 1; i <= 31; i++) {
         if (xhci->slots[slotid-1].eps[i-1]) {
@@ -1776,32 +1874,57 @@ static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
     return CC_SUCCESS;
 }
 
+static USBPort *xhci_lookup_uport(XHCIState *xhci, uint32_t *slot_ctx)
+{
+    USBPort *uport;
+    char path[32];
+    int i, pos, port;
+
+    port = (slot_ctx[1]>>16) & 0xFF;
+    port = xhci->ports[port-1].uport->index+1;
+    pos = snprintf(path, sizeof(path), "%d", port);
+    for (i = 0; i < 5; i++) {
+        port = (slot_ctx[0] >> 4*i) & 0x0f;
+        if (!port) {
+            break;
+        }
+        pos += snprintf(path + pos, sizeof(path) - pos, ".%d", port);
+    }
+
+    QTAILQ_FOREACH(uport, &xhci->bus.used, next) {
+        if (strcmp(uport->path, path) == 0) {
+            return uport;
+        }
+    }
+    return NULL;
+}
+
 static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
                                   uint64_t pictx, bool bsr)
 {
     XHCISlot *slot;
+    USBPort *uport;
     USBDevice *dev;
     dma_addr_t ictx, octx, dcbaap;
     uint64_t poctx;
     uint32_t ictl_ctx[2];
     uint32_t slot_ctx[4];
     uint32_t ep0_ctx[5];
-    unsigned int port;
     int i;
     TRBCCode res;
 
     trace_usb_xhci_slot_address(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
-    pci_dma_read(&xhci->pci_dev, dcbaap + 8*slotid, &poctx, sizeof(poctx));
+    poctx = ldq_le_pci_dma(&xhci->pci_dev, dcbaap + 8*slotid);
     ictx = xhci_mask64(pictx);
-    octx = xhci_mask64(le64_to_cpu(poctx));
+    octx = xhci_mask64(poctx);
 
     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] != 0x3) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1809,8 +1932,8 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
         return CC_TRB_ERROR;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx+32, slot_ctx, sizeof(slot_ctx));
-    pci_dma_read(&xhci->pci_dev, ictx+64, ep0_ctx, sizeof(ep0_ctx));
+    xhci_dma_read_u32s(xhci, ictx+32, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, ictx+64, ep0_ctx, sizeof(ep0_ctx));
 
     DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
@@ -1818,38 +1941,48 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-    port = (slot_ctx[1]>>16) & 0xFF;
-    dev = xhci->ports[port-1].port.dev;
-
-    if (port < 1 || port > MAXPORTS) {
-        fprintf(stderr, "xhci: bad port %d\n", port);
+    uport = xhci_lookup_uport(xhci, slot_ctx);
+    if (uport == NULL) {
+        fprintf(stderr, "xhci: port not found\n");
         return CC_TRB_ERROR;
-    } else if (!dev) {
-        fprintf(stderr, "xhci: port %d not connected\n", port);
+    }
+
+    dev = uport->dev;
+    if (!dev) {
+        fprintf(stderr, "xhci: port %s not connected\n", uport->path);
         return CC_USB_TRANSACTION_ERROR;
     }
 
-    for (i = 0; i < MAXSLOTS; i++) {
-        if (xhci->slots[i].port == port) {
-            fprintf(stderr, "xhci: port %d already assigned to slot %d\n",
-                    port, i+1);
+    for (i = 0; i < xhci->numslots; i++) {
+        if (i == slotid-1) {
+            continue;
+        }
+        if (xhci->slots[i].uport == uport) {
+            fprintf(stderr, "xhci: port %s already assigned to slot %d\n",
+                    uport->path, i+1);
             return CC_TRB_ERROR;
         }
     }
 
     slot = &xhci->slots[slotid-1];
-    slot->port = port;
+    slot->uport = uport;
     slot->ctx = octx;
 
     if (bsr) {
         slot_ctx[3] = SLOT_DEFAULT << SLOT_STATE_SHIFT;
     } else {
+        USBPacket p;
         slot->devaddr = xhci->devaddr++;
         slot_ctx[3] = (SLOT_ADDRESSED << SLOT_STATE_SHIFT) | slot->devaddr;
         DPRINTF("xhci: device address is %d\n", slot->devaddr);
-        usb_device_handle_control(dev, NULL,
+        usb_device_reset(dev);
+        usb_packet_setup(&p, USB_TOKEN_OUT,
+                         usb_ep_get(dev, USB_TOKEN_OUT, 0),
+                         0, false, false);
+        usb_device_handle_control(dev, &p,
                                   DeviceOutRequest | USB_REQ_SET_ADDRESS,
                                   slot->devaddr, 0, 0, NULL);
+        assert(p.status != USB_RET_ASYNC);
     }
 
     res = xhci_enable_ep(xhci, slotid, 1, octx+32, ep0_ctx);
@@ -1859,8 +1992,8 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
-    pci_dma_write(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
 
     return res;
 }
@@ -1878,7 +2011,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
     TRBCCode res;
 
     trace_usb_xhci_slot_configure(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     ictx = xhci_mask64(pictx);
     octx = xhci->slots[slotid-1].ctx;
@@ -1893,17 +2026,17 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             }
         }
 
-        pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
         slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
         slot_ctx[3] |= SLOT_ADDRESSED << SLOT_STATE_SHIFT;
         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
-        pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
         return CC_SUCCESS;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if ((ictl_ctx[0] & 0x3) != 0x0 || (ictl_ctx[1] & 0x3) != 0x1) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1911,8 +2044,8 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
         return CC_TRB_ERROR;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx+32, islot_ctx, sizeof(islot_ctx));
-    pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
+    xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     if (SLOT_STATE(slot_ctx[3]) < SLOT_ADDRESSED) {
         fprintf(stderr, "xhci: invalid slot state %08x\n", slot_ctx[3]);
@@ -1924,8 +2057,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             xhci_disable_ep(xhci, slotid, i);
         }
         if (ictl_ctx[1] & (1<<i)) {
-            pci_dma_read(&xhci->pci_dev, ictx+32+(32*i), ep_ctx,
-                         sizeof(ep_ctx));
+            xhci_dma_read_u32s(xhci, ictx+32+(32*i), ep_ctx, sizeof(ep_ctx));
             DPRINTF("xhci: input ep%d.%d context: %08x %08x %08x %08x %08x\n",
                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
                     ep_ctx[3], ep_ctx[4]);
@@ -1937,7 +2069,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             DPRINTF("xhci: output ep%d.%d context: %08x %08x %08x %08x %08x\n",
                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
                     ep_ctx[3], ep_ctx[4]);
-            pci_dma_write(&xhci->pci_dev, octx+(32*i), ep_ctx, sizeof(ep_ctx));
+            xhci_dma_write_u32s(xhci, octx+(32*i), ep_ctx, sizeof(ep_ctx));
         }
     }
 
@@ -1949,7 +2081,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
 
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     return CC_SUCCESS;
 }
@@ -1966,7 +2098,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     uint32_t slot_ctx[4];
 
     trace_usb_xhci_slot_evaluate(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     ictx = xhci_mask64(pictx);
     octx = xhci->slots[slotid-1].ctx;
@@ -1974,7 +2106,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] & ~0x3) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1983,12 +2115,12 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     }
 
     if (ictl_ctx[1] & 0x1) {
-        pci_dma_read(&xhci->pci_dev, ictx+32, islot_ctx, sizeof(islot_ctx));
+        xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
 
         DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
                 islot_ctx[0], islot_ctx[1], islot_ctx[2], islot_ctx[3]);
 
-        pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
         slot_ctx[1] &= ~0xFFFF; /* max exit latency */
         slot_ctx[1] |= islot_ctx[1] & 0xFFFF;
@@ -1998,17 +2130,17 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
 
-        pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
     }
 
     if (ictl_ctx[1] & 0x2) {
-        pci_dma_read(&xhci->pci_dev, ictx+64, iep0_ctx, sizeof(iep0_ctx));
+        xhci_dma_read_u32s(xhci, ictx+64, iep0_ctx, sizeof(iep0_ctx));
 
         DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
                 iep0_ctx[0], iep0_ctx[1], iep0_ctx[2],
                 iep0_ctx[3], iep0_ctx[4]);
 
-        pci_dma_read(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+        xhci_dma_read_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
 
         ep0_ctx[1] &= ~0xFFFF0000; /* max packet size*/
         ep0_ctx[1] |= iep0_ctx[1] & 0xFFFF0000;
@@ -2016,7 +2148,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
         DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
                 ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-        pci_dma_write(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+        xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
     }
 
     return CC_SUCCESS;
@@ -2029,7 +2161,7 @@ static TRBCCode xhci_reset_slot(XHCIState *xhci, unsigned int slotid)
     int i;
 
     trace_usb_xhci_slot_reset(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     octx = xhci->slots[slotid-1].ctx;
 
@@ -2041,12 +2173,12 @@ static TRBCCode xhci_reset_slot(XHCIState *xhci, unsigned int slotid)
         }
     }
 
-    pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
     slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
     slot_ctx[3] |= SLOT_DEFAULT << SLOT_STATE_SHIFT;
     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     return CC_SUCCESS;
 }
@@ -2055,7 +2187,7 @@ static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *tr
 {
     unsigned int slotid;
     slotid = (trb->control >> TRB_CR_SLOTID_SHIFT) & TRB_CR_SLOTID_MASK;
-    if (slotid < 1 || slotid > MAXSLOTS) {
+    if (slotid < 1 || slotid > xhci->numslots) {
         fprintf(stderr, "xhci: bad slot id %d\n", slotid);
         event->ccode = CC_TRB_ERROR;
         return 0;
@@ -2067,10 +2199,32 @@ static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *tr
     return slotid;
 }
 
+/* cleanup slot state on usb device detach */
+static void xhci_detach_slot(XHCIState *xhci, USBPort *uport)
+{
+    int slot, ep;
+
+    for (slot = 0; slot < xhci->numslots; slot++) {
+        if (xhci->slots[slot].uport == uport) {
+            break;
+        }
+    }
+    if (slot == xhci->numslots) {
+        return;
+    }
+
+    for (ep = 0; ep < 31; ep++) {
+        if (xhci->slots[slot].eps[ep]) {
+            xhci_ep_nuke_xfers(xhci, slot+1, ep+1);
+        }
+    }
+    xhci->slots[slot].uport = NULL;
+}
+
 static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
 {
     dma_addr_t ctx;
-    uint8_t bw_ctx[MAXPORTS+1];
+    uint8_t bw_ctx[xhci->numports+1];
 
     DPRINTF("xhci_get_port_bandwidth()\n");
 
@@ -2080,7 +2234,7 @@ static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
 
     /* TODO: actually implement real values here */
     bw_ctx[0] = 0;
-    memset(&bw_ctx[1], 80, MAXPORTS); /* 80% */
+    memset(&bw_ctx[1], 80, xhci->numports); /* 80% */
     pci_dma_write(&xhci->pci_dev, ctx, bw_ctx, sizeof(bw_ctx));
 
     return CC_SUCCESS;
@@ -2147,12 +2301,12 @@ static void xhci_process_commands(XHCIState *xhci)
         event.ptr = addr;
         switch (type) {
         case CR_ENABLE_SLOT:
-            for (i = 0; i < MAXSLOTS; i++) {
+            for (i = 0; i < xhci->numslots; i++) {
                 if (!xhci->slots[i].enabled) {
                     break;
                 }
             }
-            if (i >= MAXSLOTS) {
+            if (i >= xhci->numslots) {
                 fprintf(stderr, "xhci: no device slots available\n");
                 event.ccode = CC_NO_SLOTS_ERROR;
             } else {
@@ -2244,36 +2398,90 @@ static void xhci_process_commands(XHCIState *xhci)
             break;
         }
         event.slotid = slotid;
-        xhci_event(xhci, &event);
+        xhci_event(xhci, &event, 0);
+    }
+}
+
+static bool xhci_port_have_device(XHCIPort *port)
+{
+    if (!port->uport->dev || !port->uport->dev->attached) {
+        return false; /* no device present */
     }
+    if (!((1 << port->uport->dev->speed) & port->speedmask)) {
+        return false; /* speed mismatch */
+    }
+    return true;
+}
+
+static void xhci_port_notify(XHCIPort *port, uint32_t bits)
+{
+    XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS,
+                     port->portnr << 24 };
+
+    if ((port->portsc & bits) == bits) {
+        return;
+    }
+    port->portsc |= bits;
+    if (!xhci_running(port->xhci)) {
+        return;
+    }
+    xhci_event(port->xhci, &ev, 0);
 }
 
-static void xhci_update_port(XHCIState *xhci, XHCIPort *port, int is_detach)
+static void xhci_port_update(XHCIPort *port, int is_detach)
 {
-    int nr = port->port.index + 1;
+    uint32_t pls = PLS_RX_DETECT;
 
     port->portsc = PORTSC_PP;
-    if (port->port.dev && port->port.dev->attached && !is_detach) {
+    if (!is_detach && xhci_port_have_device(port)) {
         port->portsc |= PORTSC_CCS;
-        switch (port->port.dev->speed) {
+        switch (port->uport->dev->speed) {
         case USB_SPEED_LOW:
             port->portsc |= PORTSC_SPEED_LOW;
+            pls = PLS_POLLING;
             break;
         case USB_SPEED_FULL:
             port->portsc |= PORTSC_SPEED_FULL;
+            pls = PLS_POLLING;
             break;
         case USB_SPEED_HIGH:
             port->portsc |= PORTSC_SPEED_HIGH;
+            pls = PLS_POLLING;
+            break;
+        case USB_SPEED_SUPER:
+            port->portsc |= PORTSC_SPEED_SUPER;
+            port->portsc |= PORTSC_PED;
+            pls = PLS_U0;
             break;
         }
     }
+    set_field(&port->portsc, pls, PORTSC_PLS);
+    trace_usb_xhci_port_link(port->portnr, pls);
+    xhci_port_notify(port, PORTSC_CSC);
+}
+
+static void xhci_port_reset(XHCIPort *port)
+{
+    trace_usb_xhci_port_reset(port->portnr);
 
-    if (xhci_running(xhci)) {
-        port->portsc |= PORTSC_CSC;
-        XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS, nr << 24};
-        xhci_event(xhci, &ev);
-        DPRINTF("xhci: port change event for port %d\n", nr);
+    if (!xhci_port_have_device(port)) {
+        return;
     }
+
+    usb_device_reset(port->uport->dev);
+
+    switch (port->uport->dev->speed) {
+    case USB_SPEED_LOW:
+    case USB_SPEED_FULL:
+    case USB_SPEED_HIGH:
+        set_field(&port->portsc, PLS_U0, PORTSC_PLS);
+        trace_usb_xhci_port_link(port->portnr, PLS_U0);
+        port->portsc |= PORTSC_PED;
+        break;
+    }
+
+    port->portsc &= ~PORTSC_PR;
+    xhci_port_notify(port, PORTSC_PRC);
 }
 
 static void xhci_reset(DeviceState *dev)
@@ -2296,32 +2504,38 @@ static void xhci_reset(DeviceState *dev)
     xhci->config = 0;
     xhci->devaddr = 2;
 
-    for (i = 0; i < MAXSLOTS; i++) {
+    for (i = 0; i < xhci->numslots; i++) {
         xhci_disable_slot(xhci, i+1);
     }
 
-    for (i = 0; i < MAXPORTS; i++) {
-        xhci_update_port(xhci, xhci->ports + i, 0);
+    for (i = 0; i < xhci->numports; i++) {
+        xhci_port_update(xhci->ports + i, 0);
     }
 
-    xhci->mfindex = 0;
-    xhci->iman = 0;
-    xhci->imod = 0;
-    xhci->erstsz = 0;
-    xhci->erstba_low = 0;
-    xhci->erstba_high = 0;
-    xhci->erdp_low = 0;
-    xhci->erdp_high = 0;
+    for (i = 0; i < xhci->numintrs; i++) {
+        xhci->intr[i].iman = 0;
+        xhci->intr[i].imod = 0;
+        xhci->intr[i].erstsz = 0;
+        xhci->intr[i].erstba_low = 0;
+        xhci->intr[i].erstba_high = 0;
+        xhci->intr[i].erdp_low = 0;
+        xhci->intr[i].erdp_high = 0;
+        xhci->intr[i].msix_used = 0;
+
+        xhci->intr[i].er_ep_idx = 0;
+        xhci->intr[i].er_pcs = 1;
+        xhci->intr[i].er_full = 0;
+        xhci->intr[i].ev_buffer_put = 0;
+        xhci->intr[i].ev_buffer_get = 0;
+    }
 
-    xhci->er_ep_idx = 0;
-    xhci->er_pcs = 1;
-    xhci->er_full = 0;
-    xhci->ev_buffer_put = 0;
-    xhci->ev_buffer_get = 0;
+    xhci->mfindex_start = qemu_get_clock_ns(vm_clock);
+    xhci_mfwrap_update(xhci);
 }
 
-static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_cap_read(void *ptr, hwaddr reg, unsigned size)
 {
+    XHCIState *xhci = ptr;
     uint32_t ret;
 
     switch (reg) {
@@ -2329,7 +2543,8 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x01000000 | LEN_CAP;
         break;
     case 0x04: /* HCSPARAMS 1 */
-        ret = (MAXPORTS<<24) | (MAXINTRS<<8) | MAXSLOTS;
+        ret = ((xhci->numports_2+xhci->numports_3)<<24)
+            | (xhci->numintrs<<8) | xhci->numslots;
         break;
     case 0x08: /* HCSPARAMS 2 */
         ret = 0x0000000f;
@@ -2356,10 +2571,10 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x02000402; /* USB 2.0 */
         break;
     case 0x24: /* Supported Protocol:04 */
-        ret = 0x20425455; /* "USB " */
+        ret = 0x20425355; /* "USB " */
         break;
     case 0x28: /* Supported Protocol:08 */
-        ret = 0x00000001 | (USB2_PORTS<<8);
+        ret = 0x00000001 | (xhci->numports_2<<8);
         break;
     case 0x2c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
@@ -2368,16 +2583,16 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x03000002; /* USB 3.0 */
         break;
     case 0x34: /* Supported Protocol:04 */
-        ret = 0x20425455; /* "USB " */
+        ret = 0x20425355; /* "USB " */
         break;
     case 0x38: /* Supported Protocol:08 */
-        ret = 0x00000000 | (USB2_PORTS+1) | (USB3_PORTS<<8);
+        ret = 0x00000000 | (xhci->numports_2+1) | (xhci->numports_3<<8);
         break;
     case 0x3c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
         break;
     default:
-        fprintf(stderr, "xhci_cap_read: reg %d unimplemented\n", reg);
+        fprintf(stderr, "xhci_cap_read: reg %d unimplemented\n", (int)reg);
         ret = 0;
     }
 
@@ -2385,20 +2600,14 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
     return ret;
 }
 
-static uint32_t xhci_port_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_port_read(void *ptr, hwaddr reg, unsigned size)
 {
-    uint32_t port = reg >> 4;
+    XHCIPort *port = ptr;
     uint32_t ret;
 
-    if (port >= MAXPORTS) {
-        fprintf(stderr, "xhci_port_read: port %d out of bounds\n", port);
-        ret = 0;
-        goto out;
-    }
-
-    switch (reg & 0xf) {
+    switch (reg) {
     case 0x00: /* PORTSC */
-        ret = xhci->ports[port].portsc;
+        ret = port->portsc;
         break;
     case 0x04: /* PORTPMSC */
     case 0x08: /* PORTLI */
@@ -2407,65 +2616,56 @@ static uint32_t xhci_port_read(XHCIState *xhci, uint32_t reg)
     case 0x0c: /* reserved */
     default:
         fprintf(stderr, "xhci_port_read (port %d): reg 0x%x unimplemented\n",
-                port, reg);
+                port->portnr, (uint32_t)reg);
         ret = 0;
     }
 
-out:
-    trace_usb_xhci_port_read(port, reg & 0x0f, ret);
+    trace_usb_xhci_port_read(port->portnr, reg, ret);
     return ret;
 }
 
-static void xhci_port_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_port_write(void *ptr, hwaddr reg,
+                            uint64_t val, unsigned size)
 {
-    uint32_t port = reg >> 4;
+    XHCIPort *port = ptr;
     uint32_t portsc;
 
-    trace_usb_xhci_port_write(port, reg & 0x0f, val);
-
-    if (port >= MAXPORTS) {
-        fprintf(stderr, "xhci_port_read: port %d out of bounds\n", port);
-        return;
-    }
+    trace_usb_xhci_port_write(port->portnr, reg, val);
 
-    switch (reg & 0xf) {
+    switch (reg) {
     case 0x00: /* PORTSC */
-        portsc = xhci->ports[port].portsc;
+        portsc = port->portsc;
         /* write-1-to-clear bits*/
         portsc &= ~(val & (PORTSC_CSC|PORTSC_PEC|PORTSC_WRC|PORTSC_OCC|
                            PORTSC_PRC|PORTSC_PLC|PORTSC_CEC));
         if (val & PORTSC_LWS) {
             /* overwrite PLS only when LWS=1 */
-            portsc &= ~(PORTSC_PLS_MASK << PORTSC_PLS_SHIFT);
-            portsc |= val & (PORTSC_PLS_MASK << PORTSC_PLS_SHIFT);
+            uint32_t pls = get_field(val, PORTSC_PLS);
+            set_field(&portsc, pls, PORTSC_PLS);
+            trace_usb_xhci_port_link(port->portnr, pls);
         }
         /* read/write bits */
         portsc &= ~(PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE);
         portsc |= (val & (PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE));
+        port->portsc = portsc;
         /* write-1-to-start bits */
         if (val & PORTSC_PR) {
-            DPRINTF("xhci: port %d reset\n", port);
-            usb_device_reset(xhci->ports[port].port.dev);
-            portsc |= PORTSC_PRC | PORTSC_PED;
+            xhci_port_reset(port);
         }
-        xhci->ports[port].portsc = portsc;
         break;
     case 0x04: /* PORTPMSC */
     case 0x08: /* PORTLI */
     default:
         fprintf(stderr, "xhci_port_write (port %d): reg 0x%x unimplemented\n",
-                port, reg);
+                port->portnr, (uint32_t)reg);
     }
 }
 
-static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_oper_read(void *ptr, hwaddr reg, unsigned size)
 {
+    XHCIState *xhci = ptr;
     uint32_t ret;
 
-    if (reg >= 0x400) {
-        return xhci_port_read(xhci, reg - 0x400);
-    }
-
     switch (reg) {
     case 0x00: /* USBCMD */
         ret = xhci->usbcmd;
@@ -2495,7 +2695,7 @@ static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
         ret = xhci->config;
         break;
     default:
-        fprintf(stderr, "xhci_oper_read: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_read: reg 0x%x unimplemented\n", (int)reg);
         ret = 0;
     }
 
@@ -2503,12 +2703,10 @@ static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
     return ret;
 }
 
-static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_oper_write(void *ptr, hwaddr reg,
+                            uint64_t val, unsigned size)
 {
-    if (reg >= 0x400) {
-        xhci_port_write(xhci, reg - 0x400, val);
-        return;
-    }
+    XHCIState *xhci = ptr;
 
     trace_usb_xhci_oper_write(reg, val);
 
@@ -2520,16 +2718,17 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
             xhci_stop(xhci);
         }
         xhci->usbcmd = val & 0xc0f;
+        xhci_mfwrap_update(xhci);
         if (val & USBCMD_HCRST) {
             xhci_reset(&xhci->pci_dev.qdev);
         }
-        xhci_irq_update(xhci);
+        xhci_intx_update(xhci);
         break;
 
     case 0x04: /* USBSTS */
         /* these bits are write-1-to-clear */
         xhci->usbsts &= ~(val & (USBSTS_HSE|USBSTS_EINT|USBSTS_PCD|USBSTS_SRE));
-        xhci_irq_update(xhci);
+        xhci_intx_update(xhci);
         break;
 
     case 0x14: /* DNCTRL */
@@ -2543,7 +2742,7 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         if (xhci->crcr_low & (CRCR_CA|CRCR_CS) && (xhci->crcr_low & CRCR_CRR)) {
             XHCIEvent event = {ER_COMMAND_COMPLETE, CC_COMMAND_RING_STOPPED};
             xhci->crcr_low &= ~CRCR_CRR;
-            xhci_event(xhci, &event);
+            xhci_event(xhci, &event, 0);
             DPRINTF("xhci: command ring stopped (CRCR=%08x)\n", xhci->crcr_low);
         } else {
             dma_addr_t base = xhci_addr64(xhci->crcr_low & ~0x3f, val);
@@ -2561,101 +2760,127 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         xhci->config = val & 0xff;
         break;
     default:
-        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", (int)reg);
     }
 }
 
-static uint32_t xhci_runtime_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_runtime_read(void *ptr, hwaddr reg,
+                                  unsigned size)
 {
-    uint32_t ret;
+    XHCIState *xhci = ptr;
+    uint32_t ret = 0;
 
-    switch (reg) {
-    case 0x00: /* MFINDEX */
-        fprintf(stderr, "xhci_runtime_read: MFINDEX not yet implemented\n");
-        ret = xhci->mfindex;
-        break;
-    case 0x20: /* IMAN */
-        ret = xhci->iman;
-        break;
-    case 0x24: /* IMOD */
-        ret = xhci->imod;
-        break;
-    case 0x28: /* ERSTSZ */
-        ret = xhci->erstsz;
-        break;
-    case 0x30: /* ERSTBA low */
-        ret = xhci->erstba_low;
-        break;
-    case 0x34: /* ERSTBA high */
-        ret = xhci->erstba_high;
-        break;
-    case 0x38: /* ERDP low */
-        ret = xhci->erdp_low;
-        break;
-    case 0x3c: /* ERDP high */
-        ret = xhci->erdp_high;
-        break;
-    default:
-        fprintf(stderr, "xhci_runtime_read: reg 0x%x unimplemented\n", reg);
-        ret = 0;
+    if (reg < 0x20) {
+        switch (reg) {
+        case 0x00: /* MFINDEX */
+            ret = xhci_mfindex_get(xhci) & 0x3fff;
+            break;
+        default:
+            fprintf(stderr, "xhci_runtime_read: reg 0x%x unimplemented\n",
+                    (int)reg);
+            break;
+        }
+    } else {
+        int v = (reg - 0x20) / 0x20;
+        XHCIInterrupter *intr = &xhci->intr[v];
+        switch (reg & 0x1f) {
+        case 0x00: /* IMAN */
+            ret = intr->iman;
+            break;
+        case 0x04: /* IMOD */
+            ret = intr->imod;
+            break;
+        case 0x08: /* ERSTSZ */
+            ret = intr->erstsz;
+            break;
+        case 0x10: /* ERSTBA low */
+            ret = intr->erstba_low;
+            break;
+        case 0x14: /* ERSTBA high */
+            ret = intr->erstba_high;
+            break;
+        case 0x18: /* ERDP low */
+            ret = intr->erdp_low;
+            break;
+        case 0x1c: /* ERDP high */
+            ret = intr->erdp_high;
+            break;
+        }
     }
 
     trace_usb_xhci_runtime_read(reg, ret);
     return ret;
 }
 
-static void xhci_runtime_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_runtime_write(void *ptr, hwaddr reg,
+                               uint64_t val, unsigned size)
 {
-    trace_usb_xhci_runtime_read(reg, val);
+    XHCIState *xhci = ptr;
+    int v = (reg - 0x20) / 0x20;
+    XHCIInterrupter *intr = &xhci->intr[v];
+    trace_usb_xhci_runtime_write(reg, val);
 
-    switch (reg) {
-    case 0x20: /* IMAN */
+    if (reg < 0x20) {
+        fprintf(stderr, "%s: reg 0x%x unimplemented\n", __func__, (int)reg);
+        return;
+    }
+
+    switch (reg & 0x1f) {
+    case 0x00: /* IMAN */
         if (val & IMAN_IP) {
-            xhci->iman &= ~IMAN_IP;
+            intr->iman &= ~IMAN_IP;
+        }
+        intr->iman &= ~IMAN_IE;
+        intr->iman |= val & IMAN_IE;
+        if (v == 0) {
+            xhci_intx_update(xhci);
         }
-        xhci->iman &= ~IMAN_IE;
-        xhci->iman |= val & IMAN_IE;
-        xhci_irq_update(xhci);
+        xhci_msix_update(xhci, v);
         break;
-    case 0x24: /* IMOD */
-        xhci->imod = val;
+    case 0x04: /* IMOD */
+        intr->imod = val;
         break;
-    case 0x28: /* ERSTSZ */
-        xhci->erstsz = val & 0xffff;
+    case 0x08: /* ERSTSZ */
+        intr->erstsz = val & 0xffff;
         break;
-    case 0x30: /* ERSTBA low */
+    case 0x10: /* ERSTBA low */
         /* XXX NEC driver bug: it doesn't align this to 64 bytes
-        xhci->erstba_low = val & 0xffffffc0; */
-        xhci->erstba_low = val & 0xfffffff0;
+        intr->erstba_low = val & 0xffffffc0; */
+        intr->erstba_low = val & 0xfffffff0;
         break;
-    case 0x34: /* ERSTBA high */
-        xhci->erstba_high = val;
-        xhci_er_reset(xhci);
+    case 0x14: /* ERSTBA high */
+        intr->erstba_high = val;
+        xhci_er_reset(xhci, v);
         break;
-    case 0x38: /* ERDP low */
+    case 0x18: /* ERDP low */
         if (val & ERDP_EHB) {
-            xhci->erdp_low &= ~ERDP_EHB;
+            intr->erdp_low &= ~ERDP_EHB;
         }
-        xhci->erdp_low = (val & ~ERDP_EHB) | (xhci->erdp_low & ERDP_EHB);
+        intr->erdp_low = (val & ~ERDP_EHB) | (intr->erdp_low & ERDP_EHB);
         break;
-    case 0x3c: /* ERDP high */
-        xhci->erdp_high = val;
-        xhci_events_update(xhci);
+    case 0x1c: /* ERDP high */
+        intr->erdp_high = val;
+        xhci_events_update(xhci, v);
         break;
     default:
-        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n",
+                (int)reg);
     }
 }
 
-static uint32_t xhci_doorbell_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_doorbell_read(void *ptr, hwaddr reg,
+                                   unsigned size)
 {
     /* doorbells always read as 0 */
     trace_usb_xhci_doorbell_read(reg, 0);
     return 0;
 }
 
-static void xhci_doorbell_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_doorbell_write(void *ptr, hwaddr reg,
+                                uint64_t val, unsigned size)
 {
+    XHCIState *xhci = ptr;
+
     trace_usb_xhci_doorbell_write(reg, val);
 
     if (!xhci_running(xhci)) {
@@ -2669,69 +2894,57 @@ static void xhci_doorbell_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         if (val == 0) {
             xhci_process_commands(xhci);
         } else {
-            fprintf(stderr, "xhci: bad doorbell 0 write: 0x%x\n", val);
+            fprintf(stderr, "xhci: bad doorbell 0 write: 0x%x\n",
+                    (uint32_t)val);
         }
     } else {
-        if (reg > MAXSLOTS) {
-            fprintf(stderr, "xhci: bad doorbell %d\n", reg);
+        if (reg > xhci->numslots) {
+            fprintf(stderr, "xhci: bad doorbell %d\n", (int)reg);
         } else if (val > 31) {
-            fprintf(stderr, "xhci: bad doorbell %d write: 0x%x\n", reg, val);
+            fprintf(stderr, "xhci: bad doorbell %d write: 0x%x\n",
+                    (int)reg, (uint32_t)val);
         } else {
             xhci_kick_ep(xhci, reg, val);
         }
     }
 }
 
-static uint64_t xhci_mem_read(void *ptr, target_phys_addr_t addr,
-                              unsigned size)
-{
-    XHCIState *xhci = ptr;
-
-    /* Only aligned reads are allowed on xHCI */
-    if (addr & 3) {
-        fprintf(stderr, "xhci_mem_read: Mis-aligned read\n");
-        return 0;
-    }
-
-    if (addr < LEN_CAP) {
-        return xhci_cap_read(xhci, addr);
-    } else if (addr >= OFF_OPER && addr < (OFF_OPER + LEN_OPER)) {
-        return xhci_oper_read(xhci, addr - OFF_OPER);
-    } else if (addr >= OFF_RUNTIME && addr < (OFF_RUNTIME + LEN_RUNTIME)) {
-        return xhci_runtime_read(xhci, addr - OFF_RUNTIME);
-    } else if (addr >= OFF_DOORBELL && addr < (OFF_DOORBELL + LEN_DOORBELL)) {
-        return xhci_doorbell_read(xhci, addr - OFF_DOORBELL);
-    } else {
-        fprintf(stderr, "xhci_mem_read: Bad offset %x\n", (int)addr);
-        return 0;
-    }
-}
+static const MemoryRegionOps xhci_cap_ops = {
+    .read = xhci_cap_read,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 4,
+    .impl.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static void xhci_mem_write(void *ptr, target_phys_addr_t addr,
-                           uint64_t val, unsigned size)
-{
-    XHCIState *xhci = ptr;
+static const MemoryRegionOps xhci_oper_ops = {
+    .read = xhci_oper_read,
+    .write = xhci_oper_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-    /* Only aligned writes are allowed on xHCI */
-    if (addr & 3) {
-        fprintf(stderr, "xhci_mem_write: Mis-aligned write\n");
-        return;
-    }
+static const MemoryRegionOps xhci_port_ops = {
+    .read = xhci_port_read,
+    .write = xhci_port_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-    if (addr >= OFF_OPER && addr < (OFF_OPER + LEN_OPER)) {
-        xhci_oper_write(xhci, addr - OFF_OPER, val);
-    } else if (addr >= OFF_RUNTIME && addr < (OFF_RUNTIME + LEN_RUNTIME)) {
-        xhci_runtime_write(xhci, addr - OFF_RUNTIME, val);
-    } else if (addr >= OFF_DOORBELL && addr < (OFF_DOORBELL + LEN_DOORBELL)) {
-        xhci_doorbell_write(xhci, addr - OFF_DOORBELL, val);
-    } else {
-        fprintf(stderr, "xhci_mem_write: Bad offset %x\n", (int)addr);
-    }
-}
+static const MemoryRegionOps xhci_runtime_ops = {
+    .read = xhci_runtime_read,
+    .write = xhci_runtime_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static const MemoryRegionOps xhci_mem_ops = {
-    .read = xhci_mem_read,
-    .write = xhci_mem_write,
+static const MemoryRegionOps xhci_doorbell_ops = {
+    .read = xhci_doorbell_read,
+    .write = xhci_doorbell_write,
     .valid.min_access_size = 4,
     .valid.max_access_size = 4,
     .endianness = DEVICE_LITTLE_ENDIAN,
@@ -2740,53 +2953,53 @@ static const MemoryRegionOps xhci_mem_ops = {
 static void xhci_attach(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    xhci_update_port(xhci, port, 0);
+    xhci_port_update(port, 0);
 }
 
 static void xhci_detach(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    xhci_update_port(xhci, port, 1);
+    xhci_detach_slot(xhci, usbport);
+    xhci_port_update(port, 1);
 }
 
 static void xhci_wakeup(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
-    int nr = port->port.index + 1;
-    XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS, nr << 24};
-    uint32_t pls;
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    pls = (port->portsc >> PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK;
-    if (pls != 3) {
-        return;
-    }
-    port->portsc |= 0xf << PORTSC_PLS_SHIFT;
-    if (port->portsc & PORTSC_PLC) {
+    if (get_field(port->portsc, PORTSC_PLS) != PLS_U3) {
         return;
     }
-    port->portsc |= PORTSC_PLC;
-    xhci_event(xhci, &ev);
+    set_field(&port->portsc, PLS_RESUME, PORTSC_PLS);
+    xhci_port_notify(port, PORTSC_PLC);
 }
 
 static void xhci_complete(USBPort *port, USBPacket *packet)
 {
     XHCITransfer *xfer = container_of(packet, XHCITransfer, packet);
 
-    xhci_complete_packet(xfer, packet->result);
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        xhci_ep_nuke_one_xfer(xfer);
+        return;
+    }
+    xhci_complete_packet(xfer);
     xhci_kick_ep(xfer->xhci, xfer->slotid, xfer->epid);
 }
 
-static void xhci_child_detach(USBPort *port, USBDevice *child)
+static void xhci_child_detach(USBPort *uport, USBDevice *child)
 {
-    FIXME();
+    USBBus *bus = usb_bus_from_device(child);
+    XHCIState *xhci = container_of(bus, XHCIState, bus);
+
+    xhci_detach_slot(xhci, uport);
 }
 
-static USBPortOps xhci_port_ops = {
+static USBPortOps xhci_uport_ops = {
     .attach   = xhci_attach,
     .detach   = xhci_detach,
     .wakeup   = xhci_wakeup,
@@ -2799,7 +3012,7 @@ static int xhci_find_slotid(XHCIState *xhci, USBDevice *dev)
     XHCISlot *slot;
     int slotid;
 
-    for (slotid = 1; slotid <= MAXSLOTS; slotid++) {
+    for (slotid = 1; slotid <= xhci->numslots; slotid++) {
         slot = &xhci->slots[slotid-1];
         if (slot->devaddr == dev->addr) {
             return slotid;
@@ -2840,28 +3053,51 @@ static USBBusOps xhci_bus_ops = {
 
 static void usb_xhci_init(XHCIState *xhci, DeviceState *dev)
 {
-    int i;
+    XHCIPort *port;
+    int i, usbports, speedmask;
 
     xhci->usbsts = USBSTS_HCH;
 
+    if (xhci->numports_2 > MAXPORTS_2) {
+        xhci->numports_2 = MAXPORTS_2;
+    }
+    if (xhci->numports_3 > MAXPORTS_3) {
+        xhci->numports_3 = MAXPORTS_3;
+    }
+    usbports = MAX(xhci->numports_2, xhci->numports_3);
+    xhci->numports = xhci->numports_2 + xhci->numports_3;
+
     usb_bus_new(&xhci->bus, &xhci_bus_ops, &xhci->pci_dev.qdev);
 
-    for (i = 0; i < MAXPORTS; i++) {
-        memset(&xhci->ports[i], 0, sizeof(xhci->ports[i]));
-        usb_register_port(&xhci->bus, &xhci->ports[i].port, xhci, i,
-                          &xhci_port_ops,
-                          USB_SPEED_MASK_LOW  |
-                          USB_SPEED_MASK_FULL |
-                          USB_SPEED_MASK_HIGH);
-    }
-    for (i = 0; i < MAXSLOTS; i++) {
-        xhci->slots[i].enabled = 0;
+    for (i = 0; i < usbports; i++) {
+        speedmask = 0;
+        if (i < xhci->numports_2) {
+            port = &xhci->ports[i];
+            port->portnr = i + 1;
+            port->uport = &xhci->uports[i];
+            port->speedmask =
+                USB_SPEED_MASK_LOW  |
+                USB_SPEED_MASK_FULL |
+                USB_SPEED_MASK_HIGH;
+            snprintf(port->name, sizeof(port->name), "usb2 port #%d", i+1);
+            speedmask |= port->speedmask;
+        }
+        if (i < xhci->numports_3) {
+            port = &xhci->ports[i + xhci->numports_2];
+            port->portnr = i + 1 + xhci->numports_2;
+            port->uport = &xhci->uports[i];
+            port->speedmask = USB_SPEED_MASK_SUPER;
+            snprintf(port->name, sizeof(port->name), "usb3 port #%d", i+1);
+            speedmask |= port->speedmask;
+        }
+        usb_register_port(&xhci->bus, &xhci->uports[i], xhci, i,
+                          &xhci_uport_ops, speedmask);
     }
 }
 
 static int usb_xhci_initfn(struct PCIDevice *dev)
 {
-    int ret;
+    int i, ret;
 
     XHCIState *xhci = DO_UPCAST(XHCIState, pci_dev, dev);
 
@@ -2872,10 +3108,47 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
 
     usb_xhci_init(xhci, &dev->qdev);
 
+    if (xhci->numintrs > MAXINTRS) {
+        xhci->numintrs = MAXINTRS;
+    }
+    if (xhci->numintrs < 1) {
+        xhci->numintrs = 1;
+    }
+    if (xhci->numslots > MAXSLOTS) {
+        xhci->numslots = MAXSLOTS;
+    }
+    if (xhci->numslots < 1) {
+        xhci->numslots = 1;
+    }
+
+    xhci->mfwrap_timer = qemu_new_timer_ns(vm_clock, xhci_mfwrap_timer, xhci);
+
     xhci->irq = xhci->pci_dev.irq[0];
 
-    memory_region_init_io(&xhci->mem, &xhci_mem_ops, xhci,
-                          "xhci", LEN_REGS);
+    memory_region_init(&xhci->mem, "xhci", LEN_REGS);
+    memory_region_init_io(&xhci->mem_cap, &xhci_cap_ops, xhci,
+                          "capabilities", LEN_CAP);
+    memory_region_init_io(&xhci->mem_oper, &xhci_oper_ops, xhci,
+                          "operational", 0x400);
+    memory_region_init_io(&xhci->mem_runtime, &xhci_runtime_ops, xhci,
+                          "runtime", LEN_RUNTIME);
+    memory_region_init_io(&xhci->mem_doorbell, &xhci_doorbell_ops, xhci,
+                          "doorbell", LEN_DOORBELL);
+
+    memory_region_add_subregion(&xhci->mem, 0,            &xhci->mem_cap);
+    memory_region_add_subregion(&xhci->mem, OFF_OPER,     &xhci->mem_oper);
+    memory_region_add_subregion(&xhci->mem, OFF_RUNTIME,  &xhci->mem_runtime);
+    memory_region_add_subregion(&xhci->mem, OFF_DOORBELL, &xhci->mem_doorbell);
+
+    for (i = 0; i < xhci->numports; i++) {
+        XHCIPort *port = &xhci->ports[i];
+        uint32_t offset = OFF_OPER + 0x400 + 0x10 * i;
+        port->xhci = xhci;
+        memory_region_init_io(&port->mem, &xhci_port_ops, port,
+                              port->name, 0x10);
+        memory_region_add_subregion(&xhci->mem, offset, &port->mem);
+    }
+
     pci_register_bar(&xhci->pci_dev, 0,
                      PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &xhci->mem);
@@ -2883,32 +3156,31 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
     ret = pcie_cap_init(&xhci->pci_dev, 0xa0, PCI_EXP_TYPE_ENDPOINT, 0);
     assert(ret >= 0);
 
-    if (xhci->msi) {
-        ret = msi_init(&xhci->pci_dev, 0x70, 1, true, false);
-        assert(ret >= 0);
+    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI)) {
+        msi_init(&xhci->pci_dev, 0x70, xhci->numintrs, true, false);
+    }
+    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI_X)) {
+        msix_init(&xhci->pci_dev, xhci->numintrs,
+                  &xhci->mem, 0, OFF_MSIX_TABLE,
+                  &xhci->mem, 0, OFF_MSIX_PBA,
+                  0x90);
     }
 
     return 0;
 }
 
-static void xhci_write_config(PCIDevice *dev, uint32_t addr, uint32_t val,
-                              int len)
-{
-    XHCIState *xhci = DO_UPCAST(XHCIState, pci_dev, dev);
-
-    pci_default_write_config(dev, addr, val, len);
-    if (xhci->msi) {
-        msi_write_config(dev, addr, val, len);
-    }
-}
-
 static const VMStateDescription vmstate_xhci = {
     .name = "xhci",
     .unmigratable = 1,
 };
 
 static Property xhci_properties[] = {
-    DEFINE_PROP_UINT32("msi", XHCIState, msi, 0),
+    DEFINE_PROP_BIT("msi",      XHCIState, flags, XHCI_FLAG_USE_MSI, true),
+    DEFINE_PROP_BIT("msix",     XHCIState, flags, XHCI_FLAG_USE_MSI_X, true),
+    DEFINE_PROP_UINT32("intrs", XHCIState, numintrs, MAXINTRS),
+    DEFINE_PROP_UINT32("slots", XHCIState, numslots, MAXSLOTS),
+    DEFINE_PROP_UINT32("p2",    XHCIState, numports_2, 4),
+    DEFINE_PROP_UINT32("p3",    XHCIState, numports_3, 4),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2926,10 +3198,10 @@ static void xhci_class_init(ObjectClass *klass, void *data)
     k->class_id     = PCI_CLASS_SERIAL_USB;
     k->revision     = 0x03;
     k->is_express   = 1;
-    k->config_write = xhci_write_config;
+    k->no_hotplug   = 1;
 }
 
-static TypeInfo xhci_info = {
+static const TypeInfo xhci_info = {
     .name          = "nec-usb-xhci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XHCIState),
diff --git a/hw/usb/host-bsd.c b/hw/usb/host-bsd.c
index ec262666207..07f0e01cc00 100644
--- a/hw/usb/host-bsd.c
+++ b/hw/usb/host-bsd.c
@@ -25,7 +25,7 @@
  */
 
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "hw/usb.h"
 
 /* usb.h declares these */
@@ -121,7 +121,7 @@ static void usb_host_handle_reset(USBDevice *dev)
  * -check device states against transfer requests
  *  and return appropriate response
  */
-static int usb_host_handle_control(USBDevice *dev,
+static void usb_host_handle_control(USBDevice *dev,
                                    USBPacket *p,
                                    int request,
                                    int value,
@@ -139,7 +139,6 @@ static int usb_host_handle_control(USBDevice *dev,
 
         /* specific SET_ADDRESS support */
         dev->addr = value;
-        return 0;
     } else if ((request >> 8) == UT_WRITE_DEVICE &&
                (request & 0xff) == UR_SET_CONFIG) {
 
@@ -151,10 +150,8 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: failed to set configuration - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
-
-        return 0;
     } else if ((request >> 8) == UT_WRITE_INTERFACE &&
                (request & 0xff) == UR_SET_INTERFACE) {
 
@@ -168,10 +165,8 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: failed to set alternate interface - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
-
-        return 0;
     } else {
         req.ucr_request.bmRequestType = request >> 8;
         req.ucr_request.bRequest = request & 0xff;
@@ -201,14 +196,14 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: error after request - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_NAK; // STALL
+            p->status = USB_RET_NAK; /* STALL */
         } else {
-            return req.ucr_actlen;
+            p->actual_length = req.ucr_actlen;
         }
     }
 }
 
-static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHostDevice *s = (USBHostDevice *)dev;
     int ret, fd, mode;
@@ -232,7 +227,8 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
     fd = ensure_ep_open(s, devep, mode);
     if (fd < 0) {
         sigprocmask(SIG_SETMASK, &old_mask, NULL);
-        return USB_RET_NODEV;
+        p->status = USB_RET_NODEV;
+        return;
     }
 
     if (ioctl(fd, USB_SET_TIMEOUT, &timeout) < 0) {
@@ -267,12 +263,13 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
         switch(errno) {
         case ETIMEDOUT:
         case EINTR:
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            break;
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
     } else {
-        return ret;
+        p->actual_length = ret;
     }
 }
 
@@ -295,6 +292,7 @@ static void usb_host_handle_destroy(USBDevice *opaque)
 
 static int usb_host_initfn(USBDevice *dev)
 {
+    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
     return 0;
 }
 
@@ -409,7 +407,7 @@ static void usb_host_class_initfn(ObjectClass *klass, void *data)
     uc->handle_destroy = usb_host_handle_destroy;
 }
 
-static TypeInfo usb_host_dev_info = {
+static const TypeInfo usb_host_dev_info = {
     .name          = "usb-host",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHostDevice),
@@ -635,7 +633,7 @@ static int usb_host_info_device(void *opaque,
     return 0;
 }
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     usb_host_scan(mon, usb_host_info_device);
 }
diff --git a/hw/usb/host-linux.c b/hw/usb/host-linux.c
index 8df92074d31..a2cff8a74dd 100644
--- a/hw/usb/host-linux.c
+++ b/hw/usb/host-linux.c
@@ -31,9 +31,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 #include <dirent.h>
@@ -135,7 +135,7 @@ static int parse_filter(const char *spec, struct USBAutoFilter *f);
 static void usb_host_auto_check(void *unused);
 static int usb_host_read_file(char *line, size_t line_size,
                             const char *device_file, const char *device_name);
-static int usb_linux_update_endp_table(USBHostDevice *s);
+static void usb_linux_update_endp_table(USBHostDevice *s);
 
 static int usb_host_usbfs_type(USBHostDevice *s, USBPacket *p)
 {
@@ -366,28 +366,34 @@ static void async_complete(void *opaque)
         if (p) {
             switch (aurb->urb.status) {
             case 0:
-                p->result += aurb->urb.actual_length;
+                p->actual_length += aurb->urb.actual_length;
+                if (!aurb->more) {
+                    /* Clear previous ASYNC status */
+                    p->status = USB_RET_SUCCESS;
+                }
                 break;
 
             case -EPIPE:
                 set_halt(s, p->pid, p->ep->nr);
-                p->result = USB_RET_STALL;
+                p->status = USB_RET_STALL;
                 break;
 
             case -EOVERFLOW:
-                p->result = USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
                 break;
 
             default:
-                p->result = USB_RET_IOERROR;
+                p->status = USB_RET_IOERROR;
                 break;
             }
 
             if (aurb->urb.type == USBDEVFS_URB_TYPE_CONTROL) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p, p->result);
+                trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                            p->status, aurb->urb.actual_length);
                 usb_generic_async_ctrl_complete(&s->dev, p);
             } else if (!aurb->more) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p, p->result);
+                trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                            p->status, aurb->urb.actual_length);
                 usb_packet_complete(&s->dev, p);
             }
         }
@@ -733,27 +739,31 @@ static void usb_host_stop_n_free_iso(USBHostDevice *s, int pid, uint8_t ep)
     clear_iso_started(s, pid, ep);
 }
 
-static int urb_status_to_usb_ret(int status)
+static void urb_status_to_usb_ret(int status, USBPacket *p)
 {
     switch (status) {
     case -EPIPE:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
     case -EOVERFLOW:
-        return USB_RET_BABBLE;
+        p->status = USB_RET_BABBLE;
+        break;
     default:
-        return USB_RET_IOERROR;
+        p->status = USB_RET_IOERROR;
     }
 }
 
-static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
+static void usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
 {
     AsyncURB *aurb;
-    int i, j, ret, max_packet_size, offset, len = 0;
+    int i, j, max_packet_size, offset, len;
     uint8_t *buf;
 
     max_packet_size = p->ep->max_packet_size;
-    if (max_packet_size == 0)
-        return USB_RET_NAK;
+    if (max_packet_size == 0) {
+        p->status = USB_RET_NAK;
+        return;
+    }
 
     aurb = get_iso_urb(s, p->pid, p->ep->nr);
     if (!aurb) {
@@ -766,18 +776,17 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
         if (in) {
             /* Check urb status  */
             if (aurb[i].urb.status) {
-                len = urb_status_to_usb_ret(aurb[i].urb.status);
+                urb_status_to_usb_ret(aurb[i].urb.status, p);
                 /* Move to the next urb */
                 aurb[i].iso_frame_idx = ISO_FRAME_DESC_PER_URB - 1;
             /* Check frame status */
             } else if (aurb[i].urb.iso_frame_desc[j].status) {
-                len = urb_status_to_usb_ret(
-                                        aurb[i].urb.iso_frame_desc[j].status);
+                urb_status_to_usb_ret(aurb[i].urb.iso_frame_desc[j].status, p);
             /* Check the frame fits */
             } else if (aurb[i].urb.iso_frame_desc[j].actual_length
                        > p->iov.size) {
                 printf("husb: received iso data is larger then packet\n");
-                len = USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
             /* All good copy data over */
             } else {
                 len = aurb[i].urb.iso_frame_desc[j].actual_length;
@@ -792,7 +801,8 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
             /* Check the frame fits */
             if (len > max_packet_size) {
                 printf("husb: send iso data is larger then max packet size\n");
-                return USB_RET_NAK;
+                p->status = USB_RET_NAK;
+                return;
             }
 
             /* All good copy data over */
@@ -823,17 +833,16 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
         /* (Re)-submit all fully consumed / filled urbs */
         for (i = 0; i < s->iso_urb_count; i++) {
             if (aurb[i].iso_frame_idx == ISO_FRAME_DESC_PER_URB) {
-                ret = ioctl(s->fd, USBDEVFS_SUBMITURB, &aurb[i]);
-                if (ret < 0) {
+                if (ioctl(s->fd, USBDEVFS_SUBMITURB, &aurb[i]) < 0) {
                     perror("USBDEVFS_SUBMITURB");
-                    if (!in || len == 0) {
+                    if (!in || p->status == USB_RET_SUCCESS) {
                         switch(errno) {
                         case ETIMEDOUT:
-                            len = USB_RET_NAK;
+                            p->status = USB_RET_NAK;
                             break;
                         case EPIPE:
                         default:
-                            len = USB_RET_STALL;
+                            p->status = USB_RET_STALL;
                         }
                     }
                     break;
@@ -843,11 +852,9 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
             }
         }
     }
-
-    return len;
 }
 
-static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
     struct usbdevfs_urb *urb;
@@ -861,8 +868,10 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
                             p->ep->nr, p->iov.size);
 
     if (!is_valid(s, p->pid, p->ep->nr)) {
-        trace_usb_host_req_complete(s->bus_num, s->addr, p, USB_RET_NAK);
-        return USB_RET_NAK;
+        p->status = USB_RET_NAK;
+        trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                    p->status, p->actual_length);
+        return;
     }
 
     if (p->pid == USB_TOKEN_IN) {
@@ -876,14 +885,17 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
         ret = ioctl(s->fd, USBDEVFS_CLEAR_HALT, &arg);
         if (ret < 0) {
             perror("USBDEVFS_CLEAR_HALT");
-            trace_usb_host_req_complete(s->bus_num, s->addr, p, USB_RET_NAK);
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                        p->status, p->actual_length);
+            return;
         }
         clear_halt(s, p->pid, p->ep->nr);
     }
 
     if (is_isoc(s, p->pid, p->ep->nr)) {
-        return usb_host_handle_iso_data(s, p, p->pid == USB_TOKEN_IN);
+        usb_host_handle_iso_data(s, p, p->pid == USB_TOKEN_IN);
+        return;
     }
 
     v = 0;
@@ -931,19 +943,21 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
 
             switch(errno) {
             case ETIMEDOUT:
+                p->status = USB_RET_NAK;
                 trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            USB_RET_NAK);
-                return USB_RET_NAK;
+                                            p->status, p->actual_length);
+                break;
             case EPIPE:
             default:
+                p->status = USB_RET_STALL;
                 trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            USB_RET_STALL);
-                return USB_RET_STALL;
+                                            p->status, p->actual_length);
             }
+            return;
         }
     } while (rem > 0);
 
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
 static int ctrl_error(void)
@@ -955,14 +969,13 @@ static int ctrl_error(void)
     }
 }
 
-static int usb_host_set_address(USBHostDevice *s, int addr)
+static void usb_host_set_address(USBHostDevice *s, int addr)
 {
     trace_usb_host_set_address(s->bus_num, s->addr, addr);
     s->dev.addr = addr;
-    return 0;
 }
 
-static int usb_host_set_config(USBHostDevice *s, int config)
+static void usb_host_set_config(USBHostDevice *s, int config, USBPacket *p)
 {
     int ret, first = 1;
 
@@ -987,14 +1000,15 @@ again:
     }
 
     if (ret < 0) {
-        return ctrl_error();
+        p->status = ctrl_error();
+        return;
     }
     usb_host_claim_interfaces(s, config);
     usb_linux_update_endp_table(s);
-    return 0;
 }
 
-static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
+static void usb_host_set_interface(USBHostDevice *s, int iface, int alt,
+                                   USBPacket *p)
 {
     struct usbdevfs_setinterface si;
     int i, ret;
@@ -1011,7 +1025,8 @@ static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
     }
 
     if (iface >= USB_MAX_INTERFACES) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     si.interface  = iface;
@@ -1022,15 +1037,15 @@ static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
             iface, alt, ret, errno);
 
     if (ret < 0) {
-        return ctrl_error();
+        p->status = ctrl_error();
+        return;
     }
 
     s->dev.altsetting[iface] = alt;
     usb_linux_update_endp_table(s);
-    return 0;
 }
 
-static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_host_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
@@ -1045,23 +1060,22 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
 
     /* Note request is (bRequestType << 8) | bRequest */
     trace_usb_host_req_control(s->bus_num, s->addr, p, request, value, index);
-    assert(p->result == 0);
 
     switch (request) {
     case DeviceOutRequest | USB_REQ_SET_ADDRESS:
-        ret = usb_host_set_address(s, value);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_address(s, value);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
-        ret = usb_host_set_config(s, value & 0xff);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_config(s, value & 0xff, p);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
-        ret = usb_host_set_interface(s, index, value);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_interface(s, index, value, p);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
         if (value == 0) { /* clear halt */
@@ -1069,16 +1083,16 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
             ioctl(s->fd, USBDEVFS_CLEAR_HALT, &index);
             clear_halt(s, pid, index & 0x0f);
             trace_usb_host_req_emulated(s->bus_num, s->addr, p, 0);
-            return 0;
+            return;
         }
     }
 
     /* The rest are asynchronous */
-
     if (length > sizeof(dev->data_buf)) {
         fprintf(stderr, "husb: ctrl buffer too small (%d > %zu)\n",
                 length, sizeof(dev->data_buf));
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     aurb = async_alloc(s);
@@ -1112,18 +1126,20 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
 
         switch(errno) {
         case ETIMEDOUT:
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            break;
         case EPIPE:
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
+            break;
         }
+        return;
     }
 
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-/* returns 1 on problem encountered or 0 for success */
-static int usb_linux_update_endp_table(USBHostDevice *s)
+static void usb_linux_update_endp_table(USBHostDevice *s)
 {
     static const char *tname[] = {
         [USB_ENDPOINT_XFER_CONTROL] = "control",
@@ -1149,23 +1165,23 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
         if (d->bLength < 2) {
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "descriptor too short");
-            goto error;
+            return;
         }
         if (i + d->bLength > s->descr_len) {
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "descriptor too long");
-            goto error;
+            return;
         }
         switch (d->bDescriptorType) {
         case 0:
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "invalid descriptor type");
-            goto error;
+            return;
         case USB_DT_DEVICE:
             if (d->bLength < 0x12) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "device descriptor too short");
-                goto error;
+                return;
             }
             v = (d->u.device.idVendor_hi << 8) | d->u.device.idVendor_lo;
             p = (d->u.device.idProduct_hi << 8) | d->u.device.idProduct_lo;
@@ -1175,7 +1191,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x09) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "config descriptor too short");
-                goto error;
+                return;
             }
             configuration = d->u.config.bConfigurationValue;
             active = (configuration == s->dev.configuration);
@@ -1186,7 +1202,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x09) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "interface descriptor too short");
-                goto error;
+                return;
             }
             interface = d->u.interface.bInterfaceNumber;
             altsetting = d->u.interface.bAlternateSetting;
@@ -1199,7 +1215,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x07) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "endpoint descriptor too short");
-                goto error;
+                return;
             }
             devep = d->u.endpoint.bEndpointAddress;
             pid = (devep & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT;
@@ -1207,7 +1223,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (ep == 0) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "invalid endpoint address");
-                goto error;
+                return;
             }
 
             type = d->u.endpoint.bmAttributes & 0x3;
@@ -1224,7 +1240,8 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
                 usb_ep_set_type(&s->dev, pid, ep, type);
                 usb_ep_set_ifnum(&s->dev, pid, ep, interface);
                 if ((s->options & (1 << USB_HOST_OPT_PIPELINE)) &&
-                    (type == USB_ENDPOINT_XFER_BULK)) {
+                    (type == USB_ENDPOINT_XFER_BULK) &&
+                    (pid == USB_TOKEN_OUT)) {
                     usb_ep_set_pipeline(&s->dev, pid, ep, true);
                 }
 
@@ -1239,11 +1256,6 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             break;
         }
     }
-    return 0;
-
-error:
-    usb_ep_reset(&s->dev);
-    return 1;
 }
 
 /*
@@ -1302,7 +1314,7 @@ static int usb_host_open(USBHostDevice *dev, int bus_num,
 
     dev->bus_num = bus_num;
     dev->addr = addr;
-    strcpy(dev->port, port);
+    pstrcpy(dev->port, sizeof(dev->port), port);
     dev->fd = fd;
 
     /* read the device description */
@@ -1330,10 +1342,7 @@ static int usb_host_open(USBHostDevice *dev, int bus_num,
     }
 
     usb_ep_init(&dev->dev);
-    ret = usb_linux_update_endp_table(dev);
-    if (ret) {
-        goto fail;
-    }
+    usb_linux_update_endp_table(dev);
 
     if (speed == -1) {
         struct usbdevfs_connectinfo ci;
@@ -1467,6 +1476,7 @@ static int usb_host_initfn(USBDevice *dev)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
 
+    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
     dev->auto_attach = 0;
     s->fd = -1;
     s->hub_fd = -1;
@@ -1524,7 +1534,7 @@ static void usb_host_class_initfn(ObjectClass *klass, void *data)
     dc->props = usb_host_dev_properties;
 }
 
-static TypeInfo usb_host_dev_info = {
+static const TypeInfo usb_host_dev_info = {
     .name          = "usb-host",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHostDevice),
@@ -1727,6 +1737,7 @@ static int usb_host_scan(void *opaque, USBScanFunc *func)
 }
 
 static QEMUTimer *usb_auto_timer;
+static VMChangeStateEntry *usb_vmstate;
 
 static int usb_host_auto_scan(void *opaque, int bus_num,
                               int addr, const char *port,
@@ -1749,7 +1760,7 @@ static int usb_host_auto_scan(void *opaque, int bus_num,
         if (f->addr > 0 && f->addr != addr) {
             continue;
         }
-        if (f->port != NULL && (port == NULL || strcmp(f->port, port) != 0)) {
+        if (f->port != NULL && strcmp(f->port, port) != 0) {
             continue;
         }
 
@@ -1781,6 +1792,13 @@ static int usb_host_auto_scan(void *opaque, int bus_num,
     return 0;
 }
 
+static void usb_host_vm_state(void *unused, int running, RunState state)
+{
+    if (running) {
+        usb_host_auto_check(unused);
+    }
+}
+
 static void usb_host_auto_check(void *unused)
 {
     struct USBHostDevice *s;
@@ -1809,6 +1827,9 @@ static void usb_host_auto_check(void *unused)
         }
     }
 
+    if (!usb_vmstate) {
+        usb_vmstate = qemu_add_vm_change_state_handler(usb_host_vm_state, NULL);
+    }
     if (!usb_auto_timer) {
         usb_auto_timer = qemu_new_timer_ms(rt_clock, usb_host_auto_check, NULL);
         if (!usb_auto_timer) {
@@ -1977,7 +1998,7 @@ static void hex2str(int val, char *str, size_t size)
     }
 }
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     struct USBAutoFilter *f;
     struct USBHostDevice *s;
diff --git a/hw/usb/host-stub.c b/hw/usb/host-stub.c
index b4e10c12ca6..8affba76c1f 100644
--- a/hw/usb/host-stub.c
+++ b/hw/usb/host-stub.c
@@ -31,11 +31,11 @@
  */
 
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "USB host devices not supported\n");
 }
diff --git a/hw/usb/libhw.c b/hw/usb/libhw.c
index c0de30ea883..75f022f4ec8 100644
--- a/hw/usb/libhw.c
+++ b/hw/usb/libhw.c
@@ -20,27 +20,33 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "cpu-common.h"
+#include "exec/cpu-common.h"
 #include "hw/usb.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 int usb_packet_map(USBPacket *p, QEMUSGList *sgl)
 {
     DMADirection dir = (p->pid == USB_TOKEN_IN) ?
         DMA_DIRECTION_FROM_DEVICE : DMA_DIRECTION_TO_DEVICE;
-    dma_addr_t len;
     void *mem;
     int i;
 
     for (i = 0; i < sgl->nsg; i++) {
-        len = sgl->sg[i].len;
-        mem = dma_memory_map(sgl->dma, sgl->sg[i].base, &len, dir);
-        if (!mem) {
-            goto err;
-        }
-        qemu_iovec_add(&p->iov, mem, len);
-        if (len != sgl->sg[i].len) {
-            goto err;
+        dma_addr_t base = sgl->sg[i].base;
+        dma_addr_t len = sgl->sg[i].len;
+
+        while (len) {
+            dma_addr_t xlen = len;
+            mem = dma_memory_map(sgl->dma, base, &xlen, dir);
+            if (!mem) {
+                goto err;
+            }
+            if (xlen > len) {
+                xlen = len;
+            }
+            qemu_iovec_add(&p->iov, mem, xlen);
+            len -= xlen;
+            base += xlen;
         }
     }
     return 0;
diff --git a/hw/usb/quirks-ftdi-ids.h b/hw/usb/quirks-ftdi-ids.h
new file mode 100644
index 00000000000..57c12ef6625
--- /dev/null
+++ b/hw/usb/quirks-ftdi-ids.h
@@ -0,0 +1,1255 @@
+/*
+ * vendor/product IDs (VID/PID) of devices using FTDI USB serial converters.
+ * Please keep numerically sorted within individual areas, thanks!
+ *
+ * Philipp Gühring - pg@futureware.at - added the Device ID of the USB relais
+ * from Rudolf Gugler
+ *
+ */
+
+
+/**********************************/
+/***** devices using FTDI VID *****/
+/**********************************/
+
+
+#define FTDI_VID	0x0403	/* Vendor Id */
+
+
+/*** "original" FTDI device PIDs ***/
+
+#define FTDI_8U232AM_PID 0x6001 /* Similar device to SIO above */
+#define FTDI_8U232AM_ALT_PID 0x6006 /* FTDI's alternate PID for above */
+#define FTDI_8U2232C_PID 0x6010 /* Dual channel device */
+#define FTDI_4232H_PID 0x6011 /* Quad channel hi-speed device */
+#define FTDI_232H_PID  0x6014 /* Single channel hi-speed device */
+#define FTDI_FTX_PID   0x6015 /* FT-X series (FT201X, FT230X, FT231X, etc) */
+#define FTDI_SIO_PID	0x8372	/* Product Id SIO application of 8U100AX */
+#define FTDI_232RL_PID  0xFBFA  /* Product ID for FT232RL */
+
+
+/*** third-party PIDs (using FTDI_VID) ***/
+
+#define FTDI_LUMEL_PD12_PID	0x6002
+
+/*
+ * Marvell OpenRD Base, Client
+ * http://www.open-rd.org
+ * OpenRD Base, Client use VID 0x0403
+ */
+#define MARVELL_OPENRD_PID	0x9e90
+
+/* www.candapter.com Ewert Energy Systems CANdapter device */
+#define FTDI_CANDAPTER_PID 0x9F80 /* Product Id */
+
+/*
+ * Texas Instruments XDS100v2 JTAG / BeagleBone A3
+ * http://processors.wiki.ti.com/index.php/XDS100
+ * http://beagleboard.org/bone
+ */
+#define TI_XDS100V2_PID		0xa6d0
+
+#define FTDI_NXTCAM_PID		0xABB8 /* NXTCam for Mindstorms NXT */
+
+/* US Interface Navigator (http://www.usinterface.com/) */
+#define FTDI_USINT_CAT_PID	0xb810	/* Navigator CAT and 2nd PTT lines */
+#define FTDI_USINT_WKEY_PID	0xb811	/* Navigator WKEY and FSK lines */
+#define FTDI_USINT_RS232_PID	0xb812	/* Navigator RS232 and CONFIG lines */
+
+/* OOCDlink by Joern Kaipf <joernk@web.de>
+ * (http://www.joernonline.de/) */
+#define FTDI_OOCDLINK_PID	0xbaf8	/* Amontec JTAGkey */
+
+/* Luminary Micro Stellaris Boards, VID = FTDI_VID */
+/* FTDI 2332C Dual channel device, side A=245 FIFO (JTAG), Side B=RS232 UART */
+#define LMI_LM3S_DEVEL_BOARD_PID	0xbcd8
+#define LMI_LM3S_EVAL_BOARD_PID		0xbcd9
+#define LMI_LM3S_ICDI_BOARD_PID		0xbcda
+
+#define FTDI_TURTELIZER_PID	0xBDC8 /* JTAG/RS-232 adapter by egnite GmbH */
+
+/* OpenDCC (www.opendcc.de) product id */
+#define FTDI_OPENDCC_PID	0xBFD8
+#define FTDI_OPENDCC_SNIFFER_PID	0xBFD9
+#define FTDI_OPENDCC_THROTTLE_PID	0xBFDA
+#define FTDI_OPENDCC_GATEWAY_PID	0xBFDB
+#define FTDI_OPENDCC_GBM_PID	0xBFDC
+
+/* NZR SEM 16+ USB (http://www.nzr.de) */
+#define FTDI_NZR_SEM_USB_PID	0xC1E0	/* NZR SEM-LOG16+ */
+
+/*
+ * RR-CirKits LocoBuffer USB (http://www.rr-cirkits.com)
+ */
+#define FTDI_RRCIRKITS_LOCOBUFFER_PID	0xc7d0	/* LocoBuffer USB */
+
+/* DMX4ALL DMX Interfaces */
+#define FTDI_DMX4ALL 0xC850
+
+/*
+ * ASK.fr devices
+ */
+#define FTDI_ASK_RDR400_PID	0xC991	/* ASK RDR 400 series card reader */
+
+/* www.starting-point-systems.com µChameleon device */
+#define FTDI_MICRO_CHAMELEON_PID	0xCAA0	/* Product Id */
+
+/*
+ * Tactrix OpenPort (ECU) devices.
+ * OpenPort 1.3M submitted by Donour Sizemore.
+ * OpenPort 1.3S and 1.3U submitted by Ian Abbott.
+ */
+#define FTDI_TACTRIX_OPENPORT_13M_PID	0xCC48	/* OpenPort 1.3 Mitsubishi */
+#define FTDI_TACTRIX_OPENPORT_13S_PID	0xCC49	/* OpenPort 1.3 Subaru */
+#define FTDI_TACTRIX_OPENPORT_13U_PID	0xCC4A	/* OpenPort 1.3 Universal */
+
+#define FTDI_DISTORTEC_JTAG_LOCK_PICK_PID	0xCFF8
+
+/* SCS HF Radio Modems PID's (http://www.scs-ptc.com) */
+/* the VID is the standard ftdi vid (FTDI_VID) */
+#define FTDI_SCS_DEVICE_0_PID 0xD010    /* SCS PTC-IIusb */
+#define FTDI_SCS_DEVICE_1_PID 0xD011    /* SCS Tracker / DSP TNC */
+#define FTDI_SCS_DEVICE_2_PID 0xD012
+#define FTDI_SCS_DEVICE_3_PID 0xD013
+#define FTDI_SCS_DEVICE_4_PID 0xD014
+#define FTDI_SCS_DEVICE_5_PID 0xD015
+#define FTDI_SCS_DEVICE_6_PID 0xD016
+#define FTDI_SCS_DEVICE_7_PID 0xD017
+
+/* iPlus device */
+#define FTDI_IPLUS_PID 0xD070 /* Product Id */
+#define FTDI_IPLUS2_PID 0xD071 /* Product Id */
+
+/*
+ * Gamma Scout (http://gamma-scout.com/). Submitted by rsc@runtux.com.
+ */
+#define FTDI_GAMMA_SCOUT_PID		0xD678	/* Gamma Scout online */
+
+/* Propox devices */
+#define FTDI_PROPOX_JTAGCABLEII_PID	0xD738
+#define FTDI_PROPOX_ISPCABLEIII_PID	0xD739
+
+/* Lenz LI-USB Computer Interface. */
+#define FTDI_LENZ_LIUSB_PID	0xD780
+
+/* Vardaan Enterprises Serial Interface VEUSB422R3 */
+#define FTDI_VARDAAN_PID	0xF070
+
+/*
+ * Xsens Technologies BV products (http://www.xsens.com).
+ */
+#define XSENS_CONVERTER_0_PID	0xD388
+#define XSENS_CONVERTER_1_PID	0xD389
+#define XSENS_CONVERTER_2_PID	0xD38A
+#define XSENS_CONVERTER_3_PID	0xD38B
+#define XSENS_CONVERTER_4_PID	0xD38C
+#define XSENS_CONVERTER_5_PID	0xD38D
+#define XSENS_CONVERTER_6_PID	0xD38E
+#define XSENS_CONVERTER_7_PID	0xD38F
+
+/*
+ * NDI (www.ndigital.com) product ids
+ */
+#define FTDI_NDI_HUC_PID		0xDA70	/* NDI Host USB Converter */
+#define FTDI_NDI_SPECTRA_SCU_PID	0xDA71	/* NDI Spectra SCU */
+#define FTDI_NDI_FUTURE_2_PID		0xDA72	/* NDI future device #2 */
+#define FTDI_NDI_FUTURE_3_PID		0xDA73	/* NDI future device #3 */
+#define FTDI_NDI_AURORA_SCU_PID		0xDA74	/* NDI Aurora SCU */
+
+/*
+ * ChamSys Limited (www.chamsys.co.uk) USB wing/interface product IDs
+ */
+#define FTDI_CHAMSYS_24_MASTER_WING_PID        0xDAF8
+#define FTDI_CHAMSYS_PC_WING_PID       0xDAF9
+#define FTDI_CHAMSYS_USB_DMX_PID       0xDAFA
+#define FTDI_CHAMSYS_MIDI_TIMECODE_PID 0xDAFB
+#define FTDI_CHAMSYS_MINI_WING_PID     0xDAFC
+#define FTDI_CHAMSYS_MAXI_WING_PID     0xDAFD
+#define FTDI_CHAMSYS_MEDIA_WING_PID    0xDAFE
+#define FTDI_CHAMSYS_WING_PID  0xDAFF
+
+/*
+ * Westrex International devices submitted by Cory Lee
+ */
+#define FTDI_WESTREX_MODEL_777_PID	0xDC00	/* Model 777 */
+#define FTDI_WESTREX_MODEL_8900F_PID	0xDC01	/* Model 8900F */
+
+/*
+ * ACG Identification Technologies GmbH products (http://www.acg.de/).
+ * Submitted by anton -at- goto10 -dot- org.
+ */
+#define FTDI_ACG_HFDUAL_PID		0xDD20	/* HF Dual ISO Reader (RFID) */
+
+/*
+ * Definitions for Artemis astronomical USB based cameras
+ * Check it at http://www.artemisccd.co.uk/
+ */
+#define FTDI_ARTEMIS_PID	0xDF28	/* All Artemis Cameras */
+
+/*
+ * Definitions for ATIK Instruments astronomical USB based cameras
+ * Check it at http://www.atik-instruments.com/
+ */
+#define FTDI_ATIK_ATK16_PID	0xDF30	/* ATIK ATK-16 Grayscale Camera */
+#define FTDI_ATIK_ATK16C_PID	0xDF32	/* ATIK ATK-16C Colour Camera */
+#define FTDI_ATIK_ATK16HR_PID	0xDF31	/* ATIK ATK-16HR Grayscale Camera */
+#define FTDI_ATIK_ATK16HRC_PID	0xDF33	/* ATIK ATK-16HRC Colour Camera */
+#define FTDI_ATIK_ATK16IC_PID   0xDF35  /* ATIK ATK-16IC Grayscale Camera */
+
+/*
+ * Yost Engineering, Inc. products (www.yostengineering.com).
+ * PID 0xE050 submitted by Aaron Prose.
+ */
+#define FTDI_YEI_SERVOCENTER31_PID	0xE050	/* YEI ServoCenter3.1 USB */
+
+/*
+ * ELV USB devices submitted by Christian Abt of ELV (www.elv.de).
+ * All of these devices use FTDI's vendor ID (0x0403).
+ * Further IDs taken from ELV Windows .inf file.
+ *
+ * The previously included PID for the UO 100 module was incorrect.
+ * In fact, that PID was for ELV's UR 100 USB-RS232 converter (0xFB58).
+ *
+ * Armin Laeuger originally sent the PID for the UM 100 module.
+ */
+#define FTDI_ELV_USR_PID	0xE000	/* ELV Universal-Sound-Recorder */
+#define FTDI_ELV_MSM1_PID	0xE001	/* ELV Mini-Sound-Modul */
+#define FTDI_ELV_KL100_PID	0xE002	/* ELV Kfz-Leistungsmesser KL 100 */
+#define FTDI_ELV_WS550_PID	0xE004	/* WS 550 */
+#define FTDI_ELV_EC3000_PID	0xE006	/* ENERGY CONTROL 3000 USB */
+#define FTDI_ELV_WS888_PID	0xE008	/* WS 888 */
+#define FTDI_ELV_TWS550_PID	0xE009	/* Technoline WS 550 */
+#define FTDI_ELV_FEM_PID	0xE00A	/* Funk Energie Monitor */
+#define FTDI_ELV_FHZ1300PC_PID	0xE0E8	/* FHZ 1300 PC */
+#define FTDI_ELV_WS500_PID	0xE0E9	/* PC-Wetterstation (WS 500) */
+#define FTDI_ELV_HS485_PID	0xE0EA	/* USB to RS-485 adapter */
+#define FTDI_ELV_UMS100_PID	0xE0EB	/* ELV USB Master-Slave Schaltsteckdose UMS 100 */
+#define FTDI_ELV_TFD128_PID	0xE0EC	/* ELV Temperatur-Feuchte-Datenlogger TFD 128 */
+#define FTDI_ELV_FM3RX_PID	0xE0ED	/* ELV Messwertuebertragung FM3 RX */
+#define FTDI_ELV_WS777_PID	0xE0EE	/* Conrad WS 777 */
+#define FTDI_ELV_EM1010PC_PID	0xE0EF	/* Energy monitor EM 1010 PC */
+#define FTDI_ELV_CSI8_PID	0xE0F0	/* Computer-Schalt-Interface (CSI 8) */
+#define FTDI_ELV_EM1000DL_PID	0xE0F1	/* PC-Datenlogger fuer Energiemonitor (EM 1000 DL) */
+#define FTDI_ELV_PCK100_PID	0xE0F2	/* PC-Kabeltester (PCK 100) */
+#define FTDI_ELV_RFP500_PID	0xE0F3	/* HF-Leistungsmesser (RFP 500) */
+#define FTDI_ELV_FS20SIG_PID	0xE0F4	/* Signalgeber (FS 20 SIG) */
+#define FTDI_ELV_UTP8_PID	0xE0F5	/* ELV UTP 8 */
+#define FTDI_ELV_WS300PC_PID	0xE0F6	/* PC-Wetterstation (WS 300 PC) */
+#define FTDI_ELV_WS444PC_PID	0xE0F7	/* Conrad WS 444 PC */
+#define FTDI_PHI_FISCO_PID      0xE40B  /* PHI Fisco USB to Serial cable */
+#define FTDI_ELV_UAD8_PID	0xF068	/* USB-AD-Wandler (UAD 8) */
+#define FTDI_ELV_UDA7_PID	0xF069	/* USB-DA-Wandler (UDA 7) */
+#define FTDI_ELV_USI2_PID	0xF06A	/* USB-Schrittmotoren-Interface (USI 2) */
+#define FTDI_ELV_T1100_PID	0xF06B	/* Thermometer (T 1100) */
+#define FTDI_ELV_PCD200_PID	0xF06C	/* PC-Datenlogger (PCD 200) */
+#define FTDI_ELV_ULA200_PID	0xF06D	/* USB-LCD-Ansteuerung (ULA 200) */
+#define FTDI_ELV_ALC8500_PID	0xF06E	/* ALC 8500 Expert */
+#define FTDI_ELV_FHZ1000PC_PID	0xF06F	/* FHZ 1000 PC */
+#define FTDI_ELV_UR100_PID	0xFB58	/* USB-RS232-Umsetzer (UR 100) */
+#define FTDI_ELV_UM100_PID	0xFB5A	/* USB-Modul UM 100 */
+#define FTDI_ELV_UO100_PID	0xFB5B	/* USB-Modul UO 100 */
+/* Additional ELV PIDs that default to using the FTDI D2XX drivers on
+ * MS Windows, rather than the FTDI Virtual Com Port drivers.
+ * Maybe these will be easier to use with the libftdi/libusb user-space
+ * drivers, or possibly the Comedi drivers in some cases. */
+#define FTDI_ELV_CLI7000_PID	0xFB59	/* Computer-Light-Interface (CLI 7000) */
+#define FTDI_ELV_PPS7330_PID	0xFB5C	/* Processor-Power-Supply (PPS 7330) */
+#define FTDI_ELV_TFM100_PID	0xFB5D	/* Temperatur-Feuchte-Messgeraet (TFM 100) */
+#define FTDI_ELV_UDF77_PID	0xFB5E	/* USB DCF Funkuhr (UDF 77) */
+#define FTDI_ELV_UIO88_PID	0xFB5F	/* USB-I/O Interface (UIO 88) */
+
+/*
+ * EVER Eco Pro UPS (http://www.ever.com.pl/)
+ */
+
+#define	EVER_ECO_PRO_CDS	0xe520	/* RS-232 converter */
+
+/*
+ * Active Robots product ids.
+ */
+#define FTDI_ACTIVE_ROBOTS_PID	0xE548	/* USB comms board */
+
+/* Pyramid Computer GmbH */
+#define FTDI_PYRAMID_PID	0xE6C8	/* Pyramid Appliance Display */
+
+/* www.elsterelectricity.com Elster Unicom III Optical Probe */
+#define FTDI_ELSTER_UNICOM_PID		0xE700 /* Product Id */
+
+/*
+ * Gude Analog- und Digitalsysteme GmbH
+ */
+#define FTDI_GUDEADS_E808_PID    0xE808
+#define FTDI_GUDEADS_E809_PID    0xE809
+#define FTDI_GUDEADS_E80A_PID    0xE80A
+#define FTDI_GUDEADS_E80B_PID    0xE80B
+#define FTDI_GUDEADS_E80C_PID    0xE80C
+#define FTDI_GUDEADS_E80D_PID    0xE80D
+#define FTDI_GUDEADS_E80E_PID    0xE80E
+#define FTDI_GUDEADS_E80F_PID    0xE80F
+#define FTDI_GUDEADS_E888_PID    0xE888  /* Expert ISDN Control USB */
+#define FTDI_GUDEADS_E889_PID    0xE889  /* USB RS-232 OptoBridge */
+#define FTDI_GUDEADS_E88A_PID    0xE88A
+#define FTDI_GUDEADS_E88B_PID    0xE88B
+#define FTDI_GUDEADS_E88C_PID    0xE88C
+#define FTDI_GUDEADS_E88D_PID    0xE88D
+#define FTDI_GUDEADS_E88E_PID    0xE88E
+#define FTDI_GUDEADS_E88F_PID    0xE88F
+
+/*
+ * Eclo (http://www.eclo.pt/) product IDs.
+ * PID 0xEA90 submitted by Martin Grill.
+ */
+#define FTDI_ECLO_COM_1WIRE_PID	0xEA90	/* COM to 1-Wire USB adaptor */
+
+/* TNC-X USB-to-packet-radio adapter, versions prior to 3.0 (DLP module) */
+#define FTDI_TNC_X_PID		0xEBE0
+
+/*
+ * Teratronik product ids.
+ * Submitted by O. Wölfelschneider.
+ */
+#define FTDI_TERATRONIK_VCP_PID	 0xEC88	/* Teratronik device (preferring VCP driver on windows) */
+#define FTDI_TERATRONIK_D2XX_PID 0xEC89	/* Teratronik device (preferring D2XX driver on windows) */
+
+/* Rig Expert Ukraine devices */
+#define FTDI_REU_TINY_PID		0xED22	/* RigExpert Tiny */
+
+/*
+ * Hameg HO820 and HO870 interface (using VID 0x0403)
+ */
+#define HAMEG_HO820_PID			0xed74
+#define HAMEG_HO730_PID			0xed73
+#define HAMEG_HO720_PID			0xed72
+#define HAMEG_HO870_PID			0xed71
+
+/*
+ *  MaxStream devices	www.maxstream.net
+ */
+#define FTDI_MAXSTREAM_PID	0xEE18	/* Xbee PKG-U Module */
+
+/*
+ * microHAM product IDs (http://www.microham.com).
+ * Submitted by Justin Burket (KL1RL) <zorton@jtan.com>
+ * and Mike Studer (K6EEP) <k6eep@hamsoftware.org>.
+ * Ian Abbott <abbotti@mev.co.uk> added a few more from the driver INF file.
+ */
+#define FTDI_MHAM_KW_PID	0xEEE8	/* USB-KW interface */
+#define FTDI_MHAM_YS_PID	0xEEE9	/* USB-YS interface */
+#define FTDI_MHAM_Y6_PID	0xEEEA	/* USB-Y6 interface */
+#define FTDI_MHAM_Y8_PID	0xEEEB	/* USB-Y8 interface */
+#define FTDI_MHAM_IC_PID	0xEEEC	/* USB-IC interface */
+#define FTDI_MHAM_DB9_PID	0xEEED	/* USB-DB9 interface */
+#define FTDI_MHAM_RS232_PID	0xEEEE	/* USB-RS232 interface */
+#define FTDI_MHAM_Y9_PID	0xEEEF	/* USB-Y9 interface */
+
+/* Domintell products  http://www.domintell.com */
+#define FTDI_DOMINTELL_DGQG_PID	0xEF50	/* Master */
+#define FTDI_DOMINTELL_DUSB_PID	0xEF51	/* DUSB01 module */
+
+/*
+ * The following are the values for the Perle Systems
+ * UltraPort USB serial converters
+ */
+#define FTDI_PERLE_ULTRAPORT_PID 0xF0C0	/* Perle UltraPort Product Id */
+
+/* Sprog II (Andrew Crosland's SprogII DCC interface) */
+#define FTDI_SPROG_II		0xF0C8
+
+/* an infrared receiver for user access control with IR tags */
+#define FTDI_PIEGROUP_PID	0xF208	/* Product Id */
+
+/* ACT Solutions HomePro ZWave interface
+   (http://www.act-solutions.com/HomePro-Product-Matrix.html) */
+#define FTDI_ACTZWAVE_PID	0xF2D0
+
+/*
+ * 4N-GALAXY.DE PIDs for CAN-USB, USB-RS232, USB-RS422, USB-RS485,
+ * USB-TTY aktiv, USB-TTY passiv.  Some PIDs are used by several devices
+ * and I'm not entirely sure which are used by which.
+ */
+#define FTDI_4N_GALAXY_DE_1_PID	0xF3C0
+#define FTDI_4N_GALAXY_DE_2_PID	0xF3C1
+#define FTDI_4N_GALAXY_DE_3_PID	0xF3C2
+
+/*
+ * Linx Technologies product ids
+ */
+#define LINX_SDMUSBQSS_PID	0xF448	/* Linx SDM-USB-QS-S */
+#define LINX_MASTERDEVEL2_PID   0xF449	/* Linx Master Development 2.0 */
+#define LINX_FUTURE_0_PID   0xF44A	/* Linx future device */
+#define LINX_FUTURE_1_PID   0xF44B	/* Linx future device */
+#define LINX_FUTURE_2_PID   0xF44C	/* Linx future device */
+
+/*
+ * Oceanic product ids
+ */
+#define FTDI_OCEANIC_PID	0xF460  /* Oceanic dive instrument */
+
+/*
+ * SUUNTO product ids
+ */
+#define FTDI_SUUNTO_SPORTS_PID	0xF680	/* Suunto Sports instrument */
+
+/* USB-UIRT - An infrared receiver and transmitter using the 8U232AM chip */
+/* http://www.usbuirt.com/ */
+#define FTDI_USB_UIRT_PID	0xF850	/* Product Id */
+
+/* CCS Inc. ICDU/ICDU40 product ID -
+ * the FT232BM is used in an in-circuit-debugger unit for PIC16's/PIC18's */
+#define FTDI_CCSICDU20_0_PID    0xF9D0
+#define FTDI_CCSICDU40_1_PID    0xF9D1
+#define FTDI_CCSMACHX_2_PID     0xF9D2
+#define FTDI_CCSLOAD_N_GO_3_PID 0xF9D3
+#define FTDI_CCSICDU64_4_PID    0xF9D4
+#define FTDI_CCSPRIME8_5_PID    0xF9D5
+
+/*
+ * The following are the values for the Matrix Orbital LCD displays,
+ * which are the FT232BM ( similar to the 8U232AM )
+ */
+#define FTDI_MTXORB_0_PID      0xFA00  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_1_PID      0xFA01  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_2_PID      0xFA02  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_3_PID      0xFA03  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_4_PID      0xFA04  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_5_PID      0xFA05  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_6_PID      0xFA06  /* Matrix Orbital Product Id */
+
+/*
+ * Home Electronics (www.home-electro.com) USB gadgets
+ */
+#define FTDI_HE_TIRA1_PID	0xFA78	/* Tira-1 IR transceiver */
+
+/* Inside Accesso contactless reader (http://www.insidecontactless.com/) */
+#define INSIDE_ACCESSO		0xFAD0
+
+/*
+ * ThorLabs USB motor drivers
+ */
+#define FTDI_THORLABS_PID		0xfaf0 /* ThorLabs USB motor drivers */
+
+/*
+ * Protego product ids
+ */
+#define PROTEGO_SPECIAL_1	0xFC70	/* special/unknown device */
+#define PROTEGO_R2X0		0xFC71	/* R200-USB TRNG unit (R210, R220, and R230) */
+#define PROTEGO_SPECIAL_3	0xFC72	/* special/unknown device */
+#define PROTEGO_SPECIAL_4	0xFC73	/* special/unknown device */
+
+/*
+ * Sony Ericsson product ids
+ */
+#define FTDI_DSS20_PID		0xFC82	/* DSS-20 Sync Station for Sony Ericsson P800 */
+#define FTDI_URBAN_0_PID	0xFC8A	/* Sony Ericsson Urban, uart #0 */
+#define FTDI_URBAN_1_PID	0xFC8B	/* Sony Ericsson Urban, uart #1 */
+
+/* www.irtrans.de device */
+#define FTDI_IRTRANS_PID 0xFC60 /* Product Id */
+
+/*
+ * RM Michaelides CANview USB (http://www.rmcan.com) (FTDI_VID)
+ * CAN fieldbus interface adapter, added by port GmbH www.port.de)
+ * Ian Abbott changed the macro names for consistency.
+ */
+#define FTDI_RM_CANVIEW_PID	0xfd60	/* Product Id */
+/* www.thoughttechnology.com/ TT-USB provide with procomp use ftdi_sio */
+#define FTDI_TTUSB_PID 0xFF20 /* Product Id */
+
+#define FTDI_USBX_707_PID 0xF857	/* ADSTech IR Blaster USBX-707 (FTDI_VID) */
+
+#define FTDI_RELAIS_PID	0xFA10  /* Relais device from Rudolf Gugler */
+
+/*
+ * PCDJ use ftdi based dj-controllers. The following PID is
+ * for their DAC-2 device http://www.pcdjhardware.com/DAC2.asp
+ * (the VID is the standard ftdi vid (FTDI_VID), PID sent by Wouter Paesen)
+ */
+#define FTDI_PCDJ_DAC2_PID 0xFA88
+
+#define FTDI_R2000KU_TRUE_RNG	0xFB80  /* R2000KU TRUE RNG (FTDI_VID) */
+
+/*
+ * DIEBOLD BCS SE923 (FTDI_VID)
+ */
+#define DIEBOLD_BCS_SE923_PID	0xfb99
+
+/* www.crystalfontz.com devices
+ * - thanx for providing free devices for evaluation !
+ * they use the ftdi chipset for the USB interface
+ * and the vendor id is the same
+ */
+#define FTDI_XF_632_PID 0xFC08	/* 632: 16x2 Character Display */
+#define FTDI_XF_634_PID 0xFC09	/* 634: 20x4 Character Display */
+#define FTDI_XF_547_PID 0xFC0A	/* 547: Two line Display */
+#define FTDI_XF_633_PID 0xFC0B	/* 633: 16x2 Character Display with Keys */
+#define FTDI_XF_631_PID 0xFC0C	/* 631: 20x2 Character Display */
+#define FTDI_XF_635_PID 0xFC0D	/* 635: 20x4 Character Display */
+#define FTDI_XF_640_PID 0xFC0E	/* 640: Two line Display */
+#define FTDI_XF_642_PID 0xFC0F	/* 642: Two line Display */
+
+/*
+ * Video Networks Limited / Homechoice in the UK use an ftdi-based device
+ * for their 1Mb broadband internet service.  The following PID is exhibited
+ * by the usb device supplied (the VID is the standard ftdi vid (FTDI_VID)
+ */
+#define FTDI_VNHCPCUSB_D_PID 0xfe38 /* Product Id */
+
+/* AlphaMicro Components AMC-232USB01 device (FTDI_VID) */
+#define FTDI_AMC232_PID 0xFF00 /* Product Id */
+
+/*
+ * IBS elektronik product ids (FTDI_VID)
+ * Submitted by Thomas Schleusener
+ */
+#define FTDI_IBS_US485_PID	0xff38  /* IBS US485 (USB<-->RS422/485 interface) */
+#define FTDI_IBS_PICPRO_PID	0xff39  /* IBS PIC-Programmer */
+#define FTDI_IBS_PCMCIA_PID	0xff3a  /* IBS Card reader for PCMCIA SRAM-cards */
+#define FTDI_IBS_PK1_PID	0xff3b  /* IBS PK1 - Particel counter */
+#define FTDI_IBS_RS232MON_PID	0xff3c  /* IBS RS232 - Monitor */
+#define FTDI_IBS_APP70_PID	0xff3d  /* APP 70 (dust monitoring system) */
+#define FTDI_IBS_PEDO_PID	0xff3e  /* IBS PEDO-Modem (RF modem 868.35 MHz) */
+#define FTDI_IBS_PROD_PID	0xff3f  /* future device */
+/* www.canusb.com Lawicel CANUSB device (FTDI_VID) */
+#define FTDI_CANUSB_PID 0xFFA8 /* Product Id */
+
+/*
+ * TavIR AVR product ids (FTDI_VID)
+ */
+#define FTDI_TAVIR_STK500_PID	0xFA33	/* STK500 AVR programmer */
+
+/*
+ * TIAO product ids (FTDI_VID)
+ * http://www.tiaowiki.com/w/Main_Page
+ */
+#define FTDI_TIAO_UMPA_PID	0x8a98	/* TIAO/DIYGADGET USB Multi-Protocol Adapter */
+
+
+/********************************/
+/** third-party VID/PID combos **/
+/********************************/
+
+
+
+/*
+ * Atmel STK541
+ */
+#define ATMEL_VID		0x03eb /* Vendor ID */
+#define STK541_PID		0x2109 /* Zigbee Controller */
+
+/*
+ * Blackfin gnICE JTAG
+ * http://docs.blackfin.uclinux.org/doku.php?id=hw:jtag:gnice
+ */
+#define ADI_VID			0x0456
+#define ADI_GNICE_PID		0xF000
+#define ADI_GNICEPLUS_PID	0xF001
+
+/*
+ * Microchip Technology, Inc.
+ *
+ * MICROCHIP_VID (0x04D8) and MICROCHIP_USB_BOARD_PID (0x000A) are
+ * used by single function CDC ACM class based firmware demo
+ * applications.  The VID/PID has also been used in firmware
+ * emulating FTDI serial chips by:
+ * Hornby Elite - Digital Command Control Console
+ * http://www.hornby.com/hornby-dcc/controllers/
+ */
+#define MICROCHIP_VID		0x04D8
+#define MICROCHIP_USB_BOARD_PID	0x000A /* CDC RS-232 Emulation Demo */
+
+/*
+ * RATOC REX-USB60F
+ */
+#define RATOC_VENDOR_ID		0x0584
+#define RATOC_PRODUCT_ID_USB60F	0xb020
+
+/*
+ * Acton Research Corp.
+ */
+#define ACTON_VID		0x0647	/* Vendor ID */
+#define ACTON_SPECTRAPRO_PID	0x0100
+
+/*
+ * Contec products (http://www.contec.com)
+ * Submitted by Daniel Sangorrin
+ */
+#define CONTEC_VID		0x06CE	/* Vendor ID */
+#define CONTEC_COM1USBH_PID	0x8311	/* COM-1(USB)H */
+
+/*
+ * Definitions for B&B Electronics products.
+ */
+#define BANDB_VID		0x0856	/* B&B Electronics Vendor ID */
+#define BANDB_USOTL4_PID	0xAC01	/* USOTL4 Isolated RS-485 Converter */
+#define BANDB_USTL4_PID		0xAC02	/* USTL4 RS-485 Converter */
+#define BANDB_USO9ML2_PID	0xAC03	/* USO9ML2 Isolated RS-232 Converter */
+#define BANDB_USOPTL4_PID	0xAC11
+#define BANDB_USPTL4_PID	0xAC12
+#define BANDB_USO9ML2DR_2_PID	0xAC16
+#define BANDB_USO9ML2DR_PID	0xAC17
+#define BANDB_USOPTL4DR2_PID	0xAC18	/* USOPTL4R-2 2-port Isolated RS-232 Converter */
+#define BANDB_USOPTL4DR_PID	0xAC19
+#define BANDB_485USB9F_2W_PID	0xAC25
+#define BANDB_485USB9F_4W_PID	0xAC26
+#define BANDB_232USB9M_PID	0xAC27
+#define BANDB_485USBTB_2W_PID	0xAC33
+#define BANDB_485USBTB_4W_PID	0xAC34
+#define BANDB_TTL5USB9M_PID	0xAC49
+#define BANDB_TTL3USB9M_PID	0xAC50
+#define BANDB_ZZ_PROG1_USB_PID	0xBA02
+
+/*
+ * Intrepid Control Systems (http://www.intrepidcs.com/) ValueCAN and NeoVI
+ */
+#define INTREPID_VID		0x093C
+#define INTREPID_VALUECAN_PID	0x0601
+#define INTREPID_NEOVI_PID	0x0701
+
+/*
+ * Definitions for ID TECH (www.idt-net.com) devices
+ */
+#define IDTECH_VID		0x0ACD	/* ID TECH Vendor ID */
+#define IDTECH_IDT1221U_PID	0x0300	/* IDT1221U USB to RS-232 adapter */
+
+/*
+ * Definitions for Omnidirectional Control Technology, Inc. devices
+ */
+#define OCT_VID			0x0B39	/* OCT vendor ID */
+/* Note: OCT US101 is also rebadged as Dick Smith Electronics (NZ) XH6381 */
+/* Also rebadged as Dick Smith Electronics (Aus) XH6451 */
+/* Also rebadged as SIIG Inc. model US2308 hardware version 1 */
+#define OCT_DK201_PID		0x0103	/* OCT DK201 USB docking station */
+#define OCT_US101_PID		0x0421	/* OCT US101 USB to RS-232 */
+
+/*
+ * Definitions for Icom Inc. devices
+ */
+#define ICOM_VID		0x0C26 /* Icom vendor ID */
+/* Note: ID-1 is a communications tranceiver for HAM-radio operators */
+#define ICOM_ID_1_PID		0x0004 /* ID-1 USB to RS-232 */
+/* Note: OPC is an Optional cable to connect an Icom Tranceiver */
+#define ICOM_OPC_U_UC_PID	0x0018 /* OPC-478UC, OPC-1122U cloning cable */
+/* Note: ID-RP* devices are Icom Repeater Devices for HAM-radio */
+#define ICOM_ID_RP2C1_PID	0x0009 /* ID-RP2C Asset 1 to RS-232 */
+#define ICOM_ID_RP2C2_PID	0x000A /* ID-RP2C Asset 2 to RS-232 */
+#define ICOM_ID_RP2D_PID	0x000B /* ID-RP2D configuration port*/
+#define ICOM_ID_RP2VT_PID	0x000C /* ID-RP2V Transmit config port */
+#define ICOM_ID_RP2VR_PID	0x000D /* ID-RP2V Receive config port */
+#define ICOM_ID_RP4KVT_PID	0x0010 /* ID-RP4000V Transmit config port */
+#define ICOM_ID_RP4KVR_PID	0x0011 /* ID-RP4000V Receive config port */
+#define ICOM_ID_RP2KVT_PID	0x0012 /* ID-RP2000V Transmit config port */
+#define ICOM_ID_RP2KVR_PID	0x0013 /* ID-RP2000V Receive config port */
+
+/*
+ * GN Otometrics (http://www.otometrics.com)
+ * Submitted by Ville Sundberg.
+ */
+#define GN_OTOMETRICS_VID	0x0c33	/* Vendor ID */
+#define AURICAL_USB_PID		0x0010	/* Aurical USB Audiometer */
+
+/*
+ * The following are the values for the Sealevel SeaLINK+ adapters.
+ * (Original list sent by Tuan Hoang.  Ian Abbott renamed the macros and
+ * removed some PIDs that don't seem to match any existing products.)
+ */
+#define SEALEVEL_VID		0x0c52	/* Sealevel Vendor ID */
+#define SEALEVEL_2101_PID	0x2101	/* SeaLINK+232 (2101/2105) */
+#define SEALEVEL_2102_PID	0x2102	/* SeaLINK+485 (2102) */
+#define SEALEVEL_2103_PID	0x2103	/* SeaLINK+232I (2103) */
+#define SEALEVEL_2104_PID	0x2104	/* SeaLINK+485I (2104) */
+#define SEALEVEL_2106_PID	0x9020	/* SeaLINK+422 (2106) */
+#define SEALEVEL_2201_1_PID	0x2211	/* SeaPORT+2/232 (2201) Port 1 */
+#define SEALEVEL_2201_2_PID	0x2221	/* SeaPORT+2/232 (2201) Port 2 */
+#define SEALEVEL_2202_1_PID	0x2212	/* SeaPORT+2/485 (2202) Port 1 */
+#define SEALEVEL_2202_2_PID	0x2222	/* SeaPORT+2/485 (2202) Port 2 */
+#define SEALEVEL_2203_1_PID	0x2213	/* SeaPORT+2 (2203) Port 1 */
+#define SEALEVEL_2203_2_PID	0x2223	/* SeaPORT+2 (2203) Port 2 */
+#define SEALEVEL_2401_1_PID	0x2411	/* SeaPORT+4/232 (2401) Port 1 */
+#define SEALEVEL_2401_2_PID	0x2421	/* SeaPORT+4/232 (2401) Port 2 */
+#define SEALEVEL_2401_3_PID	0x2431	/* SeaPORT+4/232 (2401) Port 3 */
+#define SEALEVEL_2401_4_PID	0x2441	/* SeaPORT+4/232 (2401) Port 4 */
+#define SEALEVEL_2402_1_PID	0x2412	/* SeaPORT+4/485 (2402) Port 1 */
+#define SEALEVEL_2402_2_PID	0x2422	/* SeaPORT+4/485 (2402) Port 2 */
+#define SEALEVEL_2402_3_PID	0x2432	/* SeaPORT+4/485 (2402) Port 3 */
+#define SEALEVEL_2402_4_PID	0x2442	/* SeaPORT+4/485 (2402) Port 4 */
+#define SEALEVEL_2403_1_PID	0x2413	/* SeaPORT+4 (2403) Port 1 */
+#define SEALEVEL_2403_2_PID	0x2423	/* SeaPORT+4 (2403) Port 2 */
+#define SEALEVEL_2403_3_PID	0x2433	/* SeaPORT+4 (2403) Port 3 */
+#define SEALEVEL_2403_4_PID	0x2443	/* SeaPORT+4 (2403) Port 4 */
+#define SEALEVEL_2801_1_PID	0X2811	/* SeaLINK+8/232 (2801) Port 1 */
+#define SEALEVEL_2801_2_PID	0X2821	/* SeaLINK+8/232 (2801) Port 2 */
+#define SEALEVEL_2801_3_PID	0X2831	/* SeaLINK+8/232 (2801) Port 3 */
+#define SEALEVEL_2801_4_PID	0X2841	/* SeaLINK+8/232 (2801) Port 4 */
+#define SEALEVEL_2801_5_PID	0X2851	/* SeaLINK+8/232 (2801) Port 5 */
+#define SEALEVEL_2801_6_PID	0X2861	/* SeaLINK+8/232 (2801) Port 6 */
+#define SEALEVEL_2801_7_PID	0X2871	/* SeaLINK+8/232 (2801) Port 7 */
+#define SEALEVEL_2801_8_PID	0X2881	/* SeaLINK+8/232 (2801) Port 8 */
+#define SEALEVEL_2802_1_PID	0X2812	/* SeaLINK+8/485 (2802) Port 1 */
+#define SEALEVEL_2802_2_PID	0X2822	/* SeaLINK+8/485 (2802) Port 2 */
+#define SEALEVEL_2802_3_PID	0X2832	/* SeaLINK+8/485 (2802) Port 3 */
+#define SEALEVEL_2802_4_PID	0X2842	/* SeaLINK+8/485 (2802) Port 4 */
+#define SEALEVEL_2802_5_PID	0X2852	/* SeaLINK+8/485 (2802) Port 5 */
+#define SEALEVEL_2802_6_PID	0X2862	/* SeaLINK+8/485 (2802) Port 6 */
+#define SEALEVEL_2802_7_PID	0X2872	/* SeaLINK+8/485 (2802) Port 7 */
+#define SEALEVEL_2802_8_PID	0X2882	/* SeaLINK+8/485 (2802) Port 8 */
+#define SEALEVEL_2803_1_PID	0X2813	/* SeaLINK+8 (2803) Port 1 */
+#define SEALEVEL_2803_2_PID	0X2823	/* SeaLINK+8 (2803) Port 2 */
+#define SEALEVEL_2803_3_PID	0X2833	/* SeaLINK+8 (2803) Port 3 */
+#define SEALEVEL_2803_4_PID	0X2843	/* SeaLINK+8 (2803) Port 4 */
+#define SEALEVEL_2803_5_PID	0X2853	/* SeaLINK+8 (2803) Port 5 */
+#define SEALEVEL_2803_6_PID	0X2863	/* SeaLINK+8 (2803) Port 6 */
+#define SEALEVEL_2803_7_PID	0X2873	/* SeaLINK+8 (2803) Port 7 */
+#define SEALEVEL_2803_8_PID	0X2883	/* SeaLINK+8 (2803) Port 8 */
+#define SEALEVEL_2803R_1_PID	0Xa02a	/* SeaLINK+8 (2803-ROHS) Port 1+2 */
+#define SEALEVEL_2803R_2_PID	0Xa02b	/* SeaLINK+8 (2803-ROHS) Port 3+4 */
+#define SEALEVEL_2803R_3_PID	0Xa02c	/* SeaLINK+8 (2803-ROHS) Port 5+6 */
+#define SEALEVEL_2803R_4_PID	0Xa02d	/* SeaLINK+8 (2803-ROHS) Port 7+8 */
+
+/*
+ * JETI SPECTROMETER SPECBOS 1201
+ * http://www.jeti.com/cms/index.php/instruments/other-instruments/specbos-2101
+ */
+#define JETI_VID		0x0c6c
+#define JETI_SPC1201_PID	0x04b2
+
+/*
+ * FTDI USB UART chips used in construction projects from the
+ * Elektor Electronics magazine (http://www.elektor.com/)
+ */
+#define ELEKTOR_VID		0x0C7D
+#define ELEKTOR_FT323R_PID	0x0005	/* RFID-Reader, issue 09-2006 */
+
+/*
+ * Posiflex inc retail equipment (http://www.posiflex.com.tw)
+ */
+#define POSIFLEX_VID		0x0d3a  /* Vendor ID */
+#define POSIFLEX_PP7000_PID	0x0300  /* PP-7000II thermal printer */
+
+/*
+ * The following are the values for two KOBIL chipcard terminals.
+ */
+#define KOBIL_VID		0x0d46	/* KOBIL Vendor ID */
+#define KOBIL_CONV_B1_PID	0x2020	/* KOBIL Konverter for B1 */
+#define KOBIL_CONV_KAAN_PID	0x2021	/* KOBIL_Konverter for KAAN */
+
+#define FTDI_NF_RIC_VID	0x0DCD	/* Vendor Id */
+#define FTDI_NF_RIC_PID	0x0001	/* Product Id */
+
+/*
+ * Falcom Wireless Communications GmbH
+ */
+#define FALCOM_VID		0x0F94	/* Vendor Id */
+#define FALCOM_TWIST_PID	0x0001	/* Falcom Twist USB GPRS modem */
+#define FALCOM_SAMBA_PID	0x0005	/* Falcom Samba USB GPRS modem */
+
+/* Larsen and Brusgaard AltiTrack/USBtrack */
+#define LARSENBRUSGAARD_VID		0x0FD8
+#define LB_ALTITRACK_PID		0x0001
+
+/*
+ * TTi (Thurlby Thandar Instruments)
+ */
+#define TTI_VID			0x103E	/* Vendor Id */
+#define TTI_QL355P_PID		0x03E8	/* TTi QL355P power supply */
+
+/* Interbiometrics USB I/O Board */
+/* Developed for Interbiometrics by Rudolf Gugler */
+#define INTERBIOMETRICS_VID              0x1209
+#define INTERBIOMETRICS_IOBOARD_PID      0x1002
+#define INTERBIOMETRICS_MINI_IOBOARD_PID 0x1006
+
+/*
+ * Testo products (http://www.testo.com/)
+ * Submitted by Colin Leroy
+ */
+#define TESTO_VID			0x128D
+#define TESTO_USB_INTERFACE_PID		0x0001
+
+/*
+ * Mobility Electronics products.
+ */
+#define MOBILITY_VID			0x1342
+#define MOBILITY_USB_SERIAL_PID		0x0202	/* EasiDock USB 200 serial */
+
+/*
+ * FIC / OpenMoko, Inc. http://wiki.openmoko.org/wiki/Neo1973_Debug_Board_v3
+ * Submitted by Harald Welte <laforge@openmoko.org>
+ */
+#define	FIC_VID			0x1457
+#define	FIC_NEO1973_DEBUG_PID	0x5118
+
+/* Olimex */
+#define OLIMEX_VID			0x15BA
+#define OLIMEX_ARM_USB_OCD_PID		0x0003
+#define OLIMEX_ARM_USB_OCD_H_PID	0x002b
+
+/*
+ * Telldus Technologies
+ */
+#define TELLDUS_VID			0x1781	/* Vendor ID */
+#define TELLDUS_TELLSTICK_PID		0x0C30	/* RF control dongle 433 MHz using FT232RL */
+
+/*
+ * RT Systems programming cables for various ham radios
+ */
+#define RTSYSTEMS_VID			0x2100	/* Vendor ID */
+#define RTSYSTEMS_SERIAL_VX7_PID	0x9e52	/* Serial converter for VX-7 Radios using FT232RL */
+#define RTSYSTEMS_CT29B_PID		0x9e54	/* CT29B Radio Cable */
+#define RTSYSTEMS_RTS01_PID		0x9e57	/* USB-RTS01 Radio Cable */
+
+
+/*
+ * Physik Instrumente
+ * http://www.physikinstrumente.com/en/products/
+ */
+/* These two devices use the VID of FTDI */
+#define PI_C865_PID	0xe0a0  /* PI C-865 Piezomotor Controller */
+#define PI_C857_PID	0xe0a1  /* PI Encoder Trigger Box */
+
+#define PI_VID              0x1a72  /* Vendor ID */
+#define PI_C866_PID	0x1000  /* PI C-866 Piezomotor Controller */
+#define PI_C663_PID	0x1001  /* PI C-663 Mercury-Step */
+#define PI_C725_PID	0x1002  /* PI C-725 Piezomotor Controller */
+#define PI_E517_PID	0x1005  /* PI E-517 Digital Piezo Controller Operation Module */
+#define PI_C863_PID	0x1007  /* PI C-863 */
+#define PI_E861_PID	0x1008  /* PI E-861 Piezomotor Controller */
+#define PI_C867_PID	0x1009  /* PI C-867 Piezomotor Controller */
+#define PI_E609_PID	0x100D  /* PI E-609 Digital Piezo Controller */
+#define PI_E709_PID	0x100E  /* PI E-709 Digital Piezo Controller */
+#define PI_100F_PID	0x100F  /* PI Digital Piezo Controller */
+#define PI_1011_PID	0x1011  /* PI Digital Piezo Controller */
+#define PI_1012_PID	0x1012  /* PI Motion Controller */
+#define PI_1013_PID	0x1013  /* PI Motion Controller */
+#define PI_1014_PID	0x1014  /* PI Device */
+#define PI_1015_PID	0x1015  /* PI Device */
+#define PI_1016_PID	0x1016  /* PI Digital Servo Module */
+
+/*
+ * Kondo Kagaku Co.Ltd.
+ * http://www.kondo-robot.com/EN
+ */
+#define KONDO_VID 		0x165c
+#define KONDO_USB_SERIAL_PID	0x0002
+
+/*
+ * Bayer Ascensia Contour blood glucose meter USB-converter cable.
+ * http://winglucofacts.com/cables/
+ */
+#define BAYER_VID                      0x1A79
+#define BAYER_CONTOUR_CABLE_PID        0x6001
+
+/*
+ * The following are the values for the Matrix Orbital FTDI Range
+ * Anything in this range will use an FT232RL.
+ */
+#define MTXORB_VID			0x1B3D
+#define MTXORB_FTDI_RANGE_0100_PID	0x0100
+#define MTXORB_FTDI_RANGE_0101_PID	0x0101
+#define MTXORB_FTDI_RANGE_0102_PID	0x0102
+#define MTXORB_FTDI_RANGE_0103_PID	0x0103
+#define MTXORB_FTDI_RANGE_0104_PID	0x0104
+#define MTXORB_FTDI_RANGE_0105_PID	0x0105
+#define MTXORB_FTDI_RANGE_0106_PID	0x0106
+#define MTXORB_FTDI_RANGE_0107_PID	0x0107
+#define MTXORB_FTDI_RANGE_0108_PID	0x0108
+#define MTXORB_FTDI_RANGE_0109_PID	0x0109
+#define MTXORB_FTDI_RANGE_010A_PID	0x010A
+#define MTXORB_FTDI_RANGE_010B_PID	0x010B
+#define MTXORB_FTDI_RANGE_010C_PID	0x010C
+#define MTXORB_FTDI_RANGE_010D_PID	0x010D
+#define MTXORB_FTDI_RANGE_010E_PID	0x010E
+#define MTXORB_FTDI_RANGE_010F_PID	0x010F
+#define MTXORB_FTDI_RANGE_0110_PID	0x0110
+#define MTXORB_FTDI_RANGE_0111_PID	0x0111
+#define MTXORB_FTDI_RANGE_0112_PID	0x0112
+#define MTXORB_FTDI_RANGE_0113_PID	0x0113
+#define MTXORB_FTDI_RANGE_0114_PID	0x0114
+#define MTXORB_FTDI_RANGE_0115_PID	0x0115
+#define MTXORB_FTDI_RANGE_0116_PID	0x0116
+#define MTXORB_FTDI_RANGE_0117_PID	0x0117
+#define MTXORB_FTDI_RANGE_0118_PID	0x0118
+#define MTXORB_FTDI_RANGE_0119_PID	0x0119
+#define MTXORB_FTDI_RANGE_011A_PID	0x011A
+#define MTXORB_FTDI_RANGE_011B_PID	0x011B
+#define MTXORB_FTDI_RANGE_011C_PID	0x011C
+#define MTXORB_FTDI_RANGE_011D_PID	0x011D
+#define MTXORB_FTDI_RANGE_011E_PID	0x011E
+#define MTXORB_FTDI_RANGE_011F_PID	0x011F
+#define MTXORB_FTDI_RANGE_0120_PID	0x0120
+#define MTXORB_FTDI_RANGE_0121_PID	0x0121
+#define MTXORB_FTDI_RANGE_0122_PID	0x0122
+#define MTXORB_FTDI_RANGE_0123_PID	0x0123
+#define MTXORB_FTDI_RANGE_0124_PID	0x0124
+#define MTXORB_FTDI_RANGE_0125_PID	0x0125
+#define MTXORB_FTDI_RANGE_0126_PID	0x0126
+#define MTXORB_FTDI_RANGE_0127_PID	0x0127
+#define MTXORB_FTDI_RANGE_0128_PID	0x0128
+#define MTXORB_FTDI_RANGE_0129_PID	0x0129
+#define MTXORB_FTDI_RANGE_012A_PID	0x012A
+#define MTXORB_FTDI_RANGE_012B_PID	0x012B
+#define MTXORB_FTDI_RANGE_012C_PID	0x012C
+#define MTXORB_FTDI_RANGE_012D_PID	0x012D
+#define MTXORB_FTDI_RANGE_012E_PID	0x012E
+#define MTXORB_FTDI_RANGE_012F_PID	0x012F
+#define MTXORB_FTDI_RANGE_0130_PID	0x0130
+#define MTXORB_FTDI_RANGE_0131_PID	0x0131
+#define MTXORB_FTDI_RANGE_0132_PID	0x0132
+#define MTXORB_FTDI_RANGE_0133_PID	0x0133
+#define MTXORB_FTDI_RANGE_0134_PID	0x0134
+#define MTXORB_FTDI_RANGE_0135_PID	0x0135
+#define MTXORB_FTDI_RANGE_0136_PID	0x0136
+#define MTXORB_FTDI_RANGE_0137_PID	0x0137
+#define MTXORB_FTDI_RANGE_0138_PID	0x0138
+#define MTXORB_FTDI_RANGE_0139_PID	0x0139
+#define MTXORB_FTDI_RANGE_013A_PID	0x013A
+#define MTXORB_FTDI_RANGE_013B_PID	0x013B
+#define MTXORB_FTDI_RANGE_013C_PID	0x013C
+#define MTXORB_FTDI_RANGE_013D_PID	0x013D
+#define MTXORB_FTDI_RANGE_013E_PID	0x013E
+#define MTXORB_FTDI_RANGE_013F_PID	0x013F
+#define MTXORB_FTDI_RANGE_0140_PID	0x0140
+#define MTXORB_FTDI_RANGE_0141_PID	0x0141
+#define MTXORB_FTDI_RANGE_0142_PID	0x0142
+#define MTXORB_FTDI_RANGE_0143_PID	0x0143
+#define MTXORB_FTDI_RANGE_0144_PID	0x0144
+#define MTXORB_FTDI_RANGE_0145_PID	0x0145
+#define MTXORB_FTDI_RANGE_0146_PID	0x0146
+#define MTXORB_FTDI_RANGE_0147_PID	0x0147
+#define MTXORB_FTDI_RANGE_0148_PID	0x0148
+#define MTXORB_FTDI_RANGE_0149_PID	0x0149
+#define MTXORB_FTDI_RANGE_014A_PID	0x014A
+#define MTXORB_FTDI_RANGE_014B_PID	0x014B
+#define MTXORB_FTDI_RANGE_014C_PID	0x014C
+#define MTXORB_FTDI_RANGE_014D_PID	0x014D
+#define MTXORB_FTDI_RANGE_014E_PID	0x014E
+#define MTXORB_FTDI_RANGE_014F_PID	0x014F
+#define MTXORB_FTDI_RANGE_0150_PID	0x0150
+#define MTXORB_FTDI_RANGE_0151_PID	0x0151
+#define MTXORB_FTDI_RANGE_0152_PID	0x0152
+#define MTXORB_FTDI_RANGE_0153_PID	0x0153
+#define MTXORB_FTDI_RANGE_0154_PID	0x0154
+#define MTXORB_FTDI_RANGE_0155_PID	0x0155
+#define MTXORB_FTDI_RANGE_0156_PID	0x0156
+#define MTXORB_FTDI_RANGE_0157_PID	0x0157
+#define MTXORB_FTDI_RANGE_0158_PID	0x0158
+#define MTXORB_FTDI_RANGE_0159_PID	0x0159
+#define MTXORB_FTDI_RANGE_015A_PID	0x015A
+#define MTXORB_FTDI_RANGE_015B_PID	0x015B
+#define MTXORB_FTDI_RANGE_015C_PID	0x015C
+#define MTXORB_FTDI_RANGE_015D_PID	0x015D
+#define MTXORB_FTDI_RANGE_015E_PID	0x015E
+#define MTXORB_FTDI_RANGE_015F_PID	0x015F
+#define MTXORB_FTDI_RANGE_0160_PID	0x0160
+#define MTXORB_FTDI_RANGE_0161_PID	0x0161
+#define MTXORB_FTDI_RANGE_0162_PID	0x0162
+#define MTXORB_FTDI_RANGE_0163_PID	0x0163
+#define MTXORB_FTDI_RANGE_0164_PID	0x0164
+#define MTXORB_FTDI_RANGE_0165_PID	0x0165
+#define MTXORB_FTDI_RANGE_0166_PID	0x0166
+#define MTXORB_FTDI_RANGE_0167_PID	0x0167
+#define MTXORB_FTDI_RANGE_0168_PID	0x0168
+#define MTXORB_FTDI_RANGE_0169_PID	0x0169
+#define MTXORB_FTDI_RANGE_016A_PID	0x016A
+#define MTXORB_FTDI_RANGE_016B_PID	0x016B
+#define MTXORB_FTDI_RANGE_016C_PID	0x016C
+#define MTXORB_FTDI_RANGE_016D_PID	0x016D
+#define MTXORB_FTDI_RANGE_016E_PID	0x016E
+#define MTXORB_FTDI_RANGE_016F_PID	0x016F
+#define MTXORB_FTDI_RANGE_0170_PID	0x0170
+#define MTXORB_FTDI_RANGE_0171_PID	0x0171
+#define MTXORB_FTDI_RANGE_0172_PID	0x0172
+#define MTXORB_FTDI_RANGE_0173_PID	0x0173
+#define MTXORB_FTDI_RANGE_0174_PID	0x0174
+#define MTXORB_FTDI_RANGE_0175_PID	0x0175
+#define MTXORB_FTDI_RANGE_0176_PID	0x0176
+#define MTXORB_FTDI_RANGE_0177_PID	0x0177
+#define MTXORB_FTDI_RANGE_0178_PID	0x0178
+#define MTXORB_FTDI_RANGE_0179_PID	0x0179
+#define MTXORB_FTDI_RANGE_017A_PID	0x017A
+#define MTXORB_FTDI_RANGE_017B_PID	0x017B
+#define MTXORB_FTDI_RANGE_017C_PID	0x017C
+#define MTXORB_FTDI_RANGE_017D_PID	0x017D
+#define MTXORB_FTDI_RANGE_017E_PID	0x017E
+#define MTXORB_FTDI_RANGE_017F_PID	0x017F
+#define MTXORB_FTDI_RANGE_0180_PID	0x0180
+#define MTXORB_FTDI_RANGE_0181_PID	0x0181
+#define MTXORB_FTDI_RANGE_0182_PID	0x0182
+#define MTXORB_FTDI_RANGE_0183_PID	0x0183
+#define MTXORB_FTDI_RANGE_0184_PID	0x0184
+#define MTXORB_FTDI_RANGE_0185_PID	0x0185
+#define MTXORB_FTDI_RANGE_0186_PID	0x0186
+#define MTXORB_FTDI_RANGE_0187_PID	0x0187
+#define MTXORB_FTDI_RANGE_0188_PID	0x0188
+#define MTXORB_FTDI_RANGE_0189_PID	0x0189
+#define MTXORB_FTDI_RANGE_018A_PID	0x018A
+#define MTXORB_FTDI_RANGE_018B_PID	0x018B
+#define MTXORB_FTDI_RANGE_018C_PID	0x018C
+#define MTXORB_FTDI_RANGE_018D_PID	0x018D
+#define MTXORB_FTDI_RANGE_018E_PID	0x018E
+#define MTXORB_FTDI_RANGE_018F_PID	0x018F
+#define MTXORB_FTDI_RANGE_0190_PID	0x0190
+#define MTXORB_FTDI_RANGE_0191_PID	0x0191
+#define MTXORB_FTDI_RANGE_0192_PID	0x0192
+#define MTXORB_FTDI_RANGE_0193_PID	0x0193
+#define MTXORB_FTDI_RANGE_0194_PID	0x0194
+#define MTXORB_FTDI_RANGE_0195_PID	0x0195
+#define MTXORB_FTDI_RANGE_0196_PID	0x0196
+#define MTXORB_FTDI_RANGE_0197_PID	0x0197
+#define MTXORB_FTDI_RANGE_0198_PID	0x0198
+#define MTXORB_FTDI_RANGE_0199_PID	0x0199
+#define MTXORB_FTDI_RANGE_019A_PID	0x019A
+#define MTXORB_FTDI_RANGE_019B_PID	0x019B
+#define MTXORB_FTDI_RANGE_019C_PID	0x019C
+#define MTXORB_FTDI_RANGE_019D_PID	0x019D
+#define MTXORB_FTDI_RANGE_019E_PID	0x019E
+#define MTXORB_FTDI_RANGE_019F_PID	0x019F
+#define MTXORB_FTDI_RANGE_01A0_PID	0x01A0
+#define MTXORB_FTDI_RANGE_01A1_PID	0x01A1
+#define MTXORB_FTDI_RANGE_01A2_PID	0x01A2
+#define MTXORB_FTDI_RANGE_01A3_PID	0x01A3
+#define MTXORB_FTDI_RANGE_01A4_PID	0x01A4
+#define MTXORB_FTDI_RANGE_01A5_PID	0x01A5
+#define MTXORB_FTDI_RANGE_01A6_PID	0x01A6
+#define MTXORB_FTDI_RANGE_01A7_PID	0x01A7
+#define MTXORB_FTDI_RANGE_01A8_PID	0x01A8
+#define MTXORB_FTDI_RANGE_01A9_PID	0x01A9
+#define MTXORB_FTDI_RANGE_01AA_PID	0x01AA
+#define MTXORB_FTDI_RANGE_01AB_PID	0x01AB
+#define MTXORB_FTDI_RANGE_01AC_PID	0x01AC
+#define MTXORB_FTDI_RANGE_01AD_PID	0x01AD
+#define MTXORB_FTDI_RANGE_01AE_PID	0x01AE
+#define MTXORB_FTDI_RANGE_01AF_PID	0x01AF
+#define MTXORB_FTDI_RANGE_01B0_PID	0x01B0
+#define MTXORB_FTDI_RANGE_01B1_PID	0x01B1
+#define MTXORB_FTDI_RANGE_01B2_PID	0x01B2
+#define MTXORB_FTDI_RANGE_01B3_PID	0x01B3
+#define MTXORB_FTDI_RANGE_01B4_PID	0x01B4
+#define MTXORB_FTDI_RANGE_01B5_PID	0x01B5
+#define MTXORB_FTDI_RANGE_01B6_PID	0x01B6
+#define MTXORB_FTDI_RANGE_01B7_PID	0x01B7
+#define MTXORB_FTDI_RANGE_01B8_PID	0x01B8
+#define MTXORB_FTDI_RANGE_01B9_PID	0x01B9
+#define MTXORB_FTDI_RANGE_01BA_PID	0x01BA
+#define MTXORB_FTDI_RANGE_01BB_PID	0x01BB
+#define MTXORB_FTDI_RANGE_01BC_PID	0x01BC
+#define MTXORB_FTDI_RANGE_01BD_PID	0x01BD
+#define MTXORB_FTDI_RANGE_01BE_PID	0x01BE
+#define MTXORB_FTDI_RANGE_01BF_PID	0x01BF
+#define MTXORB_FTDI_RANGE_01C0_PID	0x01C0
+#define MTXORB_FTDI_RANGE_01C1_PID	0x01C1
+#define MTXORB_FTDI_RANGE_01C2_PID	0x01C2
+#define MTXORB_FTDI_RANGE_01C3_PID	0x01C3
+#define MTXORB_FTDI_RANGE_01C4_PID	0x01C4
+#define MTXORB_FTDI_RANGE_01C5_PID	0x01C5
+#define MTXORB_FTDI_RANGE_01C6_PID	0x01C6
+#define MTXORB_FTDI_RANGE_01C7_PID	0x01C7
+#define MTXORB_FTDI_RANGE_01C8_PID	0x01C8
+#define MTXORB_FTDI_RANGE_01C9_PID	0x01C9
+#define MTXORB_FTDI_RANGE_01CA_PID	0x01CA
+#define MTXORB_FTDI_RANGE_01CB_PID	0x01CB
+#define MTXORB_FTDI_RANGE_01CC_PID	0x01CC
+#define MTXORB_FTDI_RANGE_01CD_PID	0x01CD
+#define MTXORB_FTDI_RANGE_01CE_PID	0x01CE
+#define MTXORB_FTDI_RANGE_01CF_PID	0x01CF
+#define MTXORB_FTDI_RANGE_01D0_PID	0x01D0
+#define MTXORB_FTDI_RANGE_01D1_PID	0x01D1
+#define MTXORB_FTDI_RANGE_01D2_PID	0x01D2
+#define MTXORB_FTDI_RANGE_01D3_PID	0x01D3
+#define MTXORB_FTDI_RANGE_01D4_PID	0x01D4
+#define MTXORB_FTDI_RANGE_01D5_PID	0x01D5
+#define MTXORB_FTDI_RANGE_01D6_PID	0x01D6
+#define MTXORB_FTDI_RANGE_01D7_PID	0x01D7
+#define MTXORB_FTDI_RANGE_01D8_PID	0x01D8
+#define MTXORB_FTDI_RANGE_01D9_PID	0x01D9
+#define MTXORB_FTDI_RANGE_01DA_PID	0x01DA
+#define MTXORB_FTDI_RANGE_01DB_PID	0x01DB
+#define MTXORB_FTDI_RANGE_01DC_PID	0x01DC
+#define MTXORB_FTDI_RANGE_01DD_PID	0x01DD
+#define MTXORB_FTDI_RANGE_01DE_PID	0x01DE
+#define MTXORB_FTDI_RANGE_01DF_PID	0x01DF
+#define MTXORB_FTDI_RANGE_01E0_PID	0x01E0
+#define MTXORB_FTDI_RANGE_01E1_PID	0x01E1
+#define MTXORB_FTDI_RANGE_01E2_PID	0x01E2
+#define MTXORB_FTDI_RANGE_01E3_PID	0x01E3
+#define MTXORB_FTDI_RANGE_01E4_PID	0x01E4
+#define MTXORB_FTDI_RANGE_01E5_PID	0x01E5
+#define MTXORB_FTDI_RANGE_01E6_PID	0x01E6
+#define MTXORB_FTDI_RANGE_01E7_PID	0x01E7
+#define MTXORB_FTDI_RANGE_01E8_PID	0x01E8
+#define MTXORB_FTDI_RANGE_01E9_PID	0x01E9
+#define MTXORB_FTDI_RANGE_01EA_PID	0x01EA
+#define MTXORB_FTDI_RANGE_01EB_PID	0x01EB
+#define MTXORB_FTDI_RANGE_01EC_PID	0x01EC
+#define MTXORB_FTDI_RANGE_01ED_PID	0x01ED
+#define MTXORB_FTDI_RANGE_01EE_PID	0x01EE
+#define MTXORB_FTDI_RANGE_01EF_PID	0x01EF
+#define MTXORB_FTDI_RANGE_01F0_PID	0x01F0
+#define MTXORB_FTDI_RANGE_01F1_PID	0x01F1
+#define MTXORB_FTDI_RANGE_01F2_PID	0x01F2
+#define MTXORB_FTDI_RANGE_01F3_PID	0x01F3
+#define MTXORB_FTDI_RANGE_01F4_PID	0x01F4
+#define MTXORB_FTDI_RANGE_01F5_PID	0x01F5
+#define MTXORB_FTDI_RANGE_01F6_PID	0x01F6
+#define MTXORB_FTDI_RANGE_01F7_PID	0x01F7
+#define MTXORB_FTDI_RANGE_01F8_PID	0x01F8
+#define MTXORB_FTDI_RANGE_01F9_PID	0x01F9
+#define MTXORB_FTDI_RANGE_01FA_PID	0x01FA
+#define MTXORB_FTDI_RANGE_01FB_PID	0x01FB
+#define MTXORB_FTDI_RANGE_01FC_PID	0x01FC
+#define MTXORB_FTDI_RANGE_01FD_PID	0x01FD
+#define MTXORB_FTDI_RANGE_01FE_PID	0x01FE
+#define MTXORB_FTDI_RANGE_01FF_PID	0x01FF
+
+
+
+/*
+ * The Mobility Lab (TML)
+ * Submitted by Pierre Castella
+ */
+#define TML_VID			0x1B91	/* Vendor ID */
+#define TML_USB_SERIAL_PID	0x0064	/* USB - Serial Converter */
+
+/* Alti-2 products  http://www.alti-2.com */
+#define ALTI2_VID	0x1BC9
+#define ALTI2_N3_PID	0x6001	/* Neptune 3 */
+
+/*
+ * Ionics PlugComputer
+ */
+#define IONICS_VID			0x1c0c
+#define IONICS_PLUGCOMPUTER_PID		0x0102
+
+/*
+ * Dresden Elektronik Sensor Terminal Board
+ */
+#define DE_VID			0x1cf1 /* Vendor ID */
+#define STB_PID			0x0001 /* Sensor Terminal Board */
+#define WHT_PID			0x0004 /* Wireless Handheld Terminal */
+
+/*
+ * STMicroelectonics
+ */
+#define ST_VID			0x0483
+#define ST_STMCLT1030_PID	0x3747 /* ST Micro Connect Lite STMCLT1030 */
+
+/*
+ * Papouch products (http://www.papouch.com/)
+ * Submitted by Folkert van Heusden
+ */
+
+#define PAPOUCH_VID			0x5050	/* Vendor ID */
+#define PAPOUCH_SB485_PID		0x0100	/* Papouch SB485 USB-485/422 Converter */
+#define PAPOUCH_AP485_PID		0x0101	/* AP485 USB-RS485 Converter */
+#define PAPOUCH_SB422_PID		0x0102	/* Papouch SB422 USB-RS422 Converter  */
+#define PAPOUCH_SB485_2_PID		0x0103	/* Papouch SB485 USB-485/422 Converter */
+#define PAPOUCH_AP485_2_PID		0x0104	/* AP485 USB-RS485 Converter */
+#define PAPOUCH_SB422_2_PID		0x0105	/* Papouch SB422 USB-RS422 Converter  */
+#define PAPOUCH_SB485S_PID		0x0106	/* Papouch SB485S USB-485/422 Converter */
+#define PAPOUCH_SB485C_PID		0x0107	/* Papouch SB485C USB-485/422 Converter */
+#define PAPOUCH_LEC_PID			0x0300	/* LEC USB Converter */
+#define PAPOUCH_SB232_PID		0x0301	/* Papouch SB232 USB-RS232 Converter */
+#define PAPOUCH_TMU_PID			0x0400	/* TMU USB Thermometer */
+#define PAPOUCH_IRAMP_PID		0x0500	/* Papouch IRAmp Duplex */
+#define PAPOUCH_DRAK5_PID		0x0700	/* Papouch DRAK5 */
+#define PAPOUCH_QUIDO8x8_PID		0x0800	/* Papouch Quido 8/8 Module */
+#define PAPOUCH_QUIDO4x4_PID		0x0900	/* Papouch Quido 4/4 Module */
+#define PAPOUCH_QUIDO2x2_PID		0x0a00	/* Papouch Quido 2/2 Module */
+#define PAPOUCH_QUIDO10x1_PID		0x0b00	/* Papouch Quido 10/1 Module */
+#define PAPOUCH_QUIDO30x3_PID		0x0c00	/* Papouch Quido 30/3 Module */
+#define PAPOUCH_QUIDO60x3_PID		0x0d00	/* Papouch Quido 60(100)/3 Module */
+#define PAPOUCH_QUIDO2x16_PID		0x0e00	/* Papouch Quido 2/16 Module */
+#define PAPOUCH_QUIDO3x32_PID		0x0f00	/* Papouch Quido 3/32 Module */
+#define PAPOUCH_DRAK6_PID		0x1000	/* Papouch DRAK6 */
+#define PAPOUCH_UPSUSB_PID		0x8000	/* Papouch UPS-USB adapter */
+#define PAPOUCH_MU_PID			0x8001	/* MU controller */
+#define PAPOUCH_SIMUKEY_PID		0x8002	/* Papouch SimuKey */
+#define PAPOUCH_AD4USB_PID		0x8003	/* AD4USB Measurement Module */
+#define PAPOUCH_GMUX_PID		0x8004	/* Papouch GOLIATH MUX */
+#define PAPOUCH_GMSR_PID		0x8005	/* Papouch GOLIATH MSR */
+
+/*
+ * Marvell SheevaPlug
+ */
+#define MARVELL_VID		0x9e88
+#define MARVELL_SHEEVAPLUG_PID	0x9e8f
+
+/*
+ * Evolution Robotics products (http://www.evolution.com/).
+ * Submitted by Shawn M. Lavelle.
+ */
+#define EVOLUTION_VID		0xDEEE	/* Vendor ID */
+#define EVOLUTION_ER1_PID	0x0300	/* ER1 Control Module */
+#define EVO_8U232AM_PID		0x02FF	/* Evolution robotics RCM2 (FT232AM)*/
+#define EVO_HYBRID_PID		0x0302	/* Evolution robotics RCM4 PID (FT232BM)*/
+#define EVO_RCM4_PID		0x0303	/* Evolution robotics RCM4 PID */
+
+/*
+ * MJS Gadgets HD Radio / XM Radio / Sirius Radio interfaces (using VID 0x0403)
+ */
+#define MJSG_GENERIC_PID	0x9378
+#define MJSG_SR_RADIO_PID	0x9379
+#define MJSG_XM_RADIO_PID	0x937A
+#define MJSG_HD_RADIO_PID	0x937C
+
+/*
+ * D.O.Tec products (http://www.directout.eu)
+ */
+#define FTDI_DOTEC_PID 0x9868
+
+/*
+ * Xverve Signalyzer tools (http://www.signalyzer.com/)
+ */
+#define XVERVE_SIGNALYZER_ST_PID	0xBCA0
+#define XVERVE_SIGNALYZER_SLITE_PID	0xBCA1
+#define XVERVE_SIGNALYZER_SH2_PID	0xBCA2
+#define XVERVE_SIGNALYZER_SH4_PID	0xBCA4
+
+/*
+ * Segway Robotic Mobility Platform USB interface (using VID 0x0403)
+ * Submitted by John G. Rogers
+ */
+#define SEGWAY_RMP200_PID	0xe729
+
+
+/*
+ * Accesio USB Data Acquisition products (http://www.accesio.com/)
+ */
+#define ACCESIO_COM4SM_PID 	0xD578
+
+/* www.sciencescope.co.uk educational dataloggers */
+#define FTDI_SCIENCESCOPE_LOGBOOKML_PID		0xFF18
+#define FTDI_SCIENCESCOPE_LS_LOGBOOK_PID	0xFF1C
+#define FTDI_SCIENCESCOPE_HS_LOGBOOK_PID	0xFF1D
+
+/*
+ * Milkymist One JTAG/Serial
+ */
+#define QIHARDWARE_VID			0x20B7
+#define MILKYMISTONE_JTAGSERIAL_PID	0x0713
+
+/*
+ * CTI GmbH RS485 Converter http://www.cti-lean.com/
+ */
+/* USB-485-Mini*/
+#define FTDI_CTI_MINI_PID	0xF608
+/* USB-Nano-485*/
+#define FTDI_CTI_NANO_PID	0xF60B
+
+/*
+ * ZeitControl cardsystems GmbH rfid-readers http://zeitconrol.de
+ */
+/* TagTracer MIFARE*/
+#define FTDI_ZEITCONTROL_TAGTRACE_MIFARE_PID	0xF7C0
+
+/*
+ * Rainforest Automation
+ */
+/* ZigBee controller */
+#define FTDI_RF_R106		0x8A28
+
+/*
+ * Product: HCP HIT GPRS modem
+ * Manufacturer: HCP d.o.o.
+ * ATI command output: Cinterion MC55i
+ */
+#define FTDI_CINTERION_MC55I_PID	0xA951
diff --git a/hw/usb/quirks-pl2303-ids.h b/hw/usb/quirks-pl2303-ids.h
new file mode 100644
index 00000000000..8dbdb46ffe0
--- /dev/null
+++ b/hw/usb/quirks-pl2303-ids.h
@@ -0,0 +1,150 @@
+/*
+ * Prolific PL2303 USB to serial adaptor driver header file
+ *
+ *	This program is free software; you can redistribute it and/or modify
+ *	it under the terms of the GNU General Public License as published by
+ *	the Free Software Foundation; either version 2 of the License, or
+ *	(at your option) any later version.
+ *
+ */
+
+#define BENQ_VENDOR_ID			0x04a5
+#define BENQ_PRODUCT_ID_S81		0x4027
+
+#define PL2303_VENDOR_ID	0x067b
+#define PL2303_PRODUCT_ID	0x2303
+#define PL2303_PRODUCT_ID_RSAQ2		0x04bb
+#define PL2303_PRODUCT_ID_DCU11		0x1234
+#define PL2303_PRODUCT_ID_PHAROS	0xaaa0
+#define PL2303_PRODUCT_ID_RSAQ3		0xaaa2
+#define PL2303_PRODUCT_ID_ALDIGA	0x0611
+#define PL2303_PRODUCT_ID_MMX		0x0612
+#define PL2303_PRODUCT_ID_GPRS		0x0609
+#define PL2303_PRODUCT_ID_HCR331	0x331a
+#define PL2303_PRODUCT_ID_MOTOROLA	0x0307
+
+#define ATEN_VENDOR_ID		0x0557
+#define ATEN_VENDOR_ID2		0x0547
+#define ATEN_PRODUCT_ID		0x2008
+
+#define IODATA_VENDOR_ID	0x04bb
+#define IODATA_PRODUCT_ID	0x0a03
+#define IODATA_PRODUCT_ID_RSAQ5	0x0a0e
+
+#define ELCOM_VENDOR_ID		0x056e
+#define ELCOM_PRODUCT_ID	0x5003
+#define ELCOM_PRODUCT_ID_UCSGT	0x5004
+
+#define ITEGNO_VENDOR_ID	0x0eba
+#define ITEGNO_PRODUCT_ID	0x1080
+#define ITEGNO_PRODUCT_ID_2080	0x2080
+
+#define MA620_VENDOR_ID		0x0df7
+#define MA620_PRODUCT_ID	0x0620
+
+#define RATOC_VENDOR_ID		0x0584
+#define RATOC_PRODUCT_ID	0xb000
+
+#define TRIPP_VENDOR_ID		0x2478
+#define TRIPP_PRODUCT_ID	0x2008
+
+#define RADIOSHACK_VENDOR_ID	0x1453
+#define RADIOSHACK_PRODUCT_ID	0x4026
+
+#define DCU10_VENDOR_ID		0x0731
+#define DCU10_PRODUCT_ID	0x0528
+
+#define SITECOM_VENDOR_ID	0x6189
+#define SITECOM_PRODUCT_ID	0x2068
+
+/* Alcatel OT535/735 USB cable */
+#define ALCATEL_VENDOR_ID	0x11f7
+#define ALCATEL_PRODUCT_ID	0x02df
+
+/* Samsung I330 phone cradle */
+#define SAMSUNG_VENDOR_ID	0x04e8
+#define SAMSUNG_PRODUCT_ID	0x8001
+
+#define SIEMENS_VENDOR_ID	0x11f5
+#define SIEMENS_PRODUCT_ID_SX1	0x0001
+#define SIEMENS_PRODUCT_ID_X65	0x0003
+#define SIEMENS_PRODUCT_ID_X75	0x0004
+#define SIEMENS_PRODUCT_ID_EF81	0x0005
+
+#define SYNTECH_VENDOR_ID	0x0745
+#define SYNTECH_PRODUCT_ID	0x0001
+
+/* Nokia CA-42 Cable */
+#define NOKIA_CA42_VENDOR_ID	0x078b
+#define NOKIA_CA42_PRODUCT_ID	0x1234
+
+/* CA-42 CLONE Cable www.ca-42.com chipset: Prolific Technology Inc */
+#define CA_42_CA42_VENDOR_ID	0x10b5
+#define CA_42_CA42_PRODUCT_ID	0xac70
+
+#define SAGEM_VENDOR_ID		0x079b
+#define SAGEM_PRODUCT_ID	0x0027
+
+/* Leadtek GPS 9531 (ID 0413:2101) */
+#define LEADTEK_VENDOR_ID	0x0413
+#define LEADTEK_9531_PRODUCT_ID	0x2101
+
+/* USB GSM cable from Speed Dragon Multimedia, Ltd */
+#define SPEEDDRAGON_VENDOR_ID	0x0e55
+#define SPEEDDRAGON_PRODUCT_ID	0x110b
+
+/* DATAPILOT Universal-2 Phone Cable */
+#define DATAPILOT_U2_VENDOR_ID	0x0731
+#define DATAPILOT_U2_PRODUCT_ID	0x2003
+
+/* Belkin "F5U257" Serial Adapter */
+#define BELKIN_VENDOR_ID	0x050d
+#define BELKIN_PRODUCT_ID	0x0257
+
+/* Alcor Micro Corp. USB 2.0 TO RS-232 */
+#define ALCOR_VENDOR_ID		0x058F
+#define ALCOR_PRODUCT_ID	0x9720
+
+/* Willcom WS002IN Data Driver (by NetIndex Inc.) */
+#define WS002IN_VENDOR_ID	0x11f6
+#define WS002IN_PRODUCT_ID	0x2001
+
+/* Corega CG-USBRS232R Serial Adapter */
+#define COREGA_VENDOR_ID	0x07aa
+#define COREGA_PRODUCT_ID	0x002a
+
+/* Y.C. Cable U.S.A., Inc - USB to RS-232 */
+#define YCCABLE_VENDOR_ID	0x05ad
+#define YCCABLE_PRODUCT_ID	0x0fba
+
+/* "Superial" USB - Serial */
+#define SUPERIAL_VENDOR_ID	0x5372
+#define SUPERIAL_PRODUCT_ID	0x2303
+
+/* Hewlett-Packard LD220-HP POS Pole Display */
+#define HP_VENDOR_ID		0x03f0
+#define HP_LD220_PRODUCT_ID	0x3524
+
+/* Cressi Edy (diving computer) PC interface */
+#define CRESSI_VENDOR_ID	0x04b8
+#define CRESSI_EDY_PRODUCT_ID	0x0521
+
+/* Zeagle dive computer interface */
+#define ZEAGLE_VENDOR_ID	0x04b8
+#define ZEAGLE_N2ITION3_PRODUCT_ID	0x0522
+
+/* Sony, USB data cable for CMD-Jxx mobile phones */
+#define SONY_VENDOR_ID		0x054c
+#define SONY_QN3USB_PRODUCT_ID	0x0437
+
+/* Sanwa KB-USB2 multimeter cable (ID: 11ad:0001) */
+#define SANWA_VENDOR_ID		0x11ad
+#define SANWA_PRODUCT_ID	0x0001
+
+/* ADLINK ND-6530 RS232,RS485 and RS422 adapter */
+#define ADLINK_VENDOR_ID		0x0b63
+#define ADLINK_ND6530_PRODUCT_ID	0x6530
+
+/* SMART USB Serial Adapter */
+#define SMART_VENDOR_ID	0x0b8c
+#define SMART_PRODUCT_ID	0x2303
diff --git a/hw/usb/quirks.c b/hw/usb/quirks.c
new file mode 100644
index 00000000000..a761a960323
--- /dev/null
+++ b/hw/usb/quirks.c
@@ -0,0 +1,53 @@
+/*
+ * USB quirk handling
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include "quirks.h"
+#include "hw/usb.h"
+
+static bool usb_id_match(const struct usb_device_id *ids,
+                         uint16_t vendor_id, uint16_t product_id,
+                         uint8_t interface_class, uint8_t interface_subclass,
+                         uint8_t interface_protocol) {
+    int i;
+
+    for (i = 0; ids[i].vendor_id != -1; i++) {
+        if (ids[i].vendor_id  == vendor_id &&
+            ids[i].product_id == product_id &&
+            (ids[i].interface_class == -1 ||
+             (ids[i].interface_class == interface_class &&
+              ids[i].interface_subclass == interface_subclass &&
+              ids[i].interface_protocol == interface_protocol))) {
+            return true;
+        }
+    }
+    return false;
+}
+
+int usb_get_quirks(uint16_t vendor_id, uint16_t product_id,
+                   uint8_t interface_class, uint8_t interface_subclass,
+                   uint8_t interface_protocol)
+{
+    int quirks = 0;
+
+    if (usb_id_match(usbredir_raw_serial_ids, vendor_id, product_id,
+                   interface_class, interface_subclass, interface_protocol)) {
+        quirks |= USB_QUIRK_BUFFER_BULK_IN;
+    }
+    if (usb_id_match(usbredir_ftdi_serial_ids, vendor_id, product_id,
+                   interface_class, interface_subclass, interface_protocol)) {
+        quirks |= USB_QUIRK_BUFFER_BULK_IN | USB_QUIRK_IS_FTDI;
+    }
+
+    return quirks;
+}
diff --git a/hw/usb/quirks.h b/hw/usb/quirks.h
new file mode 100644
index 00000000000..8dc60655270
--- /dev/null
+++ b/hw/usb/quirks.h
@@ -0,0 +1,910 @@
+/*
+ * USB quirk handling
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+/* 1 on 1 copy of linux/drivers/usb/serial/ftdi_sio_ids.h */
+#include "quirks-ftdi-ids.h"
+/* 1 on 1 copy of linux/drivers/usb/serial/pl2303.h */
+#include "quirks-pl2303-ids.h"
+
+struct usb_device_id {
+    int vendor_id;
+    int product_id;
+    int interface_class;
+    int interface_subclass;
+    int interface_protocol;
+};
+
+#define USB_DEVICE(vendor, product) \
+    .vendor_id = vendor, .product_id = product, .interface_class = -1,
+
+#define USB_DEVICE_AND_INTERFACE_INFO(vend, prod, iclass, isubclass, iproto) \
+    .vendor_id = vend, .product_id = prod, .interface_class = iclass, \
+    .interface_subclass = isubclass, .interface_protocol = iproto
+
+static const struct usb_device_id usbredir_raw_serial_ids[] = {
+    /*
+     * Silicon Laboratories CP210x USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/cp210x.c
+     *
+     * Copyright (C) 2005 Craig Shelley (craig@microtron.org.uk)
+     */
+    { USB_DEVICE(0x045B, 0x0053) }, /* Renesas RX610 RX-Stick */
+    { USB_DEVICE(0x0471, 0x066A) }, /* AKTAKOM ACE-1001 cable */
+    { USB_DEVICE(0x0489, 0xE000) }, /* Pirelli Broadband S.p.A, DP-L10 SIP/GSM Mobile */
+    { USB_DEVICE(0x0489, 0xE003) }, /* Pirelli Broadband S.p.A, DP-L10 SIP/GSM Mobile */
+    { USB_DEVICE(0x0745, 0x1000) }, /* CipherLab USB CCD Barcode Scanner 1000 */
+    { USB_DEVICE(0x08e6, 0x5501) }, /* Gemalto Prox-PU/CU contactless smartcard reader */
+    { USB_DEVICE(0x08FD, 0x000A) }, /* Digianswer A/S , ZigBee/802.15.4 MAC Device */
+    { USB_DEVICE(0x0BED, 0x1100) }, /* MEI (TM) Cashflow-SC Bill/Voucher Acceptor */
+    { USB_DEVICE(0x0BED, 0x1101) }, /* MEI series 2000 Combo Acceptor */
+    { USB_DEVICE(0x0FCF, 0x1003) }, /* Dynastream ANT development board */
+    { USB_DEVICE(0x0FCF, 0x1004) }, /* Dynastream ANT2USB */
+    { USB_DEVICE(0x0FCF, 0x1006) }, /* Dynastream ANT development board */
+    { USB_DEVICE(0x10A6, 0xAA26) }, /* Knock-off DCU-11 cable */
+    { USB_DEVICE(0x10AB, 0x10C5) }, /* Siemens MC60 Cable */
+    { USB_DEVICE(0x10B5, 0xAC70) }, /* Nokia CA-42 USB */
+    { USB_DEVICE(0x10C4, 0x0F91) }, /* Vstabi */
+    { USB_DEVICE(0x10C4, 0x1101) }, /* Arkham Technology DS101 Bus Monitor */
+    { USB_DEVICE(0x10C4, 0x1601) }, /* Arkham Technology DS101 Adapter */
+    { USB_DEVICE(0x10C4, 0x800A) }, /* SPORTident BSM7-D-USB main station */
+    { USB_DEVICE(0x10C4, 0x803B) }, /* Pololu USB-serial converter */
+    { USB_DEVICE(0x10C4, 0x8044) }, /* Cygnal Debug Adapter */
+    { USB_DEVICE(0x10C4, 0x804E) }, /* Software Bisque Paramount ME build-in converter */
+    { USB_DEVICE(0x10C4, 0x8053) }, /* Enfora EDG1228 */
+    { USB_DEVICE(0x10C4, 0x8054) }, /* Enfora GSM2228 */
+    { USB_DEVICE(0x10C4, 0x8066) }, /* Argussoft In-System Programmer */
+    { USB_DEVICE(0x10C4, 0x806F) }, /* IMS USB to RS422 Converter Cable */
+    { USB_DEVICE(0x10C4, 0x807A) }, /* Crumb128 board */
+    { USB_DEVICE(0x10C4, 0x80C4) }, /* Cygnal Integrated Products, Inc., Optris infrared thermometer */
+    { USB_DEVICE(0x10C4, 0x80CA) }, /* Degree Controls Inc */
+    { USB_DEVICE(0x10C4, 0x80DD) }, /* Tracient RFID */
+    { USB_DEVICE(0x10C4, 0x80F6) }, /* Suunto sports instrument */
+    { USB_DEVICE(0x10C4, 0x8115) }, /* Arygon NFC/Mifare Reader */
+    { USB_DEVICE(0x10C4, 0x813D) }, /* Burnside Telecom Deskmobile */
+    { USB_DEVICE(0x10C4, 0x813F) }, /* Tams Master Easy Control */
+    { USB_DEVICE(0x10C4, 0x814A) }, /* West Mountain Radio RIGblaster P&P */
+    { USB_DEVICE(0x10C4, 0x814B) }, /* West Mountain Radio RIGtalk */
+    { USB_DEVICE(0x10C4, 0x8156) }, /* B&G H3000 link cable */
+    { USB_DEVICE(0x10C4, 0x815E) }, /* Helicomm IP-Link 1220-DVM */
+    { USB_DEVICE(0x10C4, 0x815F) }, /* Timewave HamLinkUSB */
+    { USB_DEVICE(0x10C4, 0x818B) }, /* AVIT Research USB to TTL */
+    { USB_DEVICE(0x10C4, 0x819F) }, /* MJS USB Toslink Switcher */
+    { USB_DEVICE(0x10C4, 0x81A6) }, /* ThinkOptics WavIt */
+    { USB_DEVICE(0x10C4, 0x81A9) }, /* Multiplex RC Interface */
+    { USB_DEVICE(0x10C4, 0x81AC) }, /* MSD Dash Hawk */
+    { USB_DEVICE(0x10C4, 0x81AD) }, /* INSYS USB Modem */
+    { USB_DEVICE(0x10C4, 0x81C8) }, /* Lipowsky Industrie Elektronik GmbH, Baby-JTAG */
+    { USB_DEVICE(0x10C4, 0x81E2) }, /* Lipowsky Industrie Elektronik GmbH, Baby-LIN */
+    { USB_DEVICE(0x10C4, 0x81E7) }, /* Aerocomm Radio */
+    { USB_DEVICE(0x10C4, 0x81E8) }, /* Zephyr Bioharness */
+    { USB_DEVICE(0x10C4, 0x81F2) }, /* C1007 HF band RFID controller */
+    { USB_DEVICE(0x10C4, 0x8218) }, /* Lipowsky Industrie Elektronik GmbH, HARP-1 */
+    { USB_DEVICE(0x10C4, 0x822B) }, /* Modem EDGE(GSM) Comander 2 */
+    { USB_DEVICE(0x10C4, 0x826B) }, /* Cygnal Integrated Products, Inc., Fasttrax GPS demonstration module */
+    { USB_DEVICE(0x10C4, 0x8293) }, /* Telegesis ETRX2USB */
+    { USB_DEVICE(0x10C4, 0x82F9) }, /* Procyon AVS */
+    { USB_DEVICE(0x10C4, 0x8341) }, /* Siemens MC35PU GPRS Modem */
+    { USB_DEVICE(0x10C4, 0x8382) }, /* Cygnal Integrated Products, Inc. */
+    { USB_DEVICE(0x10C4, 0x83A8) }, /* Amber Wireless AMB2560 */
+    { USB_DEVICE(0x10C4, 0x83D8) }, /* DekTec DTA Plus VHF/UHF Booster/Attenuator */
+    { USB_DEVICE(0x10C4, 0x8411) }, /* Kyocera GPS Module */
+    { USB_DEVICE(0x10C4, 0x8418) }, /* IRZ Automation Teleport SG-10 GSM/GPRS Modem */
+    { USB_DEVICE(0x10C4, 0x846E) }, /* BEI USB Sensor Interface (VCP) */
+    { USB_DEVICE(0x10C4, 0x8477) }, /* Balluff RFID */
+    { USB_DEVICE(0x10C4, 0x85EA) }, /* AC-Services IBUS-IF */
+    { USB_DEVICE(0x10C4, 0x85EB) }, /* AC-Services CIS-IBUS */
+    { USB_DEVICE(0x10C4, 0x8664) }, /* AC-Services CAN-IF */
+    { USB_DEVICE(0x10C4, 0x8665) }, /* AC-Services OBD-IF */
+    { USB_DEVICE(0x10C4, 0xEA60) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA61) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA70) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA80) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA71) }, /* Infinity GPS-MIC-1 Radio Monophone */
+    { USB_DEVICE(0x10C4, 0xF001) }, /* Elan Digital Systems USBscope50 */
+    { USB_DEVICE(0x10C4, 0xF002) }, /* Elan Digital Systems USBwave12 */
+    { USB_DEVICE(0x10C4, 0xF003) }, /* Elan Digital Systems USBpulse100 */
+    { USB_DEVICE(0x10C4, 0xF004) }, /* Elan Digital Systems USBcount50 */
+    { USB_DEVICE(0x10C5, 0xEA61) }, /* Silicon Labs MobiData GPRS USB Modem */
+    { USB_DEVICE(0x10CE, 0xEA6A) }, /* Silicon Labs MobiData GPRS USB Modem 100EU */
+    { USB_DEVICE(0x13AD, 0x9999) }, /* Baltech card reader */
+    { USB_DEVICE(0x1555, 0x0004) }, /* Owen AC4 USB-RS485 Converter */
+    { USB_DEVICE(0x166A, 0x0201) }, /* Clipsal 5500PACA C-Bus Pascal Automation Controller */
+    { USB_DEVICE(0x166A, 0x0301) }, /* Clipsal 5800PC C-Bus Wireless PC Interface */
+    { USB_DEVICE(0x166A, 0x0303) }, /* Clipsal 5500PCU C-Bus USB interface */
+    { USB_DEVICE(0x166A, 0x0304) }, /* Clipsal 5000CT2 C-Bus Black and White Touchscreen */
+    { USB_DEVICE(0x166A, 0x0305) }, /* Clipsal C-5000CT2 C-Bus Spectrum Colour Touchscreen */
+    { USB_DEVICE(0x166A, 0x0401) }, /* Clipsal L51xx C-Bus Architectural Dimmer */
+    { USB_DEVICE(0x166A, 0x0101) }, /* Clipsal 5560884 C-Bus Multi-room Audio Matrix Switcher */
+    { USB_DEVICE(0x16D6, 0x0001) }, /* Jablotron serial interface */
+    { USB_DEVICE(0x16DC, 0x0010) }, /* W-IE-NE-R Plein & Baus GmbH PL512 Power Supply */
+    { USB_DEVICE(0x16DC, 0x0011) }, /* W-IE-NE-R Plein & Baus GmbH RCM Remote Control for MARATON Power Supply */
+    { USB_DEVICE(0x16DC, 0x0012) }, /* W-IE-NE-R Plein & Baus GmbH MPOD Multi Channel Power Supply */
+    { USB_DEVICE(0x16DC, 0x0015) }, /* W-IE-NE-R Plein & Baus GmbH CML Control, Monitoring and Data Logger */
+    { USB_DEVICE(0x17A8, 0x0001) }, /* Kamstrup Optical Eye/3-wire */
+    { USB_DEVICE(0x17A8, 0x0005) }, /* Kamstrup M-Bus Master MultiPort 250D */
+    { USB_DEVICE(0x17F4, 0xAAAA) }, /* Wavesense Jazz blood glucose meter */
+    { USB_DEVICE(0x1843, 0x0200) }, /* Vaisala USB Instrument Cable */
+    { USB_DEVICE(0x18EF, 0xE00F) }, /* ELV USB-I2C-Interface */
+    { USB_DEVICE(0x1BE3, 0x07A6) }, /* WAGO 750-923 USB Service Cable */
+    { USB_DEVICE(0x1E29, 0x0102) }, /* Festo CPX-USB */
+    { USB_DEVICE(0x1E29, 0x0501) }, /* Festo CMSP */
+    { USB_DEVICE(0x3195, 0xF190) }, /* Link Instruments MSO-19 */
+    { USB_DEVICE(0x3195, 0xF280) }, /* Link Instruments MSO-28 */
+    { USB_DEVICE(0x3195, 0xF281) }, /* Link Instruments MSO-28 */
+    { USB_DEVICE(0x413C, 0x9500) }, /* DW700 GPS USB interface */
+
+    /*
+     * Prolific pl2303 USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/pl2303.c
+     *
+     * Copyright (C) 2001-2007 Greg Kroah-Hartman (greg@kroah.com)
+     * Copyright (C) 2003 IBM Corp.
+     */
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_RSAQ2) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_DCU11) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_RSAQ3) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_PHAROS) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_ALDIGA) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_MMX) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_GPRS) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_HCR331) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_MOTOROLA) },
+    { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID) },
+    { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID_RSAQ5) },
+    { USB_DEVICE(ATEN_VENDOR_ID, ATEN_PRODUCT_ID) },
+    { USB_DEVICE(ATEN_VENDOR_ID2, ATEN_PRODUCT_ID) },
+    { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID) },
+    { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID_UCSGT) },
+    { USB_DEVICE(ITEGNO_VENDOR_ID, ITEGNO_PRODUCT_ID) },
+    { USB_DEVICE(ITEGNO_VENDOR_ID, ITEGNO_PRODUCT_ID_2080) },
+    { USB_DEVICE(MA620_VENDOR_ID, MA620_PRODUCT_ID) },
+    { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID) },
+    { USB_DEVICE(TRIPP_VENDOR_ID, TRIPP_PRODUCT_ID) },
+    { USB_DEVICE(RADIOSHACK_VENDOR_ID, RADIOSHACK_PRODUCT_ID) },
+    { USB_DEVICE(DCU10_VENDOR_ID, DCU10_PRODUCT_ID) },
+    { USB_DEVICE(SITECOM_VENDOR_ID, SITECOM_PRODUCT_ID) },
+    { USB_DEVICE(ALCATEL_VENDOR_ID, ALCATEL_PRODUCT_ID) },
+    { USB_DEVICE(SAMSUNG_VENDOR_ID, SAMSUNG_PRODUCT_ID) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_SX1) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_X65) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_X75) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_EF81) },
+    { USB_DEVICE(BENQ_VENDOR_ID, BENQ_PRODUCT_ID_S81) }, /* Benq/Siemens S81 */
+    { USB_DEVICE(SYNTECH_VENDOR_ID, SYNTECH_PRODUCT_ID) },
+    { USB_DEVICE(NOKIA_CA42_VENDOR_ID, NOKIA_CA42_PRODUCT_ID) },
+    { USB_DEVICE(CA_42_CA42_VENDOR_ID, CA_42_CA42_PRODUCT_ID) },
+    { USB_DEVICE(SAGEM_VENDOR_ID, SAGEM_PRODUCT_ID) },
+    { USB_DEVICE(LEADTEK_VENDOR_ID, LEADTEK_9531_PRODUCT_ID) },
+    { USB_DEVICE(SPEEDDRAGON_VENDOR_ID, SPEEDDRAGON_PRODUCT_ID) },
+    { USB_DEVICE(DATAPILOT_U2_VENDOR_ID, DATAPILOT_U2_PRODUCT_ID) },
+    { USB_DEVICE(BELKIN_VENDOR_ID, BELKIN_PRODUCT_ID) },
+    { USB_DEVICE(ALCOR_VENDOR_ID, ALCOR_PRODUCT_ID) },
+    { USB_DEVICE(WS002IN_VENDOR_ID, WS002IN_PRODUCT_ID) },
+    { USB_DEVICE(COREGA_VENDOR_ID, COREGA_PRODUCT_ID) },
+    { USB_DEVICE(YCCABLE_VENDOR_ID, YCCABLE_PRODUCT_ID) },
+    { USB_DEVICE(SUPERIAL_VENDOR_ID, SUPERIAL_PRODUCT_ID) },
+    { USB_DEVICE(HP_VENDOR_ID, HP_LD220_PRODUCT_ID) },
+    { USB_DEVICE(CRESSI_VENDOR_ID, CRESSI_EDY_PRODUCT_ID) },
+    { USB_DEVICE(ZEAGLE_VENDOR_ID, ZEAGLE_N2ITION3_PRODUCT_ID) },
+    { USB_DEVICE(SONY_VENDOR_ID, SONY_QN3USB_PRODUCT_ID) },
+    { USB_DEVICE(SANWA_VENDOR_ID, SANWA_PRODUCT_ID) },
+    { USB_DEVICE(ADLINK_VENDOR_ID, ADLINK_ND6530_PRODUCT_ID) },
+    { USB_DEVICE(SMART_VENDOR_ID, SMART_PRODUCT_ID) },
+
+    { USB_DEVICE(-1, -1) } /* Terminating Entry */
+};
+
+static const struct usb_device_id usbredir_ftdi_serial_ids[] = {
+    /*
+     * FTDI USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/ftdi_sio.c
+     *
+     * Copyright (C) 2009 - 2010
+     *    Johan Hovold (jhovold@gmail.com)
+     * Copyright (C) 1999 - 2001
+     *    Greg Kroah-Hartman (greg@kroah.com)
+     *    Bill Ryder (bryder@sgi.com)
+     * Copyright (C) 2002
+     *    Kuba Ober (kuba@mareimbrium.org)
+     */
+    { USB_DEVICE(FTDI_VID, FTDI_ZEITCONTROL_TAGTRACE_MIFARE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CTI_MINI_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CTI_NANO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_AMC232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CANUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CANDAPTER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NXTCAM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_5_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_7_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_CAT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_WKEY_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_RS232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACTZWAVE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IRTRANS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IPLUS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IPLUS2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DMX4ALL) },
+    { USB_DEVICE(FTDI_VID, FTDI_SIO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U232AM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U232AM_ALT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_232RL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U2232C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4232H_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_232H_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_FTX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MICRO_CHAMELEON_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RELAIS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_SNIFFER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_THROTTLE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_GATEWAY_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_GBM_PID) },
+    { USB_DEVICE(INTERBIOMETRICS_VID, INTERBIOMETRICS_IOBOARD_PID) },
+    { USB_DEVICE(INTERBIOMETRICS_VID, INTERBIOMETRICS_MINI_IOBOARD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SPROG_II) },
+    { USB_DEVICE(FTDI_VID, FTDI_LENZ_LIUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_632_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_634_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_547_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_633_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_631_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_635_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_640_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_642_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DSS20_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_URBAN_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_URBAN_1_PID) },
+    { USB_DEVICE(FTDI_NF_RIC_VID, FTDI_NF_RIC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_VNHCPCUSB_D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_5_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_R2000KU_TRUE_RNG) },
+    { USB_DEVICE(FTDI_VID, FTDI_VARDAAN_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0100_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0101_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0102_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0103_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0104_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0105_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0106_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0107_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0108_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0109_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0110_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0111_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0112_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0113_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0114_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0115_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0116_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0117_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0118_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0119_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0120_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0121_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0122_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0123_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0124_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0125_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0126_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0127_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0128_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0129_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0130_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0131_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0132_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0133_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0134_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0135_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0136_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0137_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0138_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0139_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0140_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0141_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0142_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0143_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0144_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0145_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0146_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0147_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0148_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0149_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0150_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0151_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0152_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0153_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0154_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0155_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0156_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0157_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0158_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0159_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0160_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0161_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0162_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0163_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0164_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0165_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0166_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0167_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0168_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0169_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0170_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0171_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0172_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0173_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0174_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0175_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0176_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0177_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0178_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0179_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0180_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0181_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0182_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0183_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0184_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0185_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0186_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0187_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0188_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0189_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0190_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0191_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0192_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0193_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0194_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0195_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0196_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0197_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0198_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0199_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01ED_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FF_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PERLE_ULTRAPORT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PIEGROUP_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TNC_X_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USBX_707_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2101_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2102_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2103_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2104_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2106_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2201_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2201_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2202_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2202_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2203_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2203_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_4_PID) },
+    { USB_DEVICE(IDTECH_VID, IDTECH_IDT1221U_PID) },
+    { USB_DEVICE(OCT_VID, OCT_US101_PID) },
+    { USB_DEVICE(OCT_VID, OCT_DK201_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_HE_TIRA1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USB_UIRT_PID) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_1) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_R2X0) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_3) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_4) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E808_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E809_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80A_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80B_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80E_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E888_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E889_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88A_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88B_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88E_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UO100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UM100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UR100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_ALC8500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PYRAMID_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FHZ1000PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_US485_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PICPRO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PCMCIA_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PK1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_RS232MON_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_APP70_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PEDO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PROD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TAVIR_STK500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TIAO_UMPA_PID) },
+    /*
+     * ELV devices:
+     */
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_USR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_MSM1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_KL100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS550_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EC3000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS888_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TWS550_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FEM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_CLI7000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PPS7330_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TFM100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UDF77_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UIO88_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UAD8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UDA7_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_USI2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_T1100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PCD200_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_ULA200_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_CSI8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EM1000DL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PCK100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_RFP500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FS20SIG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UTP8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS300PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS444PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FHZ1300PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EM1010PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_HS485_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UMS100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TFD128_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FM3RX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS777_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_SDMUSBQSS_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_MASTERDEVEL2_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_0_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_1_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU20_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU40_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSMACHX_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSLOAD_N_GO_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU64_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSPRIME8_5_PID) },
+    { USB_DEVICE(FTDI_VID, INSIDE_ACCESSO) },
+    { USB_DEVICE(INTREPID_VID, INTREPID_VALUECAN_PID) },
+    { USB_DEVICE(INTREPID_VID, INTREPID_NEOVI_PID) },
+    { USB_DEVICE(FALCOM_VID, FALCOM_TWIST_PID) },
+    { USB_DEVICE(FALCOM_VID, FALCOM_SAMBA_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SUUNTO_SPORTS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OCEANIC_PID) },
+    { USB_DEVICE(TTI_VID, TTI_QL355P_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RM_CANVIEW_PID) },
+    { USB_DEVICE(ACTON_VID, ACTON_SPECTRAPRO_PID) },
+    { USB_DEVICE(CONTEC_VID, CONTEC_COM1USBH_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USPTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2DR_2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2DR_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4DR2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4DR_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USB9F_2W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USB9F_4W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_232USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USBTB_2W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USBTB_4W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_TTL5USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_TTL3USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_ZZ_PROG1_USB_PID) },
+    { USB_DEVICE(FTDI_VID, EVER_ECO_PRO_CDS) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_3_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_0_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_1_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_2_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_3_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_4_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_5_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_6_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_7_PID) },
+    { USB_DEVICE(MOBILITY_VID, MOBILITY_USB_SERIAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACTIVE_ROBOTS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_KW_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_YS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_IC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_DB9_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_RS232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y9_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TERATRONIK_VCP_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TERATRONIK_D2XX_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVOLUTION_ER1_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVO_HYBRID_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVO_RCM4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ARTEMIS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16HR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16HRC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16IC_PID) },
+    { USB_DEVICE(KOBIL_VID, KOBIL_CONV_B1_PID) },
+    { USB_DEVICE(KOBIL_VID, KOBIL_CONV_KAAN_PID) },
+    { USB_DEVICE(POSIFLEX_VID, POSIFLEX_PP7000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TTUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ECLO_COM_1WIRE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_WESTREX_MODEL_777_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_WESTREX_MODEL_8900F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PCDJ_DAC2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RRCIRKITS_LOCOBUFFER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ASK_RDR400_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NZR_SEM_USB_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_1_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_OPC_U_UC_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2C1_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2C2_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2D_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2VT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2VR_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP4KVT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP4KVR_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2KVT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2KVR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACG_HFDUAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_YEI_SERVOCENTER31_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_THORLABS_PID) },
+    { USB_DEVICE(TESTO_VID, TESTO_USB_INTERFACE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GAMMA_SCOUT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13M_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13S_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13U_PID) },
+    { USB_DEVICE(ELEKTOR_VID, ELEKTOR_FT323R_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_HUC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_SPECTRA_SCU_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_FUTURE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_FUTURE_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_AURORA_SCU_PID) },
+    { USB_DEVICE(TELLDUS_VID, TELLDUS_TELLSTICK_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_SERIAL_VX7_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_CT29B_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_RTS01_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MAXSTREAM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PHI_FISCO_PID) },
+    { USB_DEVICE(TML_VID, TML_USB_SERIAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELSTER_UNICOM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PROPOX_JTAGCABLEII_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PROPOX_ISPCABLEIII_PID) },
+    { USB_DEVICE(OLIMEX_VID, OLIMEX_ARM_USB_OCD_PID) },
+    { USB_DEVICE(OLIMEX_VID, OLIMEX_ARM_USB_OCD_H_PID) },
+    { USB_DEVICE(FIC_VID, FIC_NEO1973_DEBUG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OOCDLINK_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_DEVEL_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_EVAL_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_ICDI_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TURTELIZER_PID) },
+    { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID_USB60F) },
+    { USB_DEVICE(FTDI_VID, FTDI_REU_TINY_PID) },
+
+    /* Papouch devices based on FTDI chip */
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AP485_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB422_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AP485_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB422_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485S_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485C_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_LEC_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB232_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_TMU_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_IRAMP_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_DRAK5_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO8x8_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO4x4_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO2x2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO10x1_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO30x3_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO60x3_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO2x16_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO3x32_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_DRAK6_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_UPSUSB_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_MU_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SIMUKEY_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AD4USB_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_GMUX_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_GMSR_PID) },
+
+    { USB_DEVICE(FTDI_VID, FTDI_DOMINTELL_DGQG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DOMINTELL_DUSB_PID) },
+    { USB_DEVICE(ALTI2_VID, ALTI2_N3_PID) },
+    { USB_DEVICE(FTDI_VID, DIEBOLD_BCS_SE923_PID) },
+    { USB_DEVICE(ATMEL_VID, STK541_PID) },
+    { USB_DEVICE(DE_VID, STB_PID) },
+    { USB_DEVICE(DE_VID, WHT_PID) },
+    { USB_DEVICE(ADI_VID, ADI_GNICE_PID) },
+    { USB_DEVICE(ADI_VID, ADI_GNICEPLUS_PID) },
+    { USB_DEVICE_AND_INTERFACE_INFO(MICROCHIP_VID, MICROCHIP_USB_BOARD_PID,
+                                    0xff, 0xff, 0x00) },
+    { USB_DEVICE(JETI_VID, JETI_SPC1201_PID) },
+    { USB_DEVICE(MARVELL_VID, MARVELL_SHEEVAPLUG_PID) },
+    { USB_DEVICE(LARSENBRUSGAARD_VID, LB_ALTITRACK_PID) },
+    { USB_DEVICE(GN_OTOMETRICS_VID, AURICAL_USB_PID) },
+    { USB_DEVICE(FTDI_VID, PI_C865_PID) },
+    { USB_DEVICE(FTDI_VID, PI_C857_PID) },
+    { USB_DEVICE(PI_VID, PI_C866_PID) },
+    { USB_DEVICE(PI_VID, PI_C663_PID) },
+    { USB_DEVICE(PI_VID, PI_C725_PID) },
+    { USB_DEVICE(PI_VID, PI_E517_PID) },
+    { USB_DEVICE(PI_VID, PI_C863_PID) },
+    { USB_DEVICE(PI_VID, PI_E861_PID) },
+    { USB_DEVICE(PI_VID, PI_C867_PID) },
+    { USB_DEVICE(PI_VID, PI_E609_PID) },
+    { USB_DEVICE(PI_VID, PI_E709_PID) },
+    { USB_DEVICE(PI_VID, PI_100F_PID) },
+    { USB_DEVICE(PI_VID, PI_1011_PID) },
+    { USB_DEVICE(PI_VID, PI_1012_PID) },
+    { USB_DEVICE(PI_VID, PI_1013_PID) },
+    { USB_DEVICE(PI_VID, PI_1014_PID) },
+    { USB_DEVICE(PI_VID, PI_1015_PID) },
+    { USB_DEVICE(PI_VID, PI_1016_PID) },
+    { USB_DEVICE(KONDO_VID, KONDO_USB_SERIAL_PID) },
+    { USB_DEVICE(BAYER_VID, BAYER_CONTOUR_CABLE_PID) },
+    { USB_DEVICE(FTDI_VID, MARVELL_OPENRD_PID) },
+    { USB_DEVICE(FTDI_VID, TI_XDS100V2_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO820_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO720_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO730_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO870_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_GENERIC_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_SR_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_HD_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_XM_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_ST_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SLITE_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH2_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH4_PID) },
+    { USB_DEVICE(FTDI_VID, SEGWAY_RMP200_PID) },
+    { USB_DEVICE(FTDI_VID, ACCESIO_COM4SM_PID) },
+    { USB_DEVICE(IONICS_VID, IONICS_PLUGCOMPUTER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_24_MASTER_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_PC_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_USB_DMX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MIDI_TIMECODE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MINI_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MAXI_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MEDIA_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_LOGBOOKML_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_LS_LOGBOOK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_HS_LOGBOOK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CINTERION_MC55I_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DOTEC_PID) },
+    { USB_DEVICE(QIHARDWARE_VID, MILKYMISTONE_JTAGSERIAL_PID) },
+    { USB_DEVICE(ST_VID, ST_STMCLT1030_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RF_R106) },
+    { USB_DEVICE(FTDI_VID, FTDI_DISTORTEC_JTAG_LOCK_PICK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_LUMEL_PD12_PID) },
+
+    { USB_DEVICE(-1, -1) } /* Terminating Entry */
+};
+
+#undef USB_DEVICE
+#undef USB_DEVICE_AND_INTERFACE_INFO
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 10b4fbb3a7b..8c0ead07c5f 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -1,7 +1,7 @@
 /*
  * USB redirector usb-guest
  *
- * Copyright (c) 2011 Red Hat, Inc.
+ * Copyright (c) 2011-2012 Red Hat, Inc.
  *
  * Red Hat Authors:
  * Hans de Goede <hdegoede@redhat.com>
@@ -26,9 +26,11 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
+#include "char/char.h"
 
 #include <dirent.h>
 #include <sys/ioctl.h>
@@ -42,31 +44,54 @@
 #define NO_INTERFACE_INFO 255 /* Valid interface_count always <= 32 */
 #define EP2I(ep_address) (((ep_address & 0x80) >> 3) | (ep_address & 0x0f))
 #define I2EP(i) (((i & 0x10) << 3) | (i & 0x0f))
+#define USBEP2I(usb_ep) (((usb_ep)->pid == USB_TOKEN_IN) ? \
+                         ((usb_ep)->nr | 0x10) : ((usb_ep)->nr))
+#define I2USBEP(d, i) (usb_ep_get(&(d)->dev, \
+                       ((i) & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT, \
+                       (i) & 0x0f))
 
-typedef struct AsyncURB AsyncURB;
 typedef struct USBRedirDevice USBRedirDevice;
 
-/* Struct to hold buffered packets (iso or int input packets) */
+/* Struct to hold buffered packets */
 struct buf_packet {
     uint8_t *data;
-    int len;
-    int status;
+    void *free_on_destroy;
+    uint16_t len;
+    uint16_t offset;
+    uint8_t status;
     QTAILQ_ENTRY(buf_packet)next;
 };
 
 struct endp_data {
+    USBRedirDevice *dev;
     uint8_t type;
     uint8_t interval;
     uint8_t interface; /* bInterfaceNumber this ep belongs to */
+    uint16_t max_packet_size; /* In bytes, not wMaxPacketSize format !! */
     uint8_t iso_started;
     uint8_t iso_error; /* For reporting iso errors to the HC */
     uint8_t interrupt_started;
     uint8_t interrupt_error;
+    uint8_t bulk_receiving_enabled;
+    uint8_t bulk_receiving_started;
     uint8_t bufpq_prefilled;
     uint8_t bufpq_dropping_packets;
     QTAILQ_HEAD(, buf_packet) bufpq;
-    int bufpq_size;
-    int bufpq_target_size;
+    int32_t bufpq_size;
+    int32_t bufpq_target_size;
+    USBPacket *pending_async_packet;
+};
+
+struct PacketIdQueueEntry {
+    uint64_t id;
+    QTAILQ_ENTRY(PacketIdQueueEntry)next;
+};
+
+struct PacketIdQueue {
+    USBRedirDevice *dev;
+    const char *name;
+    QTAILQ_HEAD(, PacketIdQueueEntry) head;
+    int size;
 };
 
 struct USBRedirDevice {
@@ -79,33 +104,22 @@ struct USBRedirDevice {
     /* Data passed from chardev the fd_read cb to the usbredirparser read cb */
     const uint8_t *read_buf;
     int read_buf_size;
-    /* For async handling of open/close */
-    QEMUBH *open_close_bh;
+    /* For async handling of close */
+    QEMUBH *chardev_close_bh;
     /* To delay the usb attach in case of quick chardev close + open */
     QEMUTimer *attach_timer;
     int64_t next_attach_time;
     struct usbredirparser *parser;
     struct endp_data endpoint[MAX_ENDPOINTS];
-    uint32_t packet_id;
-    QTAILQ_HEAD(, AsyncURB) asyncq;
+    struct PacketIdQueue cancelled;
+    struct PacketIdQueue already_in_flight;
+    void (*buffered_bulk_in_complete)(USBRedirDevice *, USBPacket *, uint8_t);
     /* Data for device filtering */
     struct usb_redir_device_connect_header device_info;
     struct usb_redir_interface_info_header interface_info;
     struct usbredirfilter_rule *filter_rules;
     int filter_rules_count;
-};
-
-struct AsyncURB {
-    USBRedirDevice *dev;
-    USBPacket *packet;
-    uint32_t packet_id;
-    int get;
-    union {
-        struct usb_redir_control_packet_header control_packet;
-        struct usb_redir_bulk_packet_header bulk_packet;
-        struct usb_redir_interrupt_packet_header interrupt_packet;
-    };
-    QTAILQ_ENTRY(AsyncURB)next;
+    int compatible_speedmask;
 };
 
 static void usbredir_hello(void *priv, struct usb_redir_hello_header *h);
@@ -116,32 +130,39 @@ static void usbredir_interface_info(void *priv,
     struct usb_redir_interface_info_header *interface_info);
 static void usbredir_ep_info(void *priv,
     struct usb_redir_ep_info_header *ep_info);
-static void usbredir_configuration_status(void *priv, uint32_t id,
+static void usbredir_configuration_status(void *priv, uint64_t id,
     struct usb_redir_configuration_status_header *configuration_status);
-static void usbredir_alt_setting_status(void *priv, uint32_t id,
+static void usbredir_alt_setting_status(void *priv, uint64_t id,
     struct usb_redir_alt_setting_status_header *alt_setting_status);
-static void usbredir_iso_stream_status(void *priv, uint32_t id,
+static void usbredir_iso_stream_status(void *priv, uint64_t id,
     struct usb_redir_iso_stream_status_header *iso_stream_status);
-static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
+static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     struct usb_redir_interrupt_receiving_status_header
     *interrupt_receiving_status);
-static void usbredir_bulk_streams_status(void *priv, uint32_t id,
+static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status);
-static void usbredir_control_packet(void *priv, uint32_t id,
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status);
+static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len);
-static void usbredir_bulk_packet(void *priv, uint32_t id,
+static void usbredir_bulk_packet(void *priv, uint64_t id,
     struct usb_redir_bulk_packet_header *bulk_packet,
     uint8_t *data, int data_len);
-static void usbredir_iso_packet(void *priv, uint32_t id,
+static void usbredir_iso_packet(void *priv, uint64_t id,
     struct usb_redir_iso_packet_header *iso_packet,
     uint8_t *data, int data_len);
-static void usbredir_interrupt_packet(void *priv, uint32_t id,
+static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_header,
     uint8_t *data, int data_len);
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len);
 
-static int usbredir_handle_status(USBRedirDevice *dev,
-                                       int status, int actual_len);
+static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
+    int status);
+
+#define VERSION "qemu usb-redir guest " QEMU_VERSION
 
 /*
  * Logging stuff
@@ -241,66 +262,160 @@ static int usbredir_write(void *priv, uint8_t *data, int count)
         return 0;
     }
 
+    /* Don't send new data to the chardev until our state is fully synced */
+    if (!runstate_check(RUN_STATE_RUNNING)) {
+        return 0;
+    }
+
     return qemu_chr_fe_write(dev->cs, data, count);
 }
 
 /*
- * Async and buffered packets helpers
+ * Cancelled and buffered packets helpers
  */
 
-static AsyncURB *async_alloc(USBRedirDevice *dev, USBPacket *p)
+static void packet_id_queue_init(struct PacketIdQueue *q,
+    USBRedirDevice *dev, const char *name)
+{
+    q->dev = dev;
+    q->name = name;
+    QTAILQ_INIT(&q->head);
+    q->size = 0;
+}
+
+static void packet_id_queue_add(struct PacketIdQueue *q, uint64_t id)
 {
-    AsyncURB *aurb = (AsyncURB *) g_malloc0(sizeof(AsyncURB));
-    aurb->dev = dev;
-    aurb->packet = p;
-    aurb->packet_id = dev->packet_id;
-    QTAILQ_INSERT_TAIL(&dev->asyncq, aurb, next);
-    dev->packet_id++;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
 
-    return aurb;
+    DPRINTF("adding packet id %"PRIu64" to %s queue\n", id, q->name);
+
+    e = g_malloc0(sizeof(struct PacketIdQueueEntry));
+    e->id = id;
+    QTAILQ_INSERT_TAIL(&q->head, e, next);
+    q->size++;
 }
 
-static void async_free(USBRedirDevice *dev, AsyncURB *aurb)
+static int packet_id_queue_remove(struct PacketIdQueue *q, uint64_t id)
 {
-    QTAILQ_REMOVE(&dev->asyncq, aurb, next);
-    g_free(aurb);
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
+
+    QTAILQ_FOREACH(e, &q->head, next) {
+        if (e->id == id) {
+            DPRINTF("removing packet id %"PRIu64" from %s queue\n",
+                    id, q->name);
+            QTAILQ_REMOVE(&q->head, e, next);
+            q->size--;
+            g_free(e);
+            return 1;
+        }
+    }
+    return 0;
 }
 
-static AsyncURB *async_find(USBRedirDevice *dev, uint32_t packet_id)
+static void packet_id_queue_empty(struct PacketIdQueue *q)
 {
-    AsyncURB *aurb;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e, *next_e;
 
-    QTAILQ_FOREACH(aurb, &dev->asyncq, next) {
-        if (aurb->packet_id == packet_id) {
-            return aurb;
-        }
+    DPRINTF("removing %d packet-ids from %s queue\n", q->size, q->name);
+
+    QTAILQ_FOREACH_SAFE(e, &q->head, next, next_e) {
+        QTAILQ_REMOVE(&q->head, e, next);
+        g_free(e);
     }
-    DPRINTF("could not find async urb for packet_id %u\n", packet_id);
-    return NULL;
+    q->size = 0;
 }
 
 static void usbredir_cancel_packet(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
-    AsyncURB *aurb;
+    int i = USBEP2I(p->ep);
+
+    if (p->combined) {
+        usb_combined_packet_cancel(udev, p);
+        return;
+    }
+
+    if (dev->endpoint[i].pending_async_packet) {
+        assert(dev->endpoint[i].pending_async_packet == p);
+        dev->endpoint[i].pending_async_packet = NULL;
+        return;
+    }
+
+    packet_id_queue_add(&dev->cancelled, p->id);
+    usbredirparser_send_cancel_data_packet(dev->parser, p->id);
+    usbredirparser_do_write(dev->parser);
+}
+
+static int usbredir_is_cancelled(USBRedirDevice *dev, uint64_t id)
+{
+    if (!dev->dev.attached) {
+        return 1; /* Treat everything as cancelled after a disconnect */
+    }
+    return packet_id_queue_remove(&dev->cancelled, id);
+}
 
-    QTAILQ_FOREACH(aurb, &dev->asyncq, next) {
-        if (p != aurb->packet) {
+static void usbredir_fill_already_in_flight_from_ep(USBRedirDevice *dev,
+    struct USBEndpoint *ep)
+{
+    static USBPacket *p;
+
+    /* async handled packets for bulk receiving eps do not count as inflight */
+    if (dev->endpoint[USBEP2I(ep)].bulk_receiving_started) {
+        return;
+    }
+
+    QTAILQ_FOREACH(p, &ep->queue, queue) {
+        /* Skip combined packets, except for the first */
+        if (p->combined && p != p->combined->first) {
             continue;
         }
+        if (p->state == USB_PACKET_ASYNC) {
+            packet_id_queue_add(&dev->already_in_flight, p->id);
+        }
+    }
+}
 
-        DPRINTF("async cancel id %u\n", aurb->packet_id);
-        usbredirparser_send_cancel_data_packet(dev->parser, aurb->packet_id);
-        usbredirparser_do_write(dev->parser);
+static void usbredir_fill_already_in_flight(USBRedirDevice *dev)
+{
+    int ep;
+    struct USBDevice *udev = &dev->dev;
 
-        /* Mark it as dead */
-        aurb->packet = NULL;
-        break;
+    usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_ctl);
+
+    for (ep = 0; ep < USB_MAX_ENDPOINTS; ep++) {
+        usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_in[ep]);
+        usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_out[ep]);
+    }
+}
+
+static int usbredir_already_in_flight(USBRedirDevice *dev, uint64_t id)
+{
+    return packet_id_queue_remove(&dev->already_in_flight, id);
+}
+
+static USBPacket *usbredir_find_packet_by_id(USBRedirDevice *dev,
+    uint8_t ep, uint64_t id)
+{
+    USBPacket *p;
+
+    if (usbredir_is_cancelled(dev, id)) {
+        return NULL;
+    }
+
+    p = usb_ep_find_packet_by_id(&dev->dev,
+                            (ep & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT,
+                            ep & 0x0f, id);
+    if (p == NULL) {
+        ERROR("could not find packet with id %"PRIu64"\n", id);
     }
+    return p;
 }
 
-static void bufp_alloc(USBRedirDevice *dev,
-    uint8_t *data, int len, int status, uint8_t ep)
+static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
+    uint8_t status, uint8_t ep, void *free_on_destroy)
 {
     struct buf_packet *bufp;
 
@@ -324,7 +439,9 @@ static void bufp_alloc(USBRedirDevice *dev,
     bufp = g_malloc(sizeof(struct buf_packet));
     bufp->data   = data;
     bufp->len    = len;
+    bufp->offset = 0;
     bufp->status = status;
+    bufp->free_on_destroy = free_on_destroy;
     QTAILQ_INSERT_TAIL(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size++;
 }
@@ -334,7 +451,7 @@ static void bufp_free(USBRedirDevice *dev, struct buf_packet *bufp,
 {
     QTAILQ_REMOVE(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size--;
-    free(bufp->data);
+    free(bufp->free_on_destroy);
     g_free(bufp);
 }
 
@@ -360,7 +477,7 @@ static void usbredir_handle_reset(USBDevice *udev)
     usbredirparser_do_write(dev->parser);
 }
 
-static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
                                      uint8_t ep)
 {
     int status, len;
@@ -417,7 +534,7 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
                 !dev->endpoint[EP2I(ep)].bufpq_prefilled) {
             if (dev->endpoint[EP2I(ep)].bufpq_size <
                     dev->endpoint[EP2I(ep)].bufpq_target_size) {
-                return usbredir_handle_status(dev, 0, 0);
+                return;
             }
             dev->endpoint[EP2I(ep)].bufpq_prefilled = 1;
         }
@@ -431,27 +548,23 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
             /* Check iso_error for stream errors, otherwise its an underrun */
             status = dev->endpoint[EP2I(ep)].iso_error;
             dev->endpoint[EP2I(ep)].iso_error = 0;
-            return status ? USB_RET_IOERROR : 0;
+            p->status = status ? USB_RET_IOERROR : USB_RET_SUCCESS;
+            return;
         }
         DPRINTF2("iso-token-in ep %02X status %d len %d queue-size: %d\n", ep,
                  isop->status, isop->len, dev->endpoint[EP2I(ep)].bufpq_size);
 
         status = isop->status;
-        if (status != usb_redir_success) {
-            bufp_free(dev, isop, ep);
-            return USB_RET_IOERROR;
-        }
-
         len = isop->len;
         if (len > p->iov.size) {
             ERROR("received iso data is larger then packet ep %02X (%d > %d)\n",
                   ep, len, (int)p->iov.size);
-            bufp_free(dev, isop, ep);
-            return USB_RET_BABBLE;
+            len = p->iov.size;
+            status = usb_redir_babble;
         }
         usb_packet_copy(p, isop->data, len);
         bufp_free(dev, isop, ep);
-        return len;
+        usbredir_handle_status(dev, p, status);
     } else {
         /* If the stream was not started because of a pending error don't
            send the packet to the usb-host */
@@ -471,7 +584,7 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
         dev->endpoint[EP2I(ep)].iso_error = 0;
         DPRINTF2("iso-token-out ep %02X status %d len %zd\n", ep, status,
                  p->iov.size);
-        return usbredir_handle_status(dev, status, p->iov.size);
+        usbredir_handle_status(dev, p, status);
     }
 }
 
@@ -489,108 +602,265 @@ static void usbredir_stop_iso_stream(USBRedirDevice *dev, uint8_t ep)
     usbredir_free_bufpq(dev, ep);
 }
 
-static int usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
+/*
+ * The usb-host may poll the endpoint faster then our guest, resulting in lots
+ * of smaller bulkp-s. The below buffered_bulk_in_complete* functions combine
+ * data from multiple bulkp-s into a single packet, avoiding bufpq overflows.
+ */
+static void usbredir_buffered_bulk_add_data_to_packet(USBRedirDevice *dev,
+    struct buf_packet *bulkp, int count, USBPacket *p, uint8_t ep)
+{
+    usb_packet_copy(p, bulkp->data + bulkp->offset, count);
+    bulkp->offset += count;
+    if (bulkp->offset == bulkp->len) {
+        /* Store status in the last packet with data from this bulkp */
+        usbredir_handle_status(dev, p, bulkp->status);
+        bufp_free(dev, bulkp, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_raw(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        count = bulkp->len - bulkp->offset;
+        if (count > (p->iov.size - p->actual_length)) {
+            count = p->iov.size - p->actual_length;
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_ftdi(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
+    uint8_t header[2] = { 0, 0 };
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        if (bulkp->len < 2) {
+            WARNING("malformed ftdi bulk in packet\n");
+            bufp_free(dev, bulkp, ep);
+            continue;
+        }
+
+        if ((p->actual_length % maxp) == 0) {
+            usb_packet_copy(p, bulkp->data, 2);
+            memcpy(header, bulkp->data, 2);
+        } else {
+            if (bulkp->data[0] != header[0] || bulkp->data[1] != header[1]) {
+                break; /* Different header, add to next packet */
+            }
+        }
+
+        if (bulkp->offset == 0) {
+            bulkp->offset = 2; /* Skip header */
+        }
+        count = bulkp->len - bulkp->offset;
+        /* Must repeat the header at maxp interval */
+        if (count > (maxp - (p->actual_length % maxp))) {
+            count = maxp - (p->actual_length % maxp);
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
+    dev->buffered_bulk_in_complete(dev, p, ep);
+    DPRINTF("bulk-token-in ep %02X status %d len %d id %"PRIu64"\n",
+            ep, p->status, p->actual_length, p->id);
+}
+
+static void usbredir_handle_buffered_bulk_in_data(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    /* Input bulk endpoint, buffered packet input */
+    if (!dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        int bpt;
+        struct usb_redir_start_bulk_receiving_header start = {
+            .endpoint = ep,
+            .stream_id = 0,
+            .no_transfers = 5,
+        };
+        /* Round bytes_per_transfer up to a multiple of max_packet_size */
+        bpt = 512 + dev->endpoint[EP2I(ep)].max_packet_size - 1;
+        bpt /= dev->endpoint[EP2I(ep)].max_packet_size;
+        bpt *= dev->endpoint[EP2I(ep)].max_packet_size;
+        start.bytes_per_transfer = bpt;
+        /* No id, we look at the ep when receiving a status back */
+        usbredirparser_send_start_bulk_receiving(dev->parser, 0, &start);
+        usbredirparser_do_write(dev->parser);
+        DPRINTF("bulk receiving started bytes/transfer %u count %d ep %02X\n",
+                start.bytes_per_transfer, start.no_transfers, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 1;
+        /* We don't really want to drop bulk packets ever, but
+           having some upper limit to how much we buffer is good. */
+        dev->endpoint[EP2I(ep)].bufpq_target_size = 5000;
+        dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
+    }
+
+    if (QTAILQ_EMPTY(&dev->endpoint[EP2I(ep)].bufpq)) {
+        DPRINTF("bulk-token-in ep %02X, no bulkp\n", ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        dev->endpoint[EP2I(ep)].pending_async_packet = p;
+        p->status = USB_RET_ASYNC;
+        return;
+    }
+    usbredir_buffered_bulk_in_complete(dev, p, ep);
+}
+
+static void usbredir_stop_bulk_receiving(USBRedirDevice *dev, uint8_t ep)
+{
+    struct usb_redir_stop_bulk_receiving_header stop_bulk = {
+        .endpoint = ep,
+        .stream_id = 0,
+    };
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        usbredirparser_send_stop_bulk_receiving(dev->parser, 0, &stop_bulk);
+        DPRINTF("bulk receiving stopped ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
+    }
+    usbredir_free_bufpq(dev, ep);
+}
+
+static void usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
                                       uint8_t ep)
 {
-    AsyncURB *aurb = async_alloc(dev, p);
     struct usb_redir_bulk_packet_header bulk_packet;
+    size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
+
+    if (usbredir_already_in_flight(dev, p->id)) {
+        p->status = USB_RET_ASYNC;
+        return;
+    }
+
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_enabled) {
+        if (size != 0 && (size % maxp) == 0) {
+            usbredir_handle_buffered_bulk_in_data(dev, p, ep);
+            return;
+        }
+        WARNING("bulk recv invalid size %zd ep %02x, disabling\n", size, ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        usbredir_stop_bulk_receiving(dev, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_enabled = 0;
+    }
 
-    DPRINTF("bulk-out ep %02X len %zd id %u\n", ep,
-            p->iov.size, aurb->packet_id);
+    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
 
     bulk_packet.endpoint  = ep;
-    bulk_packet.length    = p->iov.size;
+    bulk_packet.length    = size;
     bulk_packet.stream_id = 0;
-    aurb->bulk_packet = bulk_packet;
+    bulk_packet.length_high = size >> 16;
+    assert(bulk_packet.length_high == 0 ||
+           usbredirparser_peer_has_cap(dev->parser,
+                                       usb_redir_cap_32bits_bulk_length));
 
     if (ep & USB_DIR_IN) {
-        usbredirparser_send_bulk_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_bulk_packet(dev->parser, p->id,
                                         &bulk_packet, NULL, 0);
     } else {
-        uint8_t buf[p->iov.size];
-        usb_packet_copy(p, buf, p->iov.size);
-        usbredir_log_data(dev, "bulk data out:", buf, p->iov.size);
-        usbredirparser_send_bulk_packet(dev->parser, aurb->packet_id,
-                                        &bulk_packet, buf, p->iov.size);
+        uint8_t buf[size];
+        if (p->combined) {
+            iov_to_buf(p->combined->iov.iov, p->combined->iov.niov,
+                       0, buf, size);
+        } else {
+            usb_packet_copy(p, buf, size);
+        }
+        usbredir_log_data(dev, "bulk data out:", buf, size);
+        usbredirparser_send_bulk_packet(dev->parser, p->id,
+                                        &bulk_packet, buf, size);
     }
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_handle_interrupt_data(USBRedirDevice *dev,
-                                           USBPacket *p, uint8_t ep)
+static void usbredir_handle_interrupt_in_data(USBRedirDevice *dev,
+                                              USBPacket *p, uint8_t ep)
 {
-    if (ep & USB_DIR_IN) {
-        /* Input interrupt endpoint, buffered packet input */
-        struct buf_packet *intp;
-        int status, len;
-
-        if (!dev->endpoint[EP2I(ep)].interrupt_started &&
-                !dev->endpoint[EP2I(ep)].interrupt_error) {
-            struct usb_redir_start_interrupt_receiving_header start_int = {
-                .endpoint = ep,
-            };
-            /* No id, we look at the ep when receiving a status back */
-            usbredirparser_send_start_interrupt_receiving(dev->parser, 0,
-                                                          &start_int);
-            usbredirparser_do_write(dev->parser);
-            DPRINTF("interrupt recv started ep %02X\n", ep);
-            dev->endpoint[EP2I(ep)].interrupt_started = 1;
-            /* We don't really want to drop interrupt packets ever, but
-               having some upper limit to how much we buffer is good. */
-            dev->endpoint[EP2I(ep)].bufpq_target_size = 1000;
-            dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
-        }
+    /* Input interrupt endpoint, buffered packet input */
+    struct buf_packet *intp;
+    int status, len;
 
-        intp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq);
-        if (intp == NULL) {
-            DPRINTF2("interrupt-token-in ep %02X, no intp\n", ep);
-            /* Check interrupt_error for stream errors */
-            status = dev->endpoint[EP2I(ep)].interrupt_error;
-            dev->endpoint[EP2I(ep)].interrupt_error = 0;
-            if (status) {
-                return usbredir_handle_status(dev, status, 0);
-            }
-            return USB_RET_NAK;
-        }
-        DPRINTF("interrupt-token-in ep %02X status %d len %d\n", ep,
-                intp->status, intp->len);
+    if (!dev->endpoint[EP2I(ep)].interrupt_started &&
+            !dev->endpoint[EP2I(ep)].interrupt_error) {
+        struct usb_redir_start_interrupt_receiving_header start_int = {
+            .endpoint = ep,
+        };
+        /* No id, we look at the ep when receiving a status back */
+        usbredirparser_send_start_interrupt_receiving(dev->parser, 0,
+                                                      &start_int);
+        usbredirparser_do_write(dev->parser);
+        DPRINTF("interrupt recv started ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].interrupt_started = 1;
+        /* We don't really want to drop interrupt packets ever, but
+           having some upper limit to how much we buffer is good. */
+        dev->endpoint[EP2I(ep)].bufpq_target_size = 1000;
+        dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
+    }
 
-        status = intp->status;
-        if (status != usb_redir_success) {
-            bufp_free(dev, intp, ep);
-            return usbredir_handle_status(dev, status, 0);
+    intp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq);
+    if (intp == NULL) {
+        DPRINTF2("interrupt-token-in ep %02X, no intp\n", ep);
+        /* Check interrupt_error for stream errors */
+        status = dev->endpoint[EP2I(ep)].interrupt_error;
+        dev->endpoint[EP2I(ep)].interrupt_error = 0;
+        if (status) {
+            usbredir_handle_status(dev, p, status);
+        } else {
+            p->status = USB_RET_NAK;
         }
+        return;
+    }
+    DPRINTF("interrupt-token-in ep %02X status %d len %d\n", ep,
+            intp->status, intp->len);
 
-        len = intp->len;
-        if (len > p->iov.size) {
-            ERROR("received int data is larger then packet ep %02X\n", ep);
-            bufp_free(dev, intp, ep);
-            return USB_RET_BABBLE;
-        }
-        usb_packet_copy(p, intp->data, len);
-        bufp_free(dev, intp, ep);
-        return len;
-    } else {
-        /* Output interrupt endpoint, normal async operation */
-        AsyncURB *aurb = async_alloc(dev, p);
-        struct usb_redir_interrupt_packet_header interrupt_packet;
-        uint8_t buf[p->iov.size];
-
-        DPRINTF("interrupt-out ep %02X len %zd id %u\n", ep, p->iov.size,
-                aurb->packet_id);
-
-        interrupt_packet.endpoint  = ep;
-        interrupt_packet.length    = p->iov.size;
-        aurb->interrupt_packet     = interrupt_packet;
-
-        usb_packet_copy(p, buf, p->iov.size);
-        usbredir_log_data(dev, "interrupt data out:", buf, p->iov.size);
-        usbredirparser_send_interrupt_packet(dev->parser, aurb->packet_id,
-                                        &interrupt_packet, buf, p->iov.size);
-        usbredirparser_do_write(dev->parser);
-        return USB_RET_ASYNC;
+    status = intp->status;
+    len = intp->len;
+    if (len > p->iov.size) {
+        ERROR("received int data is larger then packet ep %02X\n", ep);
+        len = p->iov.size;
+        status = usb_redir_babble;
     }
+    usb_packet_copy(p, intp->data, len);
+    bufp_free(dev, intp, ep);
+    usbredir_handle_status(dev, p, status);
+}
+
+/*
+ * Handle interrupt out data, the usbredir protocol expects us to do this
+ * async, so that it can report back a completion status. But guests will
+ * expect immediate completion for an interrupt endpoint, and handling this
+ * async causes migration issues. So we report success directly, counting
+ * on the fact that output interrupt packets normally always succeed.
+ */
+static void usbredir_handle_interrupt_out_data(USBRedirDevice *dev,
+                                               USBPacket *p, uint8_t ep)
+{
+    struct usb_redir_interrupt_packet_header interrupt_packet;
+    uint8_t buf[p->iov.size];
+
+    DPRINTF("interrupt-out ep %02X len %zd id %"PRIu64"\n", ep,
+            p->iov.size, p->id);
+
+    interrupt_packet.endpoint  = ep;
+    interrupt_packet.length    = p->iov.size;
+
+    usb_packet_copy(p, buf, p->iov.size);
+    usbredir_log_data(dev, "interrupt data out:", buf, p->iov.size);
+    usbredirparser_send_interrupt_packet(dev->parser, p->id,
+                                    &interrupt_packet, buf, p->iov.size);
+    usbredirparser_do_write(dev->parser);
 }
 
 static void usbredir_stop_interrupt_receiving(USBRedirDevice *dev,
@@ -609,7 +879,7 @@ static void usbredir_stop_interrupt_receiving(USBRedirDevice *dev,
     usbredir_free_bufpq(dev, ep);
 }
 
-static int usbredir_handle_data(USBDevice *udev, USBPacket *p)
+static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
     uint8_t ep;
@@ -622,142 +892,166 @@ static int usbredir_handle_data(USBDevice *udev, USBPacket *p)
     switch (dev->endpoint[EP2I(ep)].type) {
     case USB_ENDPOINT_XFER_CONTROL:
         ERROR("handle_data called for control transfer on ep %02X\n", ep);
-        return USB_RET_NAK;
-    case USB_ENDPOINT_XFER_ISOC:
-        return usbredir_handle_iso_data(dev, p, ep);
+        p->status = USB_RET_NAK;
+        break;
     case USB_ENDPOINT_XFER_BULK:
-        return usbredir_handle_bulk_data(dev, p, ep);
+        if (p->state == USB_PACKET_SETUP && p->pid == USB_TOKEN_IN &&
+                p->ep->pipeline) {
+            p->status = USB_RET_ADD_TO_QUEUE;
+            break;
+        }
+        usbredir_handle_bulk_data(dev, p, ep);
+        break;
+    case USB_ENDPOINT_XFER_ISOC:
+        usbredir_handle_iso_data(dev, p, ep);
+        break;
     case USB_ENDPOINT_XFER_INT:
-        return usbredir_handle_interrupt_data(dev, p, ep);
+        if (ep & USB_DIR_IN) {
+            usbredir_handle_interrupt_in_data(dev, p, ep);
+        } else {
+            usbredir_handle_interrupt_out_data(dev, p, ep);
+        }
+        break;
     default:
         ERROR("handle_data ep %02X has unknown type %d\n", ep,
               dev->endpoint[EP2I(ep)].type);
-        return USB_RET_NAK;
+        p->status = USB_RET_NAK;
+    }
+}
+
+static void usbredir_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
+{
+    if (ep->pid == USB_TOKEN_IN && ep->pipeline) {
+        usb_ep_combine_input_packets(ep);
+    }
+}
+
+static void usbredir_stop_ep(USBRedirDevice *dev, int i)
+{
+    uint8_t ep = I2EP(i);
+
+    switch (dev->endpoint[i].type) {
+    case USB_ENDPOINT_XFER_BULK:
+        if (ep & USB_DIR_IN) {
+            usbredir_stop_bulk_receiving(dev, ep);
+        }
+        break;
+    case USB_ENDPOINT_XFER_ISOC:
+        usbredir_stop_iso_stream(dev, ep);
+        break;
+    case USB_ENDPOINT_XFER_INT:
+        if (ep & USB_DIR_IN) {
+            usbredir_stop_interrupt_receiving(dev, ep);
+        }
+        break;
     }
+    usbredir_free_bufpq(dev, ep);
 }
 
-static int usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_ep_stopped(USBDevice *udev, USBEndpoint *uep)
+{
+    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+
+    usbredir_stop_ep(dev, USBEP2I(uep));
+    usbredirparser_do_write(dev->parser);
+}
+
+static void usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
                                 int config)
 {
     struct usb_redir_set_configuration_header set_config;
-    AsyncURB *aurb = async_alloc(dev, p);
     int i;
 
-    DPRINTF("set config %d id %u\n", config, aurb->packet_id);
+    DPRINTF("set config %d id %"PRIu64"\n", config, p->id);
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
-        switch (dev->endpoint[i].type) {
-        case USB_ENDPOINT_XFER_ISOC:
-            usbredir_stop_iso_stream(dev, I2EP(i));
-            break;
-        case USB_ENDPOINT_XFER_INT:
-            if (i & 0x10) {
-                usbredir_stop_interrupt_receiving(dev, I2EP(i));
-            }
-            break;
-        }
-        usbredir_free_bufpq(dev, I2EP(i));
+        usbredir_stop_ep(dev, i);
     }
 
     set_config.configuration = config;
-    usbredirparser_send_set_configuration(dev->parser, aurb->packet_id,
-                                          &set_config);
+    usbredirparser_send_set_configuration(dev->parser, p->id, &set_config);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_get_config(USBRedirDevice *dev, USBPacket *p)
+static void usbredir_get_config(USBRedirDevice *dev, USBPacket *p)
 {
-    AsyncURB *aurb = async_alloc(dev, p);
+    DPRINTF("get config id %"PRIu64"\n", p->id);
 
-    DPRINTF("get config id %u\n", aurb->packet_id);
-
-    aurb->get = 1;
-    usbredirparser_send_get_configuration(dev->parser, aurb->packet_id);
+    usbredirparser_send_get_configuration(dev->parser, p->id);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
                                    int interface, int alt)
 {
     struct usb_redir_set_alt_setting_header set_alt;
-    AsyncURB *aurb = async_alloc(dev, p);
     int i;
 
-    DPRINTF("set interface %d alt %d id %u\n", interface, alt,
-            aurb->packet_id);
+    DPRINTF("set interface %d alt %d id %"PRIu64"\n", interface, alt, p->id);
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         if (dev->endpoint[i].interface == interface) {
-            switch (dev->endpoint[i].type) {
-            case USB_ENDPOINT_XFER_ISOC:
-                usbredir_stop_iso_stream(dev, I2EP(i));
-                break;
-            case USB_ENDPOINT_XFER_INT:
-                if (i & 0x10) {
-                    usbredir_stop_interrupt_receiving(dev, I2EP(i));
-                }
-                break;
-            }
-            usbredir_free_bufpq(dev, I2EP(i));
+            usbredir_stop_ep(dev, i);
         }
     }
 
     set_alt.interface = interface;
     set_alt.alt = alt;
-    usbredirparser_send_set_alt_setting(dev->parser, aurb->packet_id,
-                                        &set_alt);
+    usbredirparser_send_set_alt_setting(dev->parser, p->id, &set_alt);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_get_interface(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_get_interface(USBRedirDevice *dev, USBPacket *p,
                                    int interface)
 {
     struct usb_redir_get_alt_setting_header get_alt;
-    AsyncURB *aurb = async_alloc(dev, p);
 
-    DPRINTF("get interface %d id %u\n", interface, aurb->packet_id);
+    DPRINTF("get interface %d id %"PRIu64"\n", interface, p->id);
 
     get_alt.interface = interface;
-    aurb->get = 1;
-    usbredirparser_send_get_alt_setting(dev->parser, aurb->packet_id,
-                                        &get_alt);
+    usbredirparser_send_get_alt_setting(dev->parser, p->id, &get_alt);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
+static void usbredir_handle_control(USBDevice *udev, USBPacket *p,
         int request, int value, int index, int length, uint8_t *data)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
     struct usb_redir_control_packet_header control_packet;
-    AsyncURB *aurb;
+
+    if (usbredir_already_in_flight(dev, p->id)) {
+        p->status = USB_RET_ASYNC;
+        return;
+    }
 
     /* Special cases for certain standard device requests */
     switch (request) {
     case DeviceOutRequest | USB_REQ_SET_ADDRESS:
         DPRINTF("set address %d\n", value);
         dev->dev.addr = value;
-        return 0;
+        return;
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
-        return usbredir_set_config(dev, p, value & 0xff);
+        usbredir_set_config(dev, p, value & 0xff);
+        return;
     case DeviceRequest | USB_REQ_GET_CONFIGURATION:
-        return usbredir_get_config(dev, p);
+        usbredir_get_config(dev, p);
+        return;
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
-        return usbredir_set_interface(dev, p, index, value);
+        usbredir_set_interface(dev, p, index, value);
+        return;
     case InterfaceRequest | USB_REQ_GET_INTERFACE:
-        return usbredir_get_interface(dev, p, index);
+        usbredir_get_interface(dev, p, index);
+        return;
     }
 
-    /* "Normal" ctrl requests */
-    aurb = async_alloc(dev, p);
-
-    /* Note request is (bRequestType << 8) | bRequest */
-    DPRINTF("ctrl-out type 0x%x req 0x%x val 0x%x index %d len %d id %u\n",
-            request >> 8, request & 0xff, value, index, length,
-            aurb->packet_id);
+    /* Normal ctrl requests, note request is (bRequestType << 8) | bRequest */
+    DPRINTF(
+        "ctrl-out type 0x%x req 0x%x val 0x%x index %d len %d id %"PRIu64"\n",
+        request >> 8, request & 0xff, value, index, length, p->id);
 
     control_packet.request     = request & 0xFF;
     control_packet.requesttype = request >> 8;
@@ -765,18 +1059,17 @@ static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
     control_packet.value       = value;
     control_packet.index       = index;
     control_packet.length      = length;
-    aurb->control_packet       = control_packet;
 
     if (control_packet.requesttype & USB_DIR_IN) {
-        usbredirparser_send_control_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_control_packet(dev->parser, p->id,
                                            &control_packet, NULL, 0);
     } else {
         usbredir_log_data(dev, "ctrl data out:", data, length);
-        usbredirparser_send_control_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_control_packet(dev->parser, p->id,
                                            &control_packet, data, length);
     }
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
 /*
@@ -784,53 +1077,73 @@ static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
  * from within the USBDevice data / control packet callbacks and doing a
  * usb_detach from within these callbacks is not a good idea.
  *
- * So we use a bh handler to take care of close events. We also handle
- * open events from this callback to make sure that a close directly followed
- * by an open gets handled in the right order.
+ * So we use a bh handler to take care of close events.
  */
-static void usbredir_open_close_bh(void *opaque)
+static void usbredir_chardev_close_bh(void *opaque)
 {
     USBRedirDevice *dev = opaque;
-    uint32_t caps[USB_REDIR_CAPS_SIZE] = { 0, };
-    char version[32];
-
-    strcpy(version, "qemu usb-redir guest ");
-    pstrcat(version, sizeof(version), qemu_get_version());
 
     usbredir_device_disconnect(dev);
 
     if (dev->parser) {
+        DPRINTF("destroying usbredirparser\n");
         usbredirparser_destroy(dev->parser);
         dev->parser = NULL;
     }
+}
 
-    if (dev->cs->opened) {
-        dev->parser = qemu_oom_check(usbredirparser_create());
-        dev->parser->priv = dev;
-        dev->parser->log_func = usbredir_log;
-        dev->parser->read_func = usbredir_read;
-        dev->parser->write_func = usbredir_write;
-        dev->parser->hello_func = usbredir_hello;
-        dev->parser->device_connect_func = usbredir_device_connect;
-        dev->parser->device_disconnect_func = usbredir_device_disconnect;
-        dev->parser->interface_info_func = usbredir_interface_info;
-        dev->parser->ep_info_func = usbredir_ep_info;
-        dev->parser->configuration_status_func = usbredir_configuration_status;
-        dev->parser->alt_setting_status_func = usbredir_alt_setting_status;
-        dev->parser->iso_stream_status_func = usbredir_iso_stream_status;
-        dev->parser->interrupt_receiving_status_func =
-            usbredir_interrupt_receiving_status;
-        dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
-        dev->parser->control_packet_func = usbredir_control_packet;
-        dev->parser->bulk_packet_func = usbredir_bulk_packet;
-        dev->parser->iso_packet_func = usbredir_iso_packet;
-        dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
-        dev->read_buf = NULL;
-        dev->read_buf_size = 0;
+static void usbredir_create_parser(USBRedirDevice *dev)
+{
+    uint32_t caps[USB_REDIR_CAPS_SIZE] = { 0, };
+    int flags = 0;
+
+    DPRINTF("creating usbredirparser\n");
+
+    dev->parser = qemu_oom_check(usbredirparser_create());
+    dev->parser->priv = dev;
+    dev->parser->log_func = usbredir_log;
+    dev->parser->read_func = usbredir_read;
+    dev->parser->write_func = usbredir_write;
+    dev->parser->hello_func = usbredir_hello;
+    dev->parser->device_connect_func = usbredir_device_connect;
+    dev->parser->device_disconnect_func = usbredir_device_disconnect;
+    dev->parser->interface_info_func = usbredir_interface_info;
+    dev->parser->ep_info_func = usbredir_ep_info;
+    dev->parser->configuration_status_func = usbredir_configuration_status;
+    dev->parser->alt_setting_status_func = usbredir_alt_setting_status;
+    dev->parser->iso_stream_status_func = usbredir_iso_stream_status;
+    dev->parser->interrupt_receiving_status_func =
+        usbredir_interrupt_receiving_status;
+    dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
+    dev->parser->bulk_receiving_status_func = usbredir_bulk_receiving_status;
+    dev->parser->control_packet_func = usbredir_control_packet;
+    dev->parser->bulk_packet_func = usbredir_bulk_packet;
+    dev->parser->iso_packet_func = usbredir_iso_packet;
+    dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
+    dev->parser->buffered_bulk_packet_func = usbredir_buffered_bulk_packet;
+    dev->read_buf = NULL;
+    dev->read_buf_size = 0;
+
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_connect_device_version);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_filter);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_ep_info_max_packet_size);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_64bits_ids);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_receiving);
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        flags |= usbredirparser_fl_no_hello;
+    }
+    usbredirparser_init(dev->parser, VERSION, caps, USB_REDIR_CAPS_SIZE,
+                        flags);
+    usbredirparser_do_write(dev->parser);
+}
 
-        usbredirparser_caps_set_cap(caps, usb_redir_cap_connect_device_version);
-        usbredirparser_caps_set_cap(caps, usb_redir_cap_filter);
-        usbredirparser_init(dev->parser, version, caps, USB_REDIR_CAPS_SIZE, 0);
+static void usbredir_reject_device(USBRedirDevice *dev)
+{
+    usbredir_device_disconnect(dev);
+    if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
+        usbredirparser_send_filter_reject(dev->parser);
         usbredirparser_do_write(dev->parser);
     }
 }
@@ -839,12 +1152,22 @@ static void usbredir_do_attach(void *opaque)
 {
     USBRedirDevice *dev = opaque;
 
+    /* In order to work properly with XHCI controllers we need these caps */
+    if ((dev->dev.port->speedmask & USB_SPEED_MASK_SUPER) && !(
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_ep_info_max_packet_size) &&
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_32bits_bulk_length) &&
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_64bits_ids))) {
+        ERROR("usb-redir-host lacks capabilities needed for use with XHCI\n");
+        usbredir_reject_device(dev);
+        return;
+    }
+
     if (usb_device_attach(&dev->dev) != 0) {
-        usbredir_device_disconnect(dev);
-        if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
-            usbredirparser_send_filter_reject(dev->parser);
-            usbredirparser_do_write(dev->parser);
-        }
+        WARNING("rejecting device due to speed mismatch\n");
+        usbredir_reject_device(dev);
     }
 }
 
@@ -856,13 +1179,18 @@ static int usbredir_chardev_can_read(void *opaque)
 {
     USBRedirDevice *dev = opaque;
 
-    if (dev->parser) {
-        /* usbredir_parser_do_read will consume *all* data we give it */
-        return 1024 * 1024;
-    } else {
-        /* usbredir_open_close_bh hasn't handled the open event yet */
+    if (!dev->parser) {
+        WARNING("chardev_can_read called on non open chardev!\n");
         return 0;
     }
+
+    /* Don't read new data from the chardev until our state is fully synced */
+    if (!runstate_check(RUN_STATE_RUNNING)) {
+        return 0;
+    }
+
+    /* usbredir_parser_do_read will consume *all* data we give it */
+    return 1024 * 1024;
 }
 
 static void usbredir_chardev_read(void *opaque, const uint8_t *buf, int size)
@@ -886,8 +1214,15 @@ static void usbredir_chardev_event(void *opaque, int event)
 
     switch (event) {
     case CHR_EVENT_OPENED:
+        DPRINTF("chardev open\n");
+        /* Make sure any pending closes are handled (no-op if none pending) */
+        usbredir_chardev_close_bh(dev);
+        qemu_bh_cancel(dev->chardev_close_bh);
+        usbredir_create_parser(dev);
+        break;
     case CHR_EVENT_CLOSED:
-        qemu_bh_schedule(dev->open_close_bh);
+        DPRINTF("chardev close\n");
+        qemu_bh_schedule(dev->chardev_close_bh);
         break;
     }
 }
@@ -896,6 +1231,27 @@ static void usbredir_chardev_event(void *opaque, int event)
  * init + destroy
  */
 
+static void usbredir_vm_state_change(void *priv, int running, RunState state)
+{
+    USBRedirDevice *dev = priv;
+
+    if (state == RUN_STATE_RUNNING && dev->parser != NULL) {
+        usbredirparser_do_write(dev->parser); /* Flush any pending writes */
+    }
+}
+
+static void usbredir_init_endpoints(USBRedirDevice *dev)
+{
+    int i;
+
+    usb_ep_init(&dev->dev);
+    memset(dev->endpoint, 0, sizeof(dev->endpoint));
+    for (i = 0; i < MAX_ENDPOINTS; i++) {
+        dev->endpoint[i].dev = dev;
+        QTAILQ_INIT(&dev->endpoint[i].bufpq);
+    }
+}
+
 static int usbredir_initfn(USBDevice *udev)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
@@ -917,34 +1273,35 @@ static int usbredir_initfn(USBDevice *udev)
         }
     }
 
-    dev->open_close_bh = qemu_bh_new(usbredir_open_close_bh, dev);
+    dev->chardev_close_bh = qemu_bh_new(usbredir_chardev_close_bh, dev);
     dev->attach_timer = qemu_new_timer_ms(vm_clock, usbredir_do_attach, dev);
 
-    QTAILQ_INIT(&dev->asyncq);
-    for (i = 0; i < MAX_ENDPOINTS; i++) {
-        QTAILQ_INIT(&dev->endpoint[i].bufpq);
-    }
+    packet_id_queue_init(&dev->cancelled, dev, "cancelled");
+    packet_id_queue_init(&dev->already_in_flight, dev, "already-in-flight");
+    usbredir_init_endpoints(dev);
 
     /* We'll do the attach once we receive the speed from the usb-host */
     udev->auto_attach = 0;
 
+    /* Will be cleared during setup when we find conflicts */
+    dev->compatible_speedmask = USB_SPEED_MASK_FULL | USB_SPEED_MASK_HIGH;
+
     /* Let the backend know we are ready */
     qemu_chr_fe_open(dev->cs);
     qemu_chr_add_handlers(dev->cs, usbredir_chardev_can_read,
                           usbredir_chardev_read, usbredir_chardev_event, dev);
 
+    qemu_add_vm_change_state_handler(usbredir_vm_state_change, dev);
     add_boot_device_path(dev->bootindex, &udev->qdev, NULL);
     return 0;
 }
 
 static void usbredir_cleanup_device_queues(USBRedirDevice *dev)
 {
-    AsyncURB *aurb, *next_aurb;
     int i;
 
-    QTAILQ_FOREACH_SAFE(aurb, &dev->asyncq, next, next_aurb) {
-        async_free(dev, aurb);
-    }
+    packet_id_queue_empty(&dev->cancelled);
+    packet_id_queue_empty(&dev->already_in_flight);
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         usbredir_free_bufpq(dev, I2EP(i));
     }
@@ -957,7 +1314,7 @@ static void usbredir_handle_destroy(USBDevice *udev)
     qemu_chr_fe_close(dev->cs);
     qemu_chr_delete(dev->cs);
     /* Note must be done after qemu_chr_close, as that causes a close event */
-    qemu_bh_delete(dev->open_close_bh);
+    qemu_bh_delete(dev->chardev_close_bh);
 
     qemu_del_timer(dev->attach_timer);
     qemu_free_timer(dev->attach_timer);
@@ -1007,38 +1364,88 @@ static int usbredir_check_filter(USBRedirDevice *dev)
     return 0;
 
 error:
-    usbredir_device_disconnect(dev);
-    if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
-        usbredirparser_send_filter_reject(dev->parser);
-        usbredirparser_do_write(dev->parser);
-    }
+    usbredir_reject_device(dev);
     return -1;
 }
 
+static void usbredir_check_bulk_receiving(USBRedirDevice *dev)
+{
+    int i, j, quirks;
+
+    if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_bulk_receiving)) {
+        return;
+    }
+
+    for (i = EP2I(USB_DIR_IN); i < MAX_ENDPOINTS; i++) {
+        dev->endpoint[i].bulk_receiving_enabled = 0;
+    }
+    for (i = 0; i < dev->interface_info.interface_count; i++) {
+        quirks = usb_get_quirks(dev->device_info.vendor_id,
+                                dev->device_info.product_id,
+                                dev->interface_info.interface_class[i],
+                                dev->interface_info.interface_subclass[i],
+                                dev->interface_info.interface_protocol[i]);
+        if (!(quirks & USB_QUIRK_BUFFER_BULK_IN)) {
+            continue;
+        }
+        if (quirks & USB_QUIRK_IS_FTDI) {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_ftdi;
+        } else {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_raw;
+        }
+
+        for (j = EP2I(USB_DIR_IN); j < MAX_ENDPOINTS; j++) {
+            if (dev->endpoint[j].interface ==
+                                    dev->interface_info.interface[i] &&
+                    dev->endpoint[j].type == USB_ENDPOINT_XFER_BULK &&
+                    dev->endpoint[j].max_packet_size != 0) {
+                dev->endpoint[j].bulk_receiving_enabled = 1;
+                /*
+                 * With buffering pipelining is not necessary. Also packet
+                 * combining and bulk in buffering don't play nice together!
+                 */
+                I2USBEP(dev, j)->pipeline = false;
+                break; /* Only buffer for the first ep of each intf */
+            }
+        }
+    }
+}
+
 /*
  * usbredirparser packet complete callbacks
  */
 
-static int usbredir_handle_status(USBRedirDevice *dev,
-                                       int status, int actual_len)
+static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
+    int status)
 {
     switch (status) {
     case usb_redir_success:
-        return actual_len;
+        p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
+        break;
     case usb_redir_stall:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
     case usb_redir_cancelled:
-        WARNING("returning cancelled packet to HC?\n");
-        return USB_RET_NAK;
+        /*
+         * When the usbredir-host unredirects a device, it will report a status
+         * of cancelled for all pending packets, followed by a disconnect msg.
+         */
+        p->status = USB_RET_IOERROR;
+        break;
     case usb_redir_inval:
         WARNING("got invalid param error from usb-host?\n");
-        return USB_RET_NAK;
+        p->status = USB_RET_IOERROR;
+        break;
     case usb_redir_babble:
-        return USB_RET_BABBLE;
+        p->status = USB_RET_BABBLE;
+        break;
     case usb_redir_ioerror:
     case usb_redir_timeout:
     default:
-        return USB_RET_IOERROR;
+        p->status = USB_RET_IOERROR;
     }
 }
 
@@ -1070,10 +1477,13 @@ static void usbredir_device_connect(void *priv,
     case usb_redir_speed_low:
         speed = "low speed";
         dev->dev.speed = USB_SPEED_LOW;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_FULL;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_HIGH;
         break;
     case usb_redir_speed_full:
         speed = "full speed";
         dev->dev.speed = USB_SPEED_FULL;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_HIGH;
         break;
     case usb_redir_speed_high:
         speed = "high speed";
@@ -1103,7 +1513,7 @@ static void usbredir_device_connect(void *priv,
              device_connect->device_class);
     }
 
-    dev->dev.speedmask = (1 << dev->dev.speed);
+    dev->dev.speedmask = (1 << dev->dev.speed) | dev->compatible_speedmask;
     dev->device_info = *device_connect;
 
     if (usbredir_check_filter(dev)) {
@@ -1112,18 +1522,19 @@ static void usbredir_device_connect(void *priv,
         return;
     }
 
+    usbredir_check_bulk_receiving(dev);
     qemu_mod_timer(dev->attach_timer, dev->next_attach_time);
 }
 
 static void usbredir_device_disconnect(void *priv)
 {
     USBRedirDevice *dev = priv;
-    int i;
 
     /* Stop any pending attaches */
     qemu_del_timer(dev->attach_timer);
 
     if (dev->dev.attached) {
+        DPRINTF("detaching device\n");
         usb_device_detach(&dev->dev);
         /*
          * Delay next usb device attach to give the guest a chance to see
@@ -1134,14 +1545,11 @@ static void usbredir_device_disconnect(void *priv)
 
     /* Reset state so that the next dev connected starts with a clean slate */
     usbredir_cleanup_device_queues(dev);
-    memset(dev->endpoint, 0, sizeof(dev->endpoint));
-    for (i = 0; i < MAX_ENDPOINTS; i++) {
-        QTAILQ_INIT(&dev->endpoint[i].bufpq);
-    }
-    usb_ep_init(&dev->dev);
+    usbredir_init_endpoints(dev);
     dev->interface_info.interface_count = NO_INTERFACE_INFO;
     dev->dev.addr = 0;
     dev->dev.speed = 0;
+    dev->compatible_speedmask = USB_SPEED_MASK_FULL | USB_SPEED_MASK_HIGH;
 }
 
 static void usbredir_interface_info(void *priv,
@@ -1153,9 +1561,10 @@ static void usbredir_interface_info(void *priv,
 
     /*
      * If we receive interface info after the device has already been
-     * connected (ie on a set_config), re-check the filter.
+     * connected (ie on a set_config), re-check interface dependent things.
      */
     if (qemu_timer_pending(dev->attach_timer) || dev->dev.attached) {
+        usbredir_check_bulk_receiving(dev);
         if (usbredir_check_filter(dev)) {
             ERROR("Device no longer matches filter after interface info "
                   "change, disconnecting!\n");
@@ -1163,25 +1572,77 @@ static void usbredir_interface_info(void *priv,
     }
 }
 
+static void usbredir_mark_speed_incompatible(USBRedirDevice *dev, int speed)
+{
+    dev->compatible_speedmask &= ~(1 << speed);
+    dev->dev.speedmask = (1 << dev->dev.speed) | dev->compatible_speedmask;
+}
+
+static void usbredir_set_pipeline(USBRedirDevice *dev, struct USBEndpoint *uep)
+{
+    if (uep->type != USB_ENDPOINT_XFER_BULK) {
+        return;
+    }
+    if (uep->pid == USB_TOKEN_OUT) {
+        uep->pipeline = true;
+    }
+    if (uep->pid == USB_TOKEN_IN && uep->max_packet_size != 0 &&
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_32bits_bulk_length)) {
+        uep->pipeline = true;
+    }
+}
+
+static void usbredir_setup_usb_eps(USBRedirDevice *dev)
+{
+    struct USBEndpoint *usb_ep;
+    int i;
+
+    for (i = 0; i < MAX_ENDPOINTS; i++) {
+        usb_ep = I2USBEP(dev, i);
+        usb_ep->type = dev->endpoint[i].type;
+        usb_ep->ifnum = dev->endpoint[i].interface;
+        usb_ep->max_packet_size = dev->endpoint[i].max_packet_size;
+        usbredir_set_pipeline(dev, usb_ep);
+    }
+}
+
 static void usbredir_ep_info(void *priv,
     struct usb_redir_ep_info_header *ep_info)
 {
     USBRedirDevice *dev = priv;
-    struct USBEndpoint *usb_ep;
     int i;
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         dev->endpoint[i].type = ep_info->type[i];
         dev->endpoint[i].interval = ep_info->interval[i];
         dev->endpoint[i].interface = ep_info->interface[i];
+        if (usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size)) {
+            dev->endpoint[i].max_packet_size = ep_info->max_packet_size[i];
+        }
         switch (dev->endpoint[i].type) {
         case usb_redir_type_invalid:
             break;
         case usb_redir_type_iso:
+            usbredir_mark_speed_incompatible(dev, USB_SPEED_FULL);
+            usbredir_mark_speed_incompatible(dev, USB_SPEED_HIGH);
+            /* Fall through */
         case usb_redir_type_interrupt:
+            if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size) ||
+                    ep_info->max_packet_size[i] > 64) {
+                usbredir_mark_speed_incompatible(dev, USB_SPEED_FULL);
+            }
+            if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size) ||
+                    ep_info->max_packet_size[i] > 1024) {
+                usbredir_mark_speed_incompatible(dev, USB_SPEED_HIGH);
+            }
             if (dev->endpoint[i].interval == 0) {
                 ERROR("Received 0 interval for isoc or irq endpoint\n");
-                usbredir_device_disconnect(dev);
+                usbredir_reject_device(dev);
+                return;
             }
             /* Fall through */
         case usb_redir_type_control:
@@ -1191,78 +1652,70 @@ static void usbredir_ep_info(void *priv,
             break;
         default:
             ERROR("Received invalid endpoint type\n");
-            usbredir_device_disconnect(dev);
+            usbredir_reject_device(dev);
             return;
         }
-        usb_ep = usb_ep_get(&dev->dev,
-                            (i & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT,
-                            i & 0x0f);
-        usb_ep->type = dev->endpoint[i].type;
-        usb_ep->ifnum = dev->endpoint[i].interface;
     }
+    /* The new ep info may have caused a speed incompatibility, recheck */
+    if (dev->dev.attached &&
+            !(dev->dev.port->speedmask & dev->dev.speedmask)) {
+        ERROR("Device no longer matches speed after endpoint info change, "
+              "disconnecting!\n");
+        usbredir_reject_device(dev);
+        return;
+    }
+    usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
 }
 
-static void usbredir_configuration_status(void *priv, uint32_t id,
+static void usbredir_configuration_status(void *priv, uint64_t id,
     struct usb_redir_configuration_status_header *config_status)
 {
     USBRedirDevice *dev = priv;
-    AsyncURB *aurb;
-    int len = 0;
+    USBPacket *p;
 
-    DPRINTF("set config status %d config %d id %u\n", config_status->status,
-            config_status->configuration, id);
+    DPRINTF("set config status %d config %d id %"PRIu64"\n",
+            config_status->status, config_status->configuration, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        return;
-    }
-    if (aurb->packet) {
-        if (aurb->get) {
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        if (dev->dev.setup_buf[0] & USB_DIR_IN) {
             dev->dev.data_buf[0] = config_status->configuration;
-            len = 1;
+            p->actual_length = 1;
         }
-        aurb->packet->result =
-            usbredir_handle_status(dev, config_status->status, len);
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        usbredir_handle_status(dev, p, config_status->status);
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
 }
 
-static void usbredir_alt_setting_status(void *priv, uint32_t id,
+static void usbredir_alt_setting_status(void *priv, uint64_t id,
     struct usb_redir_alt_setting_status_header *alt_setting_status)
 {
     USBRedirDevice *dev = priv;
-    AsyncURB *aurb;
-    int len = 0;
+    USBPacket *p;
 
-    DPRINTF("alt status %d intf %d alt %d id: %u\n",
-            alt_setting_status->status,
-            alt_setting_status->interface,
+    DPRINTF("alt status %d intf %d alt %d id: %"PRIu64"\n",
+            alt_setting_status->status, alt_setting_status->interface,
             alt_setting_status->alt, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        return;
-    }
-    if (aurb->packet) {
-        if (aurb->get) {
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        if (dev->dev.setup_buf[0] & USB_DIR_IN) {
             dev->dev.data_buf[0] = alt_setting_status->alt;
-            len = 1;
+            p->actual_length = 1;
         }
-        aurb->packet->result =
-            usbredir_handle_status(dev, alt_setting_status->status, len);
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        usbredir_handle_status(dev, p, alt_setting_status->status);
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
 }
 
-static void usbredir_iso_stream_status(void *priv, uint32_t id,
+static void usbredir_iso_stream_status(void *priv, uint64_t id,
     struct usb_redir_iso_stream_status_header *iso_stream_status)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = iso_stream_status->endpoint;
 
-    DPRINTF("iso status %d ep %02X id %u\n", iso_stream_status->status,
+    DPRINTF("iso status %d ep %02X id %"PRIu64"\n", iso_stream_status->status,
             ep, id);
 
     if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].iso_started) {
@@ -1276,14 +1729,14 @@ static void usbredir_iso_stream_status(void *priv, uint32_t id,
     }
 }
 
-static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
+static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     struct usb_redir_interrupt_receiving_status_header
     *interrupt_receiving_status)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = interrupt_receiving_status->endpoint;
 
-    DPRINTF("interrupt recv status %d ep %02X id %u\n",
+    DPRINTF("interrupt recv status %d ep %02X id %"PRIu64"\n",
             interrupt_receiving_status->status, ep, id);
 
     if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].interrupt_started) {
@@ -1298,107 +1751,121 @@ static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
     }
 }
 
-static void usbredir_bulk_streams_status(void *priv, uint32_t id,
+static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status)
 {
 }
 
-static void usbredir_control_packet(void *priv, uint32_t id,
-    struct usb_redir_control_packet_header *control_packet,
-    uint8_t *data, int data_len)
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status)
 {
     USBRedirDevice *dev = priv;
-    int len = control_packet->length;
-    AsyncURB *aurb;
+    uint8_t ep = bulk_receiving_status->endpoint;
 
-    DPRINTF("ctrl-in status %d len %d id %u\n", control_packet->status,
-            len, id);
+    DPRINTF("bulk recv status %d ep %02X id %"PRIu64"\n",
+            bulk_receiving_status->status, ep, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        free(data);
+    if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].bulk_receiving_started) {
         return;
     }
 
-    aurb->control_packet.status = control_packet->status;
-    aurb->control_packet.length = control_packet->length;
-    if (memcmp(&aurb->control_packet, control_packet,
-               sizeof(*control_packet))) {
-        ERROR("return control packet mismatch, please report this!\n");
-        len = USB_RET_NAK;
+    if (bulk_receiving_status->status == usb_redir_stall) {
+        DPRINTF("bulk receiving stopped by peer ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
     }
+}
+
+static void usbredir_control_packet(void *priv, uint64_t id,
+    struct usb_redir_control_packet_header *control_packet,
+    uint8_t *data, int data_len)
+{
+    USBRedirDevice *dev = priv;
+    USBPacket *p;
+    int len = control_packet->length;
+
+    DPRINTF("ctrl-in status %d len %d id %"PRIu64"\n", control_packet->status,
+            len, id);
 
-    if (aurb->packet) {
-        len = usbredir_handle_status(dev, control_packet->status, len);
-        if (len > 0) {
+    /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
+     * to work redirected to a not superspeed capable hcd */
+    if (dev->dev.speed == USB_SPEED_SUPER &&
+            !((dev->dev.port->speedmask & USB_SPEED_MASK_SUPER)) &&
+            control_packet->requesttype == 0x80 &&
+            control_packet->request == 6 &&
+            control_packet->value == 0x100 && control_packet->index == 0 &&
+            data_len >= 18 && data[7] == 9) {
+        data[7] = 64;
+    }
+
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        usbredir_handle_status(dev, p, control_packet->status);
+        if (data_len > 0) {
             usbredir_log_data(dev, "ctrl data in:", data, data_len);
-            if (data_len <= sizeof(dev->dev.data_buf)) {
-                memcpy(dev->dev.data_buf, data, data_len);
-            } else {
+            if (data_len > sizeof(dev->dev.data_buf)) {
                 ERROR("ctrl buffer too small (%d > %zu)\n",
                       data_len, sizeof(dev->dev.data_buf));
-                len = USB_RET_STALL;
+                p->status = USB_RET_STALL;
+                data_len = len = sizeof(dev->dev.data_buf);
             }
+            memcpy(dev->dev.data_buf, data, data_len);
         }
-        aurb->packet->result = len;
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        p->actual_length = len;
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
     free(data);
 }
 
-static void usbredir_bulk_packet(void *priv, uint32_t id,
+static void usbredir_bulk_packet(void *priv, uint64_t id,
     struct usb_redir_bulk_packet_header *bulk_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = bulk_packet->endpoint;
-    int len = bulk_packet->length;
-    AsyncURB *aurb;
-
-    DPRINTF("bulk-in status %d ep %02X len %d id %u\n", bulk_packet->status,
-            ep, len, id);
-
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        free(data);
-        return;
-    }
+    int len = (bulk_packet->length_high << 16) | bulk_packet->length;
+    USBPacket *p;
 
-    if (aurb->bulk_packet.endpoint != bulk_packet->endpoint ||
-            aurb->bulk_packet.stream_id != bulk_packet->stream_id) {
-        ERROR("return bulk packet mismatch, please report this!\n");
-        len = USB_RET_NAK;
-    }
+    DPRINTF("bulk-in status %d ep %02X len %d id %"PRIu64"\n",
+            bulk_packet->status, ep, len, id);
 
-    if (aurb->packet) {
-        len = usbredir_handle_status(dev, bulk_packet->status, len);
-        if (len > 0) {
+    p = usbredir_find_packet_by_id(dev, ep, id);
+    if (p) {
+        size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
+        usbredir_handle_status(dev, p, bulk_packet->status);
+        if (data_len > 0) {
             usbredir_log_data(dev, "bulk data in:", data, data_len);
-            if (data_len <= aurb->packet->iov.size) {
-                usb_packet_copy(aurb->packet, data, data_len);
+            if (data_len > size) {
+                ERROR("bulk got more data then requested (%d > %zd)\n",
+                      data_len, p->iov.size);
+                p->status = USB_RET_BABBLE;
+                data_len = len = size;
+            }
+            if (p->combined) {
+                iov_from_buf(p->combined->iov.iov, p->combined->iov.niov,
+                             0, data, data_len);
             } else {
-                ERROR("bulk buffer too small (%d > %zd)\n", data_len,
-                      aurb->packet->iov.size);
-                len = USB_RET_STALL;
+                usb_packet_copy(p, data, data_len);
             }
         }
-        aurb->packet->result = len;
-        usb_packet_complete(&dev->dev, aurb->packet);
+        p->actual_length = len;
+        if (p->pid == USB_TOKEN_IN && p->ep->pipeline) {
+            usb_combined_input_packet_complete(&dev->dev, p);
+        } else {
+            usb_packet_complete(&dev->dev, p);
+        }
     }
-    async_free(dev, aurb);
     free(data);
 }
 
-static void usbredir_iso_packet(void *priv, uint32_t id,
+static void usbredir_iso_packet(void *priv, uint64_t id,
     struct usb_redir_iso_packet_header *iso_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = iso_packet->endpoint;
 
-    DPRINTF2("iso-in status %d ep %02X len %d id %u\n", iso_packet->status, ep,
-             data_len, id);
+    DPRINTF2("iso-in status %d ep %02X len %d id %"PRIu64"\n",
+             iso_packet->status, ep, data_len, id);
 
     if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_ISOC) {
         ERROR("received iso packet for non iso endpoint %02X\n", ep);
@@ -1413,17 +1880,17 @@ static void usbredir_iso_packet(void *priv, uint32_t id,
     }
 
     /* bufp_alloc also adds the packet to the ep queue */
-    bufp_alloc(dev, data, data_len, iso_packet->status, ep);
+    bufp_alloc(dev, data, data_len, iso_packet->status, ep, data);
 }
 
-static void usbredir_interrupt_packet(void *priv, uint32_t id,
+static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = interrupt_packet->endpoint;
 
-    DPRINTF("interrupt-in status %d ep %02X len %d id %u\n",
+    DPRINTF("interrupt-in status %d ep %02X len %d id %"PRIu64"\n",
             interrupt_packet->status, ep, data_len, id);
 
     if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_INT) {
@@ -1439,33 +1906,418 @@ static void usbredir_interrupt_packet(void *priv, uint32_t id,
             return;
         }
 
+        if (QTAILQ_EMPTY(&dev->endpoint[EP2I(ep)].bufpq)) {
+            usb_wakeup(usb_ep_get(&dev->dev, USB_TOKEN_IN, ep & 0x0f));
+        }
+
         /* bufp_alloc also adds the packet to the ep queue */
-        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep);
+        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep, data);
     } else {
-        int len = interrupt_packet->length;
-
-        AsyncURB *aurb = async_find(dev, id);
-        if (!aurb) {
-            return;
+        /*
+         * We report output interrupt packets as completed directly upon
+         * submission, so all we can do here if one failed is warn.
+         */
+        if (interrupt_packet->status) {
+            WARNING("interrupt output failed status %d ep %02X id %"PRIu64"\n",
+                    interrupt_packet->status, ep, id);
         }
+    }
+}
+
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t status, ep = buffered_bulk_packet->endpoint;
+    void *free_on_destroy;
+    int i, len;
 
-        if (aurb->interrupt_packet.endpoint != interrupt_packet->endpoint) {
-            ERROR("return int packet mismatch, please report this!\n");
-            len = USB_RET_NAK;
+    DPRINTF("buffered-bulk-in status %d ep %02X len %d id %"PRIu64"\n",
+            buffered_bulk_packet->status, ep, data_len, id);
+
+    if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_BULK) {
+        ERROR("received buffered-bulk packet for non bulk ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started == 0) {
+        DPRINTF("received buffered-bulk packet on not started ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    /* Data must be in maxp chunks for buffered_bulk_add_*_data_to_packet */
+    len = dev->endpoint[EP2I(ep)].max_packet_size;
+    status = usb_redir_success;
+    free_on_destroy = NULL;
+    for (i = 0; i < data_len; i += len) {
+        if (len >= (data_len - i)) {
+            len = data_len - i;
+            status = buffered_bulk_packet->status;
+            free_on_destroy = data;
         }
+        /* bufp_alloc also adds the packet to the ep queue */
+        bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+    }
+
+    if (dev->endpoint[EP2I(ep)].pending_async_packet) {
+        USBPacket *p = dev->endpoint[EP2I(ep)].pending_async_packet;
+        dev->endpoint[EP2I(ep)].pending_async_packet = NULL;
+        usbredir_buffered_bulk_in_complete(dev, p, ep);
+        usb_packet_complete(&dev->dev, p);
+    }
+}
+
+/*
+ * Migration code
+ */
+
+static void usbredir_pre_save(void *priv)
+{
+    USBRedirDevice *dev = priv;
+
+    usbredir_fill_already_in_flight(dev);
+}
+
+static int usbredir_post_load(void *priv, int version_id)
+{
+    USBRedirDevice *dev = priv;
+
+    switch (dev->device_info.speed) {
+    case usb_redir_speed_low:
+        dev->dev.speed = USB_SPEED_LOW;
+        break;
+    case usb_redir_speed_full:
+        dev->dev.speed = USB_SPEED_FULL;
+        break;
+    case usb_redir_speed_high:
+        dev->dev.speed = USB_SPEED_HIGH;
+        break;
+    case usb_redir_speed_super:
+        dev->dev.speed = USB_SPEED_SUPER;
+        break;
+    default:
+        dev->dev.speed = USB_SPEED_FULL;
+    }
+    dev->dev.speedmask = (1 << dev->dev.speed);
+
+    usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
+
+    return 0;
+}
+
+/* For usbredirparser migration */
+static void usbredir_put_parser(QEMUFile *f, void *priv, size_t unused)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t *data;
+    int len;
+
+    if (dev->parser == NULL) {
+        qemu_put_be32(f, 0);
+        return;
+    }
+
+    usbredirparser_serialize(dev->parser, &data, &len);
+    qemu_oom_check(data);
+
+    qemu_put_be32(f, len);
+    qemu_put_buffer(f, data, len);
+
+    free(data);
+}
+
+static int usbredir_get_parser(QEMUFile *f, void *priv, size_t unused)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t *data;
+    int len, ret;
+
+    len = qemu_get_be32(f);
+    if (len == 0) {
+        return 0;
+    }
 
-        if (aurb->packet) {
-            aurb->packet->result = usbredir_handle_status(dev,
-                                               interrupt_packet->status, len);
-            usb_packet_complete(&dev->dev, aurb->packet);
+    /*
+     * If our chardev is not open already at this point the usbredir connection
+     * has been broken (non seamless migration, or restore from disk).
+     *
+     * In this case create a temporary parser to receive the migration data,
+     * and schedule the close_bh to report the device as disconnected to the
+     * guest and to destroy the parser again.
+     */
+    if (dev->parser == NULL) {
+        WARNING("usb-redir connection broken during migration\n");
+        usbredir_create_parser(dev);
+        qemu_bh_schedule(dev->chardev_close_bh);
+    }
+
+    data = g_malloc(len);
+    qemu_get_buffer(f, data, len);
+
+    ret = usbredirparser_unserialize(dev->parser, data, len);
+
+    g_free(data);
+
+    return ret;
+}
+
+static const VMStateInfo usbredir_parser_vmstate_info = {
+    .name = "usb-redir-parser",
+    .put  = usbredir_put_parser,
+    .get  = usbredir_get_parser,
+};
+
+
+/* For buffered packets (iso/irq) queue migration */
+static void usbredir_put_bufpq(QEMUFile *f, void *priv, size_t unused)
+{
+    struct endp_data *endp = priv;
+    USBRedirDevice *dev = endp->dev;
+    struct buf_packet *bufp;
+    int len, i = 0;
+
+    qemu_put_be32(f, endp->bufpq_size);
+    QTAILQ_FOREACH(bufp, &endp->bufpq, next) {
+        len = bufp->len - bufp->offset;
+        DPRINTF("put_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
+                len, bufp->status);
+        qemu_put_be32(f, len);
+        qemu_put_be32(f, bufp->status);
+        qemu_put_buffer(f, bufp->data + bufp->offset, len);
+        i++;
+    }
+    assert(i == endp->bufpq_size);
+}
+
+static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
+{
+    struct endp_data *endp = priv;
+    USBRedirDevice *dev = endp->dev;
+    struct buf_packet *bufp;
+    int i;
+
+    endp->bufpq_size = qemu_get_be32(f);
+    for (i = 0; i < endp->bufpq_size; i++) {
+        bufp = g_malloc(sizeof(struct buf_packet));
+        bufp->len = qemu_get_be32(f);
+        bufp->status = qemu_get_be32(f);
+        bufp->offset = 0;
+        bufp->data = qemu_oom_check(malloc(bufp->len)); /* regular malloc! */
+        bufp->free_on_destroy = bufp->data;
+        qemu_get_buffer(f, bufp->data, bufp->len);
+        QTAILQ_INSERT_TAIL(&endp->bufpq, bufp, next);
+        DPRINTF("get_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
+                bufp->len, bufp->status);
+    }
+    return 0;
+}
+
+static const VMStateInfo usbredir_ep_bufpq_vmstate_info = {
+    .name = "usb-redir-bufpq",
+    .put  = usbredir_put_bufpq,
+    .get  = usbredir_get_bufpq,
+};
+
+
+/* For endp_data migration */
+static const VMStateDescription usbredir_bulk_receiving_vmstate = {
+    .name = "usb-redir-ep/bulk-receiving",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(bulk_receiving_started, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool usbredir_bulk_receiving_needed(void *priv)
+{
+    struct endp_data *endp = priv;
+
+    return endp->bulk_receiving_started;
+}
+
+static const VMStateDescription usbredir_ep_vmstate = {
+    .name = "usb-redir-ep",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(type, struct endp_data),
+        VMSTATE_UINT8(interval, struct endp_data),
+        VMSTATE_UINT8(interface, struct endp_data),
+        VMSTATE_UINT16(max_packet_size, struct endp_data),
+        VMSTATE_UINT8(iso_started, struct endp_data),
+        VMSTATE_UINT8(iso_error, struct endp_data),
+        VMSTATE_UINT8(interrupt_started, struct endp_data),
+        VMSTATE_UINT8(interrupt_error, struct endp_data),
+        VMSTATE_UINT8(bufpq_prefilled, struct endp_data),
+        VMSTATE_UINT8(bufpq_dropping_packets, struct endp_data),
+        {
+            .name         = "bufpq",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_ep_bufpq_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_INT32(bufpq_target_size, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &usbredir_bulk_receiving_vmstate,
+            .needed = usbredir_bulk_receiving_needed,
+        }, {
+            /* empty */
         }
-        async_free(dev, aurb);
     }
+};
+
+
+/* For PacketIdQueue migration */
+static void usbredir_put_packet_id_q(QEMUFile *f, void *priv, size_t unused)
+{
+    struct PacketIdQueue *q = priv;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
+    int remain = q->size;
+
+    DPRINTF("put_packet_id_q %s size %d\n", q->name, q->size);
+    qemu_put_be32(f, q->size);
+    QTAILQ_FOREACH(e, &q->head, next) {
+        qemu_put_be64(f, e->id);
+        remain--;
+    }
+    assert(remain == 0);
 }
 
+static int usbredir_get_packet_id_q(QEMUFile *f, void *priv, size_t unused)
+{
+    struct PacketIdQueue *q = priv;
+    USBRedirDevice *dev = q->dev;
+    int i, size;
+    uint64_t id;
+
+    size = qemu_get_be32(f);
+    DPRINTF("get_packet_id_q %s size %d\n", q->name, size);
+    for (i = 0; i < size; i++) {
+        id = qemu_get_be64(f);
+        packet_id_queue_add(q, id);
+    }
+    assert(q->size == size);
+    return 0;
+}
+
+static const VMStateInfo usbredir_ep_packet_id_q_vmstate_info = {
+    .name = "usb-redir-packet-id-q",
+    .put  = usbredir_put_packet_id_q,
+    .get  = usbredir_get_packet_id_q,
+};
+
+static const VMStateDescription usbredir_ep_packet_id_queue_vmstate = {
+    .name = "usb-redir-packet-id-queue",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "queue",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_ep_packet_id_q_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* For usb_redir_device_connect_header migration */
+static const VMStateDescription usbredir_device_info_vmstate = {
+    .name = "usb-redir-device-info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(speed, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_class, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_subclass, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_protocol, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(vendor_id, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(product_id, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(device_version_bcd,
+                       struct usb_redir_device_connect_header),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* For usb_redir_interface_info_header migration */
+static const VMStateDescription usbredir_interface_info_vmstate = {
+    .name = "usb-redir-interface-info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(interface_count,
+                       struct usb_redir_interface_info_header),
+        VMSTATE_UINT8_ARRAY(interface,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_class,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_subclass,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_protocol,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* And finally the USBRedirDevice vmstate itself */
+static const VMStateDescription usbredir_vmstate = {
+    .name = "usb-redir",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_save = usbredir_pre_save,
+    .post_load = usbredir_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_USB_DEVICE(dev, USBRedirDevice),
+        VMSTATE_TIMER(attach_timer, USBRedirDevice),
+        {
+            .name         = "parser",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_parser_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_STRUCT_ARRAY(endpoint, USBRedirDevice, MAX_ENDPOINTS, 1,
+                             usbredir_ep_vmstate, struct endp_data),
+        VMSTATE_STRUCT(cancelled, USBRedirDevice, 1,
+                       usbredir_ep_packet_id_queue_vmstate,
+                       struct PacketIdQueue),
+        VMSTATE_STRUCT(already_in_flight, USBRedirDevice, 1,
+                       usbredir_ep_packet_id_queue_vmstate,
+                       struct PacketIdQueue),
+        VMSTATE_STRUCT(device_info, USBRedirDevice, 1,
+                       usbredir_device_info_vmstate,
+                       struct usb_redir_device_connect_header),
+        VMSTATE_STRUCT(interface_info, USBRedirDevice, 1,
+                       usbredir_interface_info_vmstate,
+                       struct usb_redir_interface_info_header),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static Property usbredir_properties[] = {
     DEFINE_PROP_CHR("chardev", USBRedirDevice, cs),
-    DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, 0),
+    DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, usbredirparser_warning),
     DEFINE_PROP_STRING("filter", USBRedirDevice, filter_str),
     DEFINE_PROP_INT32("bootindex", USBRedirDevice, bootindex, -1),
     DEFINE_PROP_END_OF_LIST(),
@@ -1483,10 +2335,13 @@ static void usbredir_class_initfn(ObjectClass *klass, void *data)
     uc->handle_reset   = usbredir_handle_reset;
     uc->handle_data    = usbredir_handle_data;
     uc->handle_control = usbredir_handle_control;
+    uc->flush_ep_queue = usbredir_flush_ep_queue;
+    uc->ep_stopped     = usbredir_ep_stopped;
+    dc->vmsd           = &usbredir_vmstate;
     dc->props          = usbredir_properties;
 }
 
-static TypeInfo usbredir_dev_info = {
+static const TypeInfo usbredir_dev_info = {
     .name          = "usb-redir",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBRedirDevice),
diff --git a/hw/versatile_i2c.c b/hw/versatile_i2c.c
index 88f530aefcf..ad71e9d92d0 100644
--- a/hw/versatile_i2c.c
+++ b/hw/versatile_i2c.c
@@ -32,7 +32,7 @@ typedef struct {
     int in;
 } VersatileI2CState;
 
-static uint64_t versatile_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t versatile_i2c_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     VersatileI2CState *s = (VersatileI2CState *)opaque;
@@ -40,12 +40,13 @@ static uint64_t versatile_i2c_read(void *opaque, target_phys_addr_t offset,
     if (offset == 0) {
         return (s->out & 1) | (s->in << 1);
     } else {
-        hw_error("%s: Bad offset 0x%x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%x\n", __func__, (int)offset);
         return -1;
     }
 }
 
-static void versatile_i2c_write(void *opaque, target_phys_addr_t offset,
+static void versatile_i2c_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     VersatileI2CState *s = (VersatileI2CState *)opaque;
@@ -58,7 +59,8 @@ static void versatile_i2c_write(void *opaque, target_phys_addr_t offset,
         s->out &= ~value;
         break;
     default:
-        hw_error("%s: Bad offset 0x%x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%x\n", __func__, (int)offset);
     }
     bitbang_i2c_set(s->bitbang, BITBANG_I2C_SCL, (s->out & 1) != 0);
     s->in = bitbang_i2c_set(s->bitbang, BITBANG_I2C_SDA, (s->out & 2) != 0);
diff --git a/hw/versatile_pci.c b/hw/versatile_pci.c
index ae53a8b37bc..9d991599cfa 100644
--- a/hw/versatile_pci.c
+++ b/hw/versatile_pci.c
@@ -8,9 +8,9 @@
  */
 
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -21,18 +21,18 @@ typedef struct {
     MemoryRegion isa;
 } PCIVPBState;
 
-static inline uint32_t vpb_pci_config_addr(target_phys_addr_t addr)
+static inline uint32_t vpb_pci_config_addr(hwaddr addr)
 {
     return addr & 0xffffff;
 }
 
-static void pci_vpb_config_write(void *opaque, target_phys_addr_t addr,
+static void pci_vpb_config_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
     pci_data_write(opaque, vpb_pci_config_addr(addr), val, size);
 }
 
-static uint64_t pci_vpb_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_vpb_config_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     uint32_t val;
@@ -119,7 +119,7 @@ static void versatile_pci_host_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_PROCESSOR_CO;
 }
 
-static TypeInfo versatile_pci_host_info = {
+static const TypeInfo versatile_pci_host_info = {
     .name          = "versatile_pci_host",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -133,7 +133,7 @@ static void pci_vpb_class_init(ObjectClass *klass, void *data)
     sdc->init = pci_vpb_init;
 }
 
-static TypeInfo pci_vpb_info = {
+static const TypeInfo pci_vpb_info = {
     .name          = "versatile_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PCIVPBState),
@@ -147,7 +147,7 @@ static void pci_realview_class_init(ObjectClass *klass, void *data)
     sdc->init = pci_realview_init;
 }
 
-static TypeInfo pci_realview_info = {
+static const TypeInfo pci_realview_info = {
     .name          = "realview_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PCIVPBState),
diff --git a/hw/versatilepb.c b/hw/versatilepb.c
index 7a920347184..e0a28f08d37 100644
--- a/hw/versatilepb.c
+++ b/hw/versatilepb.c
@@ -10,13 +10,13 @@
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
-#include "pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "pci/pci.h"
 #include "i2c.h"
 #include "boards.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 #include "flash.h"
 
 #define VERSATILE_FLASH_ADDR 0x34000000
@@ -81,7 +81,7 @@ static void vpb_sic_set_irq(void *opaque, int irq, int level)
     vpb_sic_update(s);
 }
 
-static uint64_t vpb_sic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t vpb_sic_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     vpb_sic_state *s = (vpb_sic_state *)opaque;
@@ -103,7 +103,7 @@ static uint64_t vpb_sic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void vpb_sic_write(void *opaque, target_phys_addr_t offset,
+static void vpb_sic_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     vpb_sic_state *s = (vpb_sic_state *)opaque;
@@ -167,11 +167,7 @@ static int vpb_sic_init(SysBusDevice *dev)
 
 static struct arm_boot_info versatile_binfo;
 
-static void versatile_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model,
-                     int board_id)
+static void versatile_init(QEMUMachineInitArgs *args, int board_id)
 {
     ARMCPU *cpu;
     MemoryRegion *sysmem = get_system_memory();
@@ -189,15 +185,15 @@ static void versatile_init(ram_addr_t ram_size,
     int done_smc = 0;
     DriveInfo *dinfo;
 
-    if (!cpu_model) {
-        cpu_model = "arm926";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm926";
     }
-    cpu = cpu_arm_init(cpu_model);
+    cpu = cpu_arm_init(args->cpu_model);
     if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    memory_region_init_ram(ram, "versatile.ram", ram_size);
+    memory_region_init_ram(ram, "versatile.ram", args->ram_size);
     vmstate_register_ram_global(ram);
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero.  */
@@ -207,17 +203,18 @@ static void versatile_init(ram_addr_t ram_size,
     qdev_prop_set_uint32(sysctl, "sys_id", 0x41007004);
     qdev_prop_set_uint32(sysctl, "proc_id", 0x02000000);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, 0x10000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, 0x10000000);
 
     cpu_pic = arm_pic_init_cpu(cpu);
     dev = sysbus_create_varargs("pl190", 0x10140000,
-                                cpu_pic[0], cpu_pic[1], NULL);
+                                cpu_pic[ARM_PIC_CPU_IRQ],
+                                cpu_pic[ARM_PIC_CPU_FIQ], NULL);
     for (n = 0; n < 32; n++) {
         pic[n] = qdev_get_gpio_in(dev, n);
     }
     dev = sysbus_create_simple("versatilepb_sic", 0x10003000, NULL);
     for (n = 0; n < 32; n++) {
-        sysbus_connect_irq(sysbus_from_qdev(dev), n, pic[n]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), n, pic[n]);
         sic[n] = qdev_get_gpio_in(dev, n);
     }
 
@@ -225,7 +222,7 @@ static void versatile_init(ram_addr_t ram_size,
     sysbus_create_simple("pl050_mouse", 0x10007000, sic[4]);
 
     dev = qdev_create(NULL, "versatile_pci");
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(busdev, 0, 0x41000000); /* PCI self-config */
     sysbus_mmio_map(busdev, 1, 0x42000000); /* PCI config */
@@ -247,7 +244,7 @@ static void versatile_init(ram_addr_t ram_size,
             pci_nic_init_nofail(nd, "rtl8139", NULL);
         }
     }
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
     n = drive_get_max_bus(IF_SCSI);
@@ -265,6 +262,11 @@ static void versatile_init(ram_addr_t ram_size,
     sysbus_create_simple("sp804", 0x101e2000, pic[4]);
     sysbus_create_simple("sp804", 0x101e3000, pic[5]);
 
+    sysbus_create_simple("pl061", 0x101e4000, pic[6]);
+    sysbus_create_simple("pl061", 0x101e5000, pic[7]);
+    sysbus_create_simple("pl061", 0x101e6000, pic[8]);
+    sysbus_create_simple("pl061", 0x101e7000, pic[9]);
+
     /* The versatile/PB actually has a modified Color LCD controller
        that includes hardware cursor support from the PL111.  */
     dev = sysbus_create_simple("pl110_versatile", 0x10120000, pic[16]);
@@ -285,8 +287,8 @@ static void versatile_init(ram_addr_t ram_size,
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, 0x10004000);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, sic[24]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, 0x10004000);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, sic[24]);
 
     /* Memory map for Versatile/PB:  */
     /* 0x10000000 System registers.  */
@@ -334,48 +336,38 @@ static void versatile_init(ram_addr_t ram_size,
         fprintf(stderr, "qemu: Error registering flash memory.\n");
     }
 
-    versatile_binfo.ram_size = ram_size;
-    versatile_binfo.kernel_filename = kernel_filename;
-    versatile_binfo.kernel_cmdline = kernel_cmdline;
-    versatile_binfo.initrd_filename = initrd_filename;
+    versatile_binfo.ram_size = args->ram_size;
+    versatile_binfo.kernel_filename = args->kernel_filename;
+    versatile_binfo.kernel_cmdline = args->kernel_cmdline;
+    versatile_binfo.initrd_filename = args->initrd_filename;
     versatile_binfo.board_id = board_id;
     arm_load_kernel(cpu, &versatile_binfo);
 }
 
-static void vpb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void vpb_init(QEMUMachineInitArgs *args)
 {
-    versatile_init(ram_size,
-                   boot_device,
-                   kernel_filename, kernel_cmdline,
-                   initrd_filename, cpu_model, 0x183);
+    versatile_init(args, 0x183);
 }
 
-static void vab_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void vab_init(QEMUMachineInitArgs *args)
 {
-    versatile_init(ram_size,
-                   boot_device,
-                   kernel_filename, kernel_cmdline,
-                   initrd_filename, cpu_model, 0x25e);
+    versatile_init(args, 0x25e);
 }
 
 static QEMUMachine versatilepb_machine = {
     .name = "versatilepb",
     .desc = "ARM Versatile/PB (ARM926EJ-S)",
     .init = vpb_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine versatileab_machine = {
     .name = "versatileab",
     .desc = "ARM Versatile/AB (ARM926EJ-S)",
     .init = vab_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void versatile_machine_init(void)
@@ -396,7 +388,7 @@ static void vpb_sic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_vpb_sic;
 }
 
-static TypeInfo vpb_sic_info = {
+static const TypeInfo vpb_sic_info = {
     .name          = "versatilepb_sic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(vpb_sic_state),
diff --git a/hw/vexpress.c b/hw/vexpress.c
index b6158447d72..7f0897c7731 100644
--- a/hw/vexpress.c
+++ b/hw/vexpress.c
@@ -25,12 +25,16 @@
 #include "arm-misc.h"
 #include "primecell.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
+#include "sysemu/blockdev.h"
+#include "flash.h"
 
 #define VEXPRESS_BOARD_ID 0x8e0
+#define VEXPRESS_FLASH_SIZE (64 * 1024 * 1024)
+#define VEXPRESS_FLASH_SECT_SIZE (256 * 1024)
 
 static struct arm_boot_info vexpress_binfo;
 
@@ -62,7 +66,6 @@ enum {
     VE_COMPACTFLASH,
     VE_CLCD,
     VE_NORFLASH0,
-    VE_NORFLASH0ALIAS,
     VE_NORFLASH1,
     VE_SRAM,
     VE_VIDEORAM,
@@ -71,7 +74,7 @@ enum {
     VE_DAPROM,
 };
 
-static target_phys_addr_t motherboard_legacy_map[] = {
+static hwaddr motherboard_legacy_map[] = {
     /* CS7: 0x10000000 .. 0x10020000 */
     [VE_SYSREGS] = 0x10000000,
     [VE_SP810] = 0x10001000,
@@ -103,10 +106,9 @@ static target_phys_addr_t motherboard_legacy_map[] = {
     [VE_USB] = 0x4f000000,
 };
 
-static target_phys_addr_t motherboard_aseries_map[] = {
-    /* CS0: 0x00000000 .. 0x0c000000 */
-    [VE_NORFLASH0] = 0x00000000,
-    [VE_NORFLASH0ALIAS] = 0x08000000,
+static hwaddr motherboard_aseries_map[] = {
+    /* CS0: 0x08000000 .. 0x0c000000 */
+    [VE_NORFLASH0] = 0x08000000,
     /* CS4: 0x0c000000 .. 0x10000000 */
     [VE_NORFLASH1] = 0x0c000000,
     /* CS5: 0x10000000 .. 0x14000000 */
@@ -148,9 +150,9 @@ typedef void DBoardInitFn(const VEDBoardInfo *daughterboard,
                           qemu_irq *pic, uint32_t *proc_id);
 
 struct VEDBoardInfo {
-    const target_phys_addr_t *motherboard_map;
-    target_phys_addr_t loader_start;
-    const target_phys_addr_t gic_cpu_if_addr;
+    const hwaddr *motherboard_map;
+    hwaddr loader_start;
+    const hwaddr gic_cpu_if_addr;
     DBoardInitFn *init;
 };
 
@@ -209,7 +211,7 @@ static void a9_daughterboard_init(const VEDBoardInfo *daughterboard,
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0x1e000000);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -305,7 +307,7 @@ static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
     dev = qdev_create(NULL, "a15mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0x2c000000);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -346,24 +348,21 @@ static const VEDBoardInfo a15_daughterboard = {
 };
 
 static void vexpress_common_init(const VEDBoardInfo *daughterboard,
-                                 ram_addr_t ram_size,
-                                 const char *boot_device,
-                                 const char *kernel_filename,
-                                 const char *kernel_cmdline,
-                                 const char *initrd_filename,
-                                 const char *cpu_model)
+                                 QEMUMachineInitArgs *args)
 {
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
     uint32_t proc_id;
     uint32_t sys_id;
+    DriveInfo *dinfo;
     ram_addr_t vram_size, sram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *vram = g_new(MemoryRegion, 1);
     MemoryRegion *sram = g_new(MemoryRegion, 1);
-    const target_phys_addr_t *map = daughterboard->motherboard_map;
+    const hwaddr *map = daughterboard->motherboard_map;
 
-    daughterboard->init(daughterboard, ram_size, cpu_model, pic, &proc_id);
+    daughterboard->init(daughterboard, args->ram_size, args->cpu_model,
+                        pic, &proc_id);
 
     /* Motherboard peripherals: the wiring is the same but the
      * addresses vary between the legacy and A-Series memory maps.
@@ -375,7 +374,7 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     qdev_prop_set_uint32(sysctl, "sys_id", sys_id);
     qdev_prop_set_uint32(sysctl, "proc_id", proc_id);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, map[VE_SYSREGS]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, map[VE_SYSREGS]);
 
     /* VE_SP810: not modelled */
     /* VE_SERIALPCI: not modelled */
@@ -383,8 +382,8 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, map[VE_PL041]);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, pic[11]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, map[VE_PL041]);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, pic[11]);
 
     dev = sysbus_create_varargs("pl181", map[VE_MMCI], pic[9], pic[10], NULL);
     /* Wire up MMC card detect and read-only signals */
@@ -412,9 +411,25 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
 
     sysbus_create_simple("pl111", map[VE_CLCD], pic[14]);
 
-    /* VE_NORFLASH0: not modelled */
-    /* VE_NORFLASH0ALIAS: not modelled */
-    /* VE_NORFLASH1: not modelled */
+    dinfo = drive_get_next(IF_PFLASH);
+    if (!pflash_cfi01_register(map[VE_NORFLASH0], NULL, "vexpress.flash0",
+            VEXPRESS_FLASH_SIZE, dinfo ? dinfo->bdrv : NULL,
+            VEXPRESS_FLASH_SECT_SIZE,
+            VEXPRESS_FLASH_SIZE / VEXPRESS_FLASH_SECT_SIZE, 4,
+            0x00, 0x89, 0x00, 0x18, 0)) {
+        fprintf(stderr, "vexpress: error registering flash 0.\n");
+        exit(1);
+    }
+
+    dinfo = drive_get_next(IF_PFLASH);
+    if (!pflash_cfi01_register(map[VE_NORFLASH1], NULL, "vexpress.flash1",
+            VEXPRESS_FLASH_SIZE, dinfo ? dinfo->bdrv : NULL,
+            VEXPRESS_FLASH_SECT_SIZE,
+            VEXPRESS_FLASH_SIZE / VEXPRESS_FLASH_SECT_SIZE, 4,
+            0x00, 0x89, 0x00, 0x18, 0)) {
+        fprintf(stderr, "vexpress: error registering flash 1.\n");
+        exit(1);
+    }
 
     sram_size = 0x2000000;
     memory_region_init_ram(sram, "vexpress.sram", sram_size);
@@ -435,10 +450,10 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
 
     /* VE_DAPROM: not modelled */
 
-    vexpress_binfo.ram_size = ram_size;
-    vexpress_binfo.kernel_filename = kernel_filename;
-    vexpress_binfo.kernel_cmdline = kernel_cmdline;
-    vexpress_binfo.initrd_filename = initrd_filename;
+    vexpress_binfo.ram_size = args->ram_size;
+    vexpress_binfo.kernel_filename = args->kernel_filename;
+    vexpress_binfo.kernel_cmdline = args->kernel_cmdline;
+    vexpress_binfo.initrd_filename = args->initrd_filename;
     vexpress_binfo.nb_cpus = smp_cpus;
     vexpress_binfo.board_id = VEXPRESS_BOARD_ID;
     vexpress_binfo.loader_start = daughterboard->loader_start;
@@ -448,44 +463,32 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     arm_load_kernel(arm_env_get_cpu(first_cpu), &vexpress_binfo);
 }
 
-static void vexpress_a9_init(ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void vexpress_a9_init(QEMUMachineInitArgs *args)
 {
-    vexpress_common_init(&a9_daughterboard,
-                         ram_size, boot_device, kernel_filename,
-                         kernel_cmdline, initrd_filename, cpu_model);
+    vexpress_common_init(&a9_daughterboard, args);
 }
 
-static void vexpress_a15_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void vexpress_a15_init(QEMUMachineInitArgs *args)
 {
-    vexpress_common_init(&a15_daughterboard,
-                         ram_size, boot_device, kernel_filename,
-                         kernel_cmdline, initrd_filename, cpu_model);
+    vexpress_common_init(&a15_daughterboard, args);
 }
 
 static QEMUMachine vexpress_a9_machine = {
     .name = "vexpress-a9",
     .desc = "ARM Versatile Express for Cortex-A9",
     .init = vexpress_a9_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine vexpress_a15_machine = {
     .name = "vexpress-a15",
     .desc = "ARM Versatile Express for Cortex-A15",
     .init = vexpress_a15_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void vexpress_machine_init(void)
diff --git a/hw/vfio_pci.c b/hw/vfio_pci.c
new file mode 100644
index 00000000000..c51ae6761b7
--- /dev/null
+++ b/hw/vfio_pci.c
@@ -0,0 +1,2138 @@
+/*
+ * vfio based device assignment support
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on qemu-kvm device-assignment:
+ *  Adapted for KVM by Qumranet.
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+
+#include <dirent.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <linux/vfio.h>
+
+#include "config.h"
+#include "qemu/event_notifier.h"
+#include "exec/address-spaces.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
+#include "pci/pci.h"
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/range.h"
+
+/* #define DEBUG_VFIO */
+#ifdef DEBUG_VFIO
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+typedef struct VFIOBAR {
+    off_t fd_offset; /* offset of BAR within device fd */
+    int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
+    MemoryRegion mem; /* slow, read/write access */
+    MemoryRegion mmap_mem; /* direct mapped access */
+    void *mmap;
+    size_t size;
+    uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
+    uint8_t nr; /* cache the BAR number for debug */
+} VFIOBAR;
+
+typedef struct VFIOINTx {
+    bool pending; /* interrupt pending */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+    uint8_t pin; /* which pin to pull for qemu_set_irq */
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    PCIINTxRoute route; /* routing info for QEMU bypass */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+struct VFIODevice;
+
+typedef struct VFIOMSIVector {
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    struct VFIODevice *vdev; /* back pointer to device */
+    int virq; /* KVM irqchip route for QEMU bypass */
+    bool use;
+} VFIOMSIVector;
+
+enum {
+    VFIO_INT_NONE = 0,
+    VFIO_INT_INTx = 1,
+    VFIO_INT_MSI  = 2,
+    VFIO_INT_MSIX = 3,
+};
+
+struct VFIOGroup;
+
+typedef struct VFIOContainer {
+    int fd; /* /dev/vfio/vfio, empowered by the attached groups */
+    struct {
+        /* enable abstraction to support various iommu backends */
+        union {
+            MemoryListener listener; /* Used by type1 iommu */
+        };
+        void (*release)(struct VFIOContainer *);
+    } iommu_data;
+    QLIST_HEAD(, VFIOGroup) group_list;
+    QLIST_ENTRY(VFIOContainer) next;
+} VFIOContainer;
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+    MemoryRegion mmap_mem;
+    void *mmap;
+} VFIOMSIXInfo;
+
+typedef struct VFIODevice {
+    PCIDevice pdev;
+    int fd;
+    VFIOINTx intx;
+    unsigned int config_size;
+    off_t config_offset; /* Offset of config space region within device fd */
+    unsigned int rom_size;
+    off_t rom_offset; /* Offset of ROM region within device fd */
+    int msi_cap_size;
+    VFIOMSIVector *msi_vectors;
+    VFIOMSIXInfo *msix;
+    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+    int interrupt; /* Current interrupt type */
+    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+    PCIHostDeviceAddress host;
+    QLIST_ENTRY(VFIODevice) next;
+    struct VFIOGroup *group;
+    bool reset_works;
+} VFIODevice;
+
+typedef struct VFIOGroup {
+    int fd;
+    int groupid;
+    VFIOContainer *container;
+    QLIST_HEAD(, VFIODevice) device_list;
+    QLIST_ENTRY(VFIOGroup) next;
+    QLIST_ENTRY(VFIOGroup) container_next;
+} VFIOGroup;
+
+#define MSIX_CAP_LENGTH 12
+
+static QLIST_HEAD(, VFIOContainer)
+    container_list = QLIST_HEAD_INITIALIZER(container_list);
+
+static QLIST_HEAD(, VFIOGroup)
+    group_list = QLIST_HEAD_INITIALIZER(group_list);
+
+static void vfio_disable_interrupts(VFIODevice *vdev);
+static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
+static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
+
+/*
+ * Common VFIO interrupt disable
+ */
+static void vfio_disable_irqindex(VFIODevice *vdev, int index)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
+        .index = index,
+        .start = 0,
+        .count = 0,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+/*
+ * INTx
+ */
+static void vfio_unmask_intx(VFIODevice *vdev)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
+        .index = VFIO_PCI_INTX_IRQ_INDEX,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+#ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
+static void vfio_mask_intx(VFIODevice *vdev)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
+        .index = VFIO_PCI_INTX_IRQ_INDEX,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+#endif
+
+/*
+ * Disabling BAR mmaping can be slow, but toggling it around INTx can
+ * also be a huge overhead.  We try to get the best of both worlds by
+ * waiting until an interrupt to disable mmaps (subsequent transitions
+ * to the same state are effectively no overhead).  If the interrupt has
+ * been serviced and the time gap is long enough, we re-enable mmaps for
+ * performance.  This works well for things like graphics cards, which
+ * may not use their interrupt at all and are penalized to an unusable
+ * level by read/write BAR traps.  Other devices, like NICs, have more
+ * regular interrupts and see much better latency by staying in non-mmap
+ * mode.  We therefore set the default mmap_timeout such that a ping
+ * is just enough to keep the mmap disabled.  Users can experiment with
+ * other options with the x-intx-mmap-timeout-ms parameter (a value of
+ * zero disables the timer).
+ */
+static void vfio_intx_mmap_enable(void *opaque)
+{
+    VFIODevice *vdev = opaque;
+
+    if (vdev->intx.pending) {
+        qemu_mod_timer(vdev->intx.mmap_timer,
+                       qemu_get_clock_ms(vm_clock) + vdev->intx.mmap_timeout);
+        return;
+    }
+
+    vfio_mmap_set_enabled(vdev, true);
+}
+
+static void vfio_intx_interrupt(void *opaque)
+{
+    VFIODevice *vdev = opaque;
+
+    if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function,
+            'A' + vdev->intx.pin);
+
+    vdev->intx.pending = true;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 1);
+    vfio_mmap_set_enabled(vdev, false);
+    if (vdev->intx.mmap_timeout) {
+        qemu_mod_timer(vdev->intx.mmap_timer,
+                       qemu_get_clock_ms(vm_clock) + vdev->intx.mmap_timeout);
+    }
+}
+
+static void vfio_eoi(VFIODevice *vdev)
+{
+    if (!vdev->intx.pending) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+    vfio_unmask_intx(vdev);
+}
+
+static void vfio_enable_intx_kvm(VFIODevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_RESAMPLE,
+    };
+    struct vfio_irq_set *irq_set;
+    int ret, argsz;
+    int32_t *pfd;
+
+    if (!kvm_irqfds_enabled() ||
+        vdev->intx.route.mode != PCI_INTX_ENABLED ||
+        !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+        return;
+    }
+
+    /* Get to a known interrupt state */
+    qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
+    vfio_mask_intx(vdev);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+    /* Get an eventfd for resample/unmask */
+    if (event_notifier_init(&vdev->intx.unmask, 0)) {
+        error_report("vfio: Error: event_notifier_init failed eoi\n");
+        goto fail;
+    }
+
+    /* KVM triggers it, VFIO listens for it */
+    irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
+
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to setup resample irqfd: %m\n");
+        goto fail_irqfd;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = irqfd.resamplefd;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx unmask fd: %m\n");
+        goto fail_vfio;
+    }
+
+    /* Let'em rip */
+    vfio_unmask_intx(vdev);
+
+    vdev->intx.kvm_accel = true;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
+            __func__, vdev->host.domain, vdev->host.bus,
+            vdev->host.slot, vdev->host.function);
+
+    return;
+
+fail_vfio:
+    irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
+    kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
+fail_irqfd:
+    event_notifier_cleanup(&vdev->intx.unmask);
+fail:
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+    vfio_unmask_intx(vdev);
+#endif
+}
+
+static void vfio_disable_intx_kvm(VFIODevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_DEASSIGN,
+    };
+
+    if (!vdev->intx.kvm_accel) {
+        return;
+    }
+
+    /*
+     * Get to a known state, hardware masked, QEMU ready to accept new
+     * interrupts, QEMU IRQ de-asserted.
+     */
+    vfio_mask_intx(vdev);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+    /* Tell KVM to stop listening for an INTx irqfd */
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to disable INTx irqfd: %m\n");
+    }
+
+    /* We only need to close the eventfd for VFIO to cleanup the kernel side */
+    event_notifier_cleanup(&vdev->intx.unmask);
+
+    /* QEMU starts listening for interrupt events. */
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+
+    vdev->intx.kvm_accel = false;
+
+    /* If we've missed an event, let it re-fire through QEMU */
+    vfio_unmask_intx(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
+            __func__, vdev->host.domain, vdev->host.bus,
+            vdev->host.slot, vdev->host.function);
+#endif
+}
+
+static void vfio_update_irq(PCIDevice *pdev)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    PCIINTxRoute route;
+
+    if (vdev->interrupt != VFIO_INT_INTx) {
+        return;
+    }
+
+    route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
+
+    if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
+        return; /* Nothing changed */
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, vdev->intx.route.irq, route.irq);
+
+    vfio_disable_intx_kvm(vdev);
+
+    vdev->intx.route = route;
+
+    if (route.mode != PCI_INTX_ENABLED) {
+        return;
+    }
+
+    vfio_enable_intx_kvm(vdev);
+
+    /* Re-enable the interrupt in cased we missed an EOI */
+    vfio_eoi(vdev);
+}
+
+static int vfio_enable_intx(VFIODevice *vdev)
+{
+    uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
+    int ret, argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!pin) {
+        return 0;
+    }
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
+
+#ifdef CONFIG_KVM
+    /*
+     * Only conditional to avoid generating error messages on platforms
+     * where we won't actually use the result anyway.
+     */
+    if (kvm_irqfds_enabled() &&
+        kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+        vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
+                                                        vdev->intx.pin);
+    }
+#endif
+
+    ret = event_notifier_init(&vdev->intx.interrupt, 0);
+    if (ret) {
+        error_report("vfio: Error: event_notifier_init failed\n");
+        return ret;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx fd: %m\n");
+        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+        event_notifier_cleanup(&vdev->intx.interrupt);
+        return -errno;
+    }
+
+    vfio_enable_intx_kvm(vdev);
+
+    vdev->interrupt = VFIO_INT_INTx;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    return 0;
+}
+
+static void vfio_disable_intx(VFIODevice *vdev)
+{
+    int fd;
+
+    qemu_del_timer(vdev->intx.mmap_timer);
+    vfio_disable_intx_kvm(vdev);
+    vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+    vfio_mmap_set_enabled(vdev, true);
+
+    fd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(fd, NULL, NULL, vdev);
+    event_notifier_cleanup(&vdev->intx.interrupt);
+
+    vdev->interrupt = VFIO_INT_NONE;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+/*
+ * MSI/X
+ */
+static void vfio_msi_interrupt(void *opaque)
+{
+    VFIOMSIVector *vector = opaque;
+    VFIODevice *vdev = vector->vdev;
+    int nr = vector - vdev->msi_vectors;
+
+    if (!event_notifier_test_and_clear(&vector->interrupt)) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    if (vdev->interrupt == VFIO_INT_MSIX) {
+        msix_notify(&vdev->pdev, nr);
+    } else if (vdev->interrupt == VFIO_INT_MSI) {
+        msi_notify(&vdev->pdev, nr);
+    } else {
+        error_report("vfio: MSI interrupt receieved, but not enabled?\n");
+    }
+}
+
+static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
+{
+    struct vfio_irq_set *irq_set;
+    int ret = 0, i, argsz;
+    int32_t *fds;
+
+    argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = vdev->nr_vectors;
+    fds = (int32_t *)&irq_set->data;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        if (!vdev->msi_vectors[i].use) {
+            fds[i] = -1;
+            continue;
+        }
+
+        fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
+    }
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+    g_free(irq_set);
+
+    return ret;
+}
+
+static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
+                                   MSIMessage *msg, IOHandler *handler)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOMSIVector *vector;
+    int ret;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    vector = &vdev->msi_vectors[nr];
+    vector->vdev = vdev;
+    vector->use = true;
+
+    msix_vector_use(pdev, nr);
+
+    if (event_notifier_init(&vector->interrupt, 0)) {
+        error_report("vfio: Error: event_notifier_init failed\n");
+    }
+
+    /*
+     * Attempt to enable route through KVM irqchip,
+     * default to userspace handling if unavailable.
+     */
+    vector->virq = msg ? kvm_irqchip_add_msi_route(kvm_state, *msg) : -1;
+    if (vector->virq < 0 ||
+        kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
+                                       vector->virq) < 0) {
+        if (vector->virq >= 0) {
+            kvm_irqchip_release_virq(kvm_state, vector->virq);
+            vector->virq = -1;
+        }
+        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                            handler, NULL, vector);
+    }
+
+    /*
+     * We don't want to have the host allocate all possible MSI vectors
+     * for a device if they're not in use, so we shutdown and incrementally
+     * increase them as needed.
+     */
+    if (vdev->nr_vectors < nr + 1) {
+        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
+        vdev->nr_vectors = nr + 1;
+        ret = vfio_enable_vectors(vdev, true);
+        if (ret) {
+            error_report("vfio: failed to enable vectors, %d\n", ret);
+        }
+    } else {
+        int argsz;
+        struct vfio_irq_set *irq_set;
+        int32_t *pfd;
+
+        argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+        irq_set = g_malloc0(argsz);
+        irq_set->argsz = argsz;
+        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                         VFIO_IRQ_SET_ACTION_TRIGGER;
+        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+        irq_set->start = nr;
+        irq_set->count = 1;
+        pfd = (int32_t *)&irq_set->data;
+
+        *pfd = event_notifier_get_fd(&vector->interrupt);
+
+        ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+        g_free(irq_set);
+        if (ret) {
+            error_report("vfio: failed to modify vector, %d\n", ret);
+        }
+    }
+
+    return 0;
+}
+
+static int vfio_msix_vector_use(PCIDevice *pdev,
+                                unsigned int nr, MSIMessage msg)
+{
+    return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
+}
+
+static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOMSIVector *vector = &vdev->msi_vectors[nr];
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    /*
+     * XXX What's the right thing to do here?  This turns off the interrupt
+     * completely, but do we really just want to switch the interrupt to
+     * bouncing through userspace and let msix.c drop it?  Not sure.
+     */
+    msix_vector_unuse(pdev, nr);
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+    irq_set->start = nr;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = -1;
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+    g_free(irq_set);
+
+    if (vector->virq < 0) {
+        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                            NULL, NULL, NULL);
+    } else {
+        kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
+                                          vector->virq);
+        kvm_irqchip_release_virq(kvm_state, vector->virq);
+        vector->virq = -1;
+    }
+
+    event_notifier_cleanup(&vector->interrupt);
+    vector->use = false;
+}
+
+static void vfio_enable_msix(VFIODevice *vdev)
+{
+    vfio_disable_interrupts(vdev);
+
+    vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
+
+    vdev->interrupt = VFIO_INT_MSIX;
+
+    /*
+     * Some communication channels between VF & PF or PF & fw rely on the
+     * physical state of the device and expect that enabling MSI-X from the
+     * guest enables the same on the host.  When our guest is Linux, the
+     * guest driver call to pci_enable_msix() sets the enabling bit in the
+     * MSI-X capability, but leaves the vector table masked.  We therefore
+     * can't rely on a vector_use callback (from request_irq() in the guest)
+     * to switch the physical device into MSI-X mode because that may come a
+     * long time after pci_enable_msix().  This code enables vector 0 with
+     * triggering to userspace, then immediately release the vector, leaving
+     * the physical device with no vectors enabled, but MSI-X enabled, just
+     * like the guest view.
+     */
+    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
+    vfio_msix_vector_release(&vdev->pdev, 0);
+
+    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
+                                  vfio_msix_vector_release, NULL)) {
+        error_report("vfio: msix_set_vector_notifiers failed\n");
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+static void vfio_enable_msi(VFIODevice *vdev)
+{
+    int ret, i;
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
+retry:
+    vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        MSIMessage msg;
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+        vector->vdev = vdev;
+        vector->use = true;
+
+        if (event_notifier_init(&vector->interrupt, 0)) {
+            error_report("vfio: Error: event_notifier_init failed\n");
+        }
+
+        msg = msi_get_message(&vdev->pdev, i);
+
+        /*
+         * Attempt to enable route through KVM irqchip,
+         * default to userspace handling if unavailable.
+         */
+        vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (vector->virq < 0 ||
+            kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
+                                           vector->virq) < 0) {
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                vfio_msi_interrupt, NULL, vector);
+        }
+    }
+
+    ret = vfio_enable_vectors(vdev, false);
+    if (ret) {
+        if (ret < 0) {
+            error_report("vfio: Error: Failed to setup MSI fds: %m\n");
+        } else if (ret != vdev->nr_vectors) {
+            error_report("vfio: Error: Failed to enable %d "
+                         "MSI vectors, retry with %d\n", vdev->nr_vectors, ret);
+        }
+
+        for (i = 0; i < vdev->nr_vectors; i++) {
+            VFIOMSIVector *vector = &vdev->msi_vectors[i];
+            if (vector->virq >= 0) {
+                kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
+                                                  vector->virq);
+                kvm_irqchip_release_virq(kvm_state, vector->virq);
+                vector->virq = -1;
+            } else {
+                qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                    NULL, NULL, NULL);
+            }
+            event_notifier_cleanup(&vector->interrupt);
+        }
+
+        g_free(vdev->msi_vectors);
+
+        if (ret > 0 && ret != vdev->nr_vectors) {
+            vdev->nr_vectors = ret;
+            goto retry;
+        }
+        vdev->nr_vectors = 0;
+
+        return;
+    }
+
+    vdev->interrupt = VFIO_INT_MSI;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, vdev->nr_vectors);
+}
+
+static void vfio_disable_msi_common(VFIODevice *vdev)
+{
+    g_free(vdev->msi_vectors);
+    vdev->msi_vectors = NULL;
+    vdev->nr_vectors = 0;
+    vdev->interrupt = VFIO_INT_NONE;
+
+    vfio_enable_intx(vdev);
+}
+
+static void vfio_disable_msix(VFIODevice *vdev)
+{
+    msix_unset_vector_notifiers(&vdev->pdev);
+
+    if (vdev->nr_vectors) {
+        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
+    }
+
+    vfio_disable_msi_common(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+static void vfio_disable_msi(VFIODevice *vdev)
+{
+    int i;
+
+    vfio_disable_irqindex(vdev, VFIO_PCI_MSI_IRQ_INDEX);
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+        if (!vector->use) {
+            continue;
+        }
+
+        if (vector->virq >= 0) {
+            kvm_irqchip_remove_irqfd_notifier(kvm_state,
+                                              &vector->interrupt, vector->virq);
+            kvm_irqchip_release_virq(kvm_state, vector->virq);
+            vector->virq = -1;
+        } else {
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                NULL, NULL, NULL);
+        }
+
+        event_notifier_cleanup(&vector->interrupt);
+    }
+
+    vfio_disable_msi_common(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+/*
+ * IO Port/MMIO - Beware of the endians, VFIO is always little endian
+ */
+static void vfio_bar_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size)
+{
+    VFIOBAR *bar = opaque;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+
+    switch (size) {
+    case 1:
+        buf.byte = data;
+        break;
+    case 2:
+        buf.word = cpu_to_le16(data);
+        break;
+    case 4:
+        buf.dword = cpu_to_le32(data);
+        break;
+    default:
+        hw_error("vfio: unsupported write size, %d bytes\n", size);
+        break;
+    }
+
+    if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m\n",
+                     __func__, addr, data, size);
+    }
+
+    DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
+            __func__, bar->nr, addr, data, size);
+
+    /*
+     * A read or write to a BAR always signals an INTx EOI.  This will
+     * do nothing if not pending (including not in INTx mode).  We assume
+     * that a BAR access is in response to an interrupt and that BAR
+     * accesses will service the interrupt.  Unfortunately, we don't know
+     * which access will service the interrupt, so we're potentially
+     * getting quite a few host interrupts per guest interrupt.
+     */
+    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
+}
+
+static uint64_t vfio_bar_read(void *opaque,
+                              hwaddr addr, unsigned size)
+{
+    VFIOBAR *bar = opaque;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+    uint64_t data = 0;
+
+    if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m\n",
+                     __func__, addr, size);
+        return (uint64_t)-1;
+    }
+
+    switch (size) {
+    case 1:
+        data = buf.byte;
+        break;
+    case 2:
+        data = le16_to_cpu(buf.word);
+        break;
+    case 4:
+        data = le32_to_cpu(buf.dword);
+        break;
+    default:
+        hw_error("vfio: unsupported read size, %d bytes\n", size);
+        break;
+    }
+
+    DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
+            __func__, bar->nr, addr, size, data);
+
+    /* Same as write above */
+    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_bar_ops = {
+    .read = vfio_bar_read,
+    .write = vfio_bar_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * PCI config space
+ */
+static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint32_t val = 0;
+
+    /*
+     * We only need QEMU PCI config support for the ROM BAR, the MSI and MSIX
+     * capabilities, and the multifunction bit below.  We let VFIO handle
+     * virtualizing everything else.  Performance is not a concern here.
+     */
+    if (ranges_overlap(addr, len, PCI_ROM_ADDRESS, 4) ||
+        (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
+         ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) ||
+        (pdev->cap_present & QEMU_PCI_CAP_MSI &&
+         ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size))) {
+
+        val = pci_default_read_config(pdev, addr, len);
+    } else {
+        if (pread(vdev->fd, &val, len, vdev->config_offset + addr) != len) {
+            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m\n",
+                         __func__, vdev->host.domain, vdev->host.bus,
+                         vdev->host.slot, vdev->host.function, addr, len);
+            return -errno;
+        }
+        val = le32_to_cpu(val);
+    }
+
+    /* Multifunction bit is virualized in QEMU */
+    if (unlikely(ranges_overlap(addr, len, PCI_HEADER_TYPE, 1))) {
+        uint32_t mask = PCI_HEADER_TYPE_MULTI_FUNCTION;
+
+        if (len == 4) {
+            mask <<= 16;
+        }
+
+        if (pdev->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+            val |= mask;
+        } else {
+            val &= ~mask;
+        }
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, addr, len, val);
+
+    return val;
+}
+
+static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
+                                  uint32_t val, int len)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint32_t val_le = cpu_to_le32(val);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, addr, val, len);
+
+    /* Write everything to VFIO, let it filter out what we can't write */
+    if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
+        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m\n",
+                     __func__, vdev->host.domain, vdev->host.bus,
+                     vdev->host.slot, vdev->host.function, addr, val, len);
+    }
+
+    /* Write standard header bits to emulation */
+    if (addr < PCI_CONFIG_HEADER_SIZE) {
+        pci_default_write_config(pdev, addr, val, len);
+        return;
+    }
+
+    /* MSI/MSI-X Enabling/Disabling */
+    if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
+        ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
+        int is_enabled, was_enabled = msi_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msi_enabled(pdev);
+
+        if (!was_enabled && is_enabled) {
+            vfio_enable_msi(vdev);
+        } else if (was_enabled && !is_enabled) {
+            vfio_disable_msi(vdev);
+        }
+    }
+
+    if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
+        ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
+        int is_enabled, was_enabled = msix_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msix_enabled(pdev);
+
+        if (!was_enabled && is_enabled) {
+            vfio_enable_msix(vdev);
+        } else if (was_enabled && !is_enabled) {
+            vfio_disable_msix(vdev);
+        }
+    }
+}
+
+/*
+ * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
+ */
+static int vfio_dma_unmap(VFIOContainer *container,
+                          hwaddr iova, ram_addr_t size)
+{
+    struct vfio_iommu_type1_dma_unmap unmap = {
+        .argsz = sizeof(unmap),
+        .flags = 0,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
+        DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno);
+        return -errno;
+    }
+
+    return 0;
+}
+
+static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
+                        ram_addr_t size, void *vaddr, bool readonly)
+{
+    struct vfio_iommu_type1_dma_map map = {
+        .argsz = sizeof(map),
+        .flags = VFIO_DMA_MAP_FLAG_READ,
+        .vaddr = (__u64)(uintptr_t)vaddr,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (!readonly) {
+        map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
+    }
+
+    /*
+     * Try the mapping, if it fails with EBUSY, unmap the region and try
+     * again.  This shouldn't be necessary, but we sometimes see it in
+     * the the VGA ROM space.
+     */
+    if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
+        (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
+         ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
+        return 0;
+    }
+
+    DPRINTF("VFIO_MAP_DMA: %d\n", -errno);
+    return -errno;
+}
+
+static bool vfio_listener_skipped_section(MemoryRegionSection *section)
+{
+    return !memory_region_is_ram(section->mr);
+}
+
+static void vfio_listener_region_add(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            iommu_data.listener);
+    hwaddr iova, end;
+    void *vaddr;
+    int ret;
+
+    if (vfio_listener_skipped_section(section)) {
+        DPRINTF("vfio: SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
+                section->offset_within_address_space,
+                section->offset_within_address_space + section->size - 1);
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region\n", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    end = (section->offset_within_address_space + section->size) &
+          TARGET_PAGE_MASK;
+
+    if (iova >= end) {
+        return;
+    }
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    DPRINTF("vfio: region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
+            iova, end - 1, vaddr);
+
+    ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
+    if (ret) {
+        error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx", %p) = %d (%m)\n",
+                     container, iova, end - iova, vaddr, ret);
+    }
+}
+
+static void vfio_listener_region_del(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            iommu_data.listener);
+    hwaddr iova, end;
+    int ret;
+
+    if (vfio_listener_skipped_section(section)) {
+        DPRINTF("vfio: SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
+                section->offset_within_address_space,
+                section->offset_within_address_space + section->size - 1);
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region\n", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    end = (section->offset_within_address_space + section->size) &
+          TARGET_PAGE_MASK;
+
+    if (iova >= end) {
+        return;
+    }
+
+    DPRINTF("vfio: region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+            iova, end - 1);
+
+    ret = vfio_dma_unmap(container, iova, end - iova);
+    if (ret) {
+        error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx") = %d (%m)\n",
+                     container, iova, end - iova, ret);
+    }
+}
+
+static MemoryListener vfio_memory_listener = {
+    .region_add = vfio_listener_region_add,
+    .region_del = vfio_listener_region_del,
+};
+
+static void vfio_listener_release(VFIOContainer *container)
+{
+    memory_listener_unregister(&container->iommu_data.listener);
+}
+
+/*
+ * Interrupt setup
+ */
+static void vfio_disable_interrupts(VFIODevice *vdev)
+{
+    switch (vdev->interrupt) {
+    case VFIO_INT_INTx:
+        vfio_disable_intx(vdev);
+        break;
+    case VFIO_INT_MSI:
+        vfio_disable_msi(vdev);
+        break;
+    case VFIO_INT_MSIX:
+        vfio_disable_msix(vdev);
+        break;
+    }
+}
+
+static int vfio_setup_msi(VFIODevice *vdev, int pos)
+{
+    uint16_t ctrl;
+    bool msi_64bit, msi_maskbit;
+    int ret, entries;
+
+    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+    ctrl = le16_to_cpu(ctrl);
+
+    msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
+    msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
+    entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
+
+    DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function, pos);
+
+    ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_report("vfio: msi_init failed\n");
+        return ret;
+    }
+    vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
+
+    return 0;
+}
+
+/*
+ * We don't have any control over how pci_add_capability() inserts
+ * capabilities into the chain.  In order to setup MSI-X we need a
+ * MemoryRegion for the BAR.  In order to setup the BAR and not
+ * attempt to mmap the MSI-X table area, which VFIO won't allow, we
+ * need to first look for where the MSI-X table lives.  So we
+ * unfortunately split MSI-X setup across two functions.
+ */
+static int vfio_early_setup_msix(VFIODevice *vdev)
+{
+    uint8_t pos;
+    uint16_t ctrl;
+    uint32_t table, pba;
+
+    pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
+    if (!pos) {
+        return 0;
+    }
+
+    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+
+    if (pread(vdev->fd, &table, sizeof(table),
+              vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
+        return -errno;
+    }
+
+    if (pread(vdev->fd, &pba, sizeof(pba),
+              vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
+        return -errno;
+    }
+
+    ctrl = le16_to_cpu(ctrl);
+    table = le32_to_cpu(table);
+    pba = le32_to_cpu(pba);
+
+    vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
+    vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+
+    DPRINTF("%04x:%02x:%02x.%x "
+            "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, pos, vdev->msix->table_bar,
+            vdev->msix->table_offset, vdev->msix->entries);
+
+    return 0;
+}
+
+static int vfio_setup_msix(VFIODevice *vdev, int pos)
+{
+    int ret;
+
+    ret = msix_init(&vdev->pdev, vdev->msix->entries,
+                    &vdev->bars[vdev->msix->table_bar].mem,
+                    vdev->msix->table_bar, vdev->msix->table_offset,
+                    &vdev->bars[vdev->msix->pba_bar].mem,
+                    vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_report("vfio: msix_init failed\n");
+        return ret;
+    }
+
+    return 0;
+}
+
+static void vfio_teardown_msi(VFIODevice *vdev)
+{
+    msi_uninit(&vdev->pdev);
+
+    if (vdev->msix) {
+        msix_uninit(&vdev->pdev, &vdev->bars[vdev->msix->table_bar].mem,
+                    &vdev->bars[vdev->msix->pba_bar].mem);
+    }
+}
+
+/*
+ * Resource setup
+ */
+static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        VFIOBAR *bar = &vdev->bars[i];
+
+        if (!bar->size) {
+            continue;
+        }
+
+        memory_region_set_enabled(&bar->mmap_mem, enabled);
+        if (vdev->msix && vdev->msix->table_bar == i) {
+            memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
+        }
+    }
+}
+
+static void vfio_unmap_bar(VFIODevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+
+    if (!bar->size) {
+        return;
+    }
+
+    memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
+    munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
+
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
+        munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
+    }
+
+    memory_region_destroy(&bar->mem);
+}
+
+static int vfio_mmap_bar(VFIOBAR *bar, MemoryRegion *mem, MemoryRegion *submem,
+                         void **map, size_t size, off_t offset,
+                         const char *name)
+{
+    int ret = 0;
+
+    if (size && bar->flags & VFIO_REGION_INFO_FLAG_MMAP) {
+        int prot = 0;
+
+        if (bar->flags & VFIO_REGION_INFO_FLAG_READ) {
+            prot |= PROT_READ;
+        }
+
+        if (bar->flags & VFIO_REGION_INFO_FLAG_WRITE) {
+            prot |= PROT_WRITE;
+        }
+
+        *map = mmap(NULL, size, prot, MAP_SHARED,
+                    bar->fd, bar->fd_offset + offset);
+        if (*map == MAP_FAILED) {
+            *map = NULL;
+            ret = -errno;
+            goto empty_region;
+        }
+
+        memory_region_init_ram_ptr(submem, name, size, *map);
+    } else {
+empty_region:
+        /* Create a zero sized sub-region to make cleanup easy. */
+        memory_region_init(submem, name, 0);
+    }
+
+    memory_region_add_subregion(mem, offset, submem);
+
+    return ret;
+}
+
+static void vfio_map_bar(VFIODevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    unsigned size = bar->size;
+    char name[64];
+    uint32_t pci_bar;
+    uint8_t type;
+    int ret;
+
+    /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
+    if (!size) {
+        return;
+    }
+
+    snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function, nr);
+
+    /* Determine what type of BAR this is for registration */
+    ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
+                vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
+    if (ret != sizeof(pci_bar)) {
+        error_report("vfio: Failed to read BAR %d (%m)\n", nr);
+        return;
+    }
+
+    pci_bar = le32_to_cpu(pci_bar);
+    type = pci_bar & (pci_bar & PCI_BASE_ADDRESS_SPACE_IO ?
+           ~PCI_BASE_ADDRESS_IO_MASK : ~PCI_BASE_ADDRESS_MEM_MASK);
+
+    /* A "slow" read/write mapping underlies all BARs */
+    memory_region_init_io(&bar->mem, &vfio_bar_ops, bar, name, size);
+    pci_register_bar(&vdev->pdev, nr, type, &bar->mem);
+
+    /*
+     * We can't mmap areas overlapping the MSIX vector table, so we
+     * potentially insert a direct-mapped subregion before and after it.
+     */
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        size = vdev->msix->table_offset & TARGET_PAGE_MASK;
+    }
+
+    strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
+    if (vfio_mmap_bar(bar, &bar->mem,
+                      &bar->mmap_mem, &bar->mmap, size, 0, name)) {
+        error_report("%s unsupported. Performance may be slow\n", name);
+    }
+
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        unsigned start;
+
+        start = TARGET_PAGE_ALIGN(vdev->msix->table_offset +
+                                  (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
+
+        size = start < bar->size ? bar->size - start : 0;
+        strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
+        /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
+        if (vfio_mmap_bar(bar, &bar->mem, &vdev->msix->mmap_mem,
+                          &vdev->msix->mmap, size, start, name)) {
+            error_report("%s unsupported. Performance may be slow\n", name);
+        }
+    }
+}
+
+static void vfio_map_bars(VFIODevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_map_bar(vdev, i);
+    }
+}
+
+static void vfio_unmap_bars(VFIODevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_unmap_bar(vdev, i);
+    }
+}
+
+/*
+ * General setup
+ */
+static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
+{
+    uint8_t tmp, next = 0xff;
+
+    for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
+         tmp = pdev->config[tmp + 1]) {
+        if (tmp > pos && tmp < next) {
+            next = tmp;
+        }
+    }
+
+    return next - pos;
+}
+
+static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint8_t cap_id, next, size;
+    int ret;
+
+    cap_id = pdev->config[pos];
+    next = pdev->config[pos + 1];
+
+    /*
+     * If it becomes important to configure capabilities to their actual
+     * size, use this as the default when it's something we don't recognize.
+     * Since QEMU doesn't actually handle many of the config accesses,
+     * exact size doesn't seem worthwhile.
+     */
+    size = vfio_std_cap_max_size(pdev, pos);
+
+    /*
+     * pci_add_capability always inserts the new capability at the head
+     * of the chain.  Therefore to end up with a chain that matches the
+     * physical device, we insert from the end by making this recursive.
+     * This is also why we pre-caclulate size above as cached config space
+     * will be changed as we unwind the stack.
+     */
+    if (next) {
+        ret = vfio_add_std_cap(vdev, next);
+        if (ret) {
+            return ret;
+        }
+    } else {
+        pdev->config[PCI_CAPABILITY_LIST] = 0; /* Begin the rebuild */
+    }
+
+    switch (cap_id) {
+    case PCI_CAP_ID_MSI:
+        ret = vfio_setup_msi(vdev, pos);
+        break;
+    case PCI_CAP_ID_MSIX:
+        ret = vfio_setup_msix(vdev, pos);
+        break;
+    default:
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    }
+
+    if (ret < 0) {
+        error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
+                     "0x%x[0x%x]@0x%x: %d\n", vdev->host.domain,
+                     vdev->host.bus, vdev->host.slot, vdev->host.function,
+                     cap_id, size, pos, ret);
+        return ret;
+    }
+
+    return 0;
+}
+
+static int vfio_add_capabilities(VFIODevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+
+    if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
+        !pdev->config[PCI_CAPABILITY_LIST]) {
+        return 0; /* Nothing to add */
+    }
+
+    return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
+}
+
+static int vfio_load_rom(VFIODevice *vdev)
+{
+    uint64_t size = vdev->rom_size;
+    char name[32];
+    off_t off = 0, voff = vdev->rom_offset;
+    ssize_t bytes;
+    void *ptr;
+
+    /* If loading ROM from file, pci handles it */
+    if (vdev->pdev.romfile || !vdev->pdev.rom_bar || !size) {
+        return 0;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function);
+    memory_region_init_ram(&vdev->pdev.rom, name, size);
+    ptr = memory_region_get_ram_ptr(&vdev->pdev.rom);
+    memset(ptr, 0xff, size);
+
+    while (size) {
+        bytes = pread(vdev->fd, ptr + off, size, voff + off);
+        if (bytes == 0) {
+            break; /* expect that we could get back less than the ROM BAR */
+        } else if (bytes > 0) {
+            off += bytes;
+            size -= bytes;
+        } else {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            }
+            error_report("vfio: Error reading device ROM: %m\n");
+            memory_region_destroy(&vdev->pdev.rom);
+            return -errno;
+        }
+    }
+
+    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT, 0, &vdev->pdev.rom);
+    vdev->pdev.has_rom = true;
+    return 0;
+}
+
+static int vfio_connect_container(VFIOGroup *group)
+{
+    VFIOContainer *container;
+    int ret, fd;
+
+    if (group->container) {
+        return 0;
+    }
+
+    QLIST_FOREACH(container, &container_list, next) {
+        if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
+            group->container = container;
+            QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+            return 0;
+        }
+    }
+
+    fd = qemu_open("/dev/vfio/vfio", O_RDWR);
+    if (fd < 0) {
+        error_report("vfio: failed to open /dev/vfio/vfio: %m\n");
+        return -errno;
+    }
+
+    ret = ioctl(fd, VFIO_GET_API_VERSION);
+    if (ret != VFIO_API_VERSION) {
+        error_report("vfio: supported vfio version: %d, "
+                     "reported version: %d\n", VFIO_API_VERSION, ret);
+        close(fd);
+        return -EINVAL;
+    }
+
+    container = g_malloc0(sizeof(*container));
+    container->fd = fd;
+
+    if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
+        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
+        if (ret) {
+            error_report("vfio: failed to set group container: %m\n");
+            g_free(container);
+            close(fd);
+            return -errno;
+        }
+
+        ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
+        if (ret) {
+            error_report("vfio: failed to set iommu for container: %m\n");
+            g_free(container);
+            close(fd);
+            return -errno;
+        }
+
+        container->iommu_data.listener = vfio_memory_listener;
+        container->iommu_data.release = vfio_listener_release;
+
+        memory_listener_register(&container->iommu_data.listener, &address_space_memory);
+    } else {
+        error_report("vfio: No available IOMMU models\n");
+        g_free(container);
+        close(fd);
+        return -EINVAL;
+    }
+
+    QLIST_INIT(&container->group_list);
+    QLIST_INSERT_HEAD(&container_list, container, next);
+
+    group->container = container;
+    QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+
+    return 0;
+}
+
+static void vfio_disconnect_container(VFIOGroup *group)
+{
+    VFIOContainer *container = group->container;
+
+    if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
+        error_report("vfio: error disconnecting group %d from container\n",
+                     group->groupid);
+    }
+
+    QLIST_REMOVE(group, container_next);
+    group->container = NULL;
+
+    if (QLIST_EMPTY(&container->group_list)) {
+        if (container->iommu_data.release) {
+            container->iommu_data.release(container);
+        }
+        QLIST_REMOVE(container, next);
+        DPRINTF("vfio_disconnect_container: close container->fd\n");
+        close(container->fd);
+        g_free(container);
+    }
+}
+
+static VFIOGroup *vfio_get_group(int groupid)
+{
+    VFIOGroup *group;
+    char path[32];
+    struct vfio_group_status status = { .argsz = sizeof(status) };
+
+    QLIST_FOREACH(group, &group_list, next) {
+        if (group->groupid == groupid) {
+            return group;
+        }
+    }
+
+    group = g_malloc0(sizeof(*group));
+
+    snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
+    group->fd = qemu_open(path, O_RDWR);
+    if (group->fd < 0) {
+        error_report("vfio: error opening %s: %m\n", path);
+        g_free(group);
+        return NULL;
+    }
+
+    if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
+        error_report("vfio: error getting group status: %m\n");
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
+        error_report("vfio: error, group %d is not viable, please ensure "
+                     "all devices within the iommu_group are bound to their "
+                     "vfio bus driver.\n", groupid);
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    group->groupid = groupid;
+    QLIST_INIT(&group->device_list);
+
+    if (vfio_connect_container(group)) {
+        error_report("vfio: failed to setup container for group %d\n", groupid);
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    QLIST_INSERT_HEAD(&group_list, group, next);
+
+    return group;
+}
+
+static void vfio_put_group(VFIOGroup *group)
+{
+    if (!QLIST_EMPTY(&group->device_list)) {
+        return;
+    }
+
+    vfio_disconnect_container(group);
+    QLIST_REMOVE(group, next);
+    DPRINTF("vfio_put_group: close group->fd\n");
+    close(group->fd);
+    g_free(group);
+}
+
+static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
+{
+    struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
+    struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
+    int ret, i;
+
+    ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
+    if (ret < 0) {
+        error_report("vfio: error getting device %s from group %d: %m\n",
+                     name, group->groupid);
+        error_report("Verify all devices in group %d are bound to vfio-pci "
+                     "or pci-stub and not already in use\n", group->groupid);
+        return ret;
+    }
+
+    vdev->fd = ret;
+    vdev->group = group;
+    QLIST_INSERT_HEAD(&group->device_list, vdev, next);
+
+    /* Sanity check device */
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
+    if (ret) {
+        error_report("vfio: error getting device info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name,
+            dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
+
+    if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
+        error_report("vfio: Um, this isn't a PCI device\n");
+        goto error;
+    }
+
+    vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
+    if (!vdev->reset_works) {
+        error_report("Warning, device %s does not support reset\n", name);
+    }
+
+    if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
+        error_report("vfio: unexpected number of io regions %u\n",
+                     dev_info.num_regions);
+        goto error;
+    }
+
+    if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
+        error_report("vfio: unexpected number of irqs %u\n", dev_info.num_irqs);
+        goto error;
+    }
+
+    for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
+        reg_info.index = i;
+
+        ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+        if (ret) {
+            error_report("vfio: Error getting region %d info: %m\n", i);
+            goto error;
+        }
+
+        DPRINTF("Device %s region %d:\n", name, i);
+        DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+                (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+                (unsigned long)reg_info.flags);
+
+        vdev->bars[i].flags = reg_info.flags;
+        vdev->bars[i].size = reg_info.size;
+        vdev->bars[i].fd_offset = reg_info.offset;
+        vdev->bars[i].fd = vdev->fd;
+        vdev->bars[i].nr = i;
+    }
+
+    reg_info.index = VFIO_PCI_ROM_REGION_INDEX;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+    if (ret) {
+        error_report("vfio: Error getting ROM info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s ROM:\n", name);
+    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+            (unsigned long)reg_info.flags);
+
+    vdev->rom_size = reg_info.size;
+    vdev->rom_offset = reg_info.offset;
+
+    reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+    if (ret) {
+        error_report("vfio: Error getting config info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s config:\n", name);
+    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+            (unsigned long)reg_info.flags);
+
+    vdev->config_size = reg_info.size;
+    vdev->config_offset = reg_info.offset;
+
+error:
+    if (ret) {
+        QLIST_REMOVE(vdev, next);
+        vdev->group = NULL;
+        close(vdev->fd);
+    }
+    return ret;
+}
+
+static void vfio_put_device(VFIODevice *vdev)
+{
+    QLIST_REMOVE(vdev, next);
+    vdev->group = NULL;
+    DPRINTF("vfio_put_device: close vdev->fd\n");
+    close(vdev->fd);
+    if (vdev->msix) {
+        g_free(vdev->msix);
+        vdev->msix = NULL;
+    }
+}
+
+static int vfio_initfn(PCIDevice *pdev)
+{
+    VFIODevice *pvdev, *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOGroup *group;
+    char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
+    ssize_t len;
+    struct stat st;
+    int groupid;
+    int ret;
+
+    /* Check that the host device exists */
+    snprintf(path, sizeof(path),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function);
+    if (stat(path, &st) < 0) {
+        error_report("vfio: error: no such host device: %s\n", path);
+        return -errno;
+    }
+
+    strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
+
+    len = readlink(path, iommu_group_path, PATH_MAX);
+    if (len <= 0) {
+        error_report("vfio: error no iommu_group for device\n");
+        return -errno;
+    }
+
+    iommu_group_path[len] = 0;
+    group_name = basename(iommu_group_path);
+
+    if (sscanf(group_name, "%d", &groupid) != 1) {
+        error_report("vfio: error reading %s: %m\n", path);
+        return -errno;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
+
+    group = vfio_get_group(groupid);
+    if (!group) {
+        error_report("vfio: failed to get group %d\n", groupid);
+        return -ENOENT;
+    }
+
+    snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function);
+
+    QLIST_FOREACH(pvdev, &group->device_list, next) {
+        if (pvdev->host.domain == vdev->host.domain &&
+            pvdev->host.bus == vdev->host.bus &&
+            pvdev->host.slot == vdev->host.slot &&
+            pvdev->host.function == vdev->host.function) {
+
+            error_report("vfio: error: device %s is already attached\n", path);
+            vfio_put_group(group);
+            return -EBUSY;
+        }
+    }
+
+    ret = vfio_get_device(group, path, vdev);
+    if (ret) {
+        error_report("vfio: failed to get device %s\n", path);
+        vfio_put_group(group);
+        return ret;
+    }
+
+    /* Get a copy of config space */
+    ret = pread(vdev->fd, vdev->pdev.config,
+                MIN(pci_config_size(&vdev->pdev), vdev->config_size),
+                vdev->config_offset);
+    if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
+        ret = ret < 0 ? -errno : -EFAULT;
+        error_report("vfio: Failed to read device config space\n");
+        goto out_put;
+    }
+
+    /*
+     * Clear host resource mapping info.  If we choose not to register a
+     * BAR, such as might be the case with the option ROM, we can get
+     * confusing, unwritable, residual addresses from the host here.
+     */
+    memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
+    memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
+
+    vfio_load_rom(vdev);
+
+    ret = vfio_early_setup_msix(vdev);
+    if (ret) {
+        goto out_put;
+    }
+
+    vfio_map_bars(vdev);
+
+    ret = vfio_add_capabilities(vdev);
+    if (ret) {
+        goto out_teardown;
+    }
+
+    if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
+        vdev->intx.mmap_timer = qemu_new_timer_ms(vm_clock,
+                                                  vfio_intx_mmap_enable, vdev);
+        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
+        ret = vfio_enable_intx(vdev);
+        if (ret) {
+            goto out_teardown;
+        }
+    }
+
+    return 0;
+
+out_teardown:
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_teardown_msi(vdev);
+    vfio_unmap_bars(vdev);
+out_put:
+    vfio_put_device(vdev);
+    vfio_put_group(group);
+    return ret;
+}
+
+static void vfio_exitfn(PCIDevice *pdev)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOGroup *group = vdev->group;
+
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_disable_interrupts(vdev);
+    if (vdev->intx.mmap_timer) {
+        qemu_free_timer(vdev->intx.mmap_timer);
+    }
+    vfio_teardown_msi(vdev);
+    vfio_unmap_bars(vdev);
+    vfio_put_device(vdev);
+    vfio_put_group(group);
+}
+
+static void vfio_pci_reset(DeviceState *dev)
+{
+    PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint16_t cmd;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    vfio_disable_interrupts(vdev);
+
+    /*
+     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
+     * Also put INTx Disable in known state.
+     */
+    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
+    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+             PCI_COMMAND_INTX_DISABLE);
+    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+
+    if (vdev->reset_works) {
+        if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
+            error_report("vfio: Error unable to reset physical device "
+                         "(%04x:%02x:%02x.%x): %m\n", vdev->host.domain,
+                         vdev->host.bus, vdev->host.slot, vdev->host.function);
+        }
+    }
+
+    vfio_enable_intx(vdev);
+}
+
+static Property vfio_pci_dev_properties[] = {
+    DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
+    DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
+                       intx.mmap_timeout, 1100),
+    /*
+     * TODO - support passed fds... is this necessary?
+     * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
+     * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
+     */
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vfio_pci_vmstate = {
+    .name = "vfio-pci",
+    .unmigratable = 1,
+};
+
+static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
+
+    dc->reset = vfio_pci_reset;
+    dc->props = vfio_pci_dev_properties;
+    dc->vmsd = &vfio_pci_vmstate;
+    dc->desc = "VFIO-based PCI device assignment";
+    pdc->init = vfio_initfn;
+    pdc->exit = vfio_exitfn;
+    pdc->config_read = vfio_pci_read_config;
+    pdc->config_write = vfio_pci_write_config;
+}
+
+static const TypeInfo vfio_pci_dev_info = {
+    .name = "vfio-pci",
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VFIODevice),
+    .class_init = vfio_pci_dev_class_init,
+};
+
+static void register_vfio_pci_dev_type(void)
+{
+    type_register_static(&vfio_pci_dev_info);
+}
+
+type_init(register_vfio_pci_dev_type)
diff --git a/hw/vga-isa-mm.c b/hw/vga-isa-mm.c
index 44ae7d92c8c..311c966f77a 100644
--- a/hw/vga-isa-mm.c
+++ b/hw/vga-isa-mm.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 
 #define VGA_RAM_SIZE (8192 * 1024)
 
@@ -36,7 +36,7 @@ typedef struct ISAVGAMMState {
 } ISAVGAMMState;
 
 /* Memory mapped interface */
-static uint32_t vga_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readb (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -44,14 +44,14 @@ static uint32_t vga_mm_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writeb (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
     vga_ioport_write(&s->vga, addr >> s->it_shift, value & 0xff);
 }
 
-static uint32_t vga_mm_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readw (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -59,14 +59,14 @@ static uint32_t vga_mm_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writew (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
     vga_ioport_write(&s->vga, addr >> s->it_shift, value & 0xffff);
 }
 
-static uint32_t vga_mm_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readl (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -74,7 +74,7 @@ static uint32_t vga_mm_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writel (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
@@ -97,8 +97,8 @@ static const MemoryRegionOps vga_mm_ctrl_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
-                        target_phys_addr_t ctrl_base, int it_shift,
+static void vga_mm_init(ISAVGAMMState *s, hwaddr vram_base,
+                        hwaddr ctrl_base, int it_shift,
                         MemoryRegion *address_space)
 {
     MemoryRegion *s_ioport_ctrl, *vga_io_memory;
@@ -107,6 +107,7 @@ static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
     s_ioport_ctrl = g_malloc(sizeof(*s_ioport_ctrl));
     memory_region_init_io(s_ioport_ctrl, &vga_mm_ctrl_ops, s,
                           "vga-mm-ctrl", 0x100000);
+    memory_region_set_flush_coalesced(s_ioport_ctrl);
 
     vga_io_memory = g_malloc(sizeof(*vga_io_memory));
     /* XXX: endianness? */
@@ -122,8 +123,8 @@ static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
     memory_region_set_coalescing(vga_io_memory);
 }
 
-int isa_vga_mm_init(target_phys_addr_t vram_base,
-                    target_phys_addr_t ctrl_base, int it_shift,
+int isa_vga_mm_init(hwaddr vram_base,
+                    hwaddr ctrl_base, int it_shift,
                     MemoryRegion *address_space)
 {
     ISAVGAMMState *s;
diff --git a/hw/vga-isa.c b/hw/vga-isa.c
index d2904737bc5..762e45aaeb6 100644
--- a/hw/vga-isa.c
+++ b/hw/vga-isa.c
@@ -1,6 +1,8 @@
 /*
  * QEMU ISA VGA Emulator.
  *
+ * see docs/specs/standard-vga.txt for virtual hardware specs.
+ *
  * Copyright (c) 2003 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -22,11 +24,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "loader.h"
 
 typedef struct ISAVGAState {
@@ -84,7 +86,7 @@ static void vga_class_initfn(ObjectClass *klass, void *data)
     dc->props = vga_isa_properties;
 }
 
-static TypeInfo vga_info = {
+static const TypeInfo vga_info = {
     .name          = "isa-vga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAVGAState),
diff --git a/hw/vga-pci.c b/hw/vga-pci.c
index 9abbada8f17..c491af20e4b 100644
--- a/hw/vga-pci.c
+++ b/hw/vga-pci.c
@@ -1,6 +1,8 @@
 /*
  * QEMU PCI VGA Emulator.
  *
+ * see docs/specs/standard-vga.txt for virtual hardware specs.
+ *
  * Copyright (c) 2003 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -22,17 +24,30 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
-#include "pci.h"
-#include "vga-pci.h"
+#include "ui/console.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "loader.h"
 
+#define PCI_VGA_IOPORT_OFFSET 0x400
+#define PCI_VGA_IOPORT_SIZE   (0x3e0 - 0x3c0)
+#define PCI_VGA_BOCHS_OFFSET  0x500
+#define PCI_VGA_BOCHS_SIZE    (0x0b * 2)
+#define PCI_VGA_MMIO_SIZE     0x1000
+
+enum vga_pci_flags {
+    PCI_VGA_FLAG_ENABLE_MMIO = 1,
+};
+
 typedef struct PCIVGAState {
     PCIDevice dev;
     VGACommonState vga;
+    uint32_t flags;
+    MemoryRegion mmio;
+    MemoryRegion ioport;
+    MemoryRegion bochs;
 } PCIVGAState;
 
 static const VMStateDescription vmstate_vga_pci = {
@@ -47,36 +62,126 @@ static const VMStateDescription vmstate_vga_pci = {
     }
 };
 
-static int pci_vga_initfn(PCIDevice *dev)
+static uint64_t pci_vga_ioport_read(void *ptr, hwaddr addr,
+                                    unsigned size)
 {
-     PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
-     VGACommonState *s = &d->vga;
+    PCIVGAState *d = ptr;
+    uint64_t ret = 0;
+
+    switch (size) {
+    case 1:
+        ret = vga_ioport_read(&d->vga, addr);
+        break;
+    case 2:
+        ret  = vga_ioport_read(&d->vga, addr);
+        ret |= vga_ioport_read(&d->vga, addr+1) << 8;
+        break;
+    }
+    return ret;
+}
 
-     // vga + console init
-     vga_common_init(s);
-     vga_init(s, pci_address_space(dev), pci_address_space_io(dev), true);
+static void pci_vga_ioport_write(void *ptr, hwaddr addr,
+                                 uint64_t val, unsigned size)
+{
+    PCIVGAState *d = ptr;
 
-     s->ds = graphic_console_init(s->update, s->invalidate,
-                                  s->screen_dump, s->text_update, s);
+    switch (size) {
+    case 1:
+        vga_ioport_write(&d->vga, addr + 0x3c0, val);
+        break;
+    case 2:
+        /*
+         * Update bytes in little endian order.  Allows to update
+         * indexed registers with a single word write because the
+         * index byte is updated first.
+         */
+        vga_ioport_write(&d->vga, addr + 0x3c0, val & 0xff);
+        vga_ioport_write(&d->vga, addr + 0x3c1, (val >> 8) & 0xff);
+        break;
+    }
+}
 
-     /* XXX: VGA_RAM_SIZE must be a power of two */
-     pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
+static const MemoryRegionOps pci_vga_ioport_ops = {
+    .read = pci_vga_ioport_read,
+    .write = pci_vga_ioport_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-     if (!dev->rom_bar) {
-         /* compatibility with pc-0.13 and older */
-         vga_init_vbe(s, pci_address_space(dev));
-     }
+static uint64_t pci_vga_bochs_read(void *ptr, hwaddr addr,
+                                   unsigned size)
+{
+    PCIVGAState *d = ptr;
+    int index = addr >> 1;
 
-     return 0;
+    vbe_ioport_write_index(&d->vga, 0, index);
+    return vbe_ioport_read_data(&d->vga, 0);
 }
 
-DeviceState *pci_vga_init(PCIBus *bus)
+static void pci_vga_bochs_write(void *ptr, hwaddr addr,
+                                uint64_t val, unsigned size)
 {
-    return &pci_create_simple(bus, -1, "VGA")->qdev;
+    PCIVGAState *d = ptr;
+    int index = addr >> 1;
+
+    vbe_ioport_write_index(&d->vga, 0, index);
+    vbe_ioport_write_data(&d->vga, 0, val);
+}
+
+static const MemoryRegionOps pci_vga_bochs_ops = {
+    .read = pci_vga_bochs_read,
+    .write = pci_vga_bochs_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int pci_std_vga_initfn(PCIDevice *dev)
+{
+    PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
+    VGACommonState *s = &d->vga;
+
+    /* vga + console init */
+    vga_common_init(s);
+    vga_init(s, pci_address_space(dev), pci_address_space_io(dev), true);
+
+    s->ds = graphic_console_init(s->update, s->invalidate,
+                                 s->screen_dump, s->text_update, s);
+
+    /* XXX: VGA_RAM_SIZE must be a power of two */
+    pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
+
+    /* mmio bar for vga register access */
+    if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_MMIO)) {
+        memory_region_init(&d->mmio, "vga.mmio", 4096);
+        memory_region_init_io(&d->ioport, &pci_vga_ioport_ops, d,
+                              "vga ioports remapped", PCI_VGA_IOPORT_SIZE);
+        memory_region_init_io(&d->bochs, &pci_vga_bochs_ops, d,
+                              "bochs dispi interface", PCI_VGA_BOCHS_SIZE);
+
+        memory_region_add_subregion(&d->mmio, PCI_VGA_IOPORT_OFFSET,
+                                    &d->ioport);
+        memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
+                                    &d->bochs);
+        pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
+    }
+
+    if (!dev->rom_bar) {
+        /* compatibility with pc-0.13 and older */
+        vga_init_vbe(s, pci_address_space(dev));
+    }
+
+    return 0;
 }
 
 static Property vga_pci_properties[] = {
     DEFINE_PROP_UINT32("vgamem_mb", PCIVGAState, vga.vram_size_mb, 16),
+    DEFINE_PROP_BIT("mmio", PCIVGAState, flags, PCI_VGA_FLAG_ENABLE_MMIO, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -86,7 +191,7 @@ static void vga_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->no_hotplug = 1;
-    k->init = pci_vga_initfn;
+    k->init = pci_std_vga_initfn;
     k->romfile = "vgabios-stdvga.bin";
     k->vendor_id = PCI_VENDOR_ID_QEMU;
     k->device_id = PCI_DEVICE_ID_QEMU_VGA;
@@ -95,7 +200,7 @@ static void vga_class_init(ObjectClass *klass, void *data)
     dc->props = vga_pci_properties;
 }
 
-static TypeInfo vga_info = {
+static const TypeInfo vga_info = {
     .name          = "VGA",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIVGAState),
diff --git a/hw/vga-pci.h b/hw/vga-pci.h
deleted file mode 100644
index 49abf1309dd..00000000000
--- a/hw/vga-pci.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef VGA_PCI_H
-#define VGA_PCI_H
-
-#include "qemu-common.h"
-
-/* vga-pci.c */
-DeviceState *pci_vga_init(PCIBus *bus);
-
-/* cirrus_vga.c */
-DeviceState *pci_cirrus_vga_init(PCIBus *bus);
-
-#endif
diff --git a/hw/vga.c b/hw/vga.c
index f82ced8e662..e2ba7f208c6 100644
--- a/hw/vga.c
+++ b/hw/vga.c
@@ -23,12 +23,12 @@
  */
 #include "hw.h"
 #include "vga.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "xen.h"
 #include "trace.h"
 
@@ -166,12 +166,13 @@ static uint32_t expand4[256];
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
-static void vga_screen_dump(void *opaque, const char *filename, bool cswitch);
+static void vga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
 
 static void vga_update_memory_access(VGACommonState *s)
 {
     MemoryRegion *region, *old_region = s->chain4_alias;
-    target_phys_addr_t base, offset, size;
+    hwaddr base, offset, size;
 
     s->chain4_alias = NULL;
 
@@ -360,6 +361,8 @@ uint32_t vga_ioport_read(void *opaque, uint32_t addr)
     VGACommonState *s = opaque;
     int val, index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     if (vga_ioport_invalid(s, addr)) {
         val = 0xff;
     } else {
@@ -452,6 +455,8 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     VGACommonState *s = opaque;
     int index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
         return;
@@ -577,7 +582,6 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-#ifdef CONFIG_BOCHS_VBE
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
@@ -586,7 +590,7 @@ static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
     return val;
 }
 
-static uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
+uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
     uint32_t val;
@@ -622,13 +626,13 @@ static uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
     return val;
 }
 
-static void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val)
+void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val)
 {
     VGACommonState *s = opaque;
     s->vbe_index = val;
 }
 
-static void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
+void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
 {
     VGACommonState *s = opaque;
 
@@ -779,10 +783,9 @@ static void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         }
     }
 }
-#endif
 
 /* called for accesses between 0xa0000 and 0xc0000 */
-uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr)
+uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr)
 {
     int memory_map_mode, plane;
     uint32_t ret;
@@ -839,7 +842,7 @@ uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr)
 }
 
 /* called for accesses between 0xa0000 and 0xc0000 */
-void vga_mem_writeb(VGACommonState *s, target_phys_addr_t addr, uint32_t val)
+void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
 {
     int memory_map_mode, plane, write_mode, b, func_select, mask;
     uint32_t write_mask, bit_mask, set_mask;
@@ -1124,14 +1127,12 @@ static void vga_get_offsets(VGACommonState *s,
                             uint32_t *pline_compare)
 {
     uint32_t start_addr, line_offset, line_compare;
-#ifdef CONFIG_BOCHS_VBE
+
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         line_offset = s->vbe_line_offset;
         start_addr = s->vbe_start_addr;
         line_compare = 65535;
-    } else
-#endif
-    {
+    } else {
         /* compute line_offset in bytes */
         line_offset = s->cr[VGA_CRTC_OFFSET];
         line_offset <<= 3;
@@ -1345,6 +1346,7 @@ static void vga_draw_text(VGACommonState *s, int full_update)
         s->last_scr_width = width * cw;
         s->last_scr_height = height * cheight;
         qemu_console_resize(s->ds, s->last_scr_width, s->last_scr_height);
+        dpy_text_resize(s->ds, width, height);
         s->last_depth = 0;
         s->last_width = width;
         s->last_height = height;
@@ -1358,6 +1360,14 @@ static void vga_draw_text(VGACommonState *s, int full_update)
     palette = s->last_palette;
     x_incr = cw * ((ds_get_bits_per_pixel(s->ds) + 7) >> 3);
 
+    if (full_update) {
+        s->full_update_text = 1;
+    }
+    if (s->full_update_gfx) {
+        s->full_update_gfx = 0;
+        full_update |= 1;
+    }
+
     cursor_offset = ((s->cr[VGA_CRTC_CURSOR_HI] << 8) |
                      s->cr[VGA_CRTC_CURSOR_LO]) - s->start_addr;
     if (cursor_offset != s->cursor_offset ||
@@ -1455,8 +1465,8 @@ static void vga_draw_text(VGACommonState *s, int full_update)
             ch_attr_ptr++;
         }
         if (cx_max != -1) {
-            dpy_update(s->ds, cx_min * cw, cy * cheight,
-                       (cx_max - cx_min + 1) * cw, cheight);
+            dpy_gfx_update(s->ds, cx_min * cw, cy * cheight,
+                           (cx_max - cx_min + 1) * cw, cheight);
         }
         dest += linesize * cheight;
         line1 = line + cheight;
@@ -1567,12 +1577,10 @@ static vga_draw_line_func * const vga_draw_line_table[NB_DEPTHS * VGA_DRAW_LINE_
 static int vga_get_bpp(VGACommonState *s)
 {
     int ret;
-#ifdef CONFIG_BOCHS_VBE
+
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         ret = s->vbe_regs[VBE_DISPI_INDEX_BPP];
-    } else
-#endif
-    {
+    } else {
         ret = 0;
     }
     return ret;
@@ -1582,13 +1590,10 @@ static void vga_get_resolution(VGACommonState *s, int *pwidth, int *pheight)
 {
     int width, height;
 
-#ifdef CONFIG_BOCHS_VBE
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         width = s->vbe_regs[VBE_DISPI_INDEX_XRES];
         height = s->vbe_regs[VBE_DISPI_INDEX_YRES];
-    } else
-#endif
-    {
+    } else {
         width = (s->cr[VGA_CRTC_H_DISP] + 1) * 8;
         height = s->cr[VGA_CRTC_V_DISP_END] |
             ((s->cr[VGA_CRTC_OVERFLOW] & 0x02) << 7) |
@@ -1611,7 +1616,7 @@ void vga_invalidate_scanlines(VGACommonState *s, int y1, int y2)
     }
 }
 
-static void vga_sync_dirty_bitmap(VGACommonState *s)
+void vga_sync_dirty_bitmap(VGACommonState *s)
 {
     memory_region_sync_dirty_bitmap(&s->vram);
 }
@@ -1692,7 +1697,7 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
             s->ds->surface->pf = qemu_different_endianness_pixelformat(depth);
 #endif
-            dpy_resize(s->ds);
+            dpy_gfx_resize(s->ds);
         } else {
             qemu_console_resize(s->ds, disp_width, height);
         }
@@ -1704,9 +1709,14 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         s->last_depth = depth;
         full_update = 1;
     } else if (is_buffer_shared(s->ds->surface) &&
-               (full_update || s->ds->surface->data != s->vram_ptr + (s->start_addr * 4))) {
-        s->ds->surface->data = s->vram_ptr + (s->start_addr * 4);
-        dpy_setdata(s->ds);
+               (full_update || ds_get_data(s->ds) != s->vram_ptr
+                + (s->start_addr * 4))) {
+        qemu_free_displaysurface(s->ds);
+        s->ds->surface = qemu_create_displaysurface_from(disp_width,
+                height, depth,
+                s->line_offset,
+                s->vram_ptr + (s->start_addr * 4));
+        dpy_gfx_setdata(s->ds);
     }
 
     s->rgb_to_pixel =
@@ -1811,8 +1821,8 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(s->ds, 0, y_start,
-                           disp_width, y - y_start);
+                dpy_gfx_update(s->ds, 0, y_start,
+                               disp_width, y - y_start);
                 y_start = -1;
             }
         }
@@ -1832,8 +1842,8 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(s->ds, 0, y_start,
-                   disp_width, y - y_start);
+        dpy_gfx_update(s->ds, 0, y_start,
+                       disp_width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -1867,8 +1877,8 @@ static void vga_draw_blank(VGACommonState *s, int full_update)
         memset(d, val, w);
         d += ds_get_linesize(s->ds);
     }
-    dpy_update(s->ds, 0, 0,
-               s->last_scr_width, s->last_scr_height);
+    dpy_gfx_update(s->ds, 0, 0,
+                   s->last_scr_width, s->last_scr_height);
 }
 
 #define GMODE_TEXT     0
@@ -1943,14 +1953,12 @@ void vga_common_reset(VGACommonState *s)
     s->dac_8bit = 0;
     memset(s->palette, '\0', sizeof(s->palette));
     s->bank_offset = 0;
-#ifdef CONFIG_BOCHS_VBE
     s->vbe_index = 0;
     memset(s->vbe_regs, '\0', sizeof(s->vbe_regs));
     s->vbe_regs[VBE_DISPI_INDEX_ID] = VBE_DISPI_ID5;
     s->vbe_start_addr = 0;
     s->vbe_line_offset = 0;
     s->vbe_bank_mask = (s->vram_size >> 16) - 1;
-#endif
     memset(s->font_offsets, '\0', sizeof(s->font_offsets));
     s->graphic_mode = -1; /* force full update */
     s->shift_control = 0;
@@ -2058,9 +2066,9 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             cw != s->last_cw || cheight != s->last_ch) {
             s->last_scr_width = width * cw;
             s->last_scr_height = height * cheight;
-            s->ds->surface->width = width;
-            s->ds->surface->height = height;
-            dpy_resize(s->ds);
+            qemu_console_resize(s->ds, s->last_scr_width, s->last_scr_height);
+            dpy_text_resize(s->ds, width, height);
+            s->last_depth = 0;
             s->last_width = width;
             s->last_height = height;
             s->last_ch = cheight;
@@ -2068,6 +2076,14 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             full_update = 1;
         }
 
+        if (full_update) {
+            s->full_update_gfx = 1;
+        }
+        if (s->full_update_text) {
+            s->full_update_text = 0;
+            full_update |= 1;
+        }
+
         /* Update "hardware" cursor */
         cursor_offset = ((s->cr[VGA_CRTC_CURSOR_HI] << 8) |
                          s->cr[VGA_CRTC_CURSOR_LO]) - s->start_addr;
@@ -2076,11 +2092,11 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             s->cr[VGA_CRTC_CURSOR_END] != s->cursor_end || full_update) {
             cursor_visible = !(s->cr[VGA_CRTC_CURSOR_START] & 0x20);
             if (cursor_visible && cursor_offset < size && cursor_offset >= 0)
-                dpy_cursor(s->ds,
-                           TEXTMODE_X(cursor_offset),
-                           TEXTMODE_Y(cursor_offset));
+                dpy_text_cursor(s->ds,
+                                TEXTMODE_X(cursor_offset),
+                                TEXTMODE_Y(cursor_offset));
             else
-                dpy_cursor(s->ds, -1, -1);
+                dpy_text_cursor(s->ds, -1, -1);
             s->cursor_offset = cursor_offset;
             s->cursor_start = s->cr[VGA_CRTC_CURSOR_START];
             s->cursor_end = s->cr[VGA_CRTC_CURSOR_END];
@@ -2093,7 +2109,7 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             for (i = 0; i < size; src ++, dst ++, i ++)
                 console_write_ch(dst, VMEM2CHTYPE(le32_to_cpu(*src)));
 
-            dpy_update(s->ds, 0, 0, width, height);
+            dpy_text_update(s->ds, 0, 0, width, height);
         } else {
             c_max = 0;
 
@@ -2116,7 +2132,7 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
 
             if (c_min <= c_max) {
                 i = TEXTMODE_Y(c_min);
-                dpy_update(s->ds, 0, i, width, TEXTMODE_Y(c_max) - i + 1);
+                dpy_text_update(s->ds, 0, i, width, TEXTMODE_Y(c_max) - i + 1);
             }
         }
 
@@ -2141,10 +2157,8 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
     /* Display a message */
     s->last_width = 60;
     s->last_height = height = 3;
-    dpy_cursor(s->ds, -1, -1);
-    s->ds->surface->width = s->last_width;
-    s->ds->surface->height = height;
-    dpy_resize(s->ds);
+    dpy_text_cursor(s->ds, -1, -1);
+    dpy_text_resize(s->ds, s->last_width, height);
 
     for (dst = chardata, i = 0; i < s->last_width * height; i ++)
         console_write_ch(dst ++, ' ');
@@ -2155,10 +2169,10 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
     for (i = 0; i < size; i ++)
         console_write_ch(dst ++, 0x00200100 | msg_buffer[i]);
 
-    dpy_update(s->ds, 0, 0, s->last_width, height);
+    dpy_text_update(s->ds, 0, 0, s->last_width, height);
 }
 
-static uint64_t vga_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t vga_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     VGACommonState *s = opaque;
@@ -2166,7 +2180,7 @@ static uint64_t vga_mem_read(void *opaque, target_phys_addr_t addr,
     return vga_mem_readb(s, addr);
 }
 
-static void vga_mem_write(void *opaque, target_phys_addr_t addr,
+static void vga_mem_write(void *opaque, hwaddr addr,
                           uint64_t data, unsigned size)
 {
     VGACommonState *s = opaque;
@@ -2224,13 +2238,11 @@ const VMStateDescription vmstate_vga_common = {
 
         VMSTATE_INT32(bank_offset, VGACommonState),
         VMSTATE_UINT8_EQUAL(is_vbe_vmstate, VGACommonState),
-#ifdef CONFIG_BOCHS_VBE
         VMSTATE_UINT16(vbe_index, VGACommonState),
         VMSTATE_UINT16_ARRAY(vbe_regs, VGACommonState, VBE_DISPI_INDEX_NB),
         VMSTATE_UINT32(vbe_start_addr, VGACommonState),
         VMSTATE_UINT32(vbe_line_offset, VGACommonState),
         VMSTATE_UINT32(vbe_bank_mask, VGACommonState),
-#endif
         VMSTATE_END_OF_LIST()
     }
 };
@@ -2270,11 +2282,7 @@ void vga_common_init(VGACommonState *s)
     }
     s->vram_size_mb = s->vram_size >> 20;
 
-#ifdef CONFIG_BOCHS_VBE
     s->is_vbe_vmstate = 1;
-#else
-    s->is_vbe_vmstate = 0;
-#endif
     memory_region_init_ram(&s->vram, "vga.vram", s->vram_size);
     vmstate_register_ram_global(&s->vram);
     xen_register_framebuffer(&s->vram);
@@ -2309,17 +2317,14 @@ static const MemoryRegionPortio vga_portio_list[] = {
     PORTIO_END_OF_LIST(),
 };
 
-#ifdef CONFIG_BOCHS_VBE
 static const MemoryRegionPortio vbe_portio_list[] = {
     { 0, 1, 2, .read = vbe_ioport_read_index, .write = vbe_ioport_write_index },
 # ifdef TARGET_I386
     { 1, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
-# else
-    { 2, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
 # endif
+    { 2, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
     PORTIO_END_OF_LIST(),
 };
-#endif /* CONFIG_BOCHS_VBE */
 
 /* Used by both ISA and PCI */
 MemoryRegion *vga_init_io(VGACommonState *s,
@@ -2329,14 +2334,12 @@ MemoryRegion *vga_init_io(VGACommonState *s,
     MemoryRegion *vga_mem;
 
     *vga_ports = vga_portio_list;
-    *vbe_ports = NULL;
-#ifdef CONFIG_BOCHS_VBE
     *vbe_ports = vbe_portio_list;
-#endif
 
     vga_mem = g_malloc(sizeof(*vga_mem));
     memory_region_init_io(vga_mem, &vga_mem_ops, s,
                           "vga-lowmem", 0x20000);
+    memory_region_set_flush_coalesced(vga_mem);
 
     return vga_mem;
 }
@@ -2373,7 +2376,6 @@ void vga_init(VGACommonState *s, MemoryRegion *address_space,
 
 void vga_init_vbe(VGACommonState *s, MemoryRegion *system_memory)
 {
-#ifdef CONFIG_BOCHS_VBE
     /* With pc-0.12 and below we map both the PCI BAR and the fixed VBE region,
      * so use an alias to avoid double-mapping the same region.
      */
@@ -2384,58 +2386,59 @@ void vga_init_vbe(VGACommonState *s, MemoryRegion *system_memory)
                                 VBE_DISPI_LFB_PHYSICAL_ADDRESS,
                                 &s->vram_vbe);
     s->vbe_mapped = 1;
-#endif 
 }
 /********************************************************/
 /* vga screen dump */
 
-int ppm_save(const char *filename, struct DisplaySurface *ds)
+void ppm_save(const char *filename, struct DisplaySurface *ds, Error **errp)
 {
+    int width = pixman_image_get_width(ds->image);
+    int height = pixman_image_get_height(ds->image);
     FILE *f;
-    uint8_t *d, *d1;
-    uint32_t v;
-    int y, x;
-    uint8_t r, g, b;
+    int y;
     int ret;
-    char *linebuf, *pbuf;
+    pixman_image_t *linebuf;
 
     trace_ppm_save(filename, ds);
     f = fopen(filename, "wb");
-    if (!f)
-        return -1;
-    fprintf(f, "P6\n%d %d\n%d\n",
-            ds->width, ds->height, 255);
-    linebuf = g_malloc(ds->width * 3);
-    d1 = ds->data;
-    for(y = 0; y < ds->height; y++) {
-        d = d1;
-        pbuf = linebuf;
-        for(x = 0; x < ds->width; x++) {
-            if (ds->pf.bits_per_pixel == 32)
-                v = *(uint32_t *)d;
-            else
-                v = (uint32_t) (*(uint16_t *)d);
-            /* Limited to 8 or fewer bits per channel: */
-            r = ((v >> ds->pf.rshift) & ds->pf.rmax) << (8 - ds->pf.rbits);
-            g = ((v >> ds->pf.gshift) & ds->pf.gmax) << (8 - ds->pf.gbits);
-            b = ((v >> ds->pf.bshift) & ds->pf.bmax) << (8 - ds->pf.bbits);
-            *pbuf++ = r;
-            *pbuf++ = g;
-            *pbuf++ = b;
-            d += ds->pf.bytes_per_pixel;
-        }
-        d1 += ds->linesize;
-        ret = fwrite(linebuf, 1, pbuf - linebuf, f);
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
+        return;
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", width, height, 255);
+    if (ret < 0) {
+        linebuf = NULL;
+        goto write_err;
+    }
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, width);
+    for (y = 0; y < height; y++) {
+        qemu_pixman_linebuf_fill(linebuf, ds->image, width, 0, y);
+        clearerr(f);
+        ret = fwrite(pixman_image_get_data(linebuf), 1,
+                     pixman_image_get_stride(linebuf), f);
         (void)ret;
+        if (ferror(f)) {
+            goto write_err;
+        }
     }
-    g_free(linebuf);
+
+out:
+    qemu_pixman_image_unref(linebuf);
     fclose(f);
-    return 0;
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 /* save the vga display in a PPM image even if no display is
    available */
-static void vga_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void vga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp)
 {
     VGACommonState *s = opaque;
 
@@ -2443,5 +2446,5 @@ static void vga_screen_dump(void *opaque, const char *filename, bool cswitch)
         vga_invalidate_display(s);
     }
     vga_hw_update();
-    ppm_save(filename, s->ds->surface);
+    ppm_save(filename, s->ds->surface, errp);
 }
diff --git a/hw/vga_int.h b/hw/vga_int.h
index 8938093682e..8d496ea9bf2 100644
--- a/hw/vga_int.h
+++ b/hw/vga_int.h
@@ -21,16 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_VGA_INT_H
+#define HW_VGA_INT_H 1
 
 #include <hw/hw.h>
-#include "memory.h"
+#include "qapi/error.h"
+#include "exec/memory.h"
 
 #define ST01_V_RETRACE      0x08
 #define ST01_DISP_ENABLE    0x01
 
-/* bochs VBE support */
-#define CONFIG_BOCHS_VBE
-
 #define VBE_DISPI_MAX_XRES              16000
 #define VBE_DISPI_MAX_YRES              12000
 #define VBE_DISPI_MAX_BPP               32
@@ -64,21 +64,6 @@
 
 #define VBE_DISPI_LFB_PHYSICAL_ADDRESS  0xE0000000
 
-#ifdef CONFIG_BOCHS_VBE
-
-#define VGA_STATE_COMMON_BOCHS_VBE              \
-    uint16_t vbe_index;                         \
-    uint16_t vbe_regs[VBE_DISPI_INDEX_NB];      \
-    uint32_t vbe_start_addr;                    \
-    uint32_t vbe_line_offset;                   \
-    uint32_t vbe_bank_mask;			\
-    int vbe_mapped;
-#else
-
-#define VGA_STATE_COMMON_BOCHS_VBE
-
-#endif /* !CONFIG_BOCHS_VBE */
-
 #define CH_ATTR_SIZE (160 * 100)
 #define VGA_MAX_HEIGHT 2048
 
@@ -139,7 +124,13 @@ typedef struct VGACommonState {
     void (*get_resolution)(struct VGACommonState *s,
                         int *pwidth,
                         int *pheight);
-    VGA_STATE_COMMON_BOCHS_VBE
+    /* bochs vbe state */
+    uint16_t vbe_index;
+    uint16_t vbe_regs[VBE_DISPI_INDEX_NB];
+    uint32_t vbe_start_addr;
+    uint32_t vbe_line_offset;
+    uint32_t vbe_bank_mask;
+    int vbe_mapped;
     /* display refresh support */
     DisplayState *ds;
     uint32_t font_offsets[2];
@@ -165,6 +156,8 @@ typedef struct VGACommonState {
     vga_hw_invalidate_ptr invalidate;
     vga_hw_screen_dump_ptr screen_dump;
     vga_hw_text_update_ptr text_update;
+    bool full_update_text;
+    bool full_update_gfx;
     /* hardware mouse cursor support */
     uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
     void (*cursor_invalidate)(struct VGACommonState *s);
@@ -195,19 +188,24 @@ MemoryRegion *vga_init_io(VGACommonState *s,
                           const MemoryRegionPortio **vbe_ports);
 void vga_common_reset(VGACommonState *s);
 
+void vga_sync_dirty_bitmap(VGACommonState *s);
 void vga_dirty_log_start(VGACommonState *s);
 void vga_dirty_log_stop(VGACommonState *s);
 
 extern const VMStateDescription vmstate_vga_common;
 uint32_t vga_ioport_read(void *opaque, uint32_t addr);
 void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val);
-uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr);
-void vga_mem_writeb(VGACommonState *s, target_phys_addr_t addr, uint32_t val);
+uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr);
+void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val);
 void vga_invalidate_scanlines(VGACommonState *s, int y1, int y2);
-int ppm_save(const char *filename, struct DisplaySurface *ds);
+void ppm_save(const char *filename, struct DisplaySurface *ds, Error **errp);
 
 int vga_ioport_invalid(VGACommonState *s, uint32_t addr);
+
 void vga_init_vbe(VGACommonState *s, MemoryRegion *address_space);
+uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr);
+void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val);
+void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val);
 
 extern const uint8_t sr_mask[8];
 extern const uint8_t gr_mask[16];
@@ -216,3 +214,5 @@ extern const uint8_t gr_mask[16];
 #define VGABIOS_CIRRUS_FILENAME "vgabios-cirrus.bin"
 
 extern const MemoryRegionOps vga_mem_ops;
+
+#endif
diff --git a/hw/vhost-scsi.c b/hw/vhost-scsi.c
new file mode 100644
index 00000000000..29dde3dd472
--- /dev/null
+++ b/hw/vhost-scsi.c
@@ -0,0 +1,209 @@
+/*
+ * vhost_scsi host device
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * Changes for QEMU mainline + tcm_vhost kernel upstream:
+ *  Nicholas Bellinger <nab@risingtidesystems.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include <sys/ioctl.h>
+#include "config.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "migration/migration.h"
+#include "vhost-scsi.h"
+#include "vhost.h"
+#include "virtio-scsi.h"
+
+typedef struct VHostSCSI {
+    VirtIOSCSICommon vs;
+
+    Error *migration_blocker;
+
+    struct vhost_dev dev;
+} VHostSCSI;
+
+static int vhost_scsi_set_endpoint(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    struct vhost_scsi_target backend;
+    int ret;
+
+    memset(&backend, 0, sizeof(backend));
+    pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->vs.conf->wwpn);
+    ret = ioctl(vs->dev.control, VHOST_SCSI_SET_ENDPOINT, &backend);
+    if (ret < 0) {
+        return -errno;
+    }
+    return 0;
+}
+
+static void vhost_scsi_clear_endpoint(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    struct vhost_scsi_target backend;
+
+    memset(&backend, 0, sizeof(backend));
+    pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->vs.conf->wwpn);
+    ioctl(vs->dev.control, VHOST_SCSI_CLEAR_ENDPOINT, &backend);
+}
+
+static int vhost_scsi_start(VHostSCSI *vs, VirtIODevice *vdev)
+{
+    int ret, abi_version;
+
+    ret = ioctl(vs->dev.control, VHOST_SCSI_GET_ABI_VERSION, &abi_version);
+    if (ret < 0) {
+        return -errno;
+    }
+    if (abi_version > VHOST_SCSI_ABI_VERSION) {
+        error_report("vhost-scsi: The running tcm_vhost kernel abi_version:"
+                     " %d is greater than vhost_scsi userspace supports: %d, please"
+                     " upgrade your version of QEMU\n", abi_version,
+                     VHOST_SCSI_ABI_VERSION);
+        return -ENOSYS;
+    }
+
+    ret = vhost_dev_enable_notifiers(&vs->dev, vdev);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_dev_start(&vs->dev, vdev);
+    if (ret < 0) {
+        vhost_dev_disable_notifiers(&vs->dev, vdev);
+    }
+
+    return ret;
+}
+
+static void vhost_scsi_stop(VHostSCSI *vs, VirtIODevice *vdev)
+{
+    vhost_dev_stop(&vs->dev, vdev);
+}
+
+static void vhost_scsi_set_config(VirtIODevice *vdev,
+                                  const uint8_t *config)
+{
+    VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    if ((uint32_t) ldl_raw(&scsiconf->sense_size) != vs->vs.sense_size ||
+        (uint32_t) ldl_raw(&scsiconf->cdb_size) != vs->vs.cdb_size) {
+        error_report("vhost-scsi does not support changing the sense data and CDB sizes");
+        exit(1);
+    }
+}
+
+static void vhost_scsi_set_status(VirtIODevice *vdev, uint8_t val)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    bool start = (val & (VIRTIO_CONFIG_S_DRIVER|VIRTIO_CONFIG_S_DRIVER_OK));
+
+    if (vs->dev.started == start) {
+        return;
+    }
+
+    if (start) {
+        int ret;
+
+        ret = vhost_scsi_start(vs, vdev);
+        if (ret < 0) {
+            error_report("virtio-scsi: unable to start vhost: %s\n",
+                         strerror(-ret));
+
+            /* There is no userspace virtio-scsi fallback so exit */
+            exit(1);
+        }
+    } else {
+        vhost_scsi_stop(vs, vdev);
+    }
+}
+
+static void vhost_scsi_guest_notifier_mask(VirtIODevice *vdev, int idx,
+                                           bool mask)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    printf("vhost_scsi_guest_notifier_mask vhost_started: %d\n", vs->dev.started);
+
+    vhost_virtqueue_mask(&vs->dev, vdev, idx, mask);
+}
+
+static bool vhost_scsi_guest_notifier_pending(VirtIODevice *vdev, int idx)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    printf("vhost_scsi_guest_notifier_pending vhost_started: %d\n", vs->dev.started);
+
+    return vhost_virtqueue_pending(&vs->dev, idx);
+}
+
+VirtIODevice *vhost_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf)
+{
+    VHostSCSI *vs;
+    int vhostfd = -1;
+    int ret;
+
+    if (!proxyconf->wwpn) {
+        error_report("vhost-scsi: missing wwpn\n");
+        return NULL;
+    }
+
+    if (proxyconf->vhostfd) {
+        vhostfd = monitor_handle_fd_param(cur_mon, proxyconf->vhostfd);
+        if (vhostfd == -1) {
+            error_report("vhost-scsi: unable to parse vhostfd\n");
+            return NULL;
+        }
+    }
+
+    vs = (VHostSCSI *)virtio_scsi_init_common(dev, proxyconf,
+                                              sizeof(VHostSCSI));
+
+    vs->vs.vdev.set_config = vhost_scsi_set_config;
+    vs->vs.vdev.set_status = vhost_scsi_set_status;
+    vs->vs.vdev.set_vhost_endpoint = vhost_scsi_set_endpoint;
+    vs->vs.vdev.clear_vhost_endpoint = vhost_scsi_clear_endpoint;
+    vs->vs.vdev.guest_notifier_mask = vhost_scsi_guest_notifier_mask;
+    vs->vs.vdev.guest_notifier_pending = vhost_scsi_guest_notifier_pending;
+
+    vs->dev.nvqs = VHOST_SCSI_VQ_NUM_FIXED + vs->vs.conf->num_queues;
+    vs->dev.vqs = g_new(struct vhost_virtqueue, vs->dev.nvqs);
+
+    ret = vhost_dev_init(&vs->dev, vhostfd, "/dev/vhost-scsi", true);
+    if (ret < 0) {
+        error_report("vhost-scsi: vhost initialization failed: %s\n",
+                strerror(-ret));
+        return NULL;
+    }
+    vs->dev.backend_features = 0;
+    vs->dev.acked_features = 0;
+
+    error_setg(&vs->migration_blocker,
+            "vhost-scsi does not support migration");
+    migrate_add_blocker(vs->migration_blocker);
+
+    return &vs->vs.vdev;
+}
+
+void vhost_scsi_exit(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    migrate_del_blocker(vs->migration_blocker);
+    error_free(vs->migration_blocker);
+
+    /* This will stop vhost backend. */
+    vhost_scsi_set_status(vdev, 0);
+    g_free(vs->dev.vqs);
+    virtio_cleanup(vdev);
+}
+
diff --git a/hw/vhost-scsi.h b/hw/vhost-scsi.h
new file mode 100644
index 00000000000..70f4a58d70b
--- /dev/null
+++ b/hw/vhost-scsi.h
@@ -0,0 +1,64 @@
+/*
+ * vhost_scsi host device
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_SCSI_H
+#define VHOST_SCSI_H
+
+#include "qemu-common.h"
+#include "qdev.h"
+#include "virtio-scsi.h"
+
+/*
+ * Used by QEMU userspace to ensure a consistent vhost-scsi ABI.
+ *
+ * ABI Rev 0: July 2012 version starting point for v3.6-rc merge candidate +
+ *            RFC-v2 vhost-scsi userspace.  Add GET_ABI_VERSION ioctl usage
+ * ABI Rev 1: January 2013. Ignore vhost_tpgt filed in struct vhost_scsi_target.
+ * 	      All the targets under vhost_wwpn can be seen and used by guest.
+ */
+
+#define VHOST_SCSI_ABI_VERSION 1
+
+/* TODO #include <linux/vhost.h> properly */
+/* For VHOST_SCSI_SET_ENDPOINT/VHOST_SCSI_CLEAR_ENDPOINT ioctl */
+struct vhost_scsi_target {
+    int abi_version;
+    char vhost_wwpn[224];
+    unsigned short vhost_tpgt;
+    unsigned short reserved;
+};
+
+enum vhost_scsi_vq_list {
+    VHOST_SCSI_VQ_CONTROL = 0,
+    VHOST_SCSI_VQ_EVENT = 1,
+    VHOST_SCSI_VQ_NUM_FIXED = 2,
+};
+
+#define VHOST_VIRTIO 0xAF
+#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target)
+#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
+#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
+
+#define DEFINE_VHOST_SCSI_PROPERTIES(_state, _features_field, _conf_field) \
+    DEFINE_VIRTIO_COMMON_FEATURES(_state, _features_field), \
+    DEFINE_PROP_STRING("vhostfd", _state, _conf_field.vhostfd), \
+    DEFINE_PROP_STRING("wwpn", _state, _conf_field.wwpn), \
+    DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1), \
+    DEFINE_PROP_UINT32("max_sectors", _state, _conf_field.max_sectors, 0xFFFF), \
+    DEFINE_PROP_UINT32("cmd_per_lun", _state, _conf_field.cmd_per_lun, 128)
+
+VirtIODevice *vhost_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf);
+void vhost_scsi_exit(VirtIODevice *vdev);
+
+
+#endif
diff --git a/hw/vhost.c b/hw/vhost.c
index 0fd8da84e2b..09fa4e1ad58 100644
--- a/hw/vhost.c
+++ b/hw/vhost.c
@@ -16,9 +16,9 @@
 #include <sys/ioctl.h>
 #include "vhost.h"
 #include "hw/hw.h"
-#include "range.h"
+#include "qemu/range.h"
 #include <linux/vhost.h>
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static void vhost_dev_sync_region(struct vhost_dev *dev,
                                   MemoryRegionSection *section,
@@ -65,8 +65,8 @@ static void vhost_dev_sync_region(struct vhost_dev *dev,
 
 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
                                    MemoryRegionSection *section,
-                                   target_phys_addr_t start_addr,
-                                   target_phys_addr_t end_addr)
+                                   hwaddr start_addr,
+                                   hwaddr end_addr)
 {
     int i;
 
@@ -93,8 +93,8 @@ static void vhost_log_sync(MemoryListener *listener,
 {
     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
                                          memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
-    target_phys_addr_t end_addr = start_addr + section->size;
+    hwaddr start_addr = section->offset_within_address_space;
+    hwaddr end_addr = start_addr + section->size;
 
     vhost_sync_dirty_bitmap(dev, section, start_addr, end_addr);
 }
@@ -296,7 +296,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,
     int i;
     for (i = 0; i < dev->nvqs; ++i) {
         struct vhost_virtqueue *vq = dev->vqs + i;
-        target_phys_addr_t l;
+        hwaddr l;
         void *p;
 
         if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
@@ -362,7 +362,7 @@ static void vhost_set_memory(MemoryListener *listener,
 {
     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
                                          memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     bool log_dirty = memory_region_is_logging(section->mr);
     int s = offsetof(struct vhost_memory, regions) +
@@ -434,8 +434,7 @@ static void vhost_set_memory(MemoryListener *listener,
 
 static bool vhost_section(MemoryRegionSection *section)
 {
-    return section->address_space == get_system_memory()
-        && memory_region_is_ram(section->mr);
+    return memory_region_is_ram(section->mr);
 }
 
 static void vhost_begin(MemoryListener *listener)
@@ -613,20 +612,16 @@ static void vhost_log_stop(MemoryListener *listener,
     /* FIXME: implement */
 }
 
-static int vhost_virtqueue_init(struct vhost_dev *dev,
-                                struct VirtIODevice *vdev,
-                                struct vhost_virtqueue *vq,
-                                unsigned idx)
+static int vhost_virtqueue_map(struct vhost_dev *dev,
+                               struct VirtIODevice *vdev,
+                               struct vhost_virtqueue *vq,
+                               unsigned idx)
 {
-    target_phys_addr_t s, l, a;
+    hwaddr s, l, a;
     int r;
-    struct vhost_vring_file file = {
-        .index = idx,
-    };
     struct vhost_vring_state state = {
         .index = idx,
     };
-    struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 
     vq->num = state.num = virtio_queue_get_num(vdev, idx);
     r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
@@ -675,24 +670,9 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
         r = -errno;
         goto fail_alloc;
     }
-    file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
-    r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
-    if (r) {
-        r = -errno;
-        goto fail_kick;
-    }
-
-    file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
-    r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
-    if (r) {
-        r = -errno;
-        goto fail_call;
-    }
 
     return 0;
 
-fail_call:
-fail_kick:
 fail_alloc:
     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
                               0, 0);
@@ -709,7 +689,29 @@ fail_alloc_desc:
     return r;
 }
 
-static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
+static int vhost_virtqueue_start(struct vhost_dev *dev,
+                                 struct VirtIODevice *vdev,
+                                 struct vhost_virtqueue *vq,
+                                 unsigned idx)
+{
+    struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
+    struct vhost_vring_file file = {
+        .index = idx,
+        .fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq))
+    };
+    int r;
+
+    r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
+    if (r) {
+        return -errno;
+    }
+
+    /* Clear and discard previous events if any. */
+    event_notifier_test_and_clear(&vq->masked_notifier);
+    return 0;
+}
+
+static void vhost_virtqueue_unmap(struct vhost_dev *dev,
                                     struct VirtIODevice *vdev,
                                     struct vhost_virtqueue *vq,
                                     unsigned idx)
@@ -747,14 +749,43 @@ static void vhost_eventfd_del(MemoryListener *listener,
 {
 }
 
-int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
+static int vhost_virtqueue_init(struct vhost_dev *dev,
+                                struct vhost_virtqueue *vq, int n)
+{
+    struct vhost_vring_file file = {
+        .index = n,
+    };
+    int r = event_notifier_init(&vq->masked_notifier, 0);
+    if (r < 0) {
+        return r;
+    }
+
+    file.fd = event_notifier_get_fd(&vq->masked_notifier);
+    r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
+    if (r) {
+        r = -errno;
+        goto fail_call;
+    }
+    return 0;
+fail_call:
+    event_notifier_cleanup(&vq->masked_notifier);
+    return r;
+}
+
+static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
+{
+    event_notifier_cleanup(&vq->masked_notifier);
+}
+
+int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
+                   bool force)
 {
     uint64_t features;
-    int r;
+    int i, r;
     if (devfd >= 0) {
         hdev->control = devfd;
     } else {
-        hdev->control = open("/dev/vhost-net", O_RDWR);
+        hdev->control = open(devpath, O_RDWR);
         if (hdev->control < 0) {
             return -errno;
         }
@@ -768,6 +799,13 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
     if (r < 0) {
         goto fail;
     }
+
+    for (i = 0; i < hdev->nvqs; ++i) {
+        r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
+        if (r < 0) {
+            goto fail_vq;
+        }
+    }
     hdev->features = features;
 
     hdev->memory_listener = (MemoryListener) {
@@ -792,9 +830,13 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
     hdev->log_size = 0;
     hdev->log_enabled = false;
     hdev->started = false;
-    memory_listener_register(&hdev->memory_listener, NULL);
+    memory_listener_register(&hdev->memory_listener, &address_space_memory);
     hdev->force = force;
     return 0;
+fail_vq:
+    while (--i >= 0) {
+        vhost_virtqueue_cleanup(hdev->vqs + i);
+    }
 fail:
     r = -errno;
     close(hdev->control);
@@ -803,6 +845,10 @@ fail:
 
 void vhost_dev_cleanup(struct vhost_dev *hdev)
 {
+    int i;
+    for (i = 0; i < hdev->nvqs; ++i) {
+        vhost_virtqueue_cleanup(hdev->vqs + i);
+    }
     memory_listener_unregister(&hdev->memory_listener);
     g_free(hdev->mem);
     g_free(hdev->mem_sections);
@@ -869,17 +915,53 @@ void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
 }
 
+/* Test and clear event pending status.
+ * Should be called after unmask to avoid losing events.
+ */
+bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
+{
+    struct vhost_virtqueue *vq = hdev->vqs + n;
+    assert(hdev->started);
+    return event_notifier_test_and_clear(&vq->masked_notifier);
+}
+
+/* Mask/unmask events from this vq. */
+void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
+                         bool mask)
+{
+    struct VirtQueue *vvq = virtio_get_queue(vdev, n);
+    int r;
+
+    assert(hdev->started);
+
+    struct vhost_vring_file file = {
+        .index = n,
+    };
+    if (mask) {
+        file.fd = event_notifier_get_fd(&hdev->vqs[n].masked_notifier);
+    } else {
+        file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
+    }
+    r = ioctl(hdev->control, VHOST_SET_VRING_CALL, &file);
+    assert(r >= 0);
+}
+
 /* Host notifiers must be enabled at this point. */
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
 {
-    int i, r;
+    int i, j, r;
+
+    hdev->started = true;
+
     if (!vdev->binding->set_guest_notifiers) {
         fprintf(stderr, "binding does not support guest notifiers\n");
         r = -ENOSYS;
         goto fail;
     }
 
-    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, true);
+    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque,
+                                           hdev->nvqs,
+                                           true);
     if (r < 0) {
         fprintf(stderr, "Error binding guest notifier: %d\n", -r);
         goto fail_notifiers;
@@ -895,10 +977,26 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         goto fail_mem;
     }
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vhost_virtqueue_init(hdev,
+        r = vhost_virtqueue_map(hdev,
+                                vdev,
+                                hdev->vqs + i,
+                                i);
+        if (r < 0) {
+            goto fail_vq;
+        }
+    }
+    if (vdev->set_vhost_endpoint) {
+        r = vdev->set_vhost_endpoint(vdev);
+        if (r < 0) {
+            goto fail_vq;
+        }
+    }
+
+    for (j = 0; j < hdev->nvqs; ++j) {
+        r = vhost_virtqueue_start(hdev,
                                  vdev,
-                                 hdev->vqs + i,
-                                 i);
+                                 hdev->vqs + j,
+                                 j);
         if (r < 0) {
             goto fail_vq;
         }
@@ -916,22 +1014,22 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         }
     }
 
-    hdev->started = true;
-
     return 0;
 fail_log:
 fail_vq:
     while (--i >= 0) {
-        vhost_virtqueue_cleanup(hdev,
+        vhost_virtqueue_unmap(hdev,
                                 vdev,
                                 hdev->vqs + i,
                                 i);
     }
 fail_mem:
 fail_features:
-    vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
+    vdev->binding->set_guest_notifiers(vdev->binding_opaque, hdev->nvqs, false);
 fail_notifiers:
 fail:
+
+    hdev->started = false;
     return r;
 }
 
@@ -941,22 +1039,27 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     int i, r;
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        vhost_virtqueue_cleanup(hdev,
+        vhost_virtqueue_unmap(hdev,
                                 vdev,
                                 hdev->vqs + i,
                                 i);
     }
     for (i = 0; i < hdev->n_mem_sections; ++i) {
         vhost_sync_dirty_bitmap(hdev, &hdev->mem_sections[i],
-                                0, (target_phys_addr_t)~0x0ull);
+                                0, (hwaddr)~0x0ull);
     }
-    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
+    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque,
+                                           hdev->nvqs,
+                                           false);
     if (r < 0) {
         fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
         fflush(stderr);
     }
     assert (r >= 0);
 
+    if (vdev->clear_vhost_endpoint) {
+        vdev->clear_vhost_endpoint(vdev);
+    }
     hdev->started = false;
     g_free(hdev->log);
     hdev->log = NULL;
diff --git a/hw/vhost.h b/hw/vhost.h
index 80e64df860b..44c61a58775 100644
--- a/hw/vhost.h
+++ b/hw/vhost.h
@@ -3,7 +3,7 @@
 
 #include "hw/hw.h"
 #include "hw/virtio.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Generic structures common for any vhost based device. */
 struct vhost_virtqueue {
@@ -18,6 +18,7 @@ struct vhost_virtqueue {
     void *ring;
     unsigned long long ring_phys;
     unsigned ring_size;
+    EventNotifier masked_notifier;
 };
 
 typedef unsigned long vhost_log_chunk_t;
@@ -44,7 +45,8 @@ struct vhost_dev {
     bool force;
 };
 
-int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force);
+int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
+                   bool force);
 void vhost_dev_cleanup(struct vhost_dev *hdev);
 bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev);
@@ -52,4 +54,13 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
 
+/* Test and clear masked event pending status.
+ * Should be called after unmask to avoid losing events.
+ */
+bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n);
+
+/* Mask/unmask events from this vq.
+ */
+void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
+                          bool mask);
 #endif
diff --git a/hw/vhost_net.c b/hw/vhost_net.c
index ecaa22dfb4e..d3a04caef6e 100644
--- a/hw/vhost_net.c
+++ b/hw/vhost_net.c
@@ -13,12 +13,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "net.h"
+#include "net/net.h"
 #include "net/tap.h"
 
 #include "virtio-net.h"
 #include "vhost_net.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #include "config.h"
 
@@ -109,7 +109,10 @@ struct vhost_net *vhost_net_init(NetClientState *backend, int devfd,
         (1 << VHOST_NET_F_VIRTIO_NET_HDR);
     net->backend = r;
 
-    r = vhost_dev_init(&net->dev, devfd, force);
+    net->dev.nvqs = 2;
+    net->dev.vqs = net->vqs;
+
+    r = vhost_dev_init(&net->dev, devfd, "/dev/vhost-net", force);
     if (r < 0) {
         goto fail;
     }
@@ -143,17 +146,10 @@ int vhost_net_start(struct vhost_net *net,
     struct vhost_vring_file file = { };
     int r;
 
-    net->dev.nvqs = 2;
-    net->dev.vqs = net->vqs;
-
     r = vhost_dev_enable_notifiers(&net->dev, dev);
     if (r < 0) {
         goto fail_notifiers;
     }
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc,
-                             sizeof(struct virtio_net_hdr_mrg_rxbuf));
-    }
 
     r = vhost_dev_start(&net->dev, dev);
     if (r < 0) {
@@ -179,9 +175,6 @@ fail:
     }
     net->nc->info->poll(net->nc, true);
     vhost_dev_stop(&net->dev, dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
 fail_start:
     vhost_dev_disable_notifiers(&net->dev, dev);
 fail_notifiers:
@@ -199,20 +192,25 @@ void vhost_net_stop(struct vhost_net *net,
     }
     net->nc->info->poll(net->nc, true);
     vhost_dev_stop(&net->dev, dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
     vhost_dev_disable_notifiers(&net->dev, dev);
 }
 
 void vhost_net_cleanup(struct vhost_net *net)
 {
     vhost_dev_cleanup(&net->dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
     g_free(net);
 }
+
+bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
+{
+    return vhost_virtqueue_pending(&net->dev, idx);
+}
+
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask)
+{
+    vhost_virtqueue_mask(&net->dev, dev, idx, mask);
+}
 #else
 struct vhost_net *vhost_net_init(NetClientState *backend, int devfd,
                                  bool force)
@@ -247,4 +245,14 @@ unsigned vhost_net_get_features(struct vhost_net *net, unsigned features)
 void vhost_net_ack_features(struct vhost_net *net, unsigned features)
 {
 }
+
+bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
+{
+    return -ENOSYS;
+}
+
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask)
+{
+}
 #endif
diff --git a/hw/vhost_net.h b/hw/vhost_net.h
index a9db23423c3..88912b85fda 100644
--- a/hw/vhost_net.h
+++ b/hw/vhost_net.h
@@ -1,7 +1,7 @@
 #ifndef VHOST_NET_H
 #define VHOST_NET_H
 
-#include "net.h"
+#include "net/net.h"
 
 struct vhost_net;
 typedef struct vhost_net VHostNetState;
@@ -17,4 +17,7 @@ void vhost_net_cleanup(VHostNetState *net);
 unsigned vhost_net_get_features(VHostNetState *net, unsigned features);
 void vhost_net_ack_features(VHostNetState *net, unsigned features);
 
+bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask);
 #endif
diff --git a/hw/virtex_ml507.c b/hw/virtex_ml507.c
index 79bc0d10eee..8c4e8e4313b 100644
--- a/hw/virtex_ml507.c
+++ b/hw/virtex_ml507.c
@@ -24,23 +24,22 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "pc.h"
-#include "net.h"
+#include "serial.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 #include "ppc.h"
 #include "ppc4xx.h"
 #include "ppc405.h"
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "xilinx.h"
 
 #define EPAPR_MAGIC    (0x45504150)
@@ -58,7 +57,7 @@ static struct boot_info
 /* Create reset TLB entries for BookE, spanning the 32bit addr space.  */
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa)
+                                     hwaddr pa)
 {
     ppcemb_tlb_t *tlb = &env->tlb.tlbe[0];
 
@@ -94,7 +93,7 @@ static PowerPCCPU *ppc440_init_xilinx(ram_addr_t *ram_size,
     }
     env = &cpu->env;
 
-    ppc_booke_timers_init(env, sysclk, 0/* no flags */);
+    ppc_booke_timers_init(cpu, sysclk, 0/* no flags */);
 
     ppc_dcr_init(env, NULL, NULL);
 
@@ -134,10 +133,10 @@ static void main_cpu_reset(void *opaque)
 }
 
 #define BINARY_DEVICE_TREE_FILE "virtex-ml507.dtb"
-static int xilinx_load_device_tree(target_phys_addr_t addr,
+static int xilinx_load_device_tree(hwaddr addr,
                                       uint32_t ramsize,
-                                      target_phys_addr_t initrd_base,
-                                      target_phys_addr_t initrd_size,
+                                      hwaddr initrd_base,
+                                      hwaddr initrd_size,
                                       const char *kernel_cmdline)
 {
     char *path;
@@ -183,17 +182,17 @@ static int xilinx_load_device_tree(target_phys_addr_t addr,
     return fdt_size;
 }
 
-static void virtex_init(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void virtex_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     MemoryRegion *address_space_mem = get_system_memory();
     DeviceState *dev;
     PowerPCCPU *cpu;
     CPUPPCState *env;
-    target_phys_addr_t ram_base = 0;
+    hwaddr ram_base = 0;
     DriveInfo *dinfo;
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
@@ -233,7 +232,7 @@ static void virtex_init(ram_addr_t ram_size,
 
     if (kernel_filename) {
         uint64_t entry, low, high;
-        target_phys_addr_t boot_offset;
+        hwaddr boot_offset;
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
@@ -264,6 +263,7 @@ static QEMUMachine virtex_machine = {
     .name = "virtex-ml507",
     .desc = "Xilinx Virtex ML507 reference design",
     .init = virtex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void virtex_machine_init(void)
diff --git a/hw/virtio-balloon.c b/hw/virtio-balloon.c
index dd1a6506cf2..c0a790264c9 100644
--- a/hw/virtio-balloon.c
+++ b/hw/virtio-balloon.c
@@ -13,15 +13,17 @@
  *
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "virtio.h"
 #include "pc.h"
 #include "cpu.h"
-#include "balloon.h"
+#include "sysemu/balloon.h"
 #include "virtio-balloon.h"
-#include "kvm.h"
-#include "exec-memory.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
+#include "qapi/visitor.h"
 
 #if defined(__linux__)
 #include <sys/mman.h>
@@ -36,6 +38,9 @@ typedef struct VirtIOBalloon
     uint64_t stats[VIRTIO_BALLOON_S_NR];
     VirtQueueElement stats_vq_elem;
     size_t stats_vq_offset;
+    QEMUTimer *stats_timer;
+    int64_t stats_last_update;
+    int64_t stats_poll_interval;
     DeviceState *qdev;
 } VirtIOBalloon;
 
@@ -53,6 +58,16 @@ static void balloon_page(void *addr, int deflate)
 #endif
 }
 
+static const char *balloon_stat_names[] = {
+   [VIRTIO_BALLOON_S_SWAP_IN] = "stat-swap-in",
+   [VIRTIO_BALLOON_S_SWAP_OUT] = "stat-swap-out",
+   [VIRTIO_BALLOON_S_MAJFLT] = "stat-major-faults",
+   [VIRTIO_BALLOON_S_MINFLT] = "stat-minor-faults",
+   [VIRTIO_BALLOON_S_MEMFREE] = "stat-free-memory",
+   [VIRTIO_BALLOON_S_MEMTOT] = "stat-total-memory",
+   [VIRTIO_BALLOON_S_NR] = NULL
+};
+
 /*
  * reset_stats - Mark all items in the stats array as unset
  *
@@ -67,6 +82,118 @@ static inline void reset_stats(VirtIOBalloon *dev)
     for (i = 0; i < VIRTIO_BALLOON_S_NR; dev->stats[i++] = -1);
 }
 
+static bool balloon_stats_supported(const VirtIOBalloon *s)
+{
+    return s->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ);
+}
+
+static bool balloon_stats_enabled(const VirtIOBalloon *s)
+{
+    return s->stats_poll_interval > 0;
+}
+
+static void balloon_stats_destroy_timer(VirtIOBalloon *s)
+{
+    if (balloon_stats_enabled(s)) {
+        qemu_del_timer(s->stats_timer);
+        qemu_free_timer(s->stats_timer);
+        s->stats_timer = NULL;
+        s->stats_poll_interval = 0;
+    }
+}
+
+static void balloon_stats_change_timer(VirtIOBalloon *s, int secs)
+{
+    qemu_mod_timer(s->stats_timer, qemu_get_clock_ms(vm_clock) + secs * 1000);
+}
+
+static void balloon_stats_poll_cb(void *opaque)
+{
+    VirtIOBalloon *s = opaque;
+
+    if (!balloon_stats_supported(s)) {
+        /* re-schedule */
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+        return;
+    }
+
+    virtqueue_push(s->svq, &s->stats_vq_elem, s->stats_vq_offset);
+    virtio_notify(&s->vdev, s->svq);
+}
+
+static void balloon_stats_get_all(Object *obj, struct Visitor *v,
+                                  void *opaque, const char *name, Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    int i;
+
+    if (!s->stats_last_update) {
+        error_setg(errp, "guest hasn't updated any stats yet");
+        return;
+    }
+
+    visit_start_struct(v, NULL, "guest-stats", name, 0, errp);
+    visit_type_int(v, &s->stats_last_update, "last-update", errp);
+
+    visit_start_struct(v, NULL, NULL, "stats", 0, errp);
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; i++) {
+        visit_type_int64(v, (int64_t *) &s->stats[i], balloon_stat_names[i],
+                         errp);
+    }
+    visit_end_struct(v, errp);
+
+    visit_end_struct(v, errp);
+}
+
+static void balloon_stats_get_poll_interval(Object *obj, struct Visitor *v,
+                                            void *opaque, const char *name,
+                                            Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    visit_type_int(v, &s->stats_poll_interval, name, errp);
+}
+
+static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
+                                            void *opaque, const char *name,
+                                            Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    int64_t value;
+
+    visit_type_int(v, &value, name, errp);
+    if (error_is_set(errp)) {
+        return;
+    }
+
+    if (value < 0) {
+        error_setg(errp, "timer value must be greater than zero");
+        return;
+    }
+
+    if (value == s->stats_poll_interval) {
+        return;
+    }
+
+    if (value == 0) {
+        /* timer=0 disables the timer */
+        balloon_stats_destroy_timer(s);
+        return;
+    }
+
+    if (balloon_stats_enabled(s)) {
+        /* timer interval change */
+        s->stats_poll_interval = value;
+        balloon_stats_change_timer(s, value);
+        return;
+    }
+
+    /* create a new timer */
+    g_assert(s->stats_timer == NULL);
+    s->stats_timer = qemu_new_timer_ms(vm_clock, balloon_stats_poll_cb, s);
+    s->stats_poll_interval = value;
+    balloon_stats_change_timer(s, 0);
+}
+
 static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = to_virtio_balloon(vdev);
@@ -107,9 +234,10 @@ static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
     VirtQueueElement *elem = &s->stats_vq_elem;
     VirtIOBalloonStat stat;
     size_t offset = 0;
+    qemu_timeval tv;
 
     if (!virtqueue_pop(vq, elem)) {
-        return;
+        goto out;
     }
 
     /* Initialize the stats to get rid of any stale values.  This is only
@@ -128,6 +256,18 @@ static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
             s->stats[tag] = val;
     }
     s->stats_vq_offset = offset;
+
+    if (qemu_gettimeofday(&tv) < 0) {
+        fprintf(stderr, "warning: %s: failed to get time of day\n", __func__);
+        goto out;
+    }
+
+    s->stats_last_update = tv.tv_sec;
+
+out:
+    if (balloon_stats_enabled(s)) {
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+    }
 }
 
 static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
@@ -164,28 +304,6 @@ static uint32_t virtio_balloon_get_features(VirtIODevice *vdev, uint32_t f)
 static void virtio_balloon_stat(void *opaque, BalloonInfo *info)
 {
     VirtIOBalloon *dev = opaque;
-
-#if 0
-    /* Disable guest-provided stats for now. For more details please check:
-     * https://bugzilla.redhat.com/show_bug.cgi?id=623903
-     *
-     * If you do enable it (which is probably not going to happen as we
-     * need a new command for it), remember that you also need to fill the
-     * appropriate members of the BalloonInfo structure so that the stats
-     * are returned to the client.
-     */
-    if (dev->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ)) {
-        virtqueue_push(dev->svq, &dev->stats_vq_elem, dev->stats_vq_offset);
-        virtio_notify(&dev->vdev, dev->svq);
-        return;
-    }
-#endif
-
-    /* Stats are not supported.  Clear out any stale values that might
-     * have been set by a more featureful guest kernel.
-     */
-    reset_stats(dev);
-
     info->actual = ram_size - ((uint64_t) dev->actual <<
                                VIRTIO_BALLOON_PFN_SHIFT);
 }
@@ -255,12 +373,18 @@ VirtIODevice *virtio_balloon_init(DeviceState *dev)
     s->dvq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
     s->svq = virtio_add_queue(&s->vdev, 128, virtio_balloon_receive_stats);
 
-    reset_stats(s);
-
     s->qdev = dev;
     register_savevm(dev, "virtio-balloon", -1, 1,
                     virtio_balloon_save, virtio_balloon_load, s);
 
+    object_property_add(OBJECT(dev), "guest-stats", "guest statistics",
+                        balloon_stats_get_all, NULL, NULL, s, NULL);
+
+    object_property_add(OBJECT(dev), "guest-stats-polling-interval", "int",
+                        balloon_stats_get_poll_interval,
+                        balloon_stats_set_poll_interval,
+                        NULL, s, NULL);
+
     return &s->vdev;
 }
 
@@ -268,6 +392,7 @@ void virtio_balloon_exit(VirtIODevice *vdev)
 {
     VirtIOBalloon *s = DO_UPCAST(VirtIOBalloon, vdev, vdev);
 
+    balloon_stats_destroy_timer(s);
     qemu_remove_balloon_handler(s);
     unregister_savevm(s->qdev, "virtio-balloon", s);
     virtio_cleanup(vdev);
diff --git a/hw/virtio-balloon.h b/hw/virtio-balloon.h
index 73300ddc864..b1828f4a485 100644
--- a/hw/virtio-balloon.h
+++ b/hw/virtio-balloon.h
@@ -16,7 +16,7 @@
 #define _QEMU_VIRTIO_BALLOON_H
 
 #include "virtio.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /* from Linux's linux/virtio_balloon.h */
 
diff --git a/hw/virtio-blk.c b/hw/virtio-blk.c
index 6f6d172fd0d..34913ee40ee 100644
--- a/hw/virtio-blk.c
+++ b/hw/virtio-blk.c
@@ -12,11 +12,14 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "trace.h"
 #include "hw/block-common.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "virtio-blk.h"
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+#include "hw/dataplane/virtio-blk.h"
+#endif
 #include "scsi-defs.h"
 #ifdef __linux__
 # include <scsi/sg.h>
@@ -33,6 +36,9 @@ typedef struct VirtIOBlock
     VirtIOBlkConf *blk;
     unsigned short sector_mask;
     DeviceState *qdev;
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    VirtIOBlockDataPlane *dataplane;
+#endif
 } VirtIOBlock;
 
 static VirtIOBlock *to_virtio_blk(VirtIODevice *vdev)
@@ -64,31 +70,22 @@ static void virtio_blk_req_complete(VirtIOBlockReq *req, int status)
 }
 
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
-    int is_read)
+    bool is_read)
 {
-    BlockErrorAction action = bdrv_get_on_error(req->dev->bs, is_read);
+    BlockErrorAction action = bdrv_get_error_action(req->dev->bs, is_read, error);
     VirtIOBlock *s = req->dev;
 
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
+    if (action == BDRV_ACTION_STOP) {
         req->next = s->rq;
         s->rq = req;
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->bs, error);
-    } else {
+    } else if (action == BDRV_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
         bdrv_acct_done(s->bs, &req->acct);
         g_free(req);
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_REPORT, is_read);
     }
 
-    return 1;
+    bdrv_error_action(s->bs, action, is_read, error);
+    return action != BDRV_ACTION_IGNORE;
 }
 
 static void virtio_blk_rw_complete(void *opaque, int ret)
@@ -98,7 +95,7 @@ static void virtio_blk_rw_complete(void *opaque, int ret)
     trace_virtio_blk_rw_complete(req, ret);
 
     if (ret) {
-        int is_read = !(ldl_p(&req->out->type) & VIRTIO_BLK_T_OUT);
+        bool is_read = !(ldl_p(&req->out->type) & VIRTIO_BLK_T_OUT);
         if (virtio_blk_handle_rw_error(req, -ret, is_read))
             return;
     }
@@ -401,10 +398,14 @@ static void virtio_blk_handle_request(VirtIOBlockReq *req,
         qemu_iovec_init_external(&req->qiov, &req->elem.out_sg[1],
                                  req->elem.out_num - 1);
         virtio_blk_handle_write(req, mrb);
-    } else {
+    } else if (type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_BARRIER) {
+        /* VIRTIO_BLK_T_IN is 0, so we can't just & it. */
         qemu_iovec_init_external(&req->qiov, &req->elem.in_sg[0],
                                  req->elem.in_num - 1);
         virtio_blk_handle_read(req);
+    } else {
+        virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
+        g_free(req);
     }
 }
 
@@ -416,6 +417,16 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
         .num_writes = 0,
     };
 
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * dataplane here instead of waiting for .set_status().
+     */
+    if (s->dataplane) {
+        virtio_blk_data_plane_start(s->dataplane);
+        return;
+    }
+#endif
+
     while ((req = virtio_blk_get_request(s))) {
         virtio_blk_handle_request(req, &mrb);
     }
@@ -455,8 +466,9 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
 {
     VirtIOBlock *s = opaque;
 
-    if (!running)
+    if (!running) {
         return;
+    }
 
     if (!s->bh) {
         s->bh = qemu_bh_new(virtio_blk_dma_restart_bh, s);
@@ -466,6 +478,14 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
 
 static void virtio_blk_reset(VirtIODevice *vdev)
 {
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    VirtIOBlock *s = to_virtio_blk(vdev);
+
+    if (s->dataplane) {
+        virtio_blk_data_plane_stop(s->dataplane);
+    }
+#endif
+
     /*
      * This should cancel pending requests, but can't do nicely until there
      * are per-device request lists.
@@ -533,6 +553,9 @@ static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
     features |= (1 << VIRTIO_BLK_F_BLK_SIZE);
     features |= (1 << VIRTIO_BLK_F_SCSI);
 
+    if (s->blk->config_wce) {
+        features |= (1 << VIRTIO_BLK_F_CONFIG_WCE);
+    }
     if (bdrv_enable_write_cache(s->bs))
         features |= (1 << VIRTIO_BLK_F_WCE);
 
@@ -547,6 +570,13 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
     VirtIOBlock *s = to_virtio_blk(vdev);
     uint32_t features;
 
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
+                                    VIRTIO_CONFIG_S_DRIVER_OK))) {
+        virtio_blk_data_plane_stop(s->dataplane);
+    }
+#endif
+
     if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return;
     }
@@ -644,6 +674,12 @@ VirtIODevice *virtio_blk_init(DeviceState *dev, VirtIOBlkConf *blk)
     s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
     s->vq = virtio_add_queue(&s->vdev, 128, virtio_blk_handle_output);
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    if (!virtio_blk_data_plane_create(&s->vdev, blk, &s->dataplane)) {
+        virtio_cleanup(&s->vdev);
+        return NULL;
+    }
+#endif
 
     qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
     s->qdev = dev;
@@ -661,6 +697,11 @@ VirtIODevice *virtio_blk_init(DeviceState *dev, VirtIOBlkConf *blk)
 void virtio_blk_exit(VirtIODevice *vdev)
 {
     VirtIOBlock *s = to_virtio_blk(vdev);
+
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    virtio_blk_data_plane_destroy(s->dataplane);
+    s->dataplane = NULL;
+#endif
     unregister_savevm(s->qdev, "virtio-blk", s);
     blockdev_mark_auto_del(s->bs);
     virtio_cleanup(vdev);
diff --git a/hw/virtio-blk.h b/hw/virtio-blk.h
index f0740d01af9..43ca4920808 100644
--- a/hw/virtio-blk.h
+++ b/hw/virtio-blk.h
@@ -105,10 +105,10 @@ struct VirtIOBlkConf
     char *serial;
     uint32_t scsi;
     uint32_t config_wce;
+    uint32_t data_plane;
 };
 
 #define DEFINE_VIRTIO_BLK_FEATURES(_state, _field) \
-        DEFINE_VIRTIO_COMMON_FEATURES(_state, _field), \
-        DEFINE_PROP_BIT("config-wce", _state, _field, VIRTIO_BLK_F_CONFIG_WCE, true)
+        DEFINE_VIRTIO_COMMON_FEATURES(_state, _field)
 
 #endif
diff --git a/hw/virtio-bus.c b/hw/virtio-bus.c
new file mode 100644
index 00000000000..6045d8ad86a
--- /dev/null
+++ b/hw/virtio-bus.c
@@ -0,0 +1,164 @@
+/*
+ * VirtioBus
+ *
+ *  Copyright (C) 2012 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "hw.h"
+#include "qemu/error-report.h"
+#include "qdev.h"
+#include "virtio-bus.h"
+#include "virtio.h"
+
+/* #define DEBUG_VIRTIO_BUS */
+
+#ifdef DEBUG_VIRTIO_BUS
+#define DPRINTF(fmt, ...) \
+do { printf("virtio_bus: " fmt , ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) do { } while (0)
+#endif
+
+/* Plug the VirtIODevice */
+int virtio_bus_plug_device(VirtIODevice *vdev)
+{
+    DeviceState *qdev = DEVICE(vdev);
+    BusState *qbus = BUS(qdev_get_parent_bus(qdev));
+    VirtioBusState *bus = VIRTIO_BUS(qbus);
+    VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus);
+    DPRINTF("%s: plug device.\n", qbus->name);
+
+    bus->vdev = vdev;
+
+    /*
+     * The lines below will disappear when we drop VirtIOBindings, at the end
+     * of the series.
+     */
+    bus->bindings.notify = klass->notify;
+    bus->bindings.save_config = klass->save_config;
+    bus->bindings.save_queue = klass->save_queue;
+    bus->bindings.load_config = klass->load_config;
+    bus->bindings.load_queue = klass->load_queue;
+    bus->bindings.load_done = klass->load_done;
+    bus->bindings.get_features = klass->get_features;
+    bus->bindings.query_guest_notifiers = klass->query_guest_notifiers;
+    bus->bindings.set_guest_notifiers = klass->set_guest_notifiers;
+    bus->bindings.set_host_notifier = klass->set_host_notifier;
+    bus->bindings.vmstate_change = klass->vmstate_change;
+    virtio_bind_device(bus->vdev, &bus->bindings, qbus->parent);
+
+    if (klass->device_plugged != NULL) {
+        klass->device_plugged(qbus->parent);
+    }
+
+    return 0;
+}
+
+/* Reset the virtio_bus */
+void virtio_bus_reset(VirtioBusState *bus)
+{
+    DPRINTF("%s: reset device.\n", qbus->name);
+    if (bus->vdev != NULL) {
+        virtio_reset(bus->vdev);
+    }
+}
+
+/* Destroy the VirtIODevice */
+void virtio_bus_destroy_device(VirtioBusState *bus)
+{
+    DeviceState *qdev;
+    BusState *qbus = BUS(bus);
+    VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus);
+    DPRINTF("%s: remove device.\n", qbus->name);
+
+    if (bus->vdev != NULL) {
+        if (klass->device_unplug != NULL) {
+            klass->device_unplug(qbus->parent);
+        }
+        qdev = DEVICE(bus->vdev);
+        qdev_free(qdev);
+        bus->vdev = NULL;
+    }
+}
+
+/* Get the device id of the plugged device. */
+uint16_t virtio_bus_get_vdev_id(VirtioBusState *bus)
+{
+    assert(bus->vdev != NULL);
+    return bus->vdev->device_id;
+}
+
+/* Get the config_len field of the plugged device. */
+size_t virtio_bus_get_vdev_config_len(VirtioBusState *bus)
+{
+    assert(bus->vdev != NULL);
+    return bus->vdev->config_len;
+}
+
+/* Get the features of the plugged device. */
+uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
+                                    uint32_t requested_features)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    assert(k->get_features != NULL);
+    return k->get_features(bus->vdev, requested_features);
+}
+
+/* Get bad features of the plugged device. */
+uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    if (k->bad_features != NULL) {
+        return k->bad_features(bus->vdev);
+    } else {
+        return 0;
+    }
+}
+
+/* Get config of the plugged device. */
+void virtio_bus_get_vdev_config(VirtioBusState *bus, uint8_t *config)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    if (k->get_config != NULL) {
+        k->get_config(bus->vdev, config);
+    }
+}
+
+static const TypeInfo virtio_bus_info = {
+    .name = TYPE_VIRTIO_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(VirtioBusState),
+    .abstract = true,
+    .class_size = sizeof(VirtioBusClass),
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_bus_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/virtio-bus.h b/hw/virtio-bus.h
new file mode 100644
index 00000000000..7584a0e6ae3
--- /dev/null
+++ b/hw/virtio-bus.h
@@ -0,0 +1,94 @@
+/*
+ * VirtioBus
+ *
+ *  Copyright (C) 2012 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef VIRTIO_BUS_H
+#define VIRTIO_BUS_H
+
+#include "qdev.h"
+#include "sysemu/sysemu.h"
+#include "virtio.h"
+
+#define TYPE_VIRTIO_BUS "virtio-bus"
+#define VIRTIO_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioBusClass, obj, TYPE_VIRTIO_BUS)
+#define VIRTIO_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioBusClass, klass, TYPE_VIRTIO_BUS)
+#define VIRTIO_BUS(obj) OBJECT_CHECK(VirtioBusState, (obj), TYPE_VIRTIO_BUS)
+
+typedef struct VirtioBusState VirtioBusState;
+
+typedef struct VirtioBusClass {
+    /* This is what a VirtioBus must implement */
+    BusClass parent;
+    void (*notify)(DeviceState *d, uint16_t vector);
+    void (*save_config)(DeviceState *d, QEMUFile *f);
+    void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_config)(DeviceState *d, QEMUFile *f);
+    int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_done)(DeviceState *d, QEMUFile *f);
+    unsigned (*get_features)(DeviceState *d);
+    bool (*query_guest_notifiers)(DeviceState *d);
+    int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assign);
+    int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
+    void (*vmstate_change)(DeviceState *d, bool running);
+    /*
+     * transport independent init function.
+     * This is called by virtio-bus just after the device is plugged.
+     */
+    void (*device_plugged)(DeviceState *d);
+    /*
+     * transport independent exit function.
+     * This is called by virtio-bus just before the device is unplugged.
+     */
+    void (*device_unplug)(DeviceState *d);
+} VirtioBusClass;
+
+struct VirtioBusState {
+    BusState parent_obj;
+    /*
+     * Only one VirtIODevice can be plugged on the bus.
+     */
+    VirtIODevice *vdev;
+    /*
+     * This will be removed at the end of the series.
+     */
+    VirtIOBindings bindings;
+};
+
+int virtio_bus_plug_device(VirtIODevice *vdev);
+void virtio_bus_reset(VirtioBusState *bus);
+void virtio_bus_destroy_device(VirtioBusState *bus);
+/* Get the device id of the plugged device. */
+uint16_t virtio_bus_get_vdev_id(VirtioBusState *bus);
+/* Get the config_len field of the plugged device. */
+size_t virtio_bus_get_vdev_config_len(VirtioBusState *bus);
+/* Get the features of the plugged device. */
+uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
+                                    uint32_t requested_features);
+/* Get bad features of the plugged device. */
+uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus);
+/* Get config of the plugged device. */
+void virtio_bus_get_vdev_config(VirtioBusState *bus, uint8_t *config);
+
+#endif /* VIRTIO_BUS_H */
diff --git a/hw/virtio-console.c b/hw/virtio-console.c
index cffee3d4706..46072a086fd 100644
--- a/hw/virtio-console.c
+++ b/hw/virtio-console.c
@@ -10,8 +10,8 @@
  * the COPYING file in the top-level directory.
  */
 
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 #include "trace.h"
 #include "virtio-serial.h"
 
@@ -142,7 +142,7 @@ static void virtconsole_class_init(ObjectClass *klass, void *data)
     dc->props = virtconsole_properties;
 }
 
-static TypeInfo virtconsole_info = {
+static const TypeInfo virtconsole_info = {
     .name          = "virtconsole",
     .parent        = TYPE_VIRTIO_SERIAL_PORT,
     .instance_size = sizeof(VirtConsole),
@@ -166,7 +166,7 @@ static void virtserialport_class_init(ObjectClass *klass, void *data)
     dc->props = virtserialport_properties;
 }
 
-static TypeInfo virtserialport_info = {
+static const TypeInfo virtserialport_info = {
     .name          = "virtserialport",
     .parent        = TYPE_VIRTIO_SERIAL_PORT,
     .instance_size = sizeof(VirtConsole),
diff --git a/hw/virtio-net.c b/hw/virtio-net.c
index b1998b27d36..3bb01b10374 100644
--- a/hw/virtio-net.c
+++ b/hw/virtio-net.c
@@ -11,13 +11,13 @@
  *
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
 #include "virtio.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "net/tap.h"
-#include "qemu-error.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
 #include "virtio-net.h"
 #include "vhost_net.h"
 
@@ -41,6 +41,8 @@ typedef struct VirtIONet
     int32_t tx_burst;
     int tx_waiting;
     uint32_t has_vnet_hdr;
+    size_t host_hdr_len;
+    size_t guest_hdr_len;
     uint8_t has_ufo;
     struct {
         VirtQueueElement elem;
@@ -124,12 +126,12 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
         if (!vhost_net_query(tap_get_vhost_net(n->nic->nc.peer), &n->vdev)) {
             return;
         }
+        n->vhost_started = 1;
         r = vhost_net_start(tap_get_vhost_net(n->nic->nc.peer), &n->vdev);
         if (r < 0) {
             error_report("unable to start vhost net: %d: "
                          "falling back on userspace virtio", -r);
-        } else {
-            n->vhost_started = 1;
+            n->vhost_started = 0;
         }
     } else {
         vhost_net_stop(tap_get_vhost_net(n->nic->nc.peer), &n->vdev);
@@ -200,16 +202,19 @@ static void virtio_net_reset(VirtIODevice *vdev)
     memset(n->vlans, 0, MAX_VLAN >> 3);
 }
 
-static int peer_has_vnet_hdr(VirtIONet *n)
+static void peer_test_vnet_hdr(VirtIONet *n)
 {
     if (!n->nic->nc.peer)
-        return 0;
+        return;
 
     if (n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP)
-        return 0;
+        return;
 
     n->has_vnet_hdr = tap_has_vnet_hdr(n->nic->nc.peer);
+}
 
+static int peer_has_vnet_hdr(VirtIONet *n)
+{
     return n->has_vnet_hdr;
 }
 
@@ -223,15 +228,27 @@ static int peer_has_ufo(VirtIONet *n)
     return n->has_ufo;
 }
 
+static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs)
+{
+    n->mergeable_rx_bufs = mergeable_rx_bufs;
+
+    n->guest_hdr_len = n->mergeable_rx_bufs ?
+        sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
+
+    if (peer_has_vnet_hdr(n) &&
+        tap_has_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len)) {
+        tap_set_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len);
+        n->host_hdr_len = n->guest_hdr_len;
+    }
+}
+
 static uint32_t virtio_net_get_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
     features |= (1 << VIRTIO_NET_F_MAC);
 
-    if (peer_has_vnet_hdr(n)) {
-        tap_using_vnet_hdr(n->nic->nc.peer, 1);
-    } else {
+    if (!peer_has_vnet_hdr(n)) {
         features &= ~(0x1 << VIRTIO_NET_F_CSUM);
         features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO4);
         features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO6);
@@ -277,7 +294,7 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
-    n->mergeable_rx_bufs = !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF));
+    virtio_net_set_mrg_rx_bufs(n, !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF)));
 
     if (n->has_vnet_hdr) {
         tap_set_offload(n->nic->nc.peer,
@@ -447,10 +464,6 @@ static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq)
     VirtIONet *n = to_virtio_net(vdev);
 
     qemu_flush_queued_packets(&n->nic->nc);
-
-    /* We now have RX buffers, signal to the IO thread to break out of the
-     * select to re-poll the tap file descriptor */
-    qemu_notify_event();
 }
 
 static int virtio_net_can_receive(NetClientState *nc)
@@ -503,41 +516,34 @@ static int virtio_net_has_buffers(VirtIONet *n, int bufsize)
  * cache.
  */
 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr,
-                                        const uint8_t *buf, size_t size)
+                                        uint8_t *buf, size_t size)
 {
     if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */
         (size > 27 && size < 1500) && /* normal sized MTU */
         (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */
         (buf[23] == 17) && /* ip.protocol == UDP */
         (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */
-        /* FIXME this cast is evil */
-        net_checksum_calculate((uint8_t *)buf, size);
+        net_checksum_calculate(buf, size);
         hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
     }
 }
 
-static int receive_header(VirtIONet *n, struct iovec *iov, int iovcnt,
-                          const void *buf, size_t size, size_t hdr_len)
+static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt,
+                           const void *buf, size_t size)
 {
-    struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)iov[0].iov_base;
-    int offset = 0;
-
-    hdr->flags = 0;
-    hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
-
     if (n->has_vnet_hdr) {
-        memcpy(hdr, buf, sizeof(*hdr));
-        offset = sizeof(*hdr);
-        work_around_broken_dhclient(hdr, buf + offset, size - offset);
+        /* FIXME this cast is evil */
+        void *wbuf = (void *)buf;
+        work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
+                                    size - n->host_hdr_len);
+        iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
+    } else {
+        struct virtio_net_hdr hdr = {
+            .flags = 0,
+            .gso_type = VIRTIO_NET_HDR_GSO_NONE
+        };
+        iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr);
     }
-
-    /* We only ever receive a struct virtio_net_hdr from the tapfd,
-     * but we may be passing along a larger header to the guest.
-     */
-    iov[0].iov_base += hdr_len;
-    iov[0].iov_len  -= hdr_len;
-
-    return offset;
 }
 
 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
@@ -550,9 +556,7 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
     if (n->promisc)
         return 1;
 
-    if (n->has_vnet_hdr) {
-        ptr += sizeof(struct virtio_net_hdr);
-    }
+    ptr += n->host_hdr_len;
 
     if (!memcmp(&ptr[12], vlan, sizeof(vlan))) {
         int vid = be16_to_cpup((uint16_t *)(ptr + 14)) & 0xfff;
@@ -596,19 +600,16 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
-    struct virtio_net_hdr_mrg_rxbuf *mhdr = NULL;
-    size_t guest_hdr_len, offset, i, host_hdr_len;
+    struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE];
+    struct virtio_net_hdr_mrg_rxbuf mhdr;
+    unsigned mhdr_cnt = 0;
+    size_t offset, i, guest_offset;
 
     if (!virtio_net_can_receive(&n->nic->nc))
         return -1;
 
     /* hdr_len refers to the header we supply to the guest */
-    guest_hdr_len = n->mergeable_rx_bufs ?
-        sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
-
-
-    host_hdr_len = n->has_vnet_hdr ? sizeof(struct virtio_net_hdr) : 0;
-    if (!virtio_net_has_buffers(n, size + guest_hdr_len - host_hdr_len))
+    if (!virtio_net_has_buffers(n, size + n->guest_hdr_len - n->host_hdr_len))
         return 0;
 
     if (!receive_filter(n, buf, size))
@@ -619,7 +620,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
     while (offset < size) {
         VirtQueueElement elem;
         int len, total;
-        struct iovec sg[VIRTQUEUE_MAX_SIZE];
+        const struct iovec *sg = elem.in_sg;
 
         total = 0;
 
@@ -630,7 +631,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
                     "i %zd mergeable %d offset %zd, size %zd, "
                     "guest hdr len %zd, host hdr len %zd guest features 0x%x",
                     i, n->mergeable_rx_bufs, offset, size,
-                    guest_hdr_len, host_hdr_len, n->vdev.guest_features);
+                    n->guest_hdr_len, n->host_hdr_len, n->vdev.guest_features);
             exit(1);
         }
 
@@ -639,24 +640,25 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
             exit(1);
         }
 
-        if (!n->mergeable_rx_bufs && elem.in_sg[0].iov_len != guest_hdr_len) {
-            error_report("virtio-net header not in first element");
-            exit(1);
-        }
-
-        memcpy(&sg, &elem.in_sg[0], sizeof(sg[0]) * elem.in_num);
-
         if (i == 0) {
-            if (n->mergeable_rx_bufs)
-                mhdr = (struct virtio_net_hdr_mrg_rxbuf *)sg[0].iov_base;
+            assert(offset == 0);
+            if (n->mergeable_rx_bufs) {
+                mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
+                                    sg, elem.in_num,
+                                    offsetof(typeof(mhdr), num_buffers),
+                                    sizeof(mhdr.num_buffers));
+            }
 
-            offset += receive_header(n, sg, elem.in_num,
-                                     buf + offset, size - offset, guest_hdr_len);
-            total += guest_hdr_len;
+            receive_header(n, sg, elem.in_num, buf, size);
+            offset = n->host_hdr_len;
+            total += n->guest_hdr_len;
+            guest_offset = n->guest_hdr_len;
+        } else {
+            guest_offset = 0;
         }
 
         /* copy in packet.  ugh */
-        len = iov_from_buf(sg, elem.in_num, 0,
+        len = iov_from_buf(sg, elem.in_num, guest_offset,
                            buf + offset, size - offset);
         total += len;
         offset += len;
@@ -669,7 +671,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
                          "i %zd mergeable %d offset %zd, size %zd, "
                          "guest hdr len %zd, host hdr len %zd",
                          i, n->mergeable_rx_bufs,
-                         offset, size, guest_hdr_len, host_hdr_len);
+                         offset, size, n->guest_hdr_len, n->host_hdr_len);
 #endif
             return size;
         }
@@ -678,8 +680,11 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
         virtqueue_fill(n->rx_vq, &elem, total, i++);
     }
 
-    if (mhdr) {
-        stw_p(&mhdr->num_buffers, i);
+    if (mhdr_cnt) {
+        stw_p(&mhdr.num_buffers, i);
+        iov_from_buf(mhdr_sg, mhdr_cnt,
+                     0,
+                     &mhdr.num_buffers, sizeof mhdr.num_buffers);
     }
 
     virtqueue_flush(n->rx_vq, i);
@@ -694,7 +699,7 @@ static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
 {
     VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
 
-    virtqueue_push(n->tx_vq, &n->async_tx.elem, n->async_tx.len);
+    virtqueue_push(n->tx_vq, &n->async_tx.elem, 0);
     virtio_notify(&n->vdev, n->tx_vq);
 
     n->async_tx.elem.out_num = n->async_tx.len = 0;
@@ -720,33 +725,35 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
     }
 
     while (virtqueue_pop(vq, &elem)) {
-        ssize_t ret, len = 0;
+        ssize_t ret, len;
         unsigned int out_num = elem.out_num;
         struct iovec *out_sg = &elem.out_sg[0];
-        unsigned hdr_len;
-
-        /* hdr_len refers to the header received from the guest */
-        hdr_len = n->mergeable_rx_bufs ?
-            sizeof(struct virtio_net_hdr_mrg_rxbuf) :
-            sizeof(struct virtio_net_hdr);
+        struct iovec sg[VIRTQUEUE_MAX_SIZE];
 
-        if (out_num < 1 || out_sg->iov_len != hdr_len) {
+        if (out_num < 1) {
             error_report("virtio-net header not in first element");
             exit(1);
         }
 
-        /* ignore the header if GSO is not supported */
-        if (!n->has_vnet_hdr) {
-            out_num--;
-            out_sg++;
-            len += hdr_len;
-        } else if (n->mergeable_rx_bufs) {
-            /* tapfd expects a struct virtio_net_hdr */
-            hdr_len -= sizeof(struct virtio_net_hdr);
-            out_sg->iov_len -= hdr_len;
-            len += hdr_len;
+        /*
+         * If host wants to see the guest header as is, we can
+         * pass it on unchanged. Otherwise, copy just the parts
+         * that host is interested in.
+         */
+        assert(n->host_hdr_len <= n->guest_hdr_len);
+        if (n->host_hdr_len != n->guest_hdr_len) {
+            unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg),
+                                       out_sg, out_num,
+                                       0, n->host_hdr_len);
+            sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num,
+                             out_sg, out_num,
+                             n->guest_hdr_len, -1);
+            out_num = sg_num;
+            out_sg = sg;
         }
 
+        len = n->guest_hdr_len;
+
         ret = qemu_sendv_packet_async(&n->nic->nc, out_sg, out_num,
                                       virtio_net_tx_complete);
         if (ret == 0) {
@@ -758,7 +765,7 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 
         len += ret;
 
-        virtqueue_push(vq, &elem, len);
+        virtqueue_push(vq, &elem, 0);
         virtio_notify(&n->vdev, vq);
 
         if (++num_packets >= n->tx_burst) {
@@ -903,7 +910,8 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
 
     qemu_get_buffer(f, n->mac, ETH_ALEN);
     n->tx_waiting = qemu_get_be32(f);
-    n->mergeable_rx_bufs = qemu_get_be32(f);
+
+    virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f));
 
     if (version_id >= 3)
         n->status = qemu_get_be16(f);
@@ -925,7 +933,9 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
             qemu_get_buffer(f, n->mac_table.macs,
                             n->mac_table.in_use * ETH_ALEN);
         } else if (n->mac_table.in_use) {
-            qemu_fseek(f, n->mac_table.in_use * ETH_ALEN, SEEK_CUR);
+            uint8_t *buf = g_malloc0(n->mac_table.in_use);
+            qemu_get_buffer(f, buf, n->mac_table.in_use * ETH_ALEN);
+            g_free(buf);
             n->mac_table.multi_overflow = n->mac_table.uni_overflow = 1;
             n->mac_table.in_use = 0;
         }
@@ -941,7 +951,6 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
 
         if (n->has_vnet_hdr) {
-            tap_using_vnet_hdr(n->nic->nc.peer, 1);
             tap_set_offload(n->nic->nc.peer,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_CSUM) & 1,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_TSO4) & 1,
@@ -977,6 +986,11 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
     }
     n->mac_table.first_multi = i;
+
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in n->status */
+    n->nic->nc.link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
+
     return 0;
 }
 
@@ -996,6 +1010,22 @@ static NetClientInfo net_virtio_info = {
     .link_status_changed = virtio_net_set_link_status,
 };
 
+static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx)
+{
+    VirtIONet *n = to_virtio_net(vdev);
+    assert(n->vhost_started);
+    return vhost_net_virtqueue_pending(tap_get_vhost_net(n->nic->nc.peer), idx);
+}
+
+static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx,
+                                           bool mask)
+{
+    VirtIONet *n = to_virtio_net(vdev);
+    assert(n->vhost_started);
+    vhost_net_virtqueue_mask(tap_get_vhost_net(n->nic->nc.peer),
+                             vdev, idx, mask);
+}
+
 VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
                               virtio_net_conf *net)
 {
@@ -1012,6 +1042,8 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     n->vdev.bad_features = virtio_net_bad_features;
     n->vdev.reset = virtio_net_reset;
     n->vdev.set_status = virtio_net_set_status;
+    n->vdev.guest_notifier_mask = virtio_net_guest_notifier_mask;
+    n->vdev.guest_notifier_pending = virtio_net_guest_notifier_pending;
     n->rx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_rx);
 
     if (net->tx && strcmp(net->tx, "timer") && strcmp(net->tx, "bh")) {
@@ -1035,12 +1067,19 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     n->status = VIRTIO_NET_S_LINK_UP;
 
     n->nic = qemu_new_nic(&net_virtio_info, conf, object_get_typename(OBJECT(dev)), dev->id, n);
+    peer_test_vnet_hdr(n);
+    if (peer_has_vnet_hdr(n)) {
+        tap_using_vnet_hdr(n->nic->nc.peer, 1);
+        n->host_hdr_len = sizeof(struct virtio_net_hdr);
+    } else {
+        n->host_hdr_len = 0;
+    }
 
     qemu_format_nic_info_str(&n->nic->nc, conf->macaddr.a);
 
     n->tx_waiting = 0;
     n->tx_burst = net->txburst;
-    n->mergeable_rx_bufs = 0;
+    virtio_net_set_mrg_rx_bufs(n, 0);
     n->promisc = 1; /* for compatibility */
 
     n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
diff --git a/hw/virtio-net.h b/hw/virtio-net.h
index 36aa463812e..d46fb9840f4 100644
--- a/hw/virtio-net.h
+++ b/hw/virtio-net.h
@@ -15,8 +15,7 @@
 #define _QEMU_VIRTIO_NET_H
 
 #include "virtio.h"
-#include "net.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 #define ETH_ALEN    6
 
@@ -74,33 +73,6 @@ struct virtio_net_config
     uint16_t status;
 } QEMU_PACKED;
 
-/* This is the first element of the scatter-gather list.  If you don't
- * specify GSO or CSUM features, you can simply ignore the header. */
-struct virtio_net_hdr
-{
-#define VIRTIO_NET_HDR_F_NEEDS_CSUM     1       // Use csum_start, csum_offset
-#define VIRTIO_NET_HDR_F_DATA_VALID	2	// Csum is valid
-    uint8_t flags;
-#define VIRTIO_NET_HDR_GSO_NONE         0       // Not a GSO frame
-#define VIRTIO_NET_HDR_GSO_TCPV4        1       // GSO frame, IPv4 TCP (TSO)
-#define VIRTIO_NET_HDR_GSO_UDP          3       // GSO frame, IPv4 UDP (UFO)
-#define VIRTIO_NET_HDR_GSO_TCPV6        4       // GSO frame, IPv6 TCP
-#define VIRTIO_NET_HDR_GSO_ECN          0x80    // TCP has ECN set
-    uint8_t gso_type;
-    uint16_t hdr_len;
-    uint16_t gso_size;
-    uint16_t csum_start;
-    uint16_t csum_offset;
-};
-
-/* This is the version of the header to use when the MRG_RXBUF
- * feature has been negotiated. */
-struct virtio_net_hdr_mrg_rxbuf
-{
-    struct virtio_net_hdr hdr;
-    uint16_t num_buffers;   /* Number of merged rx buffers */
-};
-
 /*
  * Control virtqueue data structures
  *
diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
index b3f0710f390..bad1aa02997 100644
--- a/hw/virtio-pci.c
+++ b/hw/virtio-pci.c
@@ -22,16 +22,17 @@
 #include "virtio-net.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
-#include "pci.h"
-#include "qemu-error.h"
-#include "msi.h"
-#include "msix.h"
-#include "net.h"
+#include "vhost-scsi.h"
+#include "pci/pci.h"
+#include "qemu/error-report.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
 #include "loader.h"
-#include "kvm.h"
-#include "blockdev.h"
+#include "sysemu/kvm.h"
+#include "sysemu/blockdev.h"
 #include "virtio-pci.h"
-#include "range.h"
+#include "qemu/range.h"
+#include "virtio-bus.h"
 
 /* from Linux's linux/virtio_pci.h */
 
@@ -97,35 +98,48 @@
 bool virtio_is_big_endian(void);
 
 /* virtio device */
+/* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
+static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d)
+{
+    return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+}
 
-static void virtio_pci_notify(void *opaque, uint16_t vector)
+/* DeviceState to VirtIOPCIProxy. Note: used on datapath,
+ * be careful and test performance if you change this.
+ */
+static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+}
+
+static void virtio_pci_notify(DeviceState *d, uint16_t vector)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
     if (msix_enabled(&proxy->pci_dev))
         msix_notify(&proxy->pci_dev, vector);
     else
         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 }
 
-static void virtio_pci_save_config(void * opaque, QEMUFile *f)
+static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     pci_device_save(&proxy->pci_dev, f);
     msix_save(&proxy->pci_dev, f);
     if (msix_present(&proxy->pci_dev))
         qemu_put_be16(f, proxy->vdev->config_vector);
 }
 
-static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
+static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     if (msix_present(&proxy->pci_dev))
         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 }
 
-static int virtio_pci_load_config(void * opaque, QEMUFile *f)
+static int virtio_pci_load_config(DeviceState *d, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     int ret;
     ret = pci_device_load(&proxy->pci_dev, f);
     if (ret) {
@@ -144,9 +158,9 @@ static int virtio_pci_load_config(void * opaque, QEMUFile *f)
     return 0;
 }
 
-static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
+static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     uint16_t vector;
     if (msix_present(&proxy->pci_dev)) {
         qemu_get_be16s(f, &vector);
@@ -244,7 +258,7 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 
 void virtio_pci_reset(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     virtio_pci_stop_ioeventfd(proxy);
     virtio_reset(proxy->vdev);
     msix_unuse_all_vectors(&proxy->pci_dev);
@@ -255,7 +269,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 {
     VirtIOPCIProxy *proxy = opaque;
     VirtIODevice *vdev = proxy->vdev;
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     switch (addr) {
     case VIRTIO_PCI_GUEST_FEATURES:
@@ -266,7 +280,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         virtio_set_features(vdev, val);
         break;
     case VIRTIO_PCI_QUEUE_PFN:
-        pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
+        pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
         if (pa == 0) {
             virtio_pci_stop_ioeventfd(proxy);
             virtio_reset(proxy->vdev);
@@ -374,79 +388,39 @@ static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
     return ret;
 }
 
-static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    return virtio_config_readb(proxy->vdev, addr);
-}
-
-static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    uint16_t val;
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    val = virtio_config_readw(proxy->vdev, addr);
-    if (virtio_is_big_endian()) {
-        /*
-         * virtio is odd, ioports are LE but config space is target native
-         * endian. However, in qemu, all PIO is LE, so we need to re-swap
-         * on BE targets
-         */
-        val = bswap16(val);
-    }
-    return val;
-}
-
-static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    uint32_t val;
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    val = virtio_config_readl(proxy->vdev, addr);
-    if (virtio_is_big_endian()) {
-        val = bswap32(val);
-    }
-    return val;
-}
-
-static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
+static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
+                                       unsigned size)
 {
     VirtIOPCIProxy *proxy = opaque;
     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
+    uint64_t val = 0;
     if (addr < config) {
-        virtio_ioport_write(proxy, addr, val);
-        return;
+        return virtio_ioport_read(proxy, addr);
     }
     addr -= config;
-    virtio_config_writeb(proxy->vdev, addr, val);
-}
 
-static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    if (addr < config) {
-        virtio_ioport_write(proxy, addr, val);
-        return;
-    }
-    addr -= config;
-    if (virtio_is_big_endian()) {
-        val = bswap16(val);
+    switch (size) {
+    case 1:
+        val = virtio_config_readb(proxy->vdev, addr);
+        break;
+    case 2:
+        val = virtio_config_readw(proxy->vdev, addr);
+        if (virtio_is_big_endian()) {
+            val = bswap16(val);
+        }
+        break;
+    case 4:
+        val = virtio_config_readl(proxy->vdev, addr);
+        if (virtio_is_big_endian()) {
+            val = bswap32(val);
+        }
+        break;
     }
-    virtio_config_writew(proxy->vdev, addr, val);
+    return val;
 }
 
-static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
+static void virtio_pci_config_write(void *opaque, hwaddr addr,
+                                    uint64_t val, unsigned size)
 {
     VirtIOPCIProxy *proxy = opaque;
     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
@@ -455,24 +429,36 @@ static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
         return;
     }
     addr -= config;
-    if (virtio_is_big_endian()) {
-        val = bswap32(val);
+    /*
+     * Virtio-PCI is odd. Ioports are LE but config space is target native
+     * endian.
+     */
+    switch (size) {
+    case 1:
+        virtio_config_writeb(proxy->vdev, addr, val);
+        break;
+    case 2:
+        if (virtio_is_big_endian()) {
+            val = bswap16(val);
+        }
+        virtio_config_writew(proxy->vdev, addr, val);
+        break;
+    case 4:
+        if (virtio_is_big_endian()) {
+            val = bswap32(val);
+        }
+        virtio_config_writel(proxy->vdev, addr, val);
+        break;
     }
-    virtio_config_writel(proxy->vdev, addr, val);
 }
 
-static const MemoryRegionPortio virtio_portio[] = {
-    { 0, 0x10000, 1, .write = virtio_pci_config_writeb, },
-    { 0, 0x10000, 2, .write = virtio_pci_config_writew, },
-    { 0, 0x10000, 4, .write = virtio_pci_config_writel, },
-    { 0, 0x10000, 1, .read = virtio_pci_config_readb, },
-    { 0, 0x10000, 2, .read = virtio_pci_config_readw, },
-    { 0, 0x10000, 4, .read = virtio_pci_config_readl, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps virtio_pci_config_ops = {
-    .old_portio = virtio_portio,
+    .read = virtio_pci_config_read,
+    .write = virtio_pci_config_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
@@ -492,9 +478,9 @@ static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
     }
 }
 
-static unsigned virtio_pci_get_features(void *opaque)
+static unsigned virtio_pci_get_features(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     return proxy->host_features;
 }
 
@@ -503,8 +489,6 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
                                         unsigned int vector,
                                         MSIMessage msg)
 {
-    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
-    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
@@ -516,53 +500,178 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
         irqfd->virq = ret;
     }
     irqfd->users++;
-
-    ret = kvm_irqchip_add_irq_notifier(kvm_state, n, irqfd->virq);
-    if (ret < 0) {
-        if (--irqfd->users == 0) {
-            kvm_irqchip_release_virq(kvm_state, irqfd->virq);
-        }
-        return ret;
-    }
-
-    virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
     return 0;
 }
 
 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
-                                             unsigned int queue_no,
                                              unsigned int vector)
 {
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    if (--irqfd->users == 0) {
+        kvm_irqchip_release_virq(kvm_state, irqfd->virq);
+    }
+}
+
+static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
+                                 unsigned int queue_no,
+                                 unsigned int vector)
+{
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
+    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
+    int ret;
+    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
+    return ret;
+}
+
+static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
+                                      unsigned int queue_no,
+                                      unsigned int vector)
+{
     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
-    ret = kvm_irqchip_remove_irq_notifier(kvm_state, n, irqfd->virq);
+    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
     assert(ret == 0);
+}
 
-    if (--irqfd->users == 0) {
-        kvm_irqchip_release_virq(kvm_state, irqfd->virq);
+static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    VirtIODevice *vdev = proxy->vdev;
+    unsigned int vector;
+    int ret, queue_no;
+    MSIMessage msg;
+
+    for (queue_no = 0; queue_no < nvqs; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        msg = msix_get_message(dev, vector);
+        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
+        if (ret < 0) {
+            goto undo;
+        }
+        /* If guest supports masking, set up irqfd now.
+         * Otherwise, delay until unmasked in the frontend.
+         */
+        if (proxy->vdev->guest_notifier_mask) {
+            ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
+            if (ret < 0) {
+                kvm_virtio_pci_vq_vector_release(proxy, vector);
+                goto undo;
+            }
+        }
+    }
+    return 0;
+
+undo:
+    while (--queue_no >= 0) {
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        if (proxy->vdev->guest_notifier_mask) {
+            kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+        }
+        kvm_virtio_pci_vq_vector_release(proxy, vector);
+    }
+    return ret;
+}
+
+static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    VirtIODevice *vdev = proxy->vdev;
+    unsigned int vector;
+    int queue_no;
+
+    for (queue_no = 0; queue_no < nvqs; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        /* If guest supports masking, clean up irqfd now.
+         * Otherwise, it was cleaned when masked in the frontend.
+         */
+        if (proxy->vdev->guest_notifier_mask) {
+            kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+        }
+        kvm_virtio_pci_vq_vector_release(proxy, vector);
+    }
+}
+
+static int kvm_virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
+                                        unsigned int queue_no,
+                                        unsigned int vector,
+                                        MSIMessage msg)
+{
+    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
+    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    int ret = 0;
+
+    if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
+        ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /* If guest supports masking, irqfd is already setup, unmask it.
+     * Otherwise, set it up now.
+     */
+    if (proxy->vdev->guest_notifier_mask) {
+        proxy->vdev->guest_notifier_mask(proxy->vdev, queue_no, false);
+        /* Test after unmasking to avoid losing events. */
+        if (proxy->vdev->guest_notifier_pending &&
+            proxy->vdev->guest_notifier_pending(proxy->vdev, queue_no)) {
+            event_notifier_set(n);
+        }
+    } else {
+        ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
     }
+    return ret;
+}
 
-    virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
+static void kvm_virtio_pci_vq_vector_mask(VirtIOPCIProxy *proxy,
+                                             unsigned int queue_no,
+                                             unsigned int vector)
+{
+    /* If guest supports masking, keep irqfd but mask it.
+     * Otherwise, clean it up now.
+     */ 
+    if (proxy->vdev->guest_notifier_mask) {
+        proxy->vdev->guest_notifier_mask(proxy->vdev, queue_no, true);
+    } else {
+        kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+    }
 }
 
-static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
+static int kvm_virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
                                      MSIMessage msg)
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = proxy->vdev;
     int ret, queue_no;
 
-    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
         if (!virtio_queue_get_num(vdev, queue_no)) {
             break;
         }
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
+        ret = kvm_virtio_pci_vq_vector_unmask(proxy, queue_no, vector, msg);
         if (ret < 0) {
             goto undo;
         }
@@ -574,31 +683,64 @@ undo:
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
+        kvm_virtio_pci_vq_vector_mask(proxy, queue_no, vector);
     }
     return ret;
 }
 
-static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
+static void kvm_virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = proxy->vdev;
     int queue_no;
 
-    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
         if (!virtio_queue_get_num(vdev, queue_no)) {
             break;
         }
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
+        kvm_virtio_pci_vq_vector_mask(proxy, queue_no, vector);
     }
 }
 
-static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
+static void kvm_virtio_pci_vector_poll(PCIDevice *dev,
+                                       unsigned int vector_start,
+                                       unsigned int vector_end)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
+    VirtIODevice *vdev = proxy->vdev;
+    int queue_no;
+    unsigned int vector;
+    EventNotifier *notifier;
+    VirtQueue *vq;
+
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector < vector_start || vector >= vector_end ||
+            !msix_is_masked(dev, vector)) {
+            continue;
+        }
+        vq = virtio_get_queue(vdev, queue_no);
+        notifier = virtio_queue_get_guest_notifier(vq);
+        if (vdev->guest_notifier_pending) {
+            if (vdev->guest_notifier_pending(vdev, queue_no)) {
+                msix_set_pending(dev, vector);
+            }
+        } else if (event_notifier_test_and_clear(notifier)) {
+            msix_set_pending(dev, vector);
+        }
+    }
+}
+
+static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
+                                         bool with_irqfd)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 
@@ -607,72 +749,94 @@ static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
         if (r < 0) {
             return r;
         }
-        virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
+        virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
     } else {
-        virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
+        virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
         event_notifier_cleanup(notifier);
     }
 
     return 0;
 }
 
-static bool virtio_pci_query_guest_notifiers(void *opaque)
+static bool virtio_pci_query_guest_notifiers(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     return msix_enabled(&proxy->pci_dev);
 }
 
-static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
+static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     VirtIODevice *vdev = proxy->vdev;
     int r, n;
+    bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
+        kvm_msi_via_irqfd_enabled();
+
+    nvqs = MIN(nvqs, VIRTIO_PCI_QUEUE_MAX);
+
+    /* When deassigning, pass a consistent nvqs value
+     * to avoid leaking notifiers.
+     */
+    assert(assign || nvqs == proxy->nvqs_with_notifiers);
+
+    proxy->nvqs_with_notifiers = nvqs;
 
     /* Must unset vector notifier while guest notifier is still assigned */
-    if (kvm_msi_via_irqfd_enabled() && !assign) {
+    if (proxy->vector_irqfd && !assign) {
         msix_unset_vector_notifiers(&proxy->pci_dev);
+        kvm_virtio_pci_vector_release(proxy, nvqs);
         g_free(proxy->vector_irqfd);
         proxy->vector_irqfd = NULL;
     }
 
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+    for (n = 0; n < nvqs; n++) {
         if (!virtio_queue_get_num(vdev, n)) {
             break;
         }
 
-        r = virtio_pci_set_guest_notifier(opaque, n, assign);
+        r = virtio_pci_set_guest_notifier(d, n, assign,
+                                          kvm_msi_via_irqfd_enabled());
         if (r < 0) {
             goto assign_error;
         }
     }
 
     /* Must set vector notifier after guest notifier has been assigned */
-    if (kvm_msi_via_irqfd_enabled() && assign) {
+    if (with_irqfd && assign) {
         proxy->vector_irqfd =
             g_malloc0(sizeof(*proxy->vector_irqfd) *
                       msix_nr_vectors_allocated(&proxy->pci_dev));
-        r = msix_set_vector_notifiers(&proxy->pci_dev,
-                                      kvm_virtio_pci_vector_use,
-                                      kvm_virtio_pci_vector_release);
+        r = kvm_virtio_pci_vector_use(proxy, nvqs);
         if (r < 0) {
             goto assign_error;
         }
+        r = msix_set_vector_notifiers(&proxy->pci_dev,
+                                      kvm_virtio_pci_vector_unmask,
+                                      kvm_virtio_pci_vector_mask,
+                                      kvm_virtio_pci_vector_poll);
+        if (r < 0) {
+            goto notifiers_error;
+        }
     }
 
     return 0;
 
+notifiers_error:
+    assert(assign);
+    kvm_virtio_pci_vector_release(proxy, nvqs);
+
 assign_error:
     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
     assert(assign);
     while (--n >= 0) {
-        virtio_pci_set_guest_notifier(opaque, n, !assign);
+        virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
     }
     return r;
 }
 
-static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
+static int virtio_pci_set_host_notifier(DeviceState *d, int n, bool assign)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 
     /* Stop using ioeventfd for virtqueue kick if the device starts using host
      * notifiers.  This makes it easy to avoid stepping on each others' toes.
@@ -688,9 +852,9 @@ static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
     return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
 }
 
-static void virtio_pci_vmstate_change(void *opaque, bool running)
+static void virtio_pci_vmstate_change(DeviceState *d, bool running)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 
     if (running) {
         /* Try to find out if the guest has bus master disabled, but is
@@ -755,7 +919,7 @@ void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
     }
 
-    virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
+    virtio_bind_device(vdev, &virtio_pci_bindings, DEVICE(proxy));
     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
@@ -880,6 +1044,41 @@ static void virtio_balloon_exit_pci(PCIDevice *pci_dev)
     virtio_exit_pci(pci_dev);
 }
 
+static int virtio_rng_init_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+    VirtIODevice *vdev;
+
+    if (proxy->rng.rng == NULL) {
+        proxy->rng.default_backend = RNG_RANDOM(object_new(TYPE_RNG_RANDOM));
+
+        object_property_add_child(OBJECT(pci_dev),
+                                  "default-backend",
+                                  OBJECT(proxy->rng.default_backend),
+                                  NULL);
+
+        object_property_set_link(OBJECT(pci_dev),
+                                 OBJECT(proxy->rng.default_backend),
+                                 "rng", NULL);
+    }
+
+    vdev = virtio_rng_init(&pci_dev->qdev, &proxy->rng);
+    if (!vdev) {
+        return -1;
+    }
+    virtio_init_pci(proxy, vdev);
+    return 0;
+}
+
+static void virtio_rng_exit_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    virtio_pci_stop_ioeventfd(proxy);
+    virtio_rng_exit(proxy->vdev);
+    virtio_exit_pci(pci_dev);
+}
+
 static Property virtio_blk_properties[] = {
     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
     DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
@@ -890,6 +1089,9 @@ static Property virtio_blk_properties[] = {
 #endif
     DEFINE_PROP_BIT("config-wce", VirtIOPCIProxy, blk.config_wce, 0, true),
     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    DEFINE_PROP_BIT("x-data-plane", VirtIOPCIProxy, blk.data_plane, 0, false),
+#endif
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
     DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
     DEFINE_PROP_END_OF_LIST(),
@@ -910,7 +1112,7 @@ static void virtio_blk_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_blk_properties;
 }
 
-static TypeInfo virtio_blk_info = {
+static const TypeInfo virtio_blk_info = {
     .name          = "virtio-blk-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -944,7 +1146,7 @@ static void virtio_net_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_net_properties;
 }
 
-static TypeInfo virtio_net_info = {
+static const TypeInfo virtio_net_info = {
     .name          = "virtio-net-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -975,7 +1177,7 @@ static void virtio_serial_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_serial_properties;
 }
 
-static TypeInfo virtio_serial_info = {
+static const TypeInfo virtio_serial_info = {
     .name          = "virtio-serial-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -1003,13 +1205,57 @@ static void virtio_balloon_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_balloon_properties;
 }
 
-static TypeInfo virtio_balloon_info = {
+static const TypeInfo virtio_balloon_info = {
     .name          = "virtio-balloon-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
     .class_init    = virtio_balloon_class_init,
 };
 
+static void virtio_rng_initfn(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&proxy->rng.rng, NULL);
+}
+
+static Property virtio_rng_properties[] = {
+    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
+    /* Set a default rate limit of 2^47 bytes per minute or roughly 2TB/s.  If
+       you have an entropy source capable of generating more entropy than this
+       and you can pass it through via virtio-rng, then hats off to you.  Until
+       then, this is unlimited for all practical purposes.
+    */
+    DEFINE_PROP_UINT64("max-bytes", VirtIOPCIProxy, rng.max_bytes, INT64_MAX),
+    DEFINE_PROP_UINT32("period", VirtIOPCIProxy, rng.period_ms, 1 << 16),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_rng_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = virtio_rng_init_pci;
+    k->exit = virtio_rng_exit_pci;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_RNG;
+    k->revision = VIRTIO_PCI_ABI_VERSION;
+    k->class_id = PCI_CLASS_OTHERS;
+    dc->reset = virtio_pci_reset;
+    dc->props = virtio_rng_properties;
+}
+
+static const TypeInfo virtio_rng_info = {
+    .name          = "virtio-rng-pci",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .instance_init = virtio_rng_initfn,
+    .class_init    = virtio_rng_class_init,
+};
+
 static int virtio_scsi_init_pci(PCIDevice *pci_dev)
 {
     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
@@ -1060,13 +1306,222 @@ static void virtio_scsi_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_scsi_properties;
 }
 
-static TypeInfo virtio_scsi_info = {
+static const TypeInfo virtio_scsi_info = {
     .name          = "virtio-scsi-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
     .class_init    = virtio_scsi_class_init,
 };
 
+static int vhost_scsi_init_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+    VirtIODevice *vdev;
+
+    vdev = vhost_scsi_init(&pci_dev->qdev, &proxy->scsi);
+    if (!vdev) {
+        return -EINVAL;
+    }
+
+    vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
+                                        ? proxy->scsi.num_queues + 3
+                                        : proxy->nvectors;
+    virtio_init_pci(proxy, vdev);
+
+    /* make the actual value visible */
+    proxy->nvectors = vdev->nvectors;
+    return 0;
+}
+
+static void vhost_scsi_exit_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    vhost_scsi_exit(proxy->vdev);
+    virtio_exit_pci(pci_dev);
+}
+
+static Property vhost_scsi_properties[] = {
+    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
+    DEFINE_VHOST_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void vhost_scsi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = vhost_scsi_init_pci;
+    k->exit = vhost_scsi_exit_pci;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
+    k->revision = 0x00;
+    k->class_id = PCI_CLASS_STORAGE_SCSI;
+    dc->reset = virtio_pci_reset;
+    dc->props = vhost_scsi_properties;
+}
+
+static const TypeInfo vhost_scsi_info = {
+    .name          = "vhost-scsi-pci",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .class_init    = vhost_scsi_class_init,
+};
+
+/*
+ * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
+ */
+
+/* This is called by virtio-bus just after the device is plugged. */
+static void virtio_pci_device_plugged(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtioBusState *bus = &proxy->bus;
+    uint8_t *config;
+    uint32_t size;
+
+    proxy->vdev = bus->vdev;
+
+    config = proxy->pci_dev.config;
+    if (proxy->class_code) {
+        pci_config_set_class(config, proxy->class_code);
+    }
+    pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
+                 pci_get_word(config + PCI_VENDOR_ID));
+    pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
+    config[PCI_INTERRUPT_PIN] = 1;
+
+    if (proxy->nvectors &&
+        msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors, 1)) {
+        proxy->nvectors = 0;
+    }
+
+    proxy->pci_dev.config_write = virtio_write_config;
+
+    size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
+         + virtio_bus_get_vdev_config_len(bus);
+    if (size & (size - 1)) {
+        size = 1 << qemu_fls(size);
+    }
+
+    memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
+                          "virtio-pci", size);
+    pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
+                     &proxy->bar);
+
+    if (!kvm_has_many_ioeventfds()) {
+        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
+    }
+
+    proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
+    proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
+    proxy->host_features = virtio_bus_get_vdev_features(bus,
+                                                      proxy->host_features);
+}
+
+/* This is called by virtio-bus just before the device is unplugged. */
+static void virtio_pci_device_unplug(DeviceState *d)
+{
+    VirtIOPCIProxy *dev = VIRTIO_PCI(d);
+    virtio_pci_stop_ioeventfd(dev);
+}
+
+static int virtio_pci_init(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *dev = VIRTIO_PCI(pci_dev);
+    VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
+    virtio_pci_bus_new(&dev->bus, dev);
+    if (k->init != NULL) {
+        return k->init(dev);
+    }
+    return 0;
+}
+
+static void virtio_pci_exit(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
+    VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    BusState *qbus = BUS(&proxy->bus);
+    virtio_bus_destroy_device(bus);
+    qbus_free(qbus);
+    virtio_exit_pci(pci_dev);
+}
+
+/*
+ * This will be renamed virtio_pci_reset at the end of the series.
+ * virtio_pci_reset is still in use at this moment.
+ */
+static void virtio_pci_rst(DeviceState *qdev)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
+    VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    virtio_pci_stop_ioeventfd(proxy);
+    virtio_bus_reset(bus);
+    msix_unuse_all_vectors(&proxy->pci_dev);
+    proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
+}
+
+static void virtio_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = virtio_pci_init;
+    k->exit = virtio_pci_exit;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->revision = VIRTIO_PCI_ABI_VERSION;
+    k->class_id = PCI_CLASS_OTHERS;
+    dc->reset = virtio_pci_rst;
+}
+
+static const TypeInfo virtio_pci_info = {
+    .name          = TYPE_VIRTIO_PCI,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .class_init    = virtio_pci_class_init,
+    .class_size    = sizeof(VirtioPCIClass),
+    .abstract      = true,
+};
+
+/* virtio-pci-bus */
+
+void virtio_pci_bus_new(VirtioBusState *bus, VirtIOPCIProxy *dev)
+{
+    DeviceState *qdev = DEVICE(dev);
+    BusState *qbus;
+    qbus_create_inplace((BusState *)bus, TYPE_VIRTIO_PCI_BUS, qdev, NULL);
+    qbus = BUS(bus);
+    qbus->allow_hotplug = 0;
+}
+
+static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
+{
+    BusClass *bus_class = BUS_CLASS(klass);
+    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
+    bus_class->max_dev = 1;
+    k->notify = virtio_pci_notify;
+    k->save_config = virtio_pci_save_config;
+    k->load_config = virtio_pci_load_config;
+    k->save_queue = virtio_pci_save_queue;
+    k->load_queue = virtio_pci_load_queue;
+    k->get_features = virtio_pci_get_features;
+    k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
+    k->set_host_notifier = virtio_pci_set_host_notifier;
+    k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
+    k->vmstate_change = virtio_pci_vmstate_change;
+    k->device_plugged = virtio_pci_device_plugged;
+    k->device_unplug = virtio_pci_device_unplug;
+}
+
+static const TypeInfo virtio_pci_bus_info = {
+    .name          = TYPE_VIRTIO_PCI_BUS,
+    .parent        = TYPE_VIRTIO_BUS,
+    .instance_size = sizeof(VirtioPCIBusState),
+    .class_init    = virtio_pci_bus_class_init,
+};
+
 static void virtio_pci_register_types(void)
 {
     type_register_static(&virtio_blk_info);
@@ -1074,6 +1529,10 @@ static void virtio_pci_register_types(void)
     type_register_static(&virtio_serial_info);
     type_register_static(&virtio_balloon_info);
     type_register_static(&virtio_scsi_info);
+    type_register_static(&vhost_scsi_info);
+    type_register_static(&virtio_rng_info);
+    type_register_static(&virtio_pci_bus_info);
+    type_register_static(&virtio_pci_info);
 }
 
 type_init(virtio_pci_register_types)
diff --git a/hw/virtio-pci.h b/hw/virtio-pci.h
index ac9d522f374..d24957cc254 100644
--- a/hw/virtio-pci.h
+++ b/hw/virtio-pci.h
@@ -15,10 +15,28 @@
 #ifndef QEMU_VIRTIO_PCI_H
 #define QEMU_VIRTIO_PCI_H
 
+#include "hw/pci/msi.h"
 #include "virtio-blk.h"
 #include "virtio-net.h"
+#include "virtio-rng.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
+#include "virtio-bus.h"
+
+typedef struct VirtIOPCIProxy VirtIOPCIProxy;
+
+/* virtio-pci-bus */
+
+typedef struct VirtioBusState VirtioPCIBusState;
+typedef struct VirtioBusClass VirtioPCIBusClass;
+
+#define TYPE_VIRTIO_PCI_BUS "virtio-pci-bus"
+#define VIRTIO_PCI_BUS(obj) \
+        OBJECT_CHECK(VirtioPCIBusState, (obj), TYPE_VIRTIO_PCI_BUS)
+#define VIRTIO_PCI_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioPCIBusClass, obj, TYPE_VIRTIO_PCI_BUS)
+#define VIRTIO_PCI_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
@@ -26,11 +44,28 @@
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
 
 typedef struct {
+    MSIMessage msg;
     int virq;
     unsigned int users;
 } VirtIOIRQFD;
 
-typedef struct {
+/*
+ * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
+ */
+#define TYPE_VIRTIO_PCI "virtio-pci"
+#define VIRTIO_PCI_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioPCIClass, obj, TYPE_VIRTIO_PCI)
+#define VIRTIO_PCI_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioPCIClass, klass, TYPE_VIRTIO_PCI)
+#define VIRTIO_PCI(obj) \
+        OBJECT_CHECK(VirtIOPCIProxy, (obj), TYPE_VIRTIO_PCI)
+
+typedef struct VirtioPCIClass {
+    PCIDeviceClass parent_class;
+    int (*init)(VirtIOPCIProxy *vpci_dev);
+} VirtioPCIClass;
+
+struct VirtIOPCIProxy {
     PCIDevice pci_dev;
     VirtIODevice *vdev;
     MemoryRegion bar;
@@ -46,13 +81,17 @@ typedef struct {
     virtio_serial_conf serial;
     virtio_net_conf net;
     VirtIOSCSIConf scsi;
+    VirtIORNGConf rng;
     bool ioeventfd_disabled;
     bool ioeventfd_started;
     VirtIOIRQFD *vector_irqfd;
-} VirtIOPCIProxy;
+    int nvqs_with_notifiers;
+    VirtioBusState bus;
+};
 
 void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev);
 void virtio_pci_reset(DeviceState *d);
+void virtio_pci_bus_new(VirtioBusState *bus, VirtIOPCIProxy *dev);
 
 /* Virtio ABI version, if we increment this, we break the guest driver. */
 #define VIRTIO_PCI_ABI_VERSION          0
diff --git a/hw/virtio-rng.c b/hw/virtio-rng.c
new file mode 100644
index 00000000000..e063127df63
--- /dev/null
+++ b/hw/virtio-rng.c
@@ -0,0 +1,205 @@
+/*
+ * A virtio device implementing a hardware random number generator.
+ *
+ * Copyright 2012 Red Hat, Inc.
+ * Copyright 2012 Amit Shah <amit.shah@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/iov.h"
+#include "qdev.h"
+#include "virtio.h"
+#include "virtio-rng.h"
+#include "qemu/rng.h"
+
+typedef struct VirtIORNG {
+    VirtIODevice vdev;
+
+    DeviceState *qdev;
+
+    /* Only one vq - guest puts buffer(s) on it when it needs entropy */
+    VirtQueue *vq;
+
+    VirtIORNGConf *conf;
+
+    RngBackend *rng;
+
+    /* We purposefully don't migrate this state.  The quota will reset on the
+     * destination as a result.  Rate limiting is host state, not guest state.
+     */
+    QEMUTimer *rate_limit_timer;
+    int64_t quota_remaining;
+} VirtIORNG;
+
+static bool is_guest_ready(VirtIORNG *vrng)
+{
+    if (virtio_queue_ready(vrng->vq)
+        && (vrng->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+        return true;
+    }
+    return false;
+}
+
+static size_t get_request_size(VirtQueue *vq, unsigned quota)
+{
+    unsigned int in, out;
+
+    virtqueue_get_avail_bytes(vq, &in, &out, quota, 0);
+    return in;
+}
+
+static void virtio_rng_process(VirtIORNG *vrng);
+
+/* Send data from a char device over to the guest */
+static void chr_read(void *opaque, const void *buf, size_t size)
+{
+    VirtIORNG *vrng = opaque;
+    VirtQueueElement elem;
+    size_t len;
+    int offset;
+
+    if (!is_guest_ready(vrng)) {
+        return;
+    }
+
+    vrng->quota_remaining -= size;
+
+    offset = 0;
+    while (offset < size) {
+        if (!virtqueue_pop(vrng->vq, &elem)) {
+            break;
+        }
+        len = iov_from_buf(elem.in_sg, elem.in_num,
+                           0, buf + offset, size - offset);
+        offset += len;
+
+        virtqueue_push(vrng->vq, &elem, len);
+    }
+    virtio_notify(&vrng->vdev, vrng->vq);
+}
+
+static void virtio_rng_process(VirtIORNG *vrng)
+{
+    size_t size;
+    unsigned quota;
+
+    if (!is_guest_ready(vrng)) {
+        return;
+    }
+
+    if (vrng->quota_remaining < 0) {
+        quota = 0;
+    } else {
+        quota = MIN((uint64_t)vrng->quota_remaining, (uint64_t)UINT32_MAX);
+    }
+    size = get_request_size(vrng->vq, quota);
+    size = MIN(vrng->quota_remaining, size);
+    if (size) {
+        rng_backend_request_entropy(vrng->rng, size, chr_read, vrng);
+    }
+}
+
+static void handle_input(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIORNG *vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+    virtio_rng_process(vrng);
+}
+
+static uint32_t get_features(VirtIODevice *vdev, uint32_t f)
+{
+    return f;
+}
+
+static void virtio_rng_save(QEMUFile *f, void *opaque)
+{
+    VirtIORNG *vrng = opaque;
+
+    virtio_save(&vrng->vdev, f);
+}
+
+static int virtio_rng_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIORNG *vrng = opaque;
+
+    if (version_id != 1) {
+        return -EINVAL;
+    }
+    virtio_load(&vrng->vdev, f);
+
+    /* We may have an element ready but couldn't process it due to a quota
+     * limit.  Make sure to try again after live migration when the quota may
+     * have been reset.
+     */
+    virtio_rng_process(vrng);
+
+    return 0;
+}
+
+static void check_rate_limit(void *opaque)
+{
+    VirtIORNG *s = opaque;
+
+    s->quota_remaining = s->conf->max_bytes;
+    virtio_rng_process(s);
+    qemu_mod_timer(s->rate_limit_timer,
+                   qemu_get_clock_ms(vm_clock) + s->conf->period_ms);
+}
+
+
+VirtIODevice *virtio_rng_init(DeviceState *dev, VirtIORNGConf *conf)
+{
+    VirtIORNG *vrng;
+    VirtIODevice *vdev;
+    Error *local_err = NULL;
+
+    vdev = virtio_common_init("virtio-rng", VIRTIO_ID_RNG, 0,
+                              sizeof(VirtIORNG));
+
+    vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+
+    vrng->rng = conf->rng;
+    if (vrng->rng == NULL) {
+        qerror_report(QERR_INVALID_PARAMETER_VALUE, "rng", "a valid object");
+        return NULL;
+    }
+
+    rng_backend_open(vrng->rng, &local_err);
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return NULL;
+    }
+
+    vrng->vq = virtio_add_queue(vdev, 8, handle_input);
+    vrng->vdev.get_features = get_features;
+
+    vrng->qdev = dev;
+    vrng->conf = conf;
+
+    assert(vrng->conf->max_bytes <= INT64_MAX);
+    vrng->quota_remaining = vrng->conf->max_bytes;
+
+    vrng->rate_limit_timer = qemu_new_timer_ms(vm_clock,
+                                               check_rate_limit, vrng);
+
+    qemu_mod_timer(vrng->rate_limit_timer,
+                   qemu_get_clock_ms(vm_clock) + vrng->conf->period_ms);
+
+    register_savevm(dev, "virtio-rng", -1, 1, virtio_rng_save,
+                    virtio_rng_load, vrng);
+
+    return vdev;
+}
+
+void virtio_rng_exit(VirtIODevice *vdev)
+{
+    VirtIORNG *vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+
+    qemu_del_timer(vrng->rate_limit_timer);
+    qemu_free_timer(vrng->rate_limit_timer);
+    unregister_savevm(vrng->qdev, "virtio-rng", vrng);
+    virtio_cleanup(vdev);
+}
diff --git a/hw/virtio-rng.h b/hw/virtio-rng.h
new file mode 100644
index 00000000000..f42d748eba3
--- /dev/null
+++ b/hw/virtio-rng.h
@@ -0,0 +1,28 @@
+/*
+ * Virtio RNG Support
+ *
+ * Copyright Red Hat, Inc. 2012
+ * Copyright Amit Shah <amit.shah@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _QEMU_VIRTIO_RNG_H
+#define _QEMU_VIRTIO_RNG_H
+
+#include "qemu/rng.h"
+#include "qemu/rng-random.h"
+
+/* The Virtio ID for the virtio rng device */
+#define VIRTIO_ID_RNG    4
+
+struct VirtIORNGConf {
+    RngBackend *rng;
+    uint64_t max_bytes;
+    uint32_t period_ms;
+    RndRandom *default_backend;
+};
+
+#endif
diff --git a/hw/virtio-scsi.c b/hw/virtio-scsi.c
index c1b47a8f4d7..363aba4a70d 100644
--- a/hw/virtio-scsi.c
+++ b/hw/virtio-scsi.c
@@ -17,131 +17,12 @@
 #include <hw/scsi.h>
 #include <hw/scsi-defs.h>
 
-#define VIRTIO_SCSI_VQ_SIZE     128
-#define VIRTIO_SCSI_CDB_SIZE    32
-#define VIRTIO_SCSI_SENSE_SIZE  96
-#define VIRTIO_SCSI_MAX_CHANNEL 0
-#define VIRTIO_SCSI_MAX_TARGET  255
-#define VIRTIO_SCSI_MAX_LUN     16383
-
-/* Response codes */
-#define VIRTIO_SCSI_S_OK                       0
-#define VIRTIO_SCSI_S_OVERRUN                  1
-#define VIRTIO_SCSI_S_ABORTED                  2
-#define VIRTIO_SCSI_S_BAD_TARGET               3
-#define VIRTIO_SCSI_S_RESET                    4
-#define VIRTIO_SCSI_S_BUSY                     5
-#define VIRTIO_SCSI_S_TRANSPORT_FAILURE        6
-#define VIRTIO_SCSI_S_TARGET_FAILURE           7
-#define VIRTIO_SCSI_S_NEXUS_FAILURE            8
-#define VIRTIO_SCSI_S_FAILURE                  9
-#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED       10
-#define VIRTIO_SCSI_S_FUNCTION_REJECTED        11
-#define VIRTIO_SCSI_S_INCORRECT_LUN            12
-
-/* Controlq type codes.  */
-#define VIRTIO_SCSI_T_TMF                      0
-#define VIRTIO_SCSI_T_AN_QUERY                 1
-#define VIRTIO_SCSI_T_AN_SUBSCRIBE             2
-
-/* Valid TMF subtypes.  */
-#define VIRTIO_SCSI_T_TMF_ABORT_TASK           0
-#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET       1
-#define VIRTIO_SCSI_T_TMF_CLEAR_ACA            2
-#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET       3
-#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET      4
-#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET   5
-#define VIRTIO_SCSI_T_TMF_QUERY_TASK           6
-#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET       7
-
-/* Events.  */
-#define VIRTIO_SCSI_T_EVENTS_MISSED            0x80000000
-#define VIRTIO_SCSI_T_NO_EVENT                 0
-#define VIRTIO_SCSI_T_TRANSPORT_RESET          1
-#define VIRTIO_SCSI_T_ASYNC_NOTIFY             2
-#define VIRTIO_SCSI_T_PARAM_CHANGE             3
-
-/* Reasons for transport reset event */
-#define VIRTIO_SCSI_EVT_RESET_HARD             0
-#define VIRTIO_SCSI_EVT_RESET_RESCAN           1
-#define VIRTIO_SCSI_EVT_RESET_REMOVED          2
-
-/* SCSI command request, followed by data-out */
 typedef struct {
-    uint8_t lun[8];              /* Logical Unit Number */
-    uint64_t tag;                /* Command identifier */
-    uint8_t task_attr;           /* Task attribute */
-    uint8_t prio;
-    uint8_t crn;
-    uint8_t cdb[];
-} QEMU_PACKED VirtIOSCSICmdReq;
-
-/* Response, followed by sense data and data-in */
-typedef struct {
-    uint32_t sense_len;          /* Sense data length */
-    uint32_t resid;              /* Residual bytes in data buffer */
-    uint16_t status_qualifier;   /* Status qualifier */
-    uint8_t status;              /* Command completion status */
-    uint8_t response;            /* Response values */
-    uint8_t sense[];
-} QEMU_PACKED VirtIOSCSICmdResp;
-
-/* Task Management Request */
-typedef struct {
-    uint32_t type;
-    uint32_t subtype;
-    uint8_t lun[8];
-    uint64_t tag;
-} QEMU_PACKED VirtIOSCSICtrlTMFReq;
-
-typedef struct {
-    uint8_t response;
-} QEMU_PACKED VirtIOSCSICtrlTMFResp;
-
-/* Asynchronous notification query/subscription */
-typedef struct {
-    uint32_t type;
-    uint8_t lun[8];
-    uint32_t event_requested;
-} QEMU_PACKED VirtIOSCSICtrlANReq;
-
-typedef struct {
-    uint32_t event_actual;
-    uint8_t response;
-} QEMU_PACKED VirtIOSCSICtrlANResp;
-
-typedef struct {
-    uint32_t event;
-    uint8_t lun[8];
-    uint32_t reason;
-} QEMU_PACKED VirtIOSCSIEvent;
-
-typedef struct {
-    uint32_t num_queues;
-    uint32_t seg_max;
-    uint32_t max_sectors;
-    uint32_t cmd_per_lun;
-    uint32_t event_info_size;
-    uint32_t sense_size;
-    uint32_t cdb_size;
-    uint16_t max_channel;
-    uint16_t max_target;
-    uint32_t max_lun;
-} QEMU_PACKED VirtIOSCSIConfig;
-
-typedef struct {
-    VirtIODevice vdev;
-    DeviceState *qdev;
-    VirtIOSCSIConf *conf;
+    VirtIOSCSICommon vs;
 
     SCSIBus bus;
-    uint32_t sense_size;
-    uint32_t cdb_size;
     int resetting;
     bool events_dropped;
-    VirtQueue *ctrl_vq;
-    VirtQueue *event_vq;
-    VirtQueue *cmd_vqs[0];
 } VirtIOSCSI;
 
 typedef struct VirtIOSCSIReq {
@@ -192,7 +73,7 @@ static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
         scsi_req_unref(req->sreq);
     }
     g_free(req);
-    virtio_notify(&s->vdev, vq);
+    virtio_notify(&s->vs.vdev, vq);
 }
 
 static void virtio_scsi_bad_req(void)
@@ -202,9 +83,9 @@ static void virtio_scsi_bad_req(void)
 }
 
 static void qemu_sgl_init_external(QEMUSGList *qsgl, struct iovec *sg,
-                                   target_phys_addr_t *addr, int num)
+                                   hwaddr *addr, int num)
 {
-    memset(qsgl, 0, sizeof(*qsgl));
+    qemu_sglist_init(qsgl, num, &dma_context_memory);
     while (num--) {
         qemu_sglist_add(qsgl, *(addr++), (sg++)->iov_len);
     }
@@ -251,7 +132,7 @@ static void virtio_scsi_save_request(QEMUFile *f, SCSIRequest *sreq)
     VirtIOSCSIReq *req = sreq->hba_private;
     uint32_t n = virtio_queue_get_id(req->vq) - 2;
 
-    assert(n < req->dev->conf->num_queues);
+    assert(n < req->dev->vs.conf->num_queues);
     qemu_put_be32s(f, &n);
     qemu_put_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
 }
@@ -265,9 +146,9 @@ static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
 
     req = g_malloc(sizeof(*req));
     qemu_get_be32s(f, &n);
-    assert(n < s->conf->num_queues);
+    assert(n < s->vs.conf->num_queues);
     qemu_get_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
-    virtio_scsi_parse_req(s, s->cmd_vqs[n], req);
+    virtio_scsi_parse_req(s, s->vs.cmd_vqs[n], req);
 
     scsi_req_ref(sreq);
     req->sreq = sreq;
@@ -424,15 +305,17 @@ static void virtio_scsi_command_complete(SCSIRequest *r, uint32_t status,
                                          size_t resid)
 {
     VirtIOSCSIReq *req = r->hba_private;
+    uint32_t sense_len;
 
     req->resp.cmd->response = VIRTIO_SCSI_S_OK;
     req->resp.cmd->status = status;
     if (req->resp.cmd->status == GOOD) {
-        req->resp.cmd->resid = resid;
+        req->resp.cmd->resid = tswap32(resid);
     } else {
         req->resp.cmd->resid = 0;
-        req->resp.cmd->sense_len =
-            scsi_req_get_sense(r, req->resp.cmd->sense, VIRTIO_SCSI_SENSE_SIZE);
+        sense_len = scsi_req_get_sense(r, req->resp.cmd->sense,
+                                       VIRTIO_SCSI_SENSE_SIZE);
+        req->resp.cmd->sense_len = tswap32(sense_len);
     }
     virtio_scsi_complete_req(req);
 }
@@ -480,8 +363,8 @@ static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
 
         out_size = req->elem.out_sg[0].iov_len;
         in_size = req->elem.in_sg[0].iov_len;
-        if (out_size < sizeof(VirtIOSCSICmdReq) + s->cdb_size ||
-            in_size < sizeof(VirtIOSCSICmdResp) + s->sense_size) {
+        if (out_size < sizeof(VirtIOSCSICmdReq) + s->vs.cdb_size ||
+            in_size < sizeof(VirtIOSCSICmdResp) + s->vs.sense_size) {
             virtio_scsi_bad_req();
         }
 
@@ -523,7 +406,7 @@ static void virtio_scsi_get_config(VirtIODevice *vdev,
                                    uint8_t *config)
 {
     VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+    VirtIOSCSICommon *s = (VirtIOSCSICommon *)vdev;
 
     stl_raw(&scsiconf->num_queues, s->conf->num_queues);
     stl_raw(&scsiconf->seg_max, 128 - 2);
@@ -532,8 +415,8 @@ static void virtio_scsi_get_config(VirtIODevice *vdev,
     stl_raw(&scsiconf->event_info_size, sizeof(VirtIOSCSIEvent));
     stl_raw(&scsiconf->sense_size, s->sense_size);
     stl_raw(&scsiconf->cdb_size, s->cdb_size);
-    stl_raw(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
-    stl_raw(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
+    stw_raw(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
+    stw_raw(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
     stl_raw(&scsiconf->max_lun, VIRTIO_SCSI_MAX_LUN);
 }
 
@@ -549,8 +432,8 @@ static void virtio_scsi_set_config(VirtIODevice *vdev,
         exit(1);
     }
 
-    s->sense_size = ldl_raw(&scsiconf->sense_size);
-    s->cdb_size = ldl_raw(&scsiconf->cdb_size);
+    s->vs.sense_size = ldl_raw(&scsiconf->sense_size);
+    s->vs.cdb_size = ldl_raw(&scsiconf->cdb_size);
 }
 
 static uint32_t virtio_scsi_get_features(VirtIODevice *vdev,
@@ -563,8 +446,12 @@ static void virtio_scsi_reset(VirtIODevice *vdev)
 {
     VirtIOSCSI *s = (VirtIOSCSI *)vdev;
 
-    s->sense_size = VIRTIO_SCSI_SENSE_SIZE;
-    s->cdb_size = VIRTIO_SCSI_CDB_SIZE;
+    s->resetting++;
+    qbus_reset_all(&s->bus.qbus);
+    s->resetting--;
+
+    s->vs.sense_size = VIRTIO_SCSI_SENSE_SIZE;
+    s->vs.cdb_size = VIRTIO_SCSI_CDB_SIZE;
     s->events_dropped = false;
 }
 
@@ -574,7 +461,7 @@ static void virtio_scsi_reset(VirtIODevice *vdev)
 static void virtio_scsi_save(QEMUFile *f, void *opaque)
 {
     VirtIOSCSI *s = opaque;
-    virtio_save(&s->vdev, f);
+    virtio_save(&s->vs.vdev, f);
 }
 
 static int virtio_scsi_load(QEMUFile *f, void *opaque, int version_id)
@@ -582,7 +469,7 @@ static int virtio_scsi_load(QEMUFile *f, void *opaque, int version_id)
     VirtIOSCSI *s = opaque;
     int ret;
 
-    ret = virtio_load(&s->vdev, f);
+    ret = virtio_load(&s->vs.vdev, f);
     if (ret) {
         return ret;
     }
@@ -592,10 +479,14 @@ static int virtio_scsi_load(QEMUFile *f, void *opaque, int version_id)
 static void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
                                    uint32_t event, uint32_t reason)
 {
-    VirtIOSCSIReq *req = virtio_scsi_pop_req(s, s->event_vq);
+    VirtIOSCSIReq *req = virtio_scsi_pop_req(s, s->vs.event_vq);
     VirtIOSCSIEvent *evt;
     int in_size;
 
+    if (!(s->vs.vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+        return;
+    }
+
     if (!req) {
         s->events_dropped = true;
         return;
@@ -647,8 +538,7 @@ static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
 {
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
 
-    if (((s->vdev.guest_features >> VIRTIO_SCSI_F_CHANGE) & 1) &&
-        (s->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK) &&
+    if (((s->vs.vdev.guest_features >> VIRTIO_SCSI_F_CHANGE) & 1) &&
         dev->type != TYPE_ROM) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_PARAM_CHANGE,
                                sense.asc | (sense.ascq << 8));
@@ -659,8 +549,7 @@ static void virtio_scsi_hotplug(SCSIBus *bus, SCSIDevice *dev)
 {
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
 
-    if (((s->vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) &&
-        (s->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+    if ((s->vs.vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_RESCAN);
     }
@@ -670,7 +559,7 @@ static void virtio_scsi_hot_unplug(SCSIBus *bus, SCSIDevice *dev)
 {
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
 
-    if ((s->vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
+    if ((s->vs.vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_REMOVED);
     }
@@ -692,35 +581,53 @@ static struct SCSIBusInfo virtio_scsi_scsi_info = {
     .load_request = virtio_scsi_load_request,
 };
 
-VirtIODevice *virtio_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf)
+VirtIOSCSICommon *virtio_scsi_init_common(DeviceState *dev, VirtIOSCSIConf *proxyconf,
+                                          size_t sz)
 {
-    VirtIOSCSI *s;
-    static int virtio_scsi_id;
-    size_t sz;
+    VirtIOSCSICommon *s;
     int i;
 
-    sz = sizeof(VirtIOSCSI) + proxyconf->num_queues * sizeof(VirtQueue *);
-    s = (VirtIOSCSI *)virtio_common_init("virtio-scsi", VIRTIO_ID_SCSI,
-                                         sizeof(VirtIOSCSIConfig), sz);
+    /* Place the cmd_vqs array at the end of the block that the caller
+     * requested.  Align the array properly, just in case.
+     */
+    sz = ROUND_UP(sz, sizeof(void *));
+    s = (VirtIOSCSICommon *)
+            virtio_common_init("virtio-scsi", VIRTIO_ID_SCSI,
+                               sizeof(VirtIOSCSIConfig),
+                               sz + sizeof(VirtQueue *) * proxyconf->num_queues);
 
     s->qdev = dev;
     s->conf = proxyconf;
+    s->sense_size = VIRTIO_SCSI_SENSE_SIZE;
+    s->cdb_size = VIRTIO_SCSI_CDB_SIZE;
 
-    /* TODO set up vdev function pointers */
     s->vdev.get_config = virtio_scsi_get_config;
-    s->vdev.set_config = virtio_scsi_set_config;
     s->vdev.get_features = virtio_scsi_get_features;
-    s->vdev.reset = virtio_scsi_reset;
 
     s->ctrl_vq = virtio_add_queue(&s->vdev, VIRTIO_SCSI_VQ_SIZE,
                                    virtio_scsi_handle_ctrl);
     s->event_vq = virtio_add_queue(&s->vdev, VIRTIO_SCSI_VQ_SIZE,
                                    virtio_scsi_handle_event);
+
+    s->cmd_vqs = (VirtQueue **) ((char *)s + sz);
     for (i = 0; i < s->conf->num_queues; i++) {
         s->cmd_vqs[i] = virtio_add_queue(&s->vdev, VIRTIO_SCSI_VQ_SIZE,
                                          virtio_scsi_handle_cmd);
     }
 
+    return s;
+}
+
+VirtIODevice *virtio_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf)
+{
+    VirtIOSCSI *s;
+    static int virtio_scsi_id;
+
+    s = (VirtIOSCSI *)virtio_scsi_init_common(dev, proxyconf, sizeof(VirtIOSCSI));
+
+    s->vs.vdev.set_config = virtio_scsi_set_config;
+    s->vs.vdev.reset = virtio_scsi_reset;
+
     scsi_bus_new(&s->bus, dev, &virtio_scsi_scsi_info);
     if (!dev->hotplugged) {
         scsi_bus_legacy_handle_cmdline(&s->bus);
@@ -729,12 +636,12 @@ VirtIODevice *virtio_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf)
     register_savevm(dev, "virtio-scsi", virtio_scsi_id++, 1,
                     virtio_scsi_save, virtio_scsi_load, s);
 
-    return &s->vdev;
+    return &s->vs.vdev;
 }
 
 void virtio_scsi_exit(VirtIODevice *vdev)
 {
     VirtIOSCSI *s = (VirtIOSCSI *)vdev;
-    unregister_savevm(s->qdev, "virtio-scsi", s);
+    unregister_savevm(s->vs.qdev, "virtio-scsi", s);
     virtio_cleanup(vdev);
 }
diff --git a/hw/virtio-scsi.h b/hw/virtio-scsi.h
index 91924f6dfcb..a5b35ce517f 100644
--- a/hw/virtio-scsi.h
+++ b/hw/virtio-scsi.h
@@ -15,8 +15,7 @@
 #define _QEMU_VIRTIO_SCSI_H
 
 #include "virtio.h"
-#include "net.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /* The ID for virtio_scsi */
 #define VIRTIO_ID_SCSI  8
@@ -26,12 +25,138 @@
 #define VIRTIO_SCSI_F_HOTPLUG                  1
 #define VIRTIO_SCSI_F_CHANGE                   2
 
+#define VIRTIO_SCSI_VQ_SIZE     128
+#define VIRTIO_SCSI_CDB_SIZE    32
+#define VIRTIO_SCSI_SENSE_SIZE  96
+#define VIRTIO_SCSI_MAX_CHANNEL 0
+#define VIRTIO_SCSI_MAX_TARGET  255
+#define VIRTIO_SCSI_MAX_LUN     16383
+
+/* Response codes */
+#define VIRTIO_SCSI_S_OK                       0
+#define VIRTIO_SCSI_S_OVERRUN                  1
+#define VIRTIO_SCSI_S_ABORTED                  2
+#define VIRTIO_SCSI_S_BAD_TARGET               3
+#define VIRTIO_SCSI_S_RESET                    4
+#define VIRTIO_SCSI_S_BUSY                     5
+#define VIRTIO_SCSI_S_TRANSPORT_FAILURE        6
+#define VIRTIO_SCSI_S_TARGET_FAILURE           7
+#define VIRTIO_SCSI_S_NEXUS_FAILURE            8
+#define VIRTIO_SCSI_S_FAILURE                  9
+#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED       10
+#define VIRTIO_SCSI_S_FUNCTION_REJECTED        11
+#define VIRTIO_SCSI_S_INCORRECT_LUN            12
+
+/* Controlq type codes.  */
+#define VIRTIO_SCSI_T_TMF                      0
+#define VIRTIO_SCSI_T_AN_QUERY                 1
+#define VIRTIO_SCSI_T_AN_SUBSCRIBE             2
+
+/* Valid TMF subtypes.  */
+#define VIRTIO_SCSI_T_TMF_ABORT_TASK           0
+#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET       1
+#define VIRTIO_SCSI_T_TMF_CLEAR_ACA            2
+#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET       3
+#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET      4
+#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET   5
+#define VIRTIO_SCSI_T_TMF_QUERY_TASK           6
+#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET       7
+
+/* Events.  */
+#define VIRTIO_SCSI_T_EVENTS_MISSED            0x80000000
+#define VIRTIO_SCSI_T_NO_EVENT                 0
+#define VIRTIO_SCSI_T_TRANSPORT_RESET          1
+#define VIRTIO_SCSI_T_ASYNC_NOTIFY             2
+#define VIRTIO_SCSI_T_PARAM_CHANGE             3
+
+/* Reasons for transport reset event */
+#define VIRTIO_SCSI_EVT_RESET_HARD             0
+#define VIRTIO_SCSI_EVT_RESET_RESCAN           1
+#define VIRTIO_SCSI_EVT_RESET_REMOVED          2
+
+/* SCSI command request, followed by data-out */
+typedef struct {
+    uint8_t lun[8];              /* Logical Unit Number */
+    uint64_t tag;                /* Command identifier */
+    uint8_t task_attr;           /* Task attribute */
+    uint8_t prio;
+    uint8_t crn;
+    uint8_t cdb[];
+} QEMU_PACKED VirtIOSCSICmdReq;
+
+/* Response, followed by sense data and data-in */
+typedef struct {
+    uint32_t sense_len;          /* Sense data length */
+    uint32_t resid;              /* Residual bytes in data buffer */
+    uint16_t status_qualifier;   /* Status qualifier */
+    uint8_t status;              /* Command completion status */
+    uint8_t response;            /* Response values */
+    uint8_t sense[];
+} QEMU_PACKED VirtIOSCSICmdResp;
+
+/* Task Management Request */
+typedef struct {
+    uint32_t type;
+    uint32_t subtype;
+    uint8_t lun[8];
+    uint64_t tag;
+} QEMU_PACKED VirtIOSCSICtrlTMFReq;
+
+typedef struct {
+    uint8_t response;
+} QEMU_PACKED VirtIOSCSICtrlTMFResp;
+
+/* Asynchronous notification query/subscription */
+typedef struct {
+    uint32_t type;
+    uint8_t lun[8];
+    uint32_t event_requested;
+} QEMU_PACKED VirtIOSCSICtrlANReq;
+
+typedef struct {
+    uint32_t event_actual;
+    uint8_t response;
+} QEMU_PACKED VirtIOSCSICtrlANResp;
+
+typedef struct {
+    uint32_t event;
+    uint8_t lun[8];
+    uint32_t reason;
+} QEMU_PACKED VirtIOSCSIEvent;
+
+typedef struct {
+    uint32_t num_queues;
+    uint32_t seg_max;
+    uint32_t max_sectors;
+    uint32_t cmd_per_lun;
+    uint32_t event_info_size;
+    uint32_t sense_size;
+    uint32_t cdb_size;
+    uint16_t max_channel;
+    uint16_t max_target;
+    uint32_t max_lun;
+} QEMU_PACKED VirtIOSCSIConfig;
+
 struct VirtIOSCSIConf {
     uint32_t num_queues;
     uint32_t max_sectors;
     uint32_t cmd_per_lun;
+    char *vhostfd;
+    char *wwpn;
 };
 
+typedef struct VirtIOSCSICommon {
+    VirtIODevice vdev;
+    DeviceState *qdev;
+    VirtIOSCSIConf *conf;
+
+    uint32_t sense_size;
+    uint32_t cdb_size;
+    VirtQueue *ctrl_vq;
+    VirtQueue *event_vq;
+    VirtQueue **cmd_vqs;
+} VirtIOSCSICommon;
+
 #define DEFINE_VIRTIO_SCSI_PROPERTIES(_state, _features_field, _conf_field) \
     DEFINE_VIRTIO_COMMON_FEATURES(_state, _features_field), \
     DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1), \
@@ -40,4 +165,7 @@ struct VirtIOSCSIConf {
     DEFINE_PROP_BIT("hotplug", _state, _features_field, VIRTIO_SCSI_F_HOTPLUG, true), \
     DEFINE_PROP_BIT("param_change", _state, _features_field, VIRTIO_SCSI_F_CHANGE, true)
 
+VirtIOSCSICommon *virtio_scsi_init_common(DeviceState *dev, VirtIOSCSIConf *proxyconf,
+                                          size_t sz);
+
 #endif /* _QEMU_VIRTIO_SCSI_H */
diff --git a/hw/virtio-serial-bus.c b/hw/virtio-serial-bus.c
index 82073f5dc26..aa7d0d7fc78 100644
--- a/hw/virtio-serial-bus.c
+++ b/hw/virtio-serial-bus.c
@@ -18,9 +18,9 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "iov.h"
-#include "monitor.h"
-#include "qemu-queue.h"
+#include "qemu/iov.h"
+#include "monitor/monitor.h"
+#include "qemu/queue.h"
 #include "sysbus.h"
 #include "trace.h"
 #include "virtio-serial.h"
@@ -36,6 +36,15 @@ struct VirtIOSerialBus {
     uint32_t max_nr_ports;
 };
 
+typedef struct VirtIOSerialPostLoad {
+    QEMUTimer *timer;
+    uint32_t nr_active_ports;
+    struct {
+        VirtIOSerialPort *port;
+        uint8_t host_connected;
+    } *connected;
+} VirtIOSerialPostLoad;
+
 struct VirtIOSerial {
     VirtIODevice vdev;
 
@@ -53,6 +62,8 @@ struct VirtIOSerial {
     uint32_t *ports_map;
 
     struct virtio_console_config config;
+
+    struct VirtIOSerialPostLoad *post_load;
 };
 
 static VirtIOSerialPort *find_port_by_id(VirtIOSerial *vser, uint32_t id)
@@ -206,13 +217,12 @@ static void flush_queued_data(VirtIOSerialPort *port)
     do_flush_queued_data(port, port->ovq, &port->vser->vdev);
 }
 
-static size_t send_control_msg(VirtIOSerialPort *port, void *buf, size_t len)
+static size_t send_control_msg(VirtIOSerial *vser, void *buf, size_t len)
 {
     VirtQueueElement elem;
     VirtQueue *vq;
-    struct virtio_console_control *cpkt;
 
-    vq = port->vser->c_ivq;
+    vq = vser->c_ivq;
     if (!virtio_queue_ready(vq)) {
         return 0;
     }
@@ -220,25 +230,24 @@ static size_t send_control_msg(VirtIOSerialPort *port, void *buf, size_t len)
         return 0;
     }
 
-    cpkt = (struct virtio_console_control *)buf;
-    stl_p(&cpkt->id, port->id);
     memcpy(elem.in_sg[0].iov_base, buf, len);
 
     virtqueue_push(vq, &elem, len);
-    virtio_notify(&port->vser->vdev, vq);
+    virtio_notify(&vser->vdev, vq);
     return len;
 }
 
-static size_t send_control_event(VirtIOSerialPort *port, uint16_t event,
-                                 uint16_t value)
+static size_t send_control_event(VirtIOSerial *vser, uint32_t port_id,
+                                 uint16_t event, uint16_t value)
 {
     struct virtio_console_control cpkt;
 
+    stl_p(&cpkt.id, port_id);
     stw_p(&cpkt.event, event);
     stw_p(&cpkt.value, value);
 
-    trace_virtio_serial_send_control_event(port->id, event, value);
-    return send_control_msg(port, &cpkt, sizeof(cpkt));
+    trace_virtio_serial_send_control_event(port_id, event, value);
+    return send_control_msg(vser, &cpkt, sizeof(cpkt));
 }
 
 /* Functions for use inside qemu to open and read from/write to ports */
@@ -250,7 +259,7 @@ int virtio_serial_open(VirtIOSerialPort *port)
     }
     /* Send port open notification to the guest */
     port->host_connected = true;
-    send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+    send_control_event(port->vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 1);
 
     return 0;
 }
@@ -265,7 +274,7 @@ int virtio_serial_close(VirtIOSerialPort *port)
     port->throttled = false;
     discard_vq_data(port->ovq, &port->vser->vdev);
 
-    send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
+    send_control_event(port->vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 0);
 
     return 0;
 }
@@ -287,6 +296,7 @@ ssize_t virtio_serial_write(VirtIOSerialPort *port, const uint8_t *buf,
 size_t virtio_serial_guest_ready(VirtIOSerialPort *port)
 {
     VirtQueue *vq = port->ivq;
+    unsigned int bytes;
 
     if (!virtio_queue_ready(vq) ||
         !(port->vser->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK) ||
@@ -296,14 +306,8 @@ size_t virtio_serial_guest_ready(VirtIOSerialPort *port)
     if (use_multiport(port->vser) && !port->guest_connected) {
         return 0;
     }
-
-    if (virtqueue_avail_bytes(vq, 4096, 0)) {
-        return 4096;
-    }
-    if (virtqueue_avail_bytes(vq, 1, 0)) {
-        return 1;
-    }
-    return 0;
+    virtqueue_get_avail_bytes(vq, &bytes, NULL, 4096, 0);
+    return bytes;
 }
 
 static void flush_queued_data_bh(void *opaque)
@@ -359,7 +363,7 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
          * ports we have here.
          */
         QTAILQ_FOREACH(port, &vser->ports, next) {
-            send_control_event(port, VIRTIO_CONSOLE_PORT_ADD, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_ADD, 1);
         }
         return;
     }
@@ -390,10 +394,11 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
          * up to hvc.
          */
         if (vsc->is_console) {
-            send_control_event(port, VIRTIO_CONSOLE_CONSOLE_PORT, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_CONSOLE_PORT, 1);
         }
 
         if (port->name) {
+            stl_p(&cpkt.id, port->id);
             stw_p(&cpkt.event, VIRTIO_CONSOLE_PORT_NAME);
             stw_p(&cpkt.value, 1);
 
@@ -404,12 +409,12 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
             memcpy(buffer + sizeof(cpkt), port->name, strlen(port->name));
             buffer[buffer_len - 1] = 0;
 
-            send_control_msg(port, buffer, buffer_len);
+            send_control_msg(vser, buffer, buffer_len);
             g_free(buffer);
         }
 
         if (port->host_connected) {
-            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 1);
         }
 
         /*
@@ -631,10 +636,93 @@ static void virtio_serial_save(QEMUFile *f, void *opaque)
     }
 }
 
-static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
+static void virtio_serial_post_load_timer_cb(void *opaque)
 {
+    uint32_t i;
     VirtIOSerial *s = opaque;
     VirtIOSerialPort *port;
+    uint8_t host_connected;
+
+    if (!s->post_load) {
+        return;
+    }
+    for (i = 0 ; i < s->post_load->nr_active_ports; ++i) {
+        port = s->post_load->connected[i].port;
+        host_connected = s->post_load->connected[i].host_connected;
+        if (host_connected != port->host_connected) {
+            /*
+             * We have to let the guest know of the host connection
+             * status change
+             */
+            send_control_event(s, port->id, VIRTIO_CONSOLE_PORT_OPEN,
+                               port->host_connected);
+        }
+    }
+    g_free(s->post_load->connected);
+    qemu_free_timer(s->post_load->timer);
+    g_free(s->post_load);
+    s->post_load = NULL;
+}
+
+static int fetch_active_ports_list(QEMUFile *f, int version_id,
+                                   VirtIOSerial *s, uint32_t nr_active_ports)
+{
+    uint32_t i;
+
+    s->post_load = g_malloc0(sizeof(*s->post_load));
+    s->post_load->nr_active_ports = nr_active_ports;
+    s->post_load->connected =
+        g_malloc0(sizeof(*s->post_load->connected) * nr_active_ports);
+
+    s->post_load->timer = qemu_new_timer_ns(vm_clock,
+                                            virtio_serial_post_load_timer_cb,
+                                            s);
+
+    /* Items in struct VirtIOSerialPort */
+    for (i = 0; i < nr_active_ports; i++) {
+        VirtIOSerialPort *port;
+        uint32_t id;
+
+        id = qemu_get_be32(f);
+        port = find_port_by_id(s, id);
+        if (!port) {
+            return -EINVAL;
+        }
+
+        port->guest_connected = qemu_get_byte(f);
+        s->post_load->connected[i].port = port;
+        s->post_load->connected[i].host_connected = qemu_get_byte(f);
+
+        if (version_id > 2) {
+            uint32_t elem_popped;
+
+            qemu_get_be32s(f, &elem_popped);
+            if (elem_popped) {
+                qemu_get_be32s(f, &port->iov_idx);
+                qemu_get_be64s(f, &port->iov_offset);
+
+                qemu_get_buffer(f, (unsigned char *)&port->elem,
+                                sizeof(port->elem));
+                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
+                                 port->elem.in_num, 1);
+                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
+                                 port->elem.out_num, 1);
+
+                /*
+                 *  Port was throttled on source machine.  Let's
+                 *  unthrottle it here so data starts flowing again.
+                 */
+                virtio_serial_throttle_port(port, false);
+            }
+        }
+    }
+    qemu_mod_timer(s->post_load->timer, 1);
+    return 0;
+}
+
+static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIOSerial *s = opaque;
     uint32_t max_nr_ports, nr_active_ports, ports_map;
     unsigned int i;
     int ret;
@@ -678,49 +766,10 @@ static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
 
     qemu_get_be32s(f, &nr_active_ports);
 
-    /* Items in struct VirtIOSerialPort */
-    for (i = 0; i < nr_active_ports; i++) {
-        uint32_t id;
-        bool host_connected;
-
-        id = qemu_get_be32(f);
-        port = find_port_by_id(s, id);
-        if (!port) {
-            return -EINVAL;
-        }
-
-        port->guest_connected = qemu_get_byte(f);
-        host_connected = qemu_get_byte(f);
-        if (host_connected != port->host_connected) {
-            /*
-             * We have to let the guest know of the host connection
-             * status change
-             */
-            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN,
-                               port->host_connected);
-        }
-
-        if (version_id > 2) {
-            uint32_t elem_popped;
-
-            qemu_get_be32s(f, &elem_popped);
-            if (elem_popped) {
-                qemu_get_be32s(f, &port->iov_idx);
-                qemu_get_be64s(f, &port->iov_offset);
-
-                qemu_get_buffer(f, (unsigned char *)&port->elem,
-                                sizeof(port->elem));
-                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
-                                 port->elem.in_num, 1);
-                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
-                                 port->elem.out_num, 1);
-
-                /*
-                 *  Port was throttled on source machine.  Let's
-                 *  unthrottle it here so data starts flowing again.
-                 */
-                virtio_serial_throttle_port(port, false);
-            }
+    if (nr_active_ports) {
+        ret = fetch_active_ports_list(f, version_id, s, nr_active_ports);
+        if (ret) {
+            return ret;
         }
     }
     return 0;
@@ -791,9 +840,7 @@ static void mark_port_added(VirtIOSerial *vser, uint32_t port_id)
 static void add_port(VirtIOSerial *vser, uint32_t port_id)
 {
     mark_port_added(vser, port_id);
-
-    send_control_event(find_port_by_id(vser, port_id),
-                       VIRTIO_CONSOLE_PORT_ADD, 1);
+    send_control_event(vser, port_id, VIRTIO_CONSOLE_PORT_ADD, 1);
 }
 
 static void remove_port(VirtIOSerial *vser, uint32_t port_id)
@@ -805,10 +852,16 @@ static void remove_port(VirtIOSerial *vser, uint32_t port_id)
     vser->ports_map[i] &= ~(1U << (port_id % 32));
 
     port = find_port_by_id(vser, port_id);
+    /*
+     * This function is only called from qdev's unplug callback; if we
+     * get a NULL port here, we're in trouble.
+     */
+    assert(port);
+
     /* Flush out any unconsumed buffers first */
     discard_vq_data(port->ovq, &port->vser->vdev);
 
-    send_control_event(port, VIRTIO_CONSOLE_PORT_REMOVE, 1);
+    send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_REMOVE, 1);
 }
 
 static int virtser_port_qdev_init(DeviceState *qdev)
@@ -965,6 +1018,8 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *conf)
 
     vser->qdev = dev;
 
+    vser->post_load = NULL;
+
     /*
      * Register for the savevm section with the virtio-console name
      * to preserve backward compat
@@ -984,7 +1039,12 @@ void virtio_serial_exit(VirtIODevice *vdev)
     g_free(vser->ivqs);
     g_free(vser->ovqs);
     g_free(vser->ports_map);
-
+    if (vser->post_load) {
+        g_free(vser->post_load->connected);
+        qemu_del_timer(vser->post_load->timer);
+        qemu_free_timer(vser->post_load->timer);
+        g_free(vser->post_load);
+    }
     virtio_cleanup(vdev);
 }
 
@@ -998,7 +1058,7 @@ static void virtio_serial_port_class_init(ObjectClass *klass, void *data)
     k->props = virtser_props;
 }
 
-static TypeInfo virtio_serial_port_type_info = {
+static const TypeInfo virtio_serial_port_type_info = {
     .name = TYPE_VIRTIO_SERIAL_PORT,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VirtIOSerialPort),
diff --git a/hw/virtio.c b/hw/virtio.c
index 209c763751b..ca170c319ef 100644
--- a/hw/virtio.c
+++ b/hw/virtio.c
@@ -14,9 +14,10 @@
 #include <inttypes.h>
 
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "virtio.h"
-#include "qemu-barrier.h"
+#include "qemu/atomic.h"
+#include "virtio-bus.h"
 
 /* The alignment to use between consumer and producer parts of vring.
  * x86 pagesize again. */
@@ -53,15 +54,15 @@ typedef struct VRingUsed
 typedef struct VRing
 {
     unsigned int num;
-    target_phys_addr_t desc;
-    target_phys_addr_t avail;
-    target_phys_addr_t used;
+    hwaddr desc;
+    hwaddr avail;
+    hwaddr used;
 } VRing;
 
 struct VirtQueue
 {
     VRing vring;
-    target_phys_addr_t pa;
+    hwaddr pa;
     uint16_t last_avail_idx;
     /* Last used index value we have signalled on */
     uint16_t signalled_used;
@@ -84,7 +85,7 @@ struct VirtQueue
 /* virt queue functions */
 static void virtqueue_init(VirtQueue *vq)
 {
-    target_phys_addr_t pa = vq->pa;
+    hwaddr pa = vq->pa;
 
     vq->vring.desc = pa;
     vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc);
@@ -93,51 +94,51 @@ static void virtqueue_init(VirtQueue *vq)
                                  VIRTIO_PCI_VRING_ALIGN);
 }
 
-static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa, int i)
+static inline uint64_t vring_desc_addr(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
     return ldq_phys(pa);
 }
 
-static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa, int i)
+static inline uint32_t vring_desc_len(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
     return ldl_phys(pa);
 }
 
-static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa, int i)
+static inline uint16_t vring_desc_flags(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
     return lduw_phys(pa);
 }
 
-static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa, int i)
+static inline uint16_t vring_desc_next(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, flags);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, idx);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
     return lduw_phys(pa);
 }
@@ -149,49 +150,49 @@ static inline uint16_t vring_used_event(VirtQueue *vq)
 
 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
     stl_phys(pa, val);
 }
 
 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
     stl_phys(pa, val);
 }
 
 static uint16_t vring_used_idx(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     return lduw_phys(pa);
 }
 
 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     stw_phys(pa, val);
 }
 
 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, flags);
     stw_phys(pa, lduw_phys(pa) | mask);
 }
 
 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, flags);
     stw_phys(pa, lduw_phys(pa) & ~mask);
 }
 
 static inline void vring_avail_event(VirtQueue *vq, uint16_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     if (!vq->notification) {
         return;
     }
@@ -241,7 +242,7 @@ void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                                   elem->in_sg[i].iov_len,
                                   1, size);
 
-        offset += elem->in_sg[i].iov_len;
+        offset += size;
     }
 
     for (i = 0; i < elem->out_num; i++)
@@ -313,7 +314,7 @@ static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
     return head;
 }
 
-static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa,
+static unsigned virtqueue_next_desc(hwaddr desc_pa,
                                     unsigned int i, unsigned int max)
 {
     unsigned int next;
@@ -335,17 +336,19 @@ static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa,
     return next;
 }
 
-int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
+void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
+                               unsigned int *out_bytes,
+                               unsigned max_in_bytes, unsigned max_out_bytes)
 {
     unsigned int idx;
-    int total_bufs, in_total, out_total;
+    unsigned int total_bufs, in_total, out_total;
 
     idx = vq->last_avail_idx;
 
     total_bufs = in_total = out_total = 0;
     while (virtqueue_num_heads(vq, idx)) {
         unsigned int max, num_bufs, indirect = 0;
-        target_phys_addr_t desc_pa;
+        hwaddr desc_pa;
         int i;
 
         max = vq->vring.num;
@@ -380,13 +383,12 @@ int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
             }
 
             if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
-                if (in_bytes > 0 &&
-                    (in_total += vring_desc_len(desc_pa, i)) >= in_bytes)
-                    return 1;
+                in_total += vring_desc_len(desc_pa, i);
             } else {
-                if (out_bytes > 0 &&
-                    (out_total += vring_desc_len(desc_pa, i)) >= out_bytes)
-                    return 1;
+                out_total += vring_desc_len(desc_pa, i);
+            }
+            if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
+                goto done;
             }
         } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
 
@@ -395,15 +397,29 @@ int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
         else
             total_bufs++;
     }
+done:
+    if (in_bytes) {
+        *in_bytes = in_total;
+    }
+    if (out_bytes) {
+        *out_bytes = out_total;
+    }
+}
 
-    return 0;
+int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
+                          unsigned int out_bytes)
+{
+    unsigned int in_total, out_total;
+
+    virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
+    return in_bytes <= in_total && out_bytes <= out_total;
 }
 
-void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
+void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
     size_t num_sg, int is_write)
 {
     unsigned int i;
-    target_phys_addr_t len;
+    hwaddr len;
 
     for (i = 0; i < num_sg; i++) {
         len = sg[i].iov_len;
@@ -418,7 +434,7 @@ void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 {
     unsigned int i, head, max;
-    target_phys_addr_t desc_pa = vq->vring.desc;
+    hwaddr desc_pa = vq->vring.desc;
 
     if (!virtqueue_num_heads(vq, vq->last_avail_idx))
         return 0;
@@ -617,13 +633,13 @@ void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
         vdev->set_config(vdev, vdev->config);
 }
 
-void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr)
+void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 {
     vdev->vq[n].pa = addr;
     virtqueue_init(&vdev->vq[n]);
 }
 
-target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].pa;
 }
@@ -860,11 +876,16 @@ int virtio_load(VirtIODevice *vdev, QEMUFile *f)
     return 0;
 }
 
-void virtio_cleanup(VirtIODevice *vdev)
+void virtio_common_cleanup(VirtIODevice *vdev)
 {
     qemu_del_vm_change_state_handler(vdev->vmstate);
     g_free(vdev->config);
     g_free(vdev->vq);
+}
+
+void virtio_cleanup(VirtIODevice *vdev)
+{
+    virtio_common_cleanup(vdev);
     g_free(vdev);
 }
 
@@ -887,14 +908,10 @@ static void virtio_vmstate_change(void *opaque, int running, RunState state)
     }
 }
 
-VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
-                                 size_t config_size, size_t struct_size)
+void virtio_init(VirtIODevice *vdev, const char *name,
+                 uint16_t device_id, size_t config_size)
 {
-    VirtIODevice *vdev;
     int i;
-
-    vdev = g_malloc0(struct_size);
-
     vdev->device_id = device_id;
     vdev->status = 0;
     vdev->isr = 0;
@@ -902,68 +919,76 @@ VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
     vdev->config_vector = VIRTIO_NO_VECTOR;
     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX);
     vdev->vm_running = runstate_is_running();
-    for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
         vdev->vq[i].vdev = vdev;
     }
 
     vdev->name = name;
     vdev->config_len = config_size;
-    if (vdev->config_len)
+    if (vdev->config_len) {
         vdev->config = g_malloc0(config_size);
-    else
+    } else {
         vdev->config = NULL;
+    }
+    vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
+                                                     vdev);
+}
 
-    vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, vdev);
-
+VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
+                                 size_t config_size, size_t struct_size)
+{
+    VirtIODevice *vdev;
+    vdev = g_malloc0(struct_size);
+    virtio_init(vdev, name, device_id, config_size);
     return vdev;
 }
 
 void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
-                        void *opaque)
+                        DeviceState *opaque)
 {
     vdev->binding = binding;
     vdev->binding_opaque = opaque;
 }
 
-target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.desc;
 }
 
-target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.avail;
 }
 
-target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.used;
 }
 
-target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.desc;
 }
 
-target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
 {
     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingAvail, ring) +
         sizeof(uint64_t) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingUsed, ring) +
         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
 	    virtio_queue_get_used_size(vdev, n);
@@ -1041,3 +1066,39 @@ EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
 {
     return &vq->host_notifier;
 }
+
+static int virtio_device_init(DeviceState *qdev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(qdev);
+    assert(k->init != NULL);
+    if (k->init(vdev) < 0) {
+        return -1;
+    }
+    virtio_bus_plug_device(vdev);
+    return 0;
+}
+
+static void virtio_device_class_init(ObjectClass *klass, void *data)
+{
+    /* Set the default value here. */
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->init = virtio_device_init;
+    dc->bus_type = TYPE_VIRTIO_BUS;
+}
+
+static const TypeInfo virtio_device_info = {
+    .name = TYPE_VIRTIO_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(VirtIODevice),
+    .class_init = virtio_device_class_init,
+    .abstract = true,
+    .class_size = sizeof(VirtioDeviceClass),
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_device_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/virtio.h b/hw/virtio.h
index 7a4f564529e..dd37d590902 100644
--- a/hw/virtio.h
+++ b/hw/virtio.h
@@ -15,10 +15,10 @@
 #define _QEMU_VIRTIO_H
 
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "qdev.h"
-#include "sysemu.h"
-#include "event_notifier.h"
+#include "sysemu/sysemu.h"
+#include "qemu/event_notifier.h"
 #ifdef CONFIG_LINUX
 #include "9p.h"
 #endif
@@ -69,7 +69,7 @@
 
 struct VirtQueue;
 
-static inline target_phys_addr_t vring_align(target_phys_addr_t addr,
+static inline hwaddr vring_align(hwaddr addr,
                                              unsigned long align)
 {
     return (addr + align - 1) & ~(align - 1);
@@ -84,32 +84,41 @@ typedef struct VirtQueueElement
     unsigned int index;
     unsigned int out_num;
     unsigned int in_num;
-    target_phys_addr_t in_addr[VIRTQUEUE_MAX_SIZE];
-    target_phys_addr_t out_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
 } VirtQueueElement;
 
 typedef struct {
-    void (*notify)(void * opaque, uint16_t vector);
-    void (*save_config)(void * opaque, QEMUFile *f);
-    void (*save_queue)(void * opaque, int n, QEMUFile *f);
-    int (*load_config)(void * opaque, QEMUFile *f);
-    int (*load_queue)(void * opaque, int n, QEMUFile *f);
-    int (*load_done)(void * opaque, QEMUFile *f);
-    unsigned (*get_features)(void * opaque);
-    bool (*query_guest_notifiers)(void * opaque);
-    int (*set_guest_notifiers)(void * opaque, bool assigned);
-    int (*set_host_notifier)(void * opaque, int n, bool assigned);
-    void (*vmstate_change)(void * opaque, bool running);
+    void (*notify)(DeviceState *d, uint16_t vector);
+    void (*save_config)(DeviceState *d, QEMUFile *f);
+    void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_config)(DeviceState *d, QEMUFile *f);
+    int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_done)(DeviceState *d, QEMUFile *f);
+    unsigned (*get_features)(DeviceState *d);
+    bool (*query_guest_notifiers)(DeviceState *d);
+    int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assigned);
+    int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
+    void (*vmstate_change)(DeviceState *d, bool running);
 } VirtIOBindings;
 
 #define VIRTIO_PCI_QUEUE_MAX 64
 
 #define VIRTIO_NO_VECTOR 0xffff
 
+#define TYPE_VIRTIO_DEVICE "virtio-device"
+#define VIRTIO_DEVICE_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioDeviceClass, obj, TYPE_VIRTIO_DEVICE)
+#define VIRTIO_DEVICE_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioDeviceClass, klass, TYPE_VIRTIO_DEVICE)
+#define VIRTIO_DEVICE(obj) \
+        OBJECT_CHECK(VirtIODevice, (obj), TYPE_VIRTIO_DEVICE)
+
 struct VirtIODevice
 {
+    DeviceState parent_obj;
     const char *name;
     uint8_t status;
     uint8_t isr;
@@ -119,21 +128,57 @@ struct VirtIODevice
     void *config;
     uint16_t config_vector;
     int nvectors;
+    /*
+     * Function pointers will be removed at the end of the series as they are in
+     * VirtioDeviceClass.
+     */
     uint32_t (*get_features)(VirtIODevice *vdev, uint32_t requested_features);
     uint32_t (*bad_features)(VirtIODevice *vdev);
+    int (*set_vhost_endpoint)(VirtIODevice *vdev);
+    void (*clear_vhost_endpoint)(VirtIODevice *vdev);
     void (*set_features)(VirtIODevice *vdev, uint32_t val);
     void (*get_config)(VirtIODevice *vdev, uint8_t *config);
     void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
     void (*reset)(VirtIODevice *vdev);
     void (*set_status)(VirtIODevice *vdev, uint8_t val);
+    /* Test and clear event pending status.
+     * Should be called after unmask to avoid losing events.
+     * If backend does not support masking,
+     * must check in frontend instead.
+     */
+    bool (*guest_notifier_pending)(VirtIODevice *vdev, int n);
+    /* Mask/unmask events from this vq. Any events reported
+     * while masked will become pending.
+     * If backend does not support masking,
+     * must mask in frontend instead.
+     */
+    void (*guest_notifier_mask)(VirtIODevice *vdev, int n, bool mask);
+
     VirtQueue *vq;
     const VirtIOBindings *binding;
-    void *binding_opaque;
+    DeviceState *binding_opaque;
     uint16_t device_id;
     bool vm_running;
     VMChangeStateEntry *vmstate;
 };
 
+typedef struct VirtioDeviceClass {
+    /* This is what a VirtioDevice must implement */
+    DeviceClass parent;
+    int (*init)(VirtIODevice *vdev);
+    uint32_t (*get_features)(VirtIODevice *vdev, uint32_t requested_features);
+    uint32_t (*bad_features)(VirtIODevice *vdev);
+    void (*set_features)(VirtIODevice *vdev, uint32_t val);
+    void (*get_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
+    void (*reset)(VirtIODevice *vdev);
+    void (*set_status)(VirtIODevice *vdev, uint8_t val);
+} VirtioDeviceClass;
+
+void virtio_init(VirtIODevice *vdev, const char *name,
+                         uint16_t device_id, size_t config_size);
+void virtio_common_cleanup(VirtIODevice *vdev);
+
 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
                             void (*handle_output)(VirtIODevice *,
                                                   VirtQueue *));
@@ -144,10 +189,14 @@ void virtqueue_flush(VirtQueue *vq, unsigned int count);
 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len, unsigned int idx);
 
-void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
+void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
     size_t num_sg, int is_write);
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
-int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes);
+int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
+                          unsigned int out_bytes);
+void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
+                               unsigned int *out_bytes,
+                               unsigned max_in_bytes, unsigned max_out_bytes);
 
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
 
@@ -175,8 +224,8 @@ uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr);
 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);
-void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr);
-target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n);
+void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr);
+hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n);
 int virtio_queue_get_num(VirtIODevice *vdev, int n);
 void virtio_queue_notify(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n);
@@ -187,7 +236,7 @@ void virtio_update_irq(VirtIODevice *vdev);
 int virtio_set_features(VirtIODevice *vdev, uint32_t val);
 
 void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
-                        void *opaque);
+                        DeviceState *opaque);
 
 /* Base devices.  */
 typedef struct VirtIOBlkConf VirtIOBlkConf;
@@ -200,6 +249,8 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *serial);
 VirtIODevice *virtio_balloon_init(DeviceState *dev);
 typedef struct VirtIOSCSIConf VirtIOSCSIConf;
 VirtIODevice *virtio_scsi_init(DeviceState *dev, VirtIOSCSIConf *conf);
+typedef struct VirtIORNGConf VirtIORNGConf;
+VirtIODevice *virtio_rng_init(DeviceState *dev, VirtIORNGConf *conf);
 #ifdef CONFIG_LINUX
 VirtIODevice *virtio_9p_init(DeviceState *dev, V9fsConf *conf);
 #endif
@@ -210,6 +261,7 @@ void virtio_blk_exit(VirtIODevice *vdev);
 void virtio_serial_exit(VirtIODevice *vdev);
 void virtio_balloon_exit(VirtIODevice *vdev);
 void virtio_scsi_exit(VirtIODevice *vdev);
+void virtio_rng_exit(VirtIODevice *vdev);
 
 #define DEFINE_VIRTIO_COMMON_FEATURES(_state, _field) \
 	DEFINE_PROP_BIT("indirect_desc", _state, _field, \
@@ -217,14 +269,14 @@ void virtio_scsi_exit(VirtIODevice *vdev);
 	DEFINE_PROP_BIT("event_idx", _state, _field, \
 			VIRTIO_RING_F_EVENT_IDX, true)
 
-target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n);
 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx);
 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n);
diff --git a/hw/vmmouse.c b/hw/vmmouse.c
index 6338efa1c39..b9afc2c4e8b 100644
--- a/hw/vmmouse.c
+++ b/hw/vmmouse.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "ps2.h"
 #include "pc.h"
 #include "qdev.h"
@@ -252,7 +252,6 @@ static void vmmouse_reset(DeviceState *d)
 {
     VMMouseState *s = container_of(d, VMMouseState, dev.qdev);
 
-    s->status = 0xffff;
     s->queue_size = VMMOUSE_QUEUE_SIZE;
 
     vmmouse_disable(s);
@@ -287,7 +286,7 @@ static void vmmouse_class_initfn(ObjectClass *klass, void *data)
     dc->props = vmmouse_properties;
 }
 
-static TypeInfo vmmouse_info = {
+static const TypeInfo vmmouse_info = {
     .name          = "vmmouse",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(VMMouseState),
diff --git a/hw/vmport.c b/hw/vmport.c
index a4f52ee5bb9..faead3a955a 100644
--- a/hw/vmport.c
+++ b/hw/vmport.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "isa.h"
 #include "pc.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "qdev.h"
 
 //#define VMPORT_DEBUG
@@ -54,7 +54,8 @@ void vmport_register(unsigned char command, IOPortReadFunc *func, void *opaque)
     port_state->opaque[command] = opaque;
 }
 
-static uint32_t vmport_ioport_read(void *opaque, uint32_t addr)
+static uint64_t vmport_ioport_read(void *opaque, hwaddr addr,
+                                   unsigned size)
 {
     VMPortState *s = opaque;
     CPUX86State *env = cpu_single_env;
@@ -81,11 +82,12 @@ static uint32_t vmport_ioport_read(void *opaque, uint32_t addr)
     return s->func[command](s->opaque[command], addr);
 }
 
-static void vmport_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void vmport_ioport_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
 {
     CPUX86State *env = cpu_single_env;
 
-    env->regs[R_EAX] = vmport_ioport_read(opaque, addr);
+    env->regs[R_EAX] = vmport_ioport_read(opaque, addr, 4);
 }
 
 static uint32_t vmport_cmd_get_version(void *opaque, uint32_t addr)
@@ -121,13 +123,14 @@ void vmmouse_set_data(const uint32_t *data)
     env->regs[R_ESI] = data[4]; env->regs[R_EDI] = data[5];
 }
 
-static const MemoryRegionPortio vmport_portio[] = {
-    {0, 1, 4, .read = vmport_ioport_read, .write = vmport_ioport_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps vmport_ops = {
-    .old_portio = vmport_portio
+    .read = vmport_ioport_read,
+    .write = vmport_ioport_write,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int vmport_initfn(ISADevice *dev)
@@ -152,7 +155,7 @@ static void vmport_class_initfn(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo vmport_info = {
+static const TypeInfo vmport_info = {
     .name          = "vmport",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(VMPortState),
diff --git a/hw/vmware_vga.c b/hw/vmware_vga.c
index f5e4f440d58..cd15ee40a87 100644
--- a/hw/vmware_vga.c
+++ b/hw/vmware_vga.c
@@ -23,22 +23,21 @@
  */
 #include "hw.h"
 #include "loader.h"
-#include "console.h"
-#include "pci.h"
-#include "vmware_vga.h"
+#include "ui/console.h"
+#include "pci/pci.h"
 
 #undef VERBOSE
 #define HW_RECT_ACCEL
 #define HW_FILL_ACCEL
 #define HW_MOUSE_ACCEL
 
-# include "vga_int.h"
+#include "vga_int.h"
+
+/* See http://vmware-svga.sf.net/ for some documentation on VMWare SVGA */
 
 struct vmsvga_state_s {
     VGACommonState vga;
 
-    int width;
-    int height;
     int invalidated;
     int depth;
     int bypp;
@@ -62,7 +61,6 @@ struct vmsvga_state_s {
     uint32_t wgreen;
     uint32_t wblue;
     int syncing;
-    int fb_size;
 
     MemoryRegion fifo_ram;
     uint8_t *fifo_ptr;
@@ -80,7 +78,7 @@ struct vmsvga_state_s {
         } *cmd;
     };
 
-#define REDRAW_FIFO_LEN	512
+#define REDRAW_FIFO_LEN  512
     struct vmsvga_rect_s {
         int x, y, w, h;
     } redraw_fifo[REDRAW_FIFO_LEN];
@@ -93,31 +91,31 @@ struct pci_vmsvga_state_s {
     MemoryRegion io_bar;
 };
 
-#define SVGA_MAGIC		0x900000UL
-#define SVGA_MAKE_ID(ver)	(SVGA_MAGIC << 8 | (ver))
-#define SVGA_ID_0		SVGA_MAKE_ID(0)
-#define SVGA_ID_1		SVGA_MAKE_ID(1)
-#define SVGA_ID_2		SVGA_MAKE_ID(2)
+#define SVGA_MAGIC              0x900000UL
+#define SVGA_MAKE_ID(ver)       (SVGA_MAGIC << 8 | (ver))
+#define SVGA_ID_0               SVGA_MAKE_ID(0)
+#define SVGA_ID_1               SVGA_MAKE_ID(1)
+#define SVGA_ID_2               SVGA_MAKE_ID(2)
 
-#define SVGA_LEGACY_BASE_PORT	0x4560
-#define SVGA_INDEX_PORT		0x0
-#define SVGA_VALUE_PORT		0x1
-#define SVGA_BIOS_PORT		0x2
+#define SVGA_LEGACY_BASE_PORT   0x4560
+#define SVGA_INDEX_PORT         0x0
+#define SVGA_VALUE_PORT         0x1
+#define SVGA_BIOS_PORT          0x2
 
 #define SVGA_VERSION_2
 
 #ifdef SVGA_VERSION_2
-# define SVGA_ID		SVGA_ID_2
-# define SVGA_IO_BASE		SVGA_LEGACY_BASE_PORT
-# define SVGA_IO_MUL		1
-# define SVGA_FIFO_SIZE		0x10000
-# define SVGA_PCI_DEVICE_ID	PCI_DEVICE_ID_VMWARE_SVGA2
+# define SVGA_ID                SVGA_ID_2
+# define SVGA_IO_BASE           SVGA_LEGACY_BASE_PORT
+# define SVGA_IO_MUL            1
+# define SVGA_FIFO_SIZE         0x10000
+# define SVGA_PCI_DEVICE_ID     PCI_DEVICE_ID_VMWARE_SVGA2
 #else
-# define SVGA_ID		SVGA_ID_1
-# define SVGA_IO_BASE		SVGA_LEGACY_BASE_PORT
-# define SVGA_IO_MUL		4
-# define SVGA_FIFO_SIZE		0x10000
-# define SVGA_PCI_DEVICE_ID	PCI_DEVICE_ID_VMWARE_SVGA
+# define SVGA_ID                SVGA_ID_1
+# define SVGA_IO_BASE           SVGA_LEGACY_BASE_PORT
+# define SVGA_IO_MUL            4
+# define SVGA_FIFO_SIZE         0x10000
+# define SVGA_PCI_DEVICE_ID     PCI_DEVICE_ID_VMWARE_SVGA
 #endif
 
 enum {
@@ -129,7 +127,7 @@ enum {
     SVGA_REG_MAX_WIDTH = 4,
     SVGA_REG_MAX_HEIGHT = 5,
     SVGA_REG_DEPTH = 6,
-    SVGA_REG_BITS_PER_PIXEL = 7,	/* Current bpp in the guest */
+    SVGA_REG_BITS_PER_PIXEL = 7,        /* Current bpp in the guest */
     SVGA_REG_PSEUDOCOLOR = 8,
     SVGA_REG_RED_MASK = 9,
     SVGA_REG_GREEN_MASK = 10,
@@ -142,46 +140,46 @@ enum {
 
     /* ID 1 and 2 registers */
     SVGA_REG_CAPABILITIES = 17,
-    SVGA_REG_MEM_START = 18,		/* Memory for command FIFO */
+    SVGA_REG_MEM_START = 18,            /* Memory for command FIFO */
     SVGA_REG_MEM_SIZE = 19,
-    SVGA_REG_CONFIG_DONE = 20,		/* Set when memory area configured */
-    SVGA_REG_SYNC = 21,			/* Write to force synchronization */
-    SVGA_REG_BUSY = 22,			/* Read to check if sync is done */
-    SVGA_REG_GUEST_ID = 23,		/* Set guest OS identifier */
-    SVGA_REG_CURSOR_ID = 24,		/* ID of cursor */
-    SVGA_REG_CURSOR_X = 25,		/* Set cursor X position */
-    SVGA_REG_CURSOR_Y = 26,		/* Set cursor Y position */
-    SVGA_REG_CURSOR_ON = 27,		/* Turn cursor on/off */
-    SVGA_REG_HOST_BITS_PER_PIXEL = 28,	/* Current bpp in the host */
-    SVGA_REG_SCRATCH_SIZE = 29,		/* Number of scratch registers */
-    SVGA_REG_MEM_REGS = 30,		/* Number of FIFO registers */
-    SVGA_REG_NUM_DISPLAYS = 31,		/* Number of guest displays */
-    SVGA_REG_PITCHLOCK = 32,		/* Fixed pitch for all modes */
-
-    SVGA_PALETTE_BASE = 1024,		/* Base of SVGA color map */
+    SVGA_REG_CONFIG_DONE = 20,          /* Set when memory area configured */
+    SVGA_REG_SYNC = 21,                 /* Write to force synchronization */
+    SVGA_REG_BUSY = 22,                 /* Read to check if sync is done */
+    SVGA_REG_GUEST_ID = 23,             /* Set guest OS identifier */
+    SVGA_REG_CURSOR_ID = 24,            /* ID of cursor */
+    SVGA_REG_CURSOR_X = 25,             /* Set cursor X position */
+    SVGA_REG_CURSOR_Y = 26,             /* Set cursor Y position */
+    SVGA_REG_CURSOR_ON = 27,            /* Turn cursor on/off */
+    SVGA_REG_HOST_BITS_PER_PIXEL = 28,  /* Current bpp in the host */
+    SVGA_REG_SCRATCH_SIZE = 29,         /* Number of scratch registers */
+    SVGA_REG_MEM_REGS = 30,             /* Number of FIFO registers */
+    SVGA_REG_NUM_DISPLAYS = 31,         /* Number of guest displays */
+    SVGA_REG_PITCHLOCK = 32,            /* Fixed pitch for all modes */
+
+    SVGA_PALETTE_BASE = 1024,           /* Base of SVGA color map */
     SVGA_PALETTE_END  = SVGA_PALETTE_BASE + 767,
     SVGA_SCRATCH_BASE = SVGA_PALETTE_BASE + 768,
 };
 
-#define SVGA_CAP_NONE			0
-#define SVGA_CAP_RECT_FILL		(1 << 0)
-#define SVGA_CAP_RECT_COPY		(1 << 1)
-#define SVGA_CAP_RECT_PAT_FILL		(1 << 2)
-#define SVGA_CAP_LEGACY_OFFSCREEN	(1 << 3)
-#define SVGA_CAP_RASTER_OP		(1 << 4)
-#define SVGA_CAP_CURSOR			(1 << 5)
-#define SVGA_CAP_CURSOR_BYPASS		(1 << 6)
-#define SVGA_CAP_CURSOR_BYPASS_2	(1 << 7)
-#define SVGA_CAP_8BIT_EMULATION		(1 << 8)
-#define SVGA_CAP_ALPHA_CURSOR		(1 << 9)
-#define SVGA_CAP_GLYPH			(1 << 10)
-#define SVGA_CAP_GLYPH_CLIPPING		(1 << 11)
-#define SVGA_CAP_OFFSCREEN_1		(1 << 12)
-#define SVGA_CAP_ALPHA_BLEND		(1 << 13)
-#define SVGA_CAP_3D			(1 << 14)
-#define SVGA_CAP_EXTENDED_FIFO		(1 << 15)
-#define SVGA_CAP_MULTIMON		(1 << 16)
-#define SVGA_CAP_PITCHLOCK		(1 << 17)
+#define SVGA_CAP_NONE                   0
+#define SVGA_CAP_RECT_FILL              (1 << 0)
+#define SVGA_CAP_RECT_COPY              (1 << 1)
+#define SVGA_CAP_RECT_PAT_FILL          (1 << 2)
+#define SVGA_CAP_LEGACY_OFFSCREEN       (1 << 3)
+#define SVGA_CAP_RASTER_OP              (1 << 4)
+#define SVGA_CAP_CURSOR                 (1 << 5)
+#define SVGA_CAP_CURSOR_BYPASS          (1 << 6)
+#define SVGA_CAP_CURSOR_BYPASS_2        (1 << 7)
+#define SVGA_CAP_8BIT_EMULATION         (1 << 8)
+#define SVGA_CAP_ALPHA_CURSOR           (1 << 9)
+#define SVGA_CAP_GLYPH                  (1 << 10)
+#define SVGA_CAP_GLYPH_CLIPPING         (1 << 11)
+#define SVGA_CAP_OFFSCREEN_1            (1 << 12)
+#define SVGA_CAP_ALPHA_BLEND            (1 << 13)
+#define SVGA_CAP_3D                     (1 << 14)
+#define SVGA_CAP_EXTENDED_FIFO          (1 << 15)
+#define SVGA_CAP_MULTIMON               (1 << 16)
+#define SVGA_CAP_PITCHLOCK              (1 << 17)
 
 /*
  * FIFO offsets (seen as an array of 32-bit words)
@@ -191,7 +189,7 @@ enum {
      * The original defined FIFO offsets
      */
     SVGA_FIFO_MIN = 0,
-    SVGA_FIFO_MAX,	/* The distance from MIN to MAX must be at least 10K */
+    SVGA_FIFO_MAX,      /* The distance from MIN to MAX must be at least 10K */
     SVGA_FIFO_NEXT_CMD,
     SVGA_FIFO_STOP,
 
@@ -205,21 +203,21 @@ enum {
     SVGA_FIFO_PITCHLOCK,
 };
 
-#define SVGA_FIFO_CAP_NONE		0
-#define SVGA_FIFO_CAP_FENCE		(1 << 0)
-#define SVGA_FIFO_CAP_ACCELFRONT	(1 << 1)
-#define SVGA_FIFO_CAP_PITCHLOCK		(1 << 2)
+#define SVGA_FIFO_CAP_NONE              0
+#define SVGA_FIFO_CAP_FENCE             (1 << 0)
+#define SVGA_FIFO_CAP_ACCELFRONT        (1 << 1)
+#define SVGA_FIFO_CAP_PITCHLOCK         (1 << 2)
 
-#define SVGA_FIFO_FLAG_NONE		0
-#define SVGA_FIFO_FLAG_ACCELFRONT	(1 << 0)
+#define SVGA_FIFO_FLAG_NONE             0
+#define SVGA_FIFO_FLAG_ACCELFRONT       (1 << 0)
 
 /* These values can probably be changed arbitrarily.  */
-#define SVGA_SCRATCH_SIZE		0x8000
-#define SVGA_MAX_WIDTH			2360
-#define SVGA_MAX_HEIGHT			1770
+#define SVGA_SCRATCH_SIZE               0x8000
+#define SVGA_MAX_WIDTH                  2360
+#define SVGA_MAX_HEIGHT                 1770
 
 #ifdef VERBOSE
-# define GUEST_OS_BASE		0x5001
+# define GUEST_OS_BASE          0x5001
 static const char *vmsvga_guest_id[] = {
     [0x00] = "Dos",
     [0x01] = "Windows 3.1",
@@ -298,44 +296,55 @@ static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
     uint8_t *src;
     uint8_t *dst;
 
-    if (x + w > s->width) {
+    if (x < 0) {
+        fprintf(stderr, "%s: update x was < 0 (%d)\n", __func__, x);
+        w += x;
+        x = 0;
+    }
+    if (w < 0) {
+        fprintf(stderr, "%s: update w was < 0 (%d)\n", __func__, w);
+        w = 0;
+    }
+    if (x + w > ds_get_width(s->vga.ds)) {
         fprintf(stderr, "%s: update width too large x: %d, w: %d\n",
-                        __FUNCTION__, x, w);
-        x = MIN(x, s->width);
-        w = s->width - x;
+                __func__, x, w);
+        x = MIN(x, ds_get_width(s->vga.ds));
+        w = ds_get_width(s->vga.ds) - x;
     }
 
-    if (y + h > s->height) {
+    if (y < 0) {
+        fprintf(stderr, "%s: update y was < 0 (%d)\n",  __func__, y);
+        h += y;
+        y = 0;
+    }
+    if (h < 0) {
+        fprintf(stderr, "%s: update h was < 0 (%d)\n",  __func__, h);
+        h = 0;
+    }
+    if (y + h > ds_get_height(s->vga.ds)) {
         fprintf(stderr, "%s: update height too large y: %d, h: %d\n",
-                        __FUNCTION__, y, h);
-        y = MIN(y, s->height);
-        h = s->height - y;
+                __func__, y, h);
+        y = MIN(y, ds_get_height(s->vga.ds));
+        h = ds_get_height(s->vga.ds) - y;
     }
 
-    line = h;
-    bypl = s->bypp * s->width;
-    width = s->bypp * w;
-    start = s->bypp * x + bypl * y;
+    bypl = ds_get_linesize(s->vga.ds);
+    width = ds_get_bytes_per_pixel(s->vga.ds) * w;
+    start = ds_get_bytes_per_pixel(s->vga.ds) * x + bypl * y;
     src = s->vga.vram_ptr + start;
     dst = ds_get_data(s->vga.ds) + start;
 
-    for (; line > 0; line --, src += bypl, dst += bypl)
+    for (line = h; line > 0; line--, src += bypl, dst += bypl) {
         memcpy(dst, src, width);
-
-    dpy_update(s->vga.ds, x, y, w, h);
-}
-
-static inline void vmsvga_update_screen(struct vmsvga_state_s *s)
-{
-    memcpy(ds_get_data(s->vga.ds), s->vga.vram_ptr,
-           s->bypp * s->width * s->height);
-    dpy_update(s->vga.ds, 0, 0, s->width, s->height);
+    }
+    dpy_gfx_update(s->vga.ds, x, y, w, h);
 }
 
 static inline void vmsvga_update_rect_delayed(struct vmsvga_state_s *s,
                 int x, int y, int w, int h)
 {
-    struct vmsvga_rect_s *rect = &s->redraw_fifo[s->redraw_fifo_last ++];
+    struct vmsvga_rect_s *rect = &s->redraw_fifo[s->redraw_fifo_last++];
+
     s->redraw_fifo_last &= REDRAW_FIFO_LEN - 1;
     rect->x = x;
     rect->y = y;
@@ -346,6 +355,7 @@ static inline void vmsvga_update_rect_delayed(struct vmsvga_state_s *s,
 static inline void vmsvga_update_rect_flush(struct vmsvga_state_s *s)
 {
     struct vmsvga_rect_s *rect;
+
     if (s->invalidated) {
         s->redraw_fifo_first = s->redraw_fifo_last;
         return;
@@ -353,7 +363,7 @@ static inline void vmsvga_update_rect_flush(struct vmsvga_state_s *s)
     /* Overlapping region updates can be optimised out here - if someone
      * knows a smart algorithm to do that, please share.  */
     while (s->redraw_fifo_first != s->redraw_fifo_last) {
-        rect = &s->redraw_fifo[s->redraw_fifo_first ++];
+        rect = &s->redraw_fifo[s->redraw_fifo_first++];
         s->redraw_fifo_first &= REDRAW_FIFO_LEN - 1;
         vmsvga_update_rect(s, rect->x, rect->y, rect->w, rect->h);
     }
@@ -364,20 +374,21 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
                 int x0, int y0, int x1, int y1, int w, int h)
 {
     uint8_t *vram = s->vga.vram_ptr;
-    int bypl = s->bypp * s->width;
-    int width = s->bypp * w;
+    int bypl = ds_get_linesize(s->vga.ds);
+    int bypp = ds_get_bytes_per_pixel(s->vga.ds);
+    int width = bypp * w;
     int line = h;
     uint8_t *ptr[2];
 
     if (y1 > y0) {
-        ptr[0] = vram + s->bypp * x0 + bypl * (y0 + h - 1);
-        ptr[1] = vram + s->bypp * x1 + bypl * (y1 + h - 1);
+        ptr[0] = vram + bypp * x0 + bypl * (y0 + h - 1);
+        ptr[1] = vram + bypp * x1 + bypl * (y1 + h - 1);
         for (; line > 0; line --, ptr[0] -= bypl, ptr[1] -= bypl) {
             memmove(ptr[1], ptr[0], width);
         }
     } else {
-        ptr[0] = vram + s->bypp * x0 + bypl * y0;
-        ptr[1] = vram + s->bypp * x1 + bypl * y1;
+        ptr[0] = vram + bypp * x0 + bypl * y0;
+        ptr[1] = vram + bypp * x1 + bypl * y1;
         for (; line > 0; line --, ptr[0] += bypl, ptr[1] += bypl) {
             memmove(ptr[1], ptr[0], width);
         }
@@ -391,13 +402,11 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
 static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
                 uint32_t c, int x, int y, int w, int h)
 {
-    uint8_t *vram = s->vga.vram_ptr;
-    int bypp = s->bypp;
-    int bypl = bypp * s->width;
-    int width = bypp * w;
+    int bypl = ds_get_linesize(s->vga.ds);
+    int width = ds_get_bytes_per_pixel(s->vga.ds) * w;
     int line = h;
     int column;
-    uint8_t *fst = vram + bypp * x + bypl * y;
+    uint8_t *fst;
     uint8_t *dst;
     uint8_t *src;
     uint8_t col[4];
@@ -407,12 +416,14 @@ static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
     col[2] = c >> 16;
     col[3] = c >> 24;
 
+    fst = s->vga.vram_ptr + ds_get_bytes_per_pixel(s->vga.ds) * x + bypl * y;
+
     if (line--) {
         dst = fst;
         src = col;
         for (column = width; column > 0; column--) {
             *(dst++) = *(src++);
-            if (src - col == bypp) {
+            if (src - col == ds_get_bytes_per_pixel(s->vga.ds)) {
                 src = col;
             }
         }
@@ -438,8 +449,8 @@ struct vmsvga_cursor_definition_s {
     uint32_t image[4096];
 };
 
-#define SVGA_BITMAP_SIZE(w, h)		((((w) + 31) >> 5) * (h))
-#define SVGA_PIXMAP_SIZE(w, h, bpp)	(((((w) * (bpp)) + 31) >> 5) * (h))
+#define SVGA_BITMAP_SIZE(w, h)          ((((w) + 31) >> 5) * (h))
+#define SVGA_PIXMAP_SIZE(w, h, bpp)     (((((w) * (bpp)) + 31) >> 5) * (h))
 
 #ifdef HW_MOUSE_ACCEL
 static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
@@ -453,16 +464,16 @@ static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
     qc->hot_y = c->hot_y;
     switch (c->bpp) {
     case 1:
-        cursor_set_mono(qc, 0xffffff, 0x000000, (void*)c->image,
-                        1, (void*)c->mask);
+        cursor_set_mono(qc, 0xffffff, 0x000000, (void *)c->image,
+                        1, (void *)c->mask);
 #ifdef DEBUG
         cursor_print_ascii_art(qc, "vmware/mono");
 #endif
         break;
     case 32:
         /* fill alpha channel from mask, set color to zero */
-        cursor_set_mono(qc, 0x000000, 0x000000, (void*)c->mask,
-                        1, (void*)c->mask);
+        cursor_set_mono(qc, 0x000000, 0x000000, (void *)c->mask,
+                        1, (void *)c->mask);
         /* add in rgb values */
         pixels = c->width * c->height;
         for (i = 0; i < pixels; i++) {
@@ -474,36 +485,40 @@ static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
         break;
     default:
         fprintf(stderr, "%s: unhandled bpp %d, using fallback cursor\n",
-                __FUNCTION__, c->bpp);
+                __func__, c->bpp);
         cursor_put(qc);
         qc = cursor_builtin_left_ptr();
     }
 
-    if (s->vga.ds->cursor_define)
-        s->vga.ds->cursor_define(qc);
+    dpy_cursor_define(s->vga.ds, qc);
     cursor_put(qc);
 }
 #endif
 
-#define CMD(f)	le32_to_cpu(s->cmd->f)
+#define CMD(f)  le32_to_cpu(s->cmd->f)
 
 static inline int vmsvga_fifo_length(struct vmsvga_state_s *s)
 {
     int num;
-    if (!s->config || !s->enable)
+
+    if (!s->config || !s->enable) {
         return 0;
+    }
     num = CMD(next_cmd) - CMD(stop);
-    if (num < 0)
+    if (num < 0) {
         num += CMD(max) - CMD(min);
+    }
     return num >> 2;
 }
 
 static inline uint32_t vmsvga_fifo_read_raw(struct vmsvga_state_s *s)
 {
     uint32_t cmd = s->fifo[CMD(stop) >> 2];
+
     s->cmd->stop = cpu_to_le32(CMD(stop) + 4);
-    if (CMD(stop) >= CMD(max))
+    if (CMD(stop) >= CMD(max)) {
         s->cmd->stop = s->cmd->min;
+    }
     return cmd;
 }
 
@@ -529,8 +544,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
         case SVGA_CMD_UPDATE:
         case SVGA_CMD_UPDATE_VERBOSE:
             len -= 5;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             x = vmsvga_fifo_read(s);
             y = vmsvga_fifo_read(s);
@@ -541,8 +557,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_RECT_FILL:
             len -= 6;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             colour = vmsvga_fifo_read(s);
             x = vmsvga_fifo_read(s);
@@ -559,8 +576,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_RECT_COPY:
             len -= 7;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             x = vmsvga_fifo_read(s);
             y = vmsvga_fifo_read(s);
@@ -578,8 +596,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_DEFINE_CURSOR:
             len -= 8;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             cursor.id = vmsvga_fifo_read(s);
             cursor.hot_x = vmsvga_fifo_read(s);
@@ -591,17 +610,21 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
             args = SVGA_BITMAP_SIZE(x, y) + SVGA_PIXMAP_SIZE(x, y, cursor.bpp);
             if (SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
-                SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image)
+                SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image) {
                     goto badcmd;
+            }
 
             len -= args;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
-            for (args = 0; args < SVGA_BITMAP_SIZE(x, y); args ++)
+            for (args = 0; args < SVGA_BITMAP_SIZE(x, y); args++) {
                 cursor.mask[args] = vmsvga_fifo_read_raw(s);
-            for (args = 0; args < SVGA_PIXMAP_SIZE(x, y, cursor.bpp); args ++)
+            }
+            for (args = 0; args < SVGA_PIXMAP_SIZE(x, y, cursor.bpp); args++) {
                 cursor.image[args] = vmsvga_fifo_read_raw(s);
+            }
 #ifdef HW_MOUSE_ACCEL
             vmsvga_cursor_define(s, &cursor);
             break;
@@ -616,9 +639,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
          */
         case SVGA_CMD_DEFINE_ALPHA_CURSOR:
             len -= 6;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-
+            }
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
@@ -634,9 +657,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             goto badcmd;
         case SVGA_CMD_DRAW_GLYPH_CLIPPED:
             len -= 4;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-
+            }
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
             args = 7 + (vmsvga_fifo_read(s) >> 2);
@@ -660,12 +683,14 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             args = 0;
         badcmd:
             len -= args;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-            while (args --)
+            }
+            while (args--) {
                 vmsvga_fifo_read(s);
+            }
             printf("%s: Unknown command 0x%02x in SVGA command FIFO\n",
-                            __FUNCTION__, cmd);
+                   __func__, cmd);
             break;
 
         rewind:
@@ -680,12 +705,14 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 static uint32_t vmsvga_index_read(void *opaque, uint32_t address)
 {
     struct vmsvga_state_s *s = opaque;
+
     return s->index;
 }
 
 static void vmsvga_index_write(void *opaque, uint32_t address, uint32_t index)
 {
     struct vmsvga_state_s *s = opaque;
+
     s->index = index;
 }
 
@@ -693,6 +720,7 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 {
     uint32_t caps;
     struct vmsvga_state_s *s = opaque;
+
     switch (s->index) {
     case SVGA_REG_ID:
         return s->svgaid;
@@ -701,10 +729,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         return s->enable;
 
     case SVGA_REG_WIDTH:
-        return s->width;
+        return ds_get_width(s->vga.ds);
 
     case SVGA_REG_HEIGHT:
-        return s->height;
+        return ds_get_height(s->vga.ds);
 
     case SVGA_REG_MAX_WIDTH:
         return SVGA_MAX_WIDTH;
@@ -723,13 +751,15 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 
     case SVGA_REG_RED_MASK:
         return s->wred;
+
     case SVGA_REG_GREEN_MASK:
         return s->wgreen;
+
     case SVGA_REG_BLUE_MASK:
         return s->wblue;
 
     case SVGA_REG_BYTES_PER_LINE:
-        return ((s->depth + 7) >> 3) * s->new_width;
+        return s->bypp * s->new_width;
 
     case SVGA_REG_FB_START: {
         struct pci_vmsvga_state_s *pci_vmsvga
@@ -741,10 +771,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         return 0x0;
 
     case SVGA_REG_VRAM_SIZE:
-        return s->vga.vram_size;
+        return s->vga.vram_size; /* No physical VRAM besides the framebuffer */
 
     case SVGA_REG_FB_SIZE:
-        return s->fb_size;
+        return s->vga.vram_size;
 
     case SVGA_REG_CAPABILITIES:
         caps = SVGA_CAP_NONE;
@@ -755,9 +785,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         caps |= SVGA_CAP_RECT_FILL;
 #endif
 #ifdef HW_MOUSE_ACCEL
-        if (s->vga.ds->mouse_set)
+        if (dpy_cursor_define_supported(s->vga.ds)) {
             caps |= SVGA_CAP_CURSOR | SVGA_CAP_CURSOR_BYPASS_2 |
                     SVGA_CAP_CURSOR_BYPASS;
+        }
 #endif
         return caps;
 
@@ -806,9 +837,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 
     default:
         if (s->index >= SVGA_SCRATCH_BASE &&
-                s->index < SVGA_SCRATCH_BASE + s->scratch_size)
+            s->index < SVGA_SCRATCH_BASE + s->scratch_size) {
             return s->scratch[s->index - SVGA_SCRATCH_BASE];
-        printf("%s: Bad register %02x\n", __FUNCTION__, s->index);
+        }
+        printf("%s: Bad register %02x\n", __func__, s->index);
     }
 
     return 0;
@@ -817,21 +849,19 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
 {
     struct vmsvga_state_s *s = opaque;
+
     switch (s->index) {
     case SVGA_REG_ID:
-        if (value == SVGA_ID_2 || value == SVGA_ID_1 || value == SVGA_ID_0)
+        if (value == SVGA_ID_2 || value == SVGA_ID_1 || value == SVGA_ID_0) {
             s->svgaid = value;
+        }
         break;
 
     case SVGA_REG_ENABLE:
-        s->enable = value;
-        s->config &= !!value;
-        s->width = -1;
-        s->height = -1;
+        s->enable = !!value;
         s->invalidated = 1;
         s->vga.invalidate(&s->vga);
-        if (s->enable) {
-            s->fb_size = ((s->depth + 7) >> 3) * s->new_width * s->new_height;
+        if (s->enable && s->config) {
             vga_dirty_log_stop(&s->vga);
         } else {
             vga_dirty_log_start(&s->vga);
@@ -839,19 +869,26 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         break;
 
     case SVGA_REG_WIDTH:
-        s->new_width = value;
-        s->invalidated = 1;
+        if (value <= SVGA_MAX_WIDTH) {
+            s->new_width = value;
+            s->invalidated = 1;
+        } else {
+            printf("%s: Bad width: %i\n", __func__, value);
+        }
         break;
 
     case SVGA_REG_HEIGHT:
-        s->new_height = value;
-        s->invalidated = 1;
+        if (value <= SVGA_MAX_HEIGHT) {
+            s->new_height = value;
+            s->invalidated = 1;
+        } else {
+            printf("%s: Bad height: %i\n", __func__, value);
+        }
         break;
 
-    case SVGA_REG_DEPTH:
     case SVGA_REG_BITS_PER_PIXEL:
         if (value != s->depth) {
-            printf("%s: Bad colour depth: %i bits\n", __FUNCTION__, value);
+            printf("%s: Bad bits per pixel: %i bits\n", __func__, value);
             s->config = 0;
         }
         break;
@@ -860,15 +897,19 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         if (value) {
             s->fifo = (uint32_t *) s->fifo_ptr;
             /* Check range and alignment.  */
-            if ((CMD(min) | CMD(max) |
-                        CMD(next_cmd) | CMD(stop)) & 3)
+            if ((CMD(min) | CMD(max) | CMD(next_cmd) | CMD(stop)) & 3) {
                 break;
-            if (CMD(min) < (uint8_t *) s->cmd->fifo - (uint8_t *) s->fifo)
+            }
+            if (CMD(min) < (uint8_t *) s->cmd->fifo - (uint8_t *) s->fifo) {
                 break;
-            if (CMD(max) > SVGA_FIFO_SIZE)
+            }
+            if (CMD(max) > SVGA_FIFO_SIZE) {
                 break;
-            if (CMD(max) < CMD(min) + 10 * 1024)
+            }
+            if (CMD(max) < CMD(min) + 10 * 1024) {
                 break;
+            }
+            vga_dirty_log_stop(&s->vga);
         }
         s->config = !!value;
         break;
@@ -882,9 +923,10 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         s->guest = value;
 #ifdef VERBOSE
         if (value >= GUEST_OS_BASE && value < GUEST_OS_BASE +
-                ARRAY_SIZE(vmsvga_guest_id))
-            printf("%s: guest runs %s.\n", __FUNCTION__,
-                            vmsvga_guest_id[value - GUEST_OS_BASE]);
+            ARRAY_SIZE(vmsvga_guest_id)) {
+            printf("%s: guest runs %s.\n", __func__,
+                   vmsvga_guest_id[value - GUEST_OS_BASE]);
+        }
 #endif
         break;
 
@@ -904,11 +946,13 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         s->cursor.on |= (value == SVGA_CURSOR_ON_SHOW);
         s->cursor.on &= (value != SVGA_CURSOR_ON_HIDE);
 #ifdef HW_MOUSE_ACCEL
-        if (s->vga.ds->mouse_set && value <= SVGA_CURSOR_ON_SHOW)
-            s->vga.ds->mouse_set(s->cursor.x, s->cursor.y, s->cursor.on);
+        if (value <= SVGA_CURSOR_ON_SHOW) {
+            dpy_mouse_set(s->vga.ds, s->cursor.x, s->cursor.y, s->cursor.on);
+        }
 #endif
         break;
 
+    case SVGA_REG_DEPTH:
     case SVGA_REG_MEM_REGS:
     case SVGA_REG_NUM_DISPLAYS:
     case SVGA_REG_PITCHLOCK:
@@ -921,28 +965,26 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
             s->scratch[s->index - SVGA_SCRATCH_BASE] = value;
             break;
         }
-        printf("%s: Bad register %02x\n", __FUNCTION__, s->index);
+        printf("%s: Bad register %02x\n", __func__, s->index);
     }
 }
 
 static uint32_t vmsvga_bios_read(void *opaque, uint32_t address)
 {
-    printf("%s: what are we supposed to return?\n", __FUNCTION__);
+    printf("%s: what are we supposed to return?\n", __func__);
     return 0xcafe;
 }
 
 static void vmsvga_bios_write(void *opaque, uint32_t address, uint32_t data)
 {
-    printf("%s: what are we supposed to do with (%08x)?\n",
-                    __FUNCTION__, data);
+    printf("%s: what are we supposed to do with (%08x)?\n", __func__, data);
 }
 
-static inline void vmsvga_size(struct vmsvga_state_s *s)
+static inline void vmsvga_check_size(struct vmsvga_state_s *s)
 {
-    if (s->new_width != s->width || s->new_height != s->height) {
-        s->width = s->new_width;
-        s->height = s->new_height;
-        qemu_console_resize(s->vga.ds, s->width, s->height);
+    if (s->new_width != ds_get_width(s->vga.ds) ||
+        s->new_height != ds_get_height(s->vga.ds)) {
+        qemu_console_resize(s->vga.ds, s->new_width, s->new_height);
         s->invalidated = 1;
     }
 }
@@ -950,12 +992,14 @@ static inline void vmsvga_size(struct vmsvga_state_s *s)
 static void vmsvga_update_display(void *opaque)
 {
     struct vmsvga_state_s *s = opaque;
+    bool dirty = false;
+
     if (!s->enable) {
         s->vga.update(&s->vga);
         return;
     }
 
-    vmsvga_size(s);
+    vmsvga_check_size(s);
 
     vmsvga_fifo_run(s);
     vmsvga_update_rect_flush(s);
@@ -964,9 +1008,23 @@ static void vmsvga_update_display(void *opaque)
      * Is it more efficient to look at vram VGA-dirty bits or wait
      * for the driver to issue SVGA_CMD_UPDATE?
      */
-    if (s->invalidated) {
+    if (memory_region_is_logging(&s->vga.vram)) {
+        vga_sync_dirty_bitmap(&s->vga);
+        dirty = memory_region_get_dirty(&s->vga.vram, 0,
+            ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds),
+            DIRTY_MEMORY_VGA);
+    }
+    if (s->invalidated || dirty) {
         s->invalidated = 0;
-        vmsvga_update_screen(s);
+        memcpy(ds_get_data(s->vga.ds), s->vga.vram_ptr,
+               ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds));
+        dpy_gfx_update(s->vga.ds, 0, 0,
+                   ds_get_width(s->vga.ds), ds_get_height(s->vga.ds));
+    }
+    if (dirty) {
+        memory_region_reset_dirty(&s->vga.vram, 0,
+            ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds),
+            DIRTY_MEMORY_VGA);
     }
 }
 
@@ -979,8 +1037,6 @@ static void vmsvga_reset(DeviceState *dev)
     s->index = 0;
     s->enable = 0;
     s->config = 0;
-    s->width = -1;
-    s->height = -1;
     s->svgaid = SVGA_ID;
     s->cursor.on = 0;
     s->redraw_fifo_first = 0;
@@ -1003,18 +1059,23 @@ static void vmsvga_invalidate_display(void *opaque)
 
 /* save the vga display in a PPM image even if no display is
    available */
-static void vmsvga_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void vmsvga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     struct vmsvga_state_s *s = opaque;
     if (!s->enable) {
-        s->vga.screen_dump(&s->vga, filename, cswitch);
+        s->vga.screen_dump(&s->vga, filename, cswitch, errp);
         return;
     }
 
-    if (s->depth == 32) {
-        DisplaySurface *ds = qemu_create_displaysurface_from(s->width,
-                s->height, 32, ds_get_linesize(s->vga.ds), s->vga.vram_ptr);
-        ppm_save(filename, ds);
+    if (ds_get_bits_per_pixel(s->vga.ds) == 32) {
+        DisplaySurface *ds = qemu_create_displaysurface_from(
+                                 ds_get_width(s->vga.ds),
+                                 ds_get_height(s->vga.ds),
+                                 32,
+                                 ds_get_linesize(s->vga.ds),
+                                 s->vga.vram_ptr);
+        ppm_save(filename, ds, errp);
         g_free(ds);
     }
 }
@@ -1023,8 +1084,9 @@ static void vmsvga_text_update(void *opaque, console_ch_t *chardata)
 {
     struct vmsvga_state_s *s = opaque;
 
-    if (s->vga.text_update)
+    if (s->vga.text_update) {
         s->vga.text_update(&s->vga, chardata);
+    }
 }
 
 static int vmsvga_post_load(void *opaque, int version_id)
@@ -1032,9 +1094,9 @@ static int vmsvga_post_load(void *opaque, int version_id)
     struct vmsvga_state_s *s = opaque;
 
     s->invalidated = 1;
-    if (s->config)
+    if (s->config) {
         s->fifo = (uint32_t *) s->fifo_ptr;
-
+    }
     return 0;
 }
 
@@ -1044,7 +1106,7 @@ static const VMStateDescription vmstate_vmware_vga_internal = {
     .minimum_version_id = 0,
     .minimum_version_id_old = 0,
     .post_load = vmsvga_post_load,
-    .fields      = (VMStateField []) {
+    .fields      = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(depth, struct vmsvga_state_s),
         VMSTATE_INT32(enable, struct vmsvga_state_s),
         VMSTATE_INT32(config, struct vmsvga_state_s),
@@ -1060,7 +1122,7 @@ static const VMStateDescription vmstate_vmware_vga_internal = {
         VMSTATE_UINT32(guest, struct vmsvga_state_s),
         VMSTATE_UINT32(svgaid, struct vmsvga_state_s),
         VMSTATE_INT32(syncing, struct vmsvga_state_s),
-        VMSTATE_INT32(fb_size, struct vmsvga_state_s),
+        VMSTATE_UNUSED(4), /* was fb_size */
         VMSTATE_END_OF_LIST()
     }
 };
@@ -1070,7 +1132,7 @@ static const VMStateDescription vmstate_vmware_vga = {
     .version_id = 0,
     .minimum_version_id = 0,
     .minimum_version_id_old = 0,
-    .fields      = (VMStateField []) {
+    .fields      = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(card, struct pci_vmsvga_state_s),
         VMSTATE_STRUCT(chip, struct pci_vmsvga_state_s, 0,
                        vmstate_vmware_vga_internal, struct vmsvga_state_s),
@@ -1098,40 +1160,16 @@ static void vmsvga_init(struct vmsvga_state_s *s,
     vga_common_init(&s->vga);
     vga_init(&s->vga, address_space, io, true);
     vmstate_register(NULL, 0, &vmstate_vga_common, &s->vga);
-
+    /* Save some values here in case they are changed later.
+     * This is suspicious and needs more though why it is needed. */
     s->depth = ds_get_bits_per_pixel(s->vga.ds);
     s->bypp = ds_get_bytes_per_pixel(s->vga.ds);
-    switch (s->depth) {
-    case 8:
-        s->wred   = 0x00000007;
-        s->wgreen = 0x00000038;
-        s->wblue  = 0x000000c0;
-        break;
-    case 15:
-        s->wred   = 0x0000001f;
-        s->wgreen = 0x000003e0;
-        s->wblue  = 0x00007c00;
-        break;
-    case 16:
-        s->wred   = 0x0000001f;
-        s->wgreen = 0x000007e0;
-        s->wblue  = 0x0000f800;
-        break;
-    case 24:
-        s->wred   = 0x00ff0000;
-        s->wgreen = 0x0000ff00;
-        s->wblue  = 0x000000ff;
-        break;
-    case 32:
-        s->wred   = 0x00ff0000;
-        s->wgreen = 0x0000ff00;
-        s->wblue  = 0x000000ff;
-        break;
-    }
+    s->wred = ds_get_rmask(s->vga.ds);
+    s->wgreen = ds_get_gmask(s->vga.ds);
+    s->wblue = ds_get_bmask(s->vga.ds);
 }
 
-static uint64_t vmsvga_io_read(void *opaque, target_phys_addr_t addr,
-                               unsigned size)
+static uint64_t vmsvga_io_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct vmsvga_state_s *s = opaque;
 
@@ -1143,7 +1181,7 @@ static uint64_t vmsvga_io_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void vmsvga_io_write(void *opaque, target_phys_addr_t addr,
+static void vmsvga_io_write(void *opaque, hwaddr addr,
                             uint64_t data, unsigned size)
 {
     struct vmsvga_state_s *s = opaque;
@@ -1175,22 +1213,20 @@ static int pci_vmsvga_initfn(PCIDevice *dev)
 {
     struct pci_vmsvga_state_s *s =
         DO_UPCAST(struct pci_vmsvga_state_s, card, dev);
-    MemoryRegion *iomem;
-
-    iomem = &s->chip.vga.vram;
 
-    s->card.config[PCI_CACHE_LINE_SIZE]	= 0x08;		/* Cache line size */
-    s->card.config[PCI_LATENCY_TIMER] = 0x40;		/* Latency timer */
-    s->card.config[PCI_INTERRUPT_LINE] = 0xff;		/* End */
+    s->card.config[PCI_CACHE_LINE_SIZE] = 0x08;         /* Cache line size */
+    s->card.config[PCI_LATENCY_TIMER] = 0x40;           /* Latency timer */
+    s->card.config[PCI_INTERRUPT_LINE] = 0xff;          /* End */
 
     memory_region_init_io(&s->io_bar, &vmsvga_io_ops, &s->chip,
                           "vmsvga-io", 0x10);
+    memory_region_set_flush_coalesced(&s->io_bar);
     pci_register_bar(&s->card, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io_bar);
 
-    vmsvga_init(&s->chip, pci_address_space(dev),
-                pci_address_space_io(dev));
+    vmsvga_init(&s->chip, pci_address_space(dev), pci_address_space_io(dev));
 
-    pci_register_bar(&s->card, 1, PCI_BASE_ADDRESS_MEM_PREFETCH, iomem);
+    pci_register_bar(&s->card, 1, PCI_BASE_ADDRESS_MEM_PREFETCH,
+                     &s->chip.vga.vram);
     pci_register_bar(&s->card, 2, PCI_BASE_ADDRESS_MEM_PREFETCH,
                      &s->chip.fifo_ram);
 
@@ -1226,7 +1262,7 @@ static void vmsvga_class_init(ObjectClass *klass, void *data)
     dc->props = vga_vmware_properties;
 }
 
-static TypeInfo vmsvga_info = {
+static const TypeInfo vmsvga_info = {
     .name          = "vmware-svga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(struct pci_vmsvga_state_s),
diff --git a/hw/vmware_vga.h b/hw/vmware_vga.h
deleted file mode 100644
index 000fbddc0f2..00000000000
--- a/hw/vmware_vga.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef QEMU_VMWARE_VGA_H
-#define QEMU_VMWARE_VGA_H
-
-#include "qemu-common.h"
-
-/* vmware_vga.c */
-static inline DeviceState *pci_vmsvga_init(PCIBus *bus)
-{
-    PCIDevice *dev;
-
-    dev = pci_create_simple(bus, -1, "vmware-svga");
-    return &dev->qdev;
-}
-
-#endif
diff --git a/hw/vt82c686.c b/hw/vt82c686.c
index 5d7c00cf4b9..2d8e3988dbb 100644
--- a/hw/vt82c686.c
+++ b/hw/vt82c686.c
@@ -15,18 +15,19 @@
 #include "vt82c686.h"
 #include "i2c.h"
 #include "smbus.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "mips.h"
 #include "apm.h"
 #include "acpi.h"
 #include "pm_smbus.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 
 typedef uint32_t pci_addr_t;
-#include "pci_host.h"
+#include "pci/pci_host.h"
 //#define DEBUG_VT82C686B
 
 #ifdef DEBUG_VT82C686B
@@ -159,6 +160,7 @@ static void vt82c686b_write_config(PCIDevice * d, uint32_t address,
 
 typedef struct VT686PMState {
     PCIDevice dev;
+    MemoryRegion io;
     ACPIREGS ar;
     APMState apm;
     PMSMBus smb;
@@ -195,92 +197,17 @@ static void pm_tmr_timer(ACPIREGS *ar)
     pm_update_sci(s);
 }
 
-static void pm_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    VT686PMState *s = opaque;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x00:
-        acpi_pm1_evt_write_sts(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x02:
-        acpi_pm1_evt_write_en(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x04:
-        acpi_pm1_cnt_write(&s->ar, val, 0);
-        break;
-    default:
-        break;
-    }
-    DPRINTF("PM writew port=0x%04x val=0x%02x\n", addr, val);
-}
-
-static uint32_t pm_ioport_readw(void *opaque, uint32_t addr)
-{
-    VT686PMState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x00:
-        val = acpi_pm1_evt_get_sts(&s->ar);
-        break;
-    case 0x02:
-        val = s->ar.pm1.evt.en;
-        break;
-    case 0x04:
-        val = s->ar.pm1.cnt.cnt;
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    DPRINTF("PM readw port=0x%04x val=0x%02x\n", addr, val);
-    return val;
-}
-
-static void pm_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    addr &= 0x0f;
-    DPRINTF("PM writel port=0x%04x val=0x%08x\n", addr, val);
-}
-
-static uint32_t pm_ioport_readl(void *opaque, uint32_t addr)
-{
-    VT686PMState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x08:
-        val = acpi_pm_tmr_get(&s->ar);
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    DPRINTF("PM readl port=0x%04x val=0x%08x\n", addr, val);
-    return val;
-}
-
 static void pm_io_space_update(VT686PMState *s)
 {
     uint32_t pm_io_base;
 
-    if (s->dev.config[0x80] & 1) {
-        pm_io_base = pci_get_long(s->dev.config + 0x40);
-        pm_io_base &= 0xffc0;
+    pm_io_base = pci_get_long(s->dev.config + 0x40);
+    pm_io_base &= 0xffc0;
 
-        /* XXX: need to improve memory and ioport allocation */
-        DPRINTF("PM: mapping to 0x%x\n", pm_io_base);
-        register_ioport_write(pm_io_base, 64, 2, pm_ioport_writew, s);
-        register_ioport_read(pm_io_base, 64, 2, pm_ioport_readw, s);
-        register_ioport_write(pm_io_base, 64, 4, pm_ioport_writel, s);
-        register_ioport_read(pm_io_base, 64, 4, pm_ioport_readl, s);
-    }
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->io, s->dev.config[0x80] & 1);
+    memory_region_set_address(&s->io, pm_io_base);
+    memory_region_transaction_commit();
 }
 
 static void pm_write_config(PCIDevice *d,
@@ -357,7 +284,7 @@ static void via_ac97_class_init(ObjectClass *klass, void *data)
     dc->desc = "AC97";
 }
 
-static TypeInfo via_ac97_info = {
+static const TypeInfo via_ac97_info = {
     .name          = "VT82C686B_AC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686AC97State),
@@ -398,7 +325,7 @@ static void via_mc97_class_init(ObjectClass *klass, void *data)
     dc->desc = "MC97";
 }
 
-static TypeInfo via_mc97_info = {
+static const TypeInfo via_mc97_info = {
     .name          = "VT82C686B_MC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686MC97State),
@@ -424,15 +351,18 @@ static int vt82c686b_pm_initfn(PCIDevice *dev)
     pci_conf[0x90] = s->smb_io_base | 1;
     pci_conf[0x91] = s->smb_io_base >> 8;
     pci_conf[0xd2] = 0x90;
-    register_ioport_write(s->smb_io_base, 0xf, 1, smb_ioport_writeb, &s->smb);
-    register_ioport_read(s->smb_io_base, 0xf, 1, smb_ioport_readb, &s->smb);
+    pm_smbus_init(&s->dev.qdev, &s->smb);
+    memory_region_add_subregion(get_system_io(), s->smb_io_base, &s->smb.io);
 
-    apm_init(&s->apm, NULL, s);
+    apm_init(dev, &s->apm, NULL, s);
 
-    acpi_pm_tmr_init(&s->ar, pm_tmr_timer);
-    acpi_pm1_cnt_init(&s->ar);
+    memory_region_init(&s->io, "vt82c686-pm", 64);
+    memory_region_set_enabled(&s->io, false);
+    memory_region_add_subregion(get_system_io(), 0, &s->io);
 
-    pm_smbus_init(&s->dev.qdev, &s->smb);
+    acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_cnt_init(&s->ar, &s->io);
 
     return 0;
 }
@@ -474,7 +404,7 @@ static void via_pm_class_init(ObjectClass *klass, void *data)
     dc->props = via_pm_properties;
 }
 
-static TypeInfo via_pm_info = {
+static const TypeInfo via_pm_info = {
     .name          = "VT82C686B_PM",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686PMState),
@@ -541,7 +471,7 @@ static void via_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_via;
 }
 
-static TypeInfo via_info = {
+static const TypeInfo via_info = {
     .name          = "VT82C686B",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT82C686BState),
diff --git a/hw/watchdog.c b/hw/watchdog.c
index b52acedd98a..072d2568820 100644
--- a/hw/watchdog.c
+++ b/hw/watchdog.c
@@ -20,12 +20,12 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-queue.h"
-#include "qemu-objects.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qemu/queue.h"
+#include "qapi/qmp/types.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "hw/watchdog.h"
 
 /* Possible values for action parameter. */
@@ -66,7 +66,7 @@ int select_watchdog(const char *p)
     QLIST_FOREACH(model, &watchdog_list, entry) {
         if (strcasecmp(model->wdt_name, p) == 0) {
             /* add the device */
-            opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+            opts = qemu_opts_create_nofail(qemu_find_opts("device"));
             qemu_opt_set(opts, "driver", p);
             return 0;
         }
diff --git a/hw/watchdog.h b/hw/watchdog.h
index c12a29311aa..3e9a9706864 100644
--- a/hw/watchdog.h
+++ b/hw/watchdog.h
@@ -22,7 +22,7 @@
 #ifndef QEMU_WATCHDOG_H
 #define QEMU_WATCHDOG_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 struct WatchdogTimerModel {
     QLIST_ENTRY(WatchdogTimerModel) entry;
diff --git a/hw/wdt_i6300esb.c b/hw/wdt_i6300esb.c
index 4a83474906c..37ce362811f 100644
--- a/hw/wdt_i6300esb.c
+++ b/hw/wdt_i6300esb.c
@@ -22,10 +22,10 @@
 #include <inttypes.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "watchdog.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /*#define I6300ESB_DEBUG 1*/
 
@@ -257,14 +257,14 @@ static uint32_t i6300esb_config_read(PCIDevice *dev, uint32_t addr, int len)
     }
 }
 
-static uint32_t i6300esb_mem_readb(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readb(void *vp, hwaddr addr)
 {
     i6300esb_debug ("addr = %x\n", (int) addr);
 
     return 0;
 }
 
-static uint32_t i6300esb_mem_readw(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readw(void *vp, hwaddr addr)
 {
     uint32_t data = 0;
     I6300State *d = vp;
@@ -282,14 +282,14 @@ static uint32_t i6300esb_mem_readw(void *vp, target_phys_addr_t addr)
     return data;
 }
 
-static uint32_t i6300esb_mem_readl(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readl(void *vp, hwaddr addr)
 {
     i6300esb_debug("addr = %x\n", (int) addr);
 
     return 0;
 }
 
-static void i6300esb_mem_writeb(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writeb(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
@@ -301,7 +301,7 @@ static void i6300esb_mem_writeb(void *vp, target_phys_addr_t addr, uint32_t val)
         d->unlock_state = 2;
 }
 
-static void i6300esb_mem_writew(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writew(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
@@ -334,7 +334,7 @@ static void i6300esb_mem_writew(void *vp, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void i6300esb_mem_writel(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writel(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
@@ -439,7 +439,7 @@ static void i6300esb_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_i6300esb;
 }
 
-static TypeInfo i6300esb_info = {
+static const TypeInfo i6300esb_info = {
     .name          = "i6300esb",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(I6300State),
diff --git a/hw/wdt_ib700.c b/hw/wdt_ib700.c
index 7f6c21d8095..599a86f5f63 100644
--- a/hw/wdt_ib700.c
+++ b/hw/wdt_ib700.c
@@ -20,7 +20,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "watchdog.h"
 #include "hw.h"
 #include "isa.h"
@@ -129,7 +129,7 @@ static void wdt_ib700_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_ib700;
 }
 
-static TypeInfo wdt_ib700_info = {
+static const TypeInfo wdt_ib700_info = {
     .name          = "ib700",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(IB700State),
diff --git a/hw/wm8750.c b/hw/wm8750.c
index 11bcec34106..bb85064c9b8 100644
--- a/hw/wm8750.c
+++ b/hw/wm8750.c
@@ -361,10 +361,10 @@ static int wm8750_tx(I2CSlave *i2c, uint8_t data)
     uint16_t value;
 
     if (s->i2c_len >= 2) {
-        printf("%s: long message (%i bytes)\n", __FUNCTION__, s->i2c_len);
 #ifdef VERBOSE
-        return 1;
+        printf("%s: long message (%i bytes)\n", __func__, s->i2c_len);
 #endif
+        return 1;
     }
     s->i2c_data[s->i2c_len ++] = data;
     if (s->i2c_len != 2)
@@ -701,7 +701,7 @@ static void wm8750_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_wm8750;
 }
 
-static TypeInfo wm8750_info = {
+static const TypeInfo wm8750_info = {
     .name          = "wm8750",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(WM8750State),
diff --git a/hw/xen-host-pci-device.c b/hw/xen-host-pci-device.c
index e7ff680ef27..743b37b9910 100644
--- a/hw/xen-host-pci-device.c
+++ b/hw/xen-host-pci-device.c
@@ -47,13 +47,13 @@ static int xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
 }
 
 
-/* This size should be enough to read the first 7 lines of a ressource file */
-#define XEN_HOST_PCI_RESSOURCE_BUFFER_SIZE 400
+/* This size should be enough to read the first 7 lines of a resource file */
+#define XEN_HOST_PCI_RESOURCE_BUFFER_SIZE 400
 static int xen_host_pci_get_resource(XenHostPCIDevice *d)
 {
     int i, rc, fd;
     char path[PATH_MAX];
-    char buf[XEN_HOST_PCI_RESSOURCE_BUFFER_SIZE];
+    char buf[XEN_HOST_PCI_RESOURCE_BUFFER_SIZE];
     unsigned long long start, end, flags, size;
     char *endptr, *s;
     uint8_t type;
diff --git a/hw/xen-host-pci-device.h b/hw/xen-host-pci-device.h
index 0079daca518..942b24dccc1 100644
--- a/hw/xen-host-pci-device.h
+++ b/hw/xen-host-pci-device.h
@@ -1,7 +1,7 @@
 #ifndef XEN_HOST_PCI_DEVICE_H
 #define XEN_HOST_PCI_DEVICE_H
 
-#include "pci.h"
+#include "pci/pci.h"
 
 enum {
     XEN_HOST_PCI_REGION_TYPE_IO = 1 << 1,
diff --git a/hw/xen.h b/hw/xen.h
index e5926b7b8ad..e3cca7fb922 100644
--- a/hw/xen.h
+++ b/hw/xen.h
@@ -8,6 +8,7 @@
  */
 #include <inttypes.h>
 
+#include "hw/irq.h"
 #include "qemu-common.h"
 
 /* xen-machine.c */
@@ -48,6 +49,7 @@ void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 struct MemoryRegion;
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
                    struct MemoryRegion *mr);
+void xen_modified_memory(ram_addr_t start, ram_addr_t length);
 #endif
 
 struct MemoryRegion;
diff --git a/hw/xen_apic.c b/hw/xen_apic.c
index a9e101f3154..1d1d15c289c 100644
--- a/hw/xen_apic.c
+++ b/hw/xen_apic.c
@@ -10,16 +10,16 @@
  * later. See the COPYING file in the top-level directory.
  */
 #include "hw/apic_internal.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 #include "xen.h"
 
-static uint64_t xen_apic_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t xen_apic_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return ~(uint64_t)0;
 }
 
-static void xen_apic_mem_write(void *opaque, target_phys_addr_t addr,
+static void xen_apic_mem_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     if (size != sizeof(uint32_t)) {
@@ -80,7 +80,7 @@ static void xen_apic_class_init(ObjectClass *klass, void *data)
     k->external_nmi = xen_apic_external_nmi;
 }
 
-static TypeInfo xen_apic_info = {
+static const TypeInfo xen_apic_info = {
     .name = "xen-apic",
     .parent = TYPE_APIC_COMMON,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/xen_backend.c b/hw/xen_backend.c
index f83a1e1d097..3fa30098ca5 100644
--- a/hw/xen_backend.c
+++ b/hw/xen_backend.c
@@ -35,8 +35,8 @@
 #include <sys/signal.h>
 
 #include "hw.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
+#include "char/char.h"
+#include "qemu/log.h"
 #include "xen_backend.h"
 
 #include <xen/grant_table.h>
diff --git a/hw/xen_backend.h b/hw/xen_backend.h
index fea86dd78bd..f37afb1f058 100644
--- a/hw/xen_backend.h
+++ b/hw/xen_backend.h
@@ -2,9 +2,8 @@
 #define QEMU_HW_XEN_BACKEND_H 1
 
 #include "xen_common.h"
-#include "sysemu.h"
-#include "net.h"
-#include "net/hub.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 
 /* ------------------------------------------------------------- */
 
diff --git a/hw/xen_common.h b/hw/xen_common.h
index 727757afb44..95bc9a7825b 100644
--- a/hw/xen_common.h
+++ b/hw/xen_common.h
@@ -16,7 +16,7 @@
 
 #include "hw.h"
 #include "xen.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /*
  * We don't support Xen prior to 3.3.0.
diff --git a/hw/xen_console.c b/hw/xen_console.c
index 9426d7374f5..44141f8692a 100644
--- a/hw/xen_console.c
+++ b/hw/xen_console.c
@@ -30,7 +30,7 @@
 #include <sys/mman.h>
 
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "xen_backend.h"
 
 #include <xen/io/console.h>
@@ -184,7 +184,11 @@ static int con_init(struct XenDevice *xendev)
 
     /* setup */
     dom = xs_get_domain_path(xenstore, con->xendev.dom);
-    snprintf(con->console, sizeof(con->console), "%s/console", dom);
+    if (!xendev->dev) {
+        snprintf(con->console, sizeof(con->console), "%s/console", dom);
+    } else {
+        snprintf(con->console, sizeof(con->console), "%s/device/console/%d", dom, xendev->dev);
+    }
     free(dom);
 
     type = xenstore_read_str(con->console, "type");
@@ -223,10 +227,16 @@ static int con_initialise(struct XenDevice *xendev)
     if (xenstore_read_int(con->console, "limit", &limit) == 0)
 	con->buffer.max_capacity = limit;
 
-    con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
-				      XC_PAGE_SIZE,
-				      PROT_READ|PROT_WRITE,
-				      con->ring_ref);
+    if (!xendev->dev) {
+        con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
+                                          XC_PAGE_SIZE,
+                                          PROT_READ|PROT_WRITE,
+                                          con->ring_ref);
+    } else {
+        con->sring = xc_gnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
+                                             con->ring_ref,
+                                             PROT_READ|PROT_WRITE);
+    }
     if (!con->sring)
 	return -1;
 
@@ -255,7 +265,11 @@ static void con_disconnect(struct XenDevice *xendev)
     xen_be_unbind_evtchn(&con->xendev);
 
     if (con->sring) {
-	munmap(con->sring, XC_PAGE_SIZE);
+        if (!xendev->gnttabdev) {
+            munmap(con->sring, XC_PAGE_SIZE);
+        } else {
+            xc_gnttab_munmap(xendev->gnttabdev, con->sring, 1);
+        }
 	con->sring = NULL;
     }
 }
@@ -273,7 +287,7 @@ static void con_event(struct XenDevice *xendev)
 
 struct XenDevOps xen_console_ops = {
     .size       = sizeof(struct XenConsole),
-    .flags      = DEVOPS_FLAG_IGNORE_STATE,
+    .flags      = DEVOPS_FLAG_IGNORE_STATE|DEVOPS_FLAG_NEED_GNTDEV,
     .init       = con_init,
     .initialise = con_initialise,
     .event      = con_event,
diff --git a/hw/xen_devconfig.c b/hw/xen_devconfig.c
index d83e8d0f64c..e2ba741d542 100644
--- a/hw/xen_devconfig.c
+++ b/hw/xen_devconfig.c
@@ -1,5 +1,5 @@
 #include "xen_backend.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
 
diff --git a/hw/xen_disk.c b/hw/xen_disk.c
index e6bb2f20b94..7fea87156d7 100644
--- a/hw/xen_disk.c
+++ b/hw/xen_disk.c
@@ -36,10 +36,9 @@
 #include <sys/uio.h>
 
 #include "hw.h"
-#include "qemu-char.h"
 #include "xen_backend.h"
 #include "xen_blkif.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
 
@@ -52,6 +51,13 @@ static int max_requests = 32;
 #define BLOCK_SIZE  512
 #define IOCB_COUNT  (BLKIF_MAX_SEGMENTS_PER_REQUEST + 2)
 
+struct PersistentGrant {
+    void *page;
+    struct XenBlkDev *blkdev;
+};
+
+typedef struct PersistentGrant PersistentGrant;
+
 struct ioreq {
     blkif_request_t     req;
     int16_t             status;
@@ -69,6 +75,7 @@ struct ioreq {
     int                 prot;
     void                *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     void                *pages;
+    int                 num_unmap;
 
     /* aio status */
     int                 aio_inflight;
@@ -105,6 +112,12 @@ struct XenBlkDev {
     int                 requests_inflight;
     int                 requests_finished;
 
+    /* Persistent grants extension */
+    gboolean            feature_persistent;
+    GTree               *persistent_gnts;
+    unsigned int        persistent_gnt_count;
+    unsigned int        max_grants;
+
     /* qemu block driver */
     DriveInfo           *dinfo;
     BlockDriverState    *bs;
@@ -113,6 +126,54 @@ struct XenBlkDev {
 
 /* ------------------------------------------------------------- */
 
+static void ioreq_reset(struct ioreq *ioreq)
+{
+    memset(&ioreq->req, 0, sizeof(ioreq->req));
+    ioreq->status = 0;
+    ioreq->start = 0;
+    ioreq->presync = 0;
+    ioreq->postsync = 0;
+    ioreq->mapped = 0;
+
+    memset(ioreq->domids, 0, sizeof(ioreq->domids));
+    memset(ioreq->refs, 0, sizeof(ioreq->refs));
+    ioreq->prot = 0;
+    memset(ioreq->page, 0, sizeof(ioreq->page));
+    ioreq->pages = NULL;
+
+    ioreq->aio_inflight = 0;
+    ioreq->aio_errors = 0;
+
+    ioreq->blkdev = NULL;
+    memset(&ioreq->list, 0, sizeof(ioreq->list));
+    memset(&ioreq->acct, 0, sizeof(ioreq->acct));
+
+    qemu_iovec_reset(&ioreq->v);
+}
+
+static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
+{
+    uint ua = GPOINTER_TO_UINT(a);
+    uint ub = GPOINTER_TO_UINT(b);
+    return (ua > ub) - (ua < ub);
+}
+
+static void destroy_grant(gpointer pgnt)
+{
+    PersistentGrant *grant = pgnt;
+    XenGnttab gnt = grant->blkdev->xendev.gnttabdev;
+
+    if (xc_gnttab_munmap(gnt, grant->page, 1) != 0) {
+        xen_be_printf(&grant->blkdev->xendev, 0,
+                      "xc_gnttab_munmap failed: %s\n",
+                      strerror(errno));
+    }
+    grant->blkdev->persistent_gnt_count--;
+    xen_be_printf(&grant->blkdev->xendev, 3,
+                  "unmapped grant %p\n", grant->page);
+    g_free(grant);
+}
+
 static struct ioreq *ioreq_start(struct XenBlkDev *blkdev)
 {
     struct ioreq *ioreq = NULL;
@@ -130,7 +191,6 @@ static struct ioreq *ioreq_start(struct XenBlkDev *blkdev)
         /* get one from freelist */
         ioreq = QLIST_FIRST(&blkdev->freelist);
         QLIST_REMOVE(ioreq, list);
-        qemu_iovec_reset(&ioreq->v);
     }
     QLIST_INSERT_HEAD(&blkdev->inflight, ioreq, list);
     blkdev->requests_inflight++;
@@ -154,7 +214,7 @@ static void ioreq_release(struct ioreq *ioreq, bool finish)
     struct XenBlkDev *blkdev = ioreq->blkdev;
 
     QLIST_REMOVE(ioreq, list);
-    memset(ioreq, 0, sizeof(*ioreq));
+    ioreq_reset(ioreq);
     ioreq->blkdev = blkdev;
     QLIST_INSERT_HEAD(&blkdev->freelist, ioreq, list);
     if (finish) {
@@ -183,12 +243,11 @@ static int ioreq_parse(struct ioreq *ioreq)
     case BLKIF_OP_READ:
         ioreq->prot = PROT_WRITE; /* to memory */
         break;
-    case BLKIF_OP_WRITE_BARRIER:
+    case BLKIF_OP_FLUSH_DISKCACHE:
+        ioreq->presync = 1;
         if (!ioreq->req.nr_segments) {
-            ioreq->presync = 1;
             return 0;
         }
-        ioreq->presync = ioreq->postsync = 1;
         /* fall through */
     case BLKIF_OP_WRITE:
         ioreq->prot = PROT_READ; /* from memory */
@@ -242,21 +301,21 @@ static void ioreq_unmap(struct ioreq *ioreq)
     XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
     int i;
 
-    if (ioreq->v.niov == 0 || ioreq->mapped == 0) {
+    if (ioreq->num_unmap == 0 || ioreq->mapped == 0) {
         return;
     }
     if (batch_maps) {
         if (!ioreq->pages) {
             return;
         }
-        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->v.niov) != 0) {
+        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
             xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
                           strerror(errno));
         }
-        ioreq->blkdev->cnt_map -= ioreq->v.niov;
+        ioreq->blkdev->cnt_map -= ioreq->num_unmap;
         ioreq->pages = NULL;
     } else {
-        for (i = 0; i < ioreq->v.niov; i++) {
+        for (i = 0; i < ioreq->num_unmap; i++) {
             if (!ioreq->page[i]) {
                 continue;
             }
@@ -274,41 +333,120 @@ static void ioreq_unmap(struct ioreq *ioreq)
 static int ioreq_map(struct ioreq *ioreq)
 {
     XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
-    int i;
+    uint32_t domids[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    uint32_t refs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    void *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    int i, j, new_maps = 0;
+    PersistentGrant *grant;
+    /* domids and refs variables will contain the information necessary
+     * to map the grants that are needed to fulfill this request.
+     *
+     * After mapping the needed grants, the page array will contain the
+     * memory address of each granted page in the order specified in ioreq
+     * (disregarding if it's a persistent grant or not).
+     */
 
     if (ioreq->v.niov == 0 || ioreq->mapped == 1) {
         return 0;
     }
-    if (batch_maps) {
+    if (ioreq->blkdev->feature_persistent) {
+        for (i = 0; i < ioreq->v.niov; i++) {
+            grant = g_tree_lookup(ioreq->blkdev->persistent_gnts,
+                                    GUINT_TO_POINTER(ioreq->refs[i]));
+
+            if (grant != NULL) {
+                page[i] = grant->page;
+                xen_be_printf(&ioreq->blkdev->xendev, 3,
+                              "using persistent-grant %" PRIu32 "\n",
+                              ioreq->refs[i]);
+            } else {
+                    /* Add the grant to the list of grants that
+                     * should be mapped
+                     */
+                    domids[new_maps] = ioreq->domids[i];
+                    refs[new_maps] = ioreq->refs[i];
+                    page[i] = NULL;
+                    new_maps++;
+            }
+        }
+        /* Set the protection to RW, since grants may be reused later
+         * with a different protection than the one needed for this request
+         */
+        ioreq->prot = PROT_WRITE | PROT_READ;
+    } else {
+        /* All grants in the request should be mapped */
+        memcpy(refs, ioreq->refs, sizeof(refs));
+        memcpy(domids, ioreq->domids, sizeof(domids));
+        memset(page, 0, sizeof(page));
+        new_maps = ioreq->v.niov;
+    }
+
+    if (batch_maps && new_maps) {
         ioreq->pages = xc_gnttab_map_grant_refs
-            (gnt, ioreq->v.niov, ioreq->domids, ioreq->refs, ioreq->prot);
+            (gnt, new_maps, domids, refs, ioreq->prot);
         if (ioreq->pages == NULL) {
             xen_be_printf(&ioreq->blkdev->xendev, 0,
                           "can't map %d grant refs (%s, %d maps)\n",
-                          ioreq->v.niov, strerror(errno), ioreq->blkdev->cnt_map);
+                          new_maps, strerror(errno), ioreq->blkdev->cnt_map);
             return -1;
         }
-        for (i = 0; i < ioreq->v.niov; i++) {
-            ioreq->v.iov[i].iov_base = ioreq->pages + i * XC_PAGE_SIZE +
-                (uintptr_t)ioreq->v.iov[i].iov_base;
+        for (i = 0, j = 0; i < ioreq->v.niov; i++) {
+            if (page[i] == NULL) {
+                page[i] = ioreq->pages + (j++) * XC_PAGE_SIZE;
+            }
         }
-        ioreq->blkdev->cnt_map += ioreq->v.niov;
-    } else  {
-        for (i = 0; i < ioreq->v.niov; i++) {
+        ioreq->blkdev->cnt_map += new_maps;
+    } else if (new_maps)  {
+        for (i = 0; i < new_maps; i++) {
             ioreq->page[i] = xc_gnttab_map_grant_ref
-                (gnt, ioreq->domids[i], ioreq->refs[i], ioreq->prot);
+                (gnt, domids[i], refs[i], ioreq->prot);
             if (ioreq->page[i] == NULL) {
                 xen_be_printf(&ioreq->blkdev->xendev, 0,
                               "can't map grant ref %d (%s, %d maps)\n",
-                              ioreq->refs[i], strerror(errno), ioreq->blkdev->cnt_map);
+                              refs[i], strerror(errno), ioreq->blkdev->cnt_map);
                 ioreq_unmap(ioreq);
                 return -1;
             }
-            ioreq->v.iov[i].iov_base = ioreq->page[i] + (uintptr_t)ioreq->v.iov[i].iov_base;
             ioreq->blkdev->cnt_map++;
         }
+        for (i = 0, j = 0; i < ioreq->v.niov; i++) {
+            if (page[i] == NULL) {
+                page[i] = ioreq->page[j++];
+            }
+        }
+    }
+    if (ioreq->blkdev->feature_persistent) {
+        while ((ioreq->blkdev->persistent_gnt_count < ioreq->blkdev->max_grants)
+              && new_maps) {
+            /* Go through the list of newly mapped grants and add as many
+             * as possible to the list of persistently mapped grants.
+             *
+             * Since we start at the end of ioreq->page(s), we only need
+             * to decrease new_maps to prevent this granted pages from
+             * being unmapped in ioreq_unmap.
+             */
+            grant = g_malloc0(sizeof(*grant));
+            new_maps--;
+            if (batch_maps) {
+                grant->page = ioreq->pages + (new_maps) * XC_PAGE_SIZE;
+            } else {
+                grant->page = ioreq->page[new_maps];
+            }
+            grant->blkdev = ioreq->blkdev;
+            xen_be_printf(&ioreq->blkdev->xendev, 3,
+                          "adding grant %" PRIu32 " page: %p\n",
+                          refs[new_maps], grant->page);
+            g_tree_insert(ioreq->blkdev->persistent_gnts,
+                          GUINT_TO_POINTER(refs[new_maps]),
+                          grant);
+            ioreq->blkdev->persistent_gnt_count++;
+        }
+    }
+    for (i = 0; i < ioreq->v.niov; i++) {
+        ioreq->v.iov[i].iov_base += (uintptr_t)page[i];
     }
     ioreq->mapped = 1;
+    ioreq->num_unmap = new_maps;
     return 0;
 }
 
@@ -370,7 +508,7 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
                        qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_WRITE:
-    case BLKIF_OP_WRITE_BARRIER:
+    case BLKIF_OP_FLUSH_DISKCACHE:
         if (!ioreq->req.nr_segments) {
             break;
         }
@@ -655,7 +793,8 @@ static int blk_init(struct XenDevice *xendev)
                   blkdev->file_size, blkdev->file_size >> 20);
 
     /* fill info */
-    xenstore_write_be_int(&blkdev->xendev, "feature-barrier", 1);
+    xenstore_write_be_int(&blkdev->xendev, "feature-flush-cache", 1);
+    xenstore_write_be_int(&blkdev->xendev, "feature-persistent", 1);
     xenstore_write_be_int(&blkdev->xendev, "info",            info);
     xenstore_write_be_int(&blkdev->xendev, "sector-size",     blkdev->file_blk);
     xenstore_write_be_int(&blkdev->xendev, "sectors",
@@ -679,6 +818,7 @@ out_error:
 static int blk_connect(struct XenDevice *xendev)
 {
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
+    int pers;
 
     if (xenstore_read_fe_int(&blkdev->xendev, "ring-ref", &blkdev->ring_ref) == -1) {
         return -1;
@@ -687,6 +827,11 @@ static int blk_connect(struct XenDevice *xendev)
                              &blkdev->xendev.remote_port) == -1) {
         return -1;
     }
+    if (xenstore_read_fe_int(&blkdev->xendev, "feature-persistent", &pers)) {
+        blkdev->feature_persistent = FALSE;
+    } else {
+        blkdev->feature_persistent = !!pers;
+    }
 
     blkdev->protocol = BLKIF_PROTOCOL_NATIVE;
     if (blkdev->xendev.protocol) {
@@ -730,6 +875,15 @@ static int blk_connect(struct XenDevice *xendev)
     }
     }
 
+    if (blkdev->feature_persistent) {
+        /* Init persistent grants */
+        blkdev->max_grants = max_requests * BLKIF_MAX_SEGMENTS_PER_REQUEST;
+        blkdev->persistent_gnts = g_tree_new_full((GCompareDataFunc)int_cmp,
+                                             NULL, NULL,
+                                             (GDestroyNotify)destroy_grant);
+        blkdev->persistent_gnt_count = 0;
+    }
+
     xen_be_bind_evtchn(&blkdev->xendev);
 
     xen_be_printf(&blkdev->xendev, 1, "ok: proto %s, ring-ref %d, "
@@ -770,6 +924,11 @@ static int blk_free(struct XenDevice *xendev)
         blk_disconnect(xendev);
     }
 
+    /* Free persistent grants */
+    if (blkdev->feature_persistent) {
+        g_tree_destroy(blkdev->persistent_gnts);
+    }
+
     while (!QLIST_EMPTY(&blkdev->freelist)) {
         ioreq = QLIST_FIRST(&blkdev->freelist);
         QLIST_REMOVE(ioreq, list);
diff --git a/hw/xen_domainbuild.c b/hw/xen_domainbuild.c
index a6a12e5930d..a4272f0680c 100644
--- a/hw/xen_domainbuild.c
+++ b/hw/xen_domainbuild.c
@@ -1,8 +1,8 @@
 #include <signal.h>
 #include "xen_backend.h"
 #include "xen_domainbuild.h"
-#include "qemu-timer.h"
-#include "qemu-log.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
 
 #include <xenguest.h>
 
@@ -153,7 +153,6 @@ static void xen_domain_poll(void *opaque)
 
 quit:
     qemu_system_shutdown_request();
-    return;
 }
 
 static int xen_domain_watcher(void)
diff --git a/hw/xen_machine_pv.c b/hw/xen_machine_pv.c
index 4b72aa75576..66e898123ea 100644
--- a/hw/xen_machine_pv.c
+++ b/hw/xen_machine_pv.c
@@ -27,15 +27,14 @@
 #include "boards.h"
 #include "xen_backend.h"
 #include "xen_domainbuild.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
-static void xen_init_pv(ram_addr_t ram_size,
-			const char *boot_device,
-			const char *kernel_filename,
-			const char *kernel_cmdline,
-			const char *initrd_filename,
-			const char *cpu_model)
+static void xen_init_pv(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     X86CPU *cpu;
     CPUX86State *env;
     DriveInfo *dinfo;
@@ -116,6 +115,7 @@ static QEMUMachine xenpv_machine = {
     .init = xen_init_pv,
     .max_cpus = 1,
     .default_machine_opts = "accel=xen",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xenpv_machine_init(void)
diff --git a/hw/xen_nic.c b/hw/xen_nic.c
index 8b79bfb73eb..dc12110dba7 100644
--- a/hw/xen_nic.c
+++ b/hw/xen_nic.c
@@ -36,10 +36,9 @@
 #include <sys/wait.h>
 
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "net/util.h"
-#include "qemu-char.h"
 #include "xen_backend.h"
 
 #include <xen/io/netif.h>
@@ -415,6 +414,7 @@ static void net_event(struct XenDevice *xendev)
 {
     struct XenNetDev *netdev = container_of(xendev, struct XenNetDev, xendev);
     net_tx_packets(netdev);
+    qemu_flush_queued_packets(&netdev->nic->nc);
 }
 
 static int net_free(struct XenDevice *xendev)
diff --git a/hw/xen_platform.c b/hw/xen_platform.c
index 0d6c2ff8c35..8866468c994 100644
--- a/hw/xen_platform.c
+++ b/hw/xen_platform.c
@@ -27,13 +27,12 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "irq.h"
 #include "xen_common.h"
-#include "net.h"
 #include "xen_backend.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include <xenguest.h>
 
@@ -85,8 +84,10 @@ static void log_writeb(PCIXenPlatformState *s, char val)
 
 static void unplug_nic(PCIBus *b, PCIDevice *d, void *o)
 {
+    /* We have to ignore passthrough devices */
     if (pci_get_word(d->config + PCI_CLASS_DEVICE) ==
-            PCI_CLASS_NETWORK_ETHERNET) {
+            PCI_CLASS_NETWORK_ETHERNET
+            && strcmp(d->name, "xen-pci-passthrough") != 0) {
         qdev_free(&d->qdev);
     }
 }
@@ -98,8 +99,10 @@ static void pci_unplug_nics(PCIBus *bus)
 
 static void unplug_disks(PCIBus *b, PCIDevice *d, void *o)
 {
+    /* We have to ignore passthrough devices */
     if (pci_get_word(d->config + PCI_CLASS_DEVICE) ==
-            PCI_CLASS_STORAGE_IDE) {
+            PCI_CLASS_STORAGE_IDE
+            && strcmp(d->name, "xen-pci-passthrough") != 0) {
         qdev_unplug(&(d->qdev), NULL);
     }
 }
@@ -224,18 +227,46 @@ static void platform_fixed_ioport_reset(void *opaque)
     platform_fixed_ioport_writeb(s, 0, 0);
 }
 
-const MemoryRegionPortio xen_platform_ioport[] = {
-    { 0, 16, 4, .write = platform_fixed_ioport_writel, },
-    { 0, 16, 2, .write = platform_fixed_ioport_writew, },
-    { 0, 16, 1, .write = platform_fixed_ioport_writeb, },
-    { 0, 16, 2, .read = platform_fixed_ioport_readw, },
-    { 0, 16, 1, .read = platform_fixed_ioport_readb, },
-    PORTIO_END_OF_LIST()
-};
+static uint64_t platform_fixed_ioport_read(void *opaque,
+                                           hwaddr addr,
+                                           unsigned size)
+{
+    switch (size) {
+    case 1:
+        return platform_fixed_ioport_readb(opaque, addr);
+    case 2:
+        return platform_fixed_ioport_readw(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void platform_fixed_ioport_write(void *opaque, hwaddr addr,
+
+                                        uint64_t val, unsigned size)
+{
+    switch (size) {
+    case 1:
+        platform_fixed_ioport_writeb(opaque, addr, val);
+        break;
+    case 2:
+        platform_fixed_ioport_writew(opaque, addr, val);
+        break;
+    case 4:
+        platform_fixed_ioport_writel(opaque, addr, val);
+        break;
+    }
+}
+
 
 static const MemoryRegionOps platform_fixed_io_ops = {
-    .old_portio = xen_platform_ioport,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .read = platform_fixed_ioport_read,
+    .write = platform_fixed_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void platform_fixed_ioport_init(PCIXenPlatformState* s)
@@ -248,7 +279,8 @@ static void platform_fixed_ioport_init(PCIXenPlatformState* s)
 
 /* Xen Platform PCI Device */
 
-static uint32_t xen_platform_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t xen_platform_ioport_readb(void *opaque, hwaddr addr,
+                                          unsigned int size)
 {
     if (addr == 0) {
         return platform_fixed_ioport_readb(opaque, 0);
@@ -257,30 +289,28 @@ static uint32_t xen_platform_ioport_readb(void *opaque, uint32_t addr)
     }
 }
 
-static void xen_platform_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void xen_platform_ioport_writeb(void *opaque, hwaddr addr,
+                                       uint64_t val, unsigned int size)
 {
     PCIXenPlatformState *s = opaque;
 
     switch (addr) {
     case 0: /* Platform flags */
-        platform_fixed_ioport_writeb(opaque, 0, val);
+        platform_fixed_ioport_writeb(opaque, 0, (uint32_t)val);
         break;
     case 8:
-        log_writeb(s, val);
+        log_writeb(s, (uint32_t)val);
         break;
     default:
         break;
     }
 }
 
-static MemoryRegionPortio xen_pci_portio[] = {
-    { 0, 0x100, 1, .read = xen_platform_ioport_readb, },
-    { 0, 0x100, 1, .write = xen_platform_ioport_writeb, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps xen_pci_io_ops = {
-    .old_portio = xen_pci_portio,
+    .read  = xen_platform_ioport_readb,
+    .write = xen_platform_ioport_writeb,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
 };
 
 static void platform_ioport_bar_setup(PCIXenPlatformState *d)
@@ -288,7 +318,7 @@ static void platform_ioport_bar_setup(PCIXenPlatformState *d)
     memory_region_init_io(&d->bar, &xen_pci_io_ops, d, "xen-pci", 0x100);
 }
 
-static uint64_t platform_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t platform_mmio_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     DPRINTF("Warning: attempted read from physical address "
@@ -297,7 +327,7 @@ static uint64_t platform_mmio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void platform_mmio_write(void *opaque, target_phys_addr_t addr,
+static void platform_mmio_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned size)
 {
     DPRINTF("Warning: attempted write of 0x%"PRIx64" to physical "
@@ -389,7 +419,7 @@ static void xen_platform_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_xen_platform;
 }
 
-static TypeInfo xen_platform_info = {
+static const TypeInfo xen_platform_info = {
     .name          = "xen-platform",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIXenPlatformState),
diff --git a/hw/xen_pt.c b/hw/xen_pt.c
index 307119a12f7..9db5f6e9645 100644
--- a/hw/xen_pt.c
+++ b/hw/xen_pt.c
@@ -54,11 +54,12 @@
 
 #include <sys/ioctl.h>
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "xen.h"
 #include "xen_backend.h"
 #include "xen_pt.h"
-#include "range.h"
+#include "qemu/range.h"
+#include "exec/address-spaces.h"
 
 #define XEN_PT_NR_IRQS (256)
 static uint8_t xen_pt_mapped_machine_irq[XEN_PT_NR_IRQS] = {0};
@@ -362,7 +363,7 @@ out:
 
 /* register regions */
 
-static uint64_t xen_pt_bar_read(void *o, target_phys_addr_t addr,
+static uint64_t xen_pt_bar_read(void *o, hwaddr addr,
                                 unsigned size)
 {
     PCIDevice *d = o;
@@ -372,7 +373,7 @@ static uint64_t xen_pt_bar_read(void *o, target_phys_addr_t addr,
                addr);
     return 0;
 }
-static void xen_pt_bar_write(void *o, target_phys_addr_t addr, uint64_t val,
+static void xen_pt_bar_write(void *o, hwaddr addr, uint64_t val,
                              unsigned size)
 {
     PCIDevice *d = o;
@@ -410,14 +411,17 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s)
             if (r->type & XEN_HOST_PCI_REGION_TYPE_PREFETCH) {
                 type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
             }
+            if (r->type & XEN_HOST_PCI_REGION_TYPE_MEM_64) {
+                type |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+            }
         }
 
         memory_region_init_io(&s->bar[i], &ops, &s->dev,
                               "xen-pci-pt-bar", r->size);
         pci_register_bar(&s->dev, i, type, &s->bar[i]);
 
-        XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%08"PRIx64
-                   " base_addr=0x%08"PRIx64" type: %#x)\n",
+        XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%lx"PRIx64
+                   " base_addr=0x%lx"PRIx64" type: %#x)\n",
                    i, r->size, r->base_addr, type);
     }
 
@@ -597,14 +601,6 @@ static void xen_pt_region_update(XenPCIPassthroughState *s,
     }
 }
 
-static void xen_pt_begin(MemoryListener *l)
-{
-}
-
-static void xen_pt_commit(MemoryListener *l)
-{
-}
-
 static void xen_pt_region_add(MemoryListener *l, MemoryRegionSection *sec)
 {
     XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
@@ -621,36 +617,31 @@ static void xen_pt_region_del(MemoryListener *l, MemoryRegionSection *sec)
     xen_pt_region_update(s, sec, false);
 }
 
-static void xen_pt_region_nop(MemoryListener *l, MemoryRegionSection *s)
+static void xen_pt_io_region_add(MemoryListener *l, MemoryRegionSection *sec)
 {
-}
+    XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
+                                             io_listener);
 
-static void xen_pt_log_fns(MemoryListener *l, MemoryRegionSection *s)
-{
+    xen_pt_region_update(s, sec, true);
 }
 
-static void xen_pt_log_global_fns(MemoryListener *l)
+static void xen_pt_io_region_del(MemoryListener *l, MemoryRegionSection *sec)
 {
-}
+    XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
+                                             io_listener);
 
-static void xen_pt_eventfd_fns(MemoryListener *l, MemoryRegionSection *s,
-                               bool match_data, uint64_t data, EventNotifier *n)
-{
+    xen_pt_region_update(s, sec, false);
 }
 
 static const MemoryListener xen_pt_memory_listener = {
-    .begin = xen_pt_begin,
-    .commit = xen_pt_commit,
     .region_add = xen_pt_region_add,
-    .region_nop = xen_pt_region_nop,
     .region_del = xen_pt_region_del,
-    .log_start = xen_pt_log_fns,
-    .log_stop = xen_pt_log_fns,
-    .log_sync = xen_pt_log_fns,
-    .log_global_start = xen_pt_log_global_fns,
-    .log_global_stop = xen_pt_log_global_fns,
-    .eventfd_add = xen_pt_eventfd_fns,
-    .eventfd_del = xen_pt_eventfd_fns,
+    .priority = 10,
+};
+
+static const MemoryListener xen_pt_io_listener = {
+    .region_add = xen_pt_io_region_add,
+    .region_del = xen_pt_io_region_del,
     .priority = 10,
 };
 
@@ -680,7 +671,8 @@ static int xen_pt_initfn(PCIDevice *d)
     s->is_virtfn = s->real_device.is_virtfn;
     if (s->is_virtfn) {
         XEN_PT_LOG(d, "%04x:%02x:%02x.%d is a SR-IOV Virtual Function\n",
-                   s->real_device.domain, bus, slot, func);
+                   s->real_device.domain, s->real_device.bus,
+                   s->real_device.dev, s->real_device.func);
     }
 
     /* Initialize virtualized PCI configuration (Extended 256 Bytes) */
@@ -691,6 +683,7 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
     s->memory_listener = xen_pt_memory_listener;
+    s->io_listener = xen_pt_io_listener;
 
     /* Handle real device's MMIO/PIO BARs */
     xen_pt_register_regions(s);
@@ -757,9 +750,10 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
 out:
-    memory_listener_register(&s->memory_listener, NULL);
+    memory_listener_register(&s->memory_listener, &address_space_memory);
+    memory_listener_register(&s->io_listener, &address_space_io);
     XEN_PT_LOG(d, "Real physical device %02x:%02x.%d registered successfuly!\n",
-               bus, slot, func);
+               s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
 
     return 0;
 }
@@ -812,6 +806,7 @@ static void xen_pt_unregister_device(PCIDevice *d)
 
     xen_pt_unregister_regions(s);
     memory_listener_unregister(&s->memory_listener);
+    memory_listener_unregister(&s->io_listener);
 
     xen_host_pci_device_put(&s->real_device);
 }
@@ -834,7 +829,7 @@ static void xen_pci_passthrough_class_init(ObjectClass *klass, void *data)
     dc->props = xen_pci_passthrough_properties;
 };
 
-static TypeInfo xen_pci_passthrough_info = {
+static const TypeInfo xen_pci_passthrough_info = {
     .name = "xen-pci-passthrough",
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XenPCIPassthroughState),
diff --git a/hw/xen_pt.h b/hw/xen_pt.h
index 41904ece931..e3497302cf4 100644
--- a/hw/xen_pt.h
+++ b/hw/xen_pt.h
@@ -3,7 +3,7 @@
 
 #include "qemu-common.h"
 #include "xen_common.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "xen-host-pci-device.h"
 
 void xen_pt_log(const PCIDevice *d, const char *f, ...) GCC_FMT_ATTR(2, 3);
@@ -96,7 +96,7 @@ typedef struct XenPTRegion {
  * - do NOT use ALL F for init_val, otherwise the tbl will not be registered.
  */
 
-/* emulated register infomation */
+/* emulated register information */
 struct XenPTRegInfo {
     uint32_t offset;
     uint32_t size;
@@ -140,7 +140,7 @@ typedef int (*xen_pt_reg_size_init_fn)
     (XenPCIPassthroughState *, const XenPTRegGroupInfo *,
      uint32_t base_offset, uint8_t *size);
 
-/* emulated register group infomation */
+/* emulated register group information */
 struct XenPTRegGroupInfo {
     uint8_t grp_id;
     XenPTRegisterGroupType grp_type;
@@ -209,6 +209,7 @@ struct XenPCIPassthroughState {
     MemoryRegion rom;
 
     MemoryListener memory_listener;
+    MemoryListener io_listener;
 };
 
 int xen_pt_config_init(XenPCIPassthroughState *s);
diff --git a/hw/xen_pt_config_init.c b/hw/xen_pt_config_init.c
index 00eb3d997d3..54a179af90a 100644
--- a/hw/xen_pt_config_init.c
+++ b/hw/xen_pt_config_init.c
@@ -12,7 +12,7 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "xen_backend.h"
 #include "xen_pt.h"
 
@@ -342,6 +342,23 @@ static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 #define XEN_PT_BAR_IO_RO_MASK     0x00000003  /* BAR ReadOnly mask(I/O) */
 #define XEN_PT_BAR_IO_EMU_MASK    0xFFFFFFFC  /* BAR emul mask(I/O) */
 
+static bool is_64bit_bar(PCIIORegion *r)
+{
+    return !!(r->type & PCI_BASE_ADDRESS_MEM_TYPE_64);
+}
+
+static uint64_t xen_pt_get_bar_size(PCIIORegion *r)
+{
+    if (is_64bit_bar(r)) {
+        uint64_t size64;
+        size64 = (r + 1)->size;
+        size64 <<= 32;
+        size64 += r->size;
+        return size64;
+    }
+    return r->size;
+}
+
 static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s,
                                          XenPTRegInfo *reg)
 {
@@ -366,7 +383,7 @@ static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s,
 
     /* check unused BAR */
     r = &d->io_regions[index];
-    if (r->size == 0) {
+    if (!xen_pt_get_bar_size(r)) {
         return XEN_PT_BAR_FLAG_UNUSED;
     }
 
@@ -481,7 +498,12 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     switch (s->bases[index].bar_flag) {
     case XEN_PT_BAR_FLAG_MEM:
         bar_emu_mask = XEN_PT_BAR_MEM_EMU_MASK;
-        bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1);
+        if (!r_size) {
+            /* low 32 bits mask for 64 bit bars */
+            bar_ro_mask = XEN_PT_BAR_ALLF;
+        } else {
+            bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1);
+        }
         break;
     case XEN_PT_BAR_FLAG_IO:
         bar_emu_mask = XEN_PT_BAR_IO_EMU_MASK;
@@ -489,7 +511,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
         break;
     case XEN_PT_BAR_FLAG_UPPER:
         bar_emu_mask = XEN_PT_BAR_ALLF;
-        bar_ro_mask = 0;    /* all upper 32bit are R/W */
+        bar_ro_mask = r_size ? r_size - 1 : 0;
         break;
     default:
         break;
@@ -501,22 +523,13 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* check whether we need to update the virtual region address or not */
     switch (s->bases[index].bar_flag) {
+    case XEN_PT_BAR_FLAG_UPPER:
     case XEN_PT_BAR_FLAG_MEM:
         /* nothing to do */
         break;
     case XEN_PT_BAR_FLAG_IO:
         /* nothing to do */
         break;
-    case XEN_PT_BAR_FLAG_UPPER:
-        if (cfg_entry->data) {
-            if (cfg_entry->data != (XEN_PT_BAR_ALLF & ~bar_ro_mask)) {
-                XEN_PT_WARN(d, "Guest attempt to set high MMIO Base Address. "
-                            "Ignore mapping. "
-                            "(offset: 0x%02x, high address: 0x%08x)\n",
-                            reg->offset, cfg_entry->data);
-            }
-        }
-        break;
     default:
         break;
     }
@@ -562,7 +575,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* Header Type0 reg static infomation table */
+/* Header Type0 reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_header0[] = {
     /* Vendor ID reg */
     {
@@ -753,7 +766,7 @@ static XenPTRegInfo xen_pt_emu_reg_header0[] = {
  * Vital Product Data Capability
  */
 
-/* Vital Product Data Capability Structure reg static infomation table */
+/* Vital Product Data Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_vpd[] = {
     {
         .offset     = PCI_CAP_LIST_NEXT,
@@ -775,7 +788,7 @@ static XenPTRegInfo xen_pt_emu_reg_vpd[] = {
  * Vendor Specific Capability
  */
 
-/* Vendor Specific Capability Structure reg static infomation table */
+/* Vendor Specific Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_vendor[] = {
     {
         .offset     = PCI_CAP_LIST_NEXT,
@@ -866,7 +879,7 @@ static int xen_pt_linkctrl2_reg_init(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* PCI Express Capability Structure reg static infomation table */
+/* PCI Express Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pcie[] = {
     /* Next Pointer reg */
     {
@@ -981,7 +994,7 @@ static int xen_pt_pmcsr_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* Power Management Capability reg static infomation table */
+/* Power Management Capability reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pm[] = {
     /* Next Pointer reg */
     {
@@ -1259,7 +1272,7 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* MSI Capability Structure reg static infomation table */
+/* MSI Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_msi[] = {
     /* Next Pointer reg */
     {
@@ -1396,7 +1409,7 @@ static int xen_pt_msixctrl_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* MSI-X Capability Structure reg static infomation table */
+/* MSI-X Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_msix[] = {
     /* Next Pointer reg */
     {
diff --git a/hw/xen_pt_msi.c b/hw/xen_pt_msi.c
index 2299cc7772f..db757cd1f10 100644
--- a/hw/xen_pt_msi.c
+++ b/hw/xen_pt_msi.c
@@ -321,7 +321,7 @@ static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr)
 
     pirq = entry->pirq;
 
-    rc = msi_msix_setup(s, entry->data, entry->data, &pirq, true, entry_nr,
+    rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr,
                         entry->pirq == XEN_PT_UNASSIGNED_PIRQ);
     if (rc) {
         return rc;
@@ -427,7 +427,7 @@ static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val)
     }
 }
 
-static void pci_msix_write(void *opaque, target_phys_addr_t addr,
+static void pci_msix_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     XenPCIPassthroughState *s = opaque;
@@ -475,7 +475,7 @@ static void pci_msix_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t pci_msix_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_msix_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     XenPCIPassthroughState *s = opaque;
diff --git a/hw/xenfb.c b/hw/xenfb.c
index 338800a4d9d..903efd30736 100644
--- a/hw/xenfb.c
+++ b/hw/xenfb.c
@@ -36,8 +36,8 @@
 #include <time.h>
 
 #include "hw.h"
-#include "console.h"
-#include "qemu-char.h"
+#include "ui/console.h"
+#include "char/char.h"
 #include "xen_backend.h"
 
 #include <xen/event_channel.h>
@@ -648,7 +648,7 @@ static void xenfb_guest_copy(struct XenFB *xenfb, int x, int y, int w, int h)
         xen_be_printf(&xenfb->c.xendev, 0, "%s: oops: convert %d -> %d bpp?\n",
                       __FUNCTION__, xenfb->depth, bpp);
 
-    dpy_update(xenfb->c.ds, x, y, w, h);
+    dpy_gfx_update(xenfb->c.ds, x, y, w, h);
 }
 
 #ifdef XENFB_TYPE_REFRESH_PERIOD
@@ -717,7 +717,7 @@ static void xenfb_update(void *opaque)
 	if (xenfb_queue_full(xenfb))
 	    return;
 
-        for (l = xenfb->c.ds->listeners; l != NULL; l = l->next) {
+        QLIST_FOREACH(l, &xenfb->c.ds->listeners, next) {
             if (l->idle)
                 continue;
             idle = 0;
@@ -766,7 +766,7 @@ static void xenfb_update(void *opaque)
         xen_be_printf(&xenfb->c.xendev, 1, "update: resizing: %dx%d @ %d bpp%s\n",
                       xenfb->width, xenfb->height, xenfb->depth,
                       is_buffer_shared(xenfb->c.ds->surface) ? " (shared)" : "");
-        dpy_resize(xenfb->c.ds);
+        dpy_gfx_resize(xenfb->c.ds);
         xenfb->up_fullscreen = 1;
     }
 
diff --git a/hw/xgmac.c b/hw/xgmac.c
index a91ef608f18..00dae7789c0 100644
--- a/hw/xgmac.c
+++ b/hw/xgmac.c
@@ -25,9 +25,9 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "net.h"
+#include "char/char.h"
+#include "qemu/log.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #ifdef DEBUG_XGMAC
@@ -252,7 +252,7 @@ static void enet_update_irq(struct XgmacState *s)
     qemu_set_irq(s->sbd_irq, !!stat);
 }
 
-static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t enet_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct XgmacState *s = opaque;
     uint64_t r = 0;
@@ -271,7 +271,7 @@ static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void enet_write(void *opaque, target_phys_addr_t addr,
+static void enet_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     struct XgmacState *s = opaque;
@@ -418,7 +418,7 @@ static void xgmac_enet_class_init(ObjectClass *klass, void *data)
     dc->props = xgmac_properties;
 }
 
-static TypeInfo xgmac_enet_info = {
+static const TypeInfo xgmac_enet_info = {
     .name          = "xgmac",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XgmacState),
diff --git a/hw/xics.c b/hw/xics.c
index b674771dc4c..9ef0d61377b 100644
--- a/hw/xics.c
+++ b/hw/xics.c
@@ -26,6 +26,7 @@
  */
 
 #include "hw.h"
+#include "trace.h"
 #include "hw/spapr.h"
 #include "hw/xics.h"
 
@@ -66,6 +67,8 @@ static void icp_check_ipi(struct icp_state *icp, int server)
         return;
     }
 
+    trace_xics_icp_check_ipi(server, ss->mfrr);
+
     if (XISR(ss)) {
         ics_reject(icp->ics, XISR(ss));
     }
@@ -108,23 +111,25 @@ static void icp_set_cppr(struct icp_state *icp, int server, uint8_t cppr)
     }
 }
 
-static void icp_set_mfrr(struct icp_state *icp, int nr, uint8_t mfrr)
+static void icp_set_mfrr(struct icp_state *icp, int server, uint8_t mfrr)
 {
-    struct icp_server_state *ss = icp->ss + nr;
+    struct icp_server_state *ss = icp->ss + server;
 
     ss->mfrr = mfrr;
     if (mfrr < CPPR(ss)) {
-        icp_check_ipi(icp, nr);
+        icp_check_ipi(icp, server);
     }
 }
 
 static uint32_t icp_accept(struct icp_server_state *ss)
 {
-    uint32_t xirr;
+    uint32_t xirr = ss->xirr;
 
     qemu_irq_lower(ss->output);
-    xirr = ss->xirr;
     ss->xirr = ss->pending_priority << 24;
+
+    trace_xics_icp_accept(xirr, ss->xirr);
+
     return xirr;
 }
 
@@ -134,6 +139,7 @@ static void icp_eoi(struct icp_state *icp, int server, uint32_t xirr)
 
     /* Send EOI -> ICS */
     ss->xirr = (ss->xirr & ~CPPR_MASK) | (xirr & CPPR_MASK);
+    trace_xics_icp_eoi(server, xirr, ss->xirr);
     ics_eoi(icp->ics, xirr & XISR_MASK);
     if (!XISR(ss)) {
         icp_resend(icp, server);
@@ -144,6 +150,8 @@ static void icp_irq(struct icp_state *icp, int server, int nr, uint8_t priority)
 {
     struct icp_server_state *ss = icp->ss + server;
 
+    trace_xics_icp_irq(server, nr, priority);
+
     if ((priority >= CPPR(ss))
         || (XISR(ss) && (ss->pending_priority <= priority))) {
         ics_reject(icp->ics, nr);
@@ -153,6 +161,7 @@ static void icp_irq(struct icp_state *icp, int server, int nr, uint8_t priority)
         }
         ss->xirr = (ss->xirr & ~XISR_MASK) | (nr & XISR_MASK);
         ss->pending_priority = priority;
+        trace_xics_icp_raise(ss->xirr, ss->pending_priority);
         qemu_irq_raise(ss->output);
     }
 }
@@ -165,17 +174,18 @@ struct ics_irq_state {
     int server;
     uint8_t priority;
     uint8_t saved_priority;
-    enum xics_irq_type type;
-    int asserted:1;
-    int sent:1;
-    int rejected:1;
-    int masked_pending:1;
+#define XICS_STATUS_ASSERTED           0x1
+#define XICS_STATUS_SENT               0x2
+#define XICS_STATUS_REJECTED           0x4
+#define XICS_STATUS_MASKED_PENDING     0x8
+    uint8_t status;
 };
 
 struct ics_state {
     int nr_irqs;
     int offset;
     qemu_irq *qirqs;
+    bool *islsi;
     struct ics_irq_state *irqs;
     struct icp_state *icp;
 };
@@ -191,8 +201,8 @@ static void resend_msi(struct ics_state *ics, int srcno)
     struct ics_irq_state *irq = ics->irqs + srcno;
 
     /* FIXME: filter by server#? */
-    if (irq->rejected) {
-        irq->rejected = 0;
+    if (irq->status & XICS_STATUS_REJECTED) {
+        irq->status &= ~XICS_STATUS_REJECTED;
         if (irq->priority != 0xff) {
             icp_irq(ics->icp, irq->server, srcno + ics->offset,
                     irq->priority);
@@ -204,8 +214,10 @@ static void resend_lsi(struct ics_state *ics, int srcno)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if ((irq->priority != 0xff) && irq->asserted && !irq->sent) {
-        irq->sent = 1;
+    if ((irq->priority != 0xff)
+        && (irq->status & XICS_STATUS_ASSERTED)
+        && !(irq->status & XICS_STATUS_SENT)) {
+        irq->status |= XICS_STATUS_SENT;
         icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
     }
 }
@@ -214,10 +226,12 @@ static void set_irq_msi(struct ics_state *ics, int srcno, int val)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
+    trace_xics_set_irq_msi(srcno, srcno + ics->offset);
+
     if (val) {
         if (irq->priority == 0xff) {
-            irq->masked_pending = 1;
-            /* masked pending */ ;
+            irq->status |= XICS_STATUS_MASKED_PENDING;
+            trace_xics_masked_pending();
         } else  {
             icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
         }
@@ -228,16 +242,20 @@ static void set_irq_lsi(struct ics_state *ics, int srcno, int val)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    irq->asserted = val;
+    trace_xics_set_irq_lsi(srcno, srcno + ics->offset);
+    if (val) {
+        irq->status |= XICS_STATUS_ASSERTED;
+    } else {
+        irq->status &= ~XICS_STATUS_ASSERTED;
+    }
     resend_lsi(ics, srcno);
 }
 
 static void ics_set_irq(void *opaque, int srcno, int val)
 {
     struct ics_state *ics = (struct ics_state *)opaque;
-    struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->type == XICS_LSI) {
+    if (ics->islsi[srcno]) {
         set_irq_lsi(ics, srcno, val);
     } else {
         set_irq_msi(ics, srcno, val);
@@ -248,11 +266,12 @@ static void write_xive_msi(struct ics_state *ics, int srcno)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (!irq->masked_pending || (irq->priority == 0xff)) {
+    if (!(irq->status & XICS_STATUS_MASKED_PENDING)
+        || (irq->priority == 0xff)) {
         return;
     }
 
-    irq->masked_pending = 0;
+    irq->status &= ~XICS_STATUS_MASKED_PENDING;
     icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
 }
 
@@ -262,15 +281,18 @@ static void write_xive_lsi(struct ics_state *ics, int srcno)
 }
 
 static void ics_write_xive(struct ics_state *ics, int nr, int server,
-                           uint8_t priority)
+                           uint8_t priority, uint8_t saved_priority)
 {
     int srcno = nr - ics->offset;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
     irq->server = server;
     irq->priority = priority;
+    irq->saved_priority = saved_priority;
+
+    trace_xics_ics_write_xive(nr, srcno, server, priority);
 
-    if (irq->type == XICS_LSI) {
+    if (ics->islsi[srcno]) {
         write_xive_lsi(ics, srcno);
     } else {
         write_xive_msi(ics, srcno);
@@ -281,8 +303,9 @@ static void ics_reject(struct ics_state *ics, int nr)
 {
     struct ics_irq_state *irq = ics->irqs + nr - ics->offset;
 
-    irq->rejected = 1; /* Irrelevant but harmless for LSI */
-    irq->sent = 0; /* Irrelevant but harmless for MSI */
+    trace_xics_ics_reject(nr, nr - ics->offset);
+    irq->status |= XICS_STATUS_REJECTED; /* Irrelevant but harmless for LSI */
+    irq->status &= ~XICS_STATUS_SENT; /* Irrelevant but harmless for MSI */
 }
 
 static void ics_resend(struct ics_state *ics)
@@ -290,10 +313,8 @@ static void ics_resend(struct ics_state *ics)
     int i;
 
     for (i = 0; i < ics->nr_irqs; i++) {
-        struct ics_irq_state *irq = ics->irqs + i;
-
         /* FIXME: filter by server#? */
-        if (irq->type == XICS_LSI) {
+        if (ics->islsi[i]) {
             resend_lsi(ics, i);
         } else {
             resend_msi(ics, i);
@@ -306,8 +327,10 @@ static void ics_eoi(struct ics_state *ics, int nr)
     int srcno = nr - ics->offset;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->type == XICS_LSI) {
-        irq->sent = 0;
+    trace_xics_ics_eoi(nr);
+
+    if (ics->islsi[srcno]) {
+        irq->status &= ~XICS_STATUS_SENT;
     }
 }
 
@@ -317,34 +340,31 @@ static void ics_eoi(struct ics_state *ics, int nr)
 
 qemu_irq xics_get_qirq(struct icp_state *icp, int irq)
 {
-    if ((irq < icp->ics->offset)
-        || (irq >= (icp->ics->offset + icp->ics->nr_irqs))) {
+    if (!ics_valid_irq(icp->ics, irq)) {
         return NULL;
     }
 
     return icp->ics->qirqs[irq - icp->ics->offset];
 }
 
-void xics_set_irq_type(struct icp_state *icp, int irq,
-                       enum xics_irq_type type)
+void xics_set_irq_type(struct icp_state *icp, int irq, bool lsi)
 {
-    assert((irq >= icp->ics->offset)
-           && (irq < (icp->ics->offset + icp->ics->nr_irqs)));
-    assert((type == XICS_MSI) || (type == XICS_LSI));
+    assert(ics_valid_irq(icp->ics, irq));
 
-    icp->ics->irqs[irq - icp->ics->offset].type = type;
+    icp->ics->islsi[irq - icp->ics->offset] = lsi;
 }
 
-static target_ulong h_cppr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_cppr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
+    CPUState *cs = CPU(cpu);
     target_ulong cppr = args[0];
 
-    icp_set_cppr(spapr->icp, env->cpu_index, cppr);
+    icp_set_cppr(spapr->icp, cs->cpu_index, cppr);
     return H_SUCCESS;
 }
 
-static target_ulong h_ipi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_ipi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                           target_ulong opcode, target_ulong *args)
 {
     target_ulong server = args[0];
@@ -356,24 +376,25 @@ static target_ulong h_ipi(CPUPPCState *env, sPAPREnvironment *spapr,
 
     icp_set_mfrr(spapr->icp, server, mfrr);
     return H_SUCCESS;
-
 }
 
-static target_ulong h_xirr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_xirr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
-    uint32_t xirr = icp_accept(spapr->icp->ss + env->cpu_index);
+    CPUState *cs = CPU(cpu);
+    uint32_t xirr = icp_accept(spapr->icp->ss + cs->cpu_index);
 
     args[0] = xirr;
     return H_SUCCESS;
 }
 
-static target_ulong h_eoi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_eoi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                           target_ulong opcode, target_ulong *args)
 {
+    CPUState *cs = CPU(cpu);
     target_ulong xirr = args[0];
 
-    icp_eoi(spapr->icp, env->cpu_index, xirr);
+    icp_eoi(spapr->icp, cs->cpu_index, xirr);
     return H_SUCCESS;
 }
 
@@ -399,7 +420,7 @@ static void rtas_set_xive(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    ics_write_xive(ics, nr, server, priority);
+    ics_write_xive(ics, nr, server, priority, priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
@@ -447,14 +468,8 @@ static void rtas_int_off(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    /* This is a NOP for now, since the described PAPR semantics don't
-     * seem to gel with what Linux does */
-#if 0
-    struct ics_irq_state *irq = xics->irqs + (nr - xics->offset);
-
-    irq->saved_priority = irq->priority;
-    ics_write_xive_msi(xics, nr, irq->server, 0xff);
-#endif
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server, 0xff,
+                   ics->irqs[nr - ics->offset].priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
@@ -478,29 +493,47 @@ static void rtas_int_on(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    /* This is a NOP for now, since the described PAPR semantics don't
-     * seem to gel with what Linux does */
-#if 0
-    struct ics_irq_state *irq = xics->irqs + (nr - xics->offset);
-
-    ics_write_xive_msi(xics, nr, irq->server, irq->saved_priority);
-#endif
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server,
+                   ics->irqs[nr - ics->offset].saved_priority,
+                   ics->irqs[nr - ics->offset].saved_priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
 
+static void xics_reset(void *opaque)
+{
+    struct icp_state *icp = (struct icp_state *)opaque;
+    struct ics_state *ics = icp->ics;
+    int i;
+
+    for (i = 0; i < icp->nr_servers; i++) {
+        icp->ss[i].xirr = 0;
+        icp->ss[i].pending_priority = 0xff;
+        icp->ss[i].mfrr = 0xff;
+        /* Make all outputs are deasserted */
+        qemu_set_irq(icp->ss[i].output, 0);
+    }
+
+    memset(ics->irqs, 0, sizeof(struct ics_irq_state) * ics->nr_irqs);
+    for (i = 0; i < ics->nr_irqs; i++) {
+        ics->irqs[i].priority = 0xff;
+        ics->irqs[i].saved_priority = 0xff;
+    }
+}
+
 struct icp_state *xics_system_init(int nr_irqs)
 {
     CPUPPCState *env;
+    CPUState *cpu;
     int max_server_num;
-    int i;
     struct icp_state *icp;
     struct ics_state *ics;
 
     max_server_num = -1;
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index > max_server_num) {
-            max_server_num = env->cpu_index;
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index > max_server_num) {
+            max_server_num = cpu->cpu_index;
         }
     }
 
@@ -508,12 +541,9 @@ struct icp_state *xics_system_init(int nr_irqs)
     icp->nr_servers = max_server_num + 1;
     icp->ss = g_malloc0(icp->nr_servers*sizeof(struct icp_server_state));
 
-    for (i = 0; i < icp->nr_servers; i++) {
-        icp->ss[i].mfrr = 0xff;
-    }
-
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        struct icp_server_state *ss = &icp->ss[env->cpu_index];
+        cpu = CPU(ppc_env_get_cpu(env));
+        struct icp_server_state *ss = &icp->ss[cpu->cpu_index];
 
         switch (PPC_INPUT(env)) {
         case PPC_FLAGS_INPUT_POWER7:
@@ -533,17 +563,13 @@ struct icp_state *xics_system_init(int nr_irqs)
 
     ics = g_malloc0(sizeof(*ics));
     ics->nr_irqs = nr_irqs;
-    ics->offset = 16;
+    ics->offset = XICS_IRQ_BASE;
     ics->irqs = g_malloc0(nr_irqs * sizeof(struct ics_irq_state));
+    ics->islsi = g_malloc0(nr_irqs * sizeof(bool));
 
     icp->ics = ics;
     ics->icp = icp;
 
-    for (i = 0; i < nr_irqs; i++) {
-        ics->irqs[i].priority = 0xff;
-        ics->irqs[i].saved_priority = 0xff;
-    }
-
     ics->qirqs = qemu_allocate_irqs(ics_set_irq, ics, nr_irqs);
 
     spapr_register_hypercall(H_CPPR, h_cppr);
@@ -556,5 +582,7 @@ struct icp_state *xics_system_init(int nr_irqs)
     spapr_rtas_register("ibm,int-off", rtas_int_off);
     spapr_rtas_register("ibm,int-on", rtas_int_on);
 
+    qemu_register_reset(xics_reset, icp);
+
     return icp;
 }
diff --git a/hw/xics.h b/hw/xics.h
index 99b96ac85aa..c3bf0083e2e 100644
--- a/hw/xics.h
+++ b/hw/xics.h
@@ -28,17 +28,12 @@
 #define __XICS_H__
 
 #define XICS_IPI        0x2
+#define XICS_IRQ_BASE   0x10
 
 struct icp_state;
 
-enum xics_irq_type {
-    XICS_MSI,        /* Message-signalled (edge) interrupt */
-    XICS_LSI,        /* Level-signalled interrupt */
-};
-
 qemu_irq xics_get_qirq(struct icp_state *icp, int irq);
-void xics_set_irq_type(struct icp_state *icp, int irq,
-                       enum xics_irq_type type);
+void xics_set_irq_type(struct icp_state *icp, int irq, bool lsi);
 
 struct icp_state *xics_system_init(int nr_irqs);
 
diff --git a/hw/xilinx.h b/hw/xilinx.h
index 556c5aa9f15..725f2f48987 100644
--- a/hw/xilinx.h
+++ b/hw/xilinx.h
@@ -1,38 +1,42 @@
+#ifndef HW_XILINX_H
+#define HW_XILINX_H 1
+
+
 #include "stream.h"
 #include "qemu-common.h"
-#include "net.h"
+#include "net/net.h"
 
 static inline DeviceState *
-xilinx_intc_create(target_phys_addr_t base, qemu_irq irq, int kind_of_intr)
+xilinx_intc_create(hwaddr base, qemu_irq irq, int kind_of_intr)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "xlnx.xps-intc");
     qdev_prop_set_uint32(dev, "kind-of-intr", kind_of_intr);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
 /* OPB Timer/Counter.  */
 static inline DeviceState *
-xilinx_timer_create(target_phys_addr_t base, qemu_irq irq, int oto, int freq)
+xilinx_timer_create(hwaddr base, qemu_irq irq, int oto, int freq)
 {
     DeviceState *dev;
 
-    dev = qdev_create(NULL, "xlnx,xps-timer");
+    dev = qdev_create(NULL, "xlnx.xps-timer");
     qdev_prop_set_uint32(dev, "one-timer-only", oto);
-    qdev_prop_set_uint32(dev, "frequency", freq);
+    qdev_prop_set_uint32(dev, "clock-frequency", freq);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
 /* XPS Ethernet Lite MAC.  */
 static inline DeviceState *
-xilinx_ethlite_create(NICInfo *nd, target_phys_addr_t base, qemu_irq irq,
+xilinx_ethlite_create(NICInfo *nd, hwaddr base, qemu_irq irq,
                       int txpingpong, int rxpingpong)
 {
     DeviceState *dev;
@@ -44,41 +48,51 @@ xilinx_ethlite_create(NICInfo *nd, target_phys_addr_t base, qemu_irq irq,
     qdev_prop_set_uint32(dev, "tx-ping-pong", txpingpong);
     qdev_prop_set_uint32(dev, "rx-ping-pong", rxpingpong);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
 static inline DeviceState *
 xilinx_axiethernet_create(NICInfo *nd, StreamSlave *peer,
-                          target_phys_addr_t base, qemu_irq irq,
+                          hwaddr base, qemu_irq irq,
                           int txmem, int rxmem)
 {
     DeviceState *dev;
+    Error *errp = NULL;
+
     qemu_check_nic_model(nd, "xlnx.axi-ethernet");
 
     dev = qdev_create(NULL, "xlnx.axi-ethernet");
     qdev_set_nic_properties(dev, nd);
     qdev_prop_set_uint32(dev, "rxmem", rxmem);
     qdev_prop_set_uint32(dev, "txmem", txmem);
-    object_property_set_link(OBJECT(dev), OBJECT(peer), "tx_dev", NULL);
+    object_property_set_link(OBJECT(dev), OBJECT(peer), "axistream-connected",
+                             &errp);
+    assert_no_error(errp);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
 
 static inline void
 xilinx_axiethernetdma_init(DeviceState *dev, StreamSlave *peer,
-                           target_phys_addr_t base, qemu_irq irq,
+                           hwaddr base, qemu_irq irq,
                            qemu_irq irq2, int freqhz)
 {
+    Error *errp = NULL;
+
     qdev_prop_set_uint32(dev, "freqhz", freqhz);
-    object_property_set_link(OBJECT(dev), OBJECT(peer), "tx_dev", NULL);
+    object_property_set_link(OBJECT(dev), OBJECT(peer), "axistream-connected",
+                             &errp);
+    assert_no_error(errp);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, irq2);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, irq2);
 }
+
+#endif
diff --git a/hw/xilinx_axidma.c b/hw/xilinx_axidma.c
index 0e28c517385..d0ee566a28e 100644
--- a/hw/xilinx_axidma.c
+++ b/hw/xilinx_axidma.c
@@ -23,10 +23,9 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "qdev-addr.h"
 
 #include "stream.h"
@@ -140,7 +139,7 @@ static void stream_reset(struct Stream *s)
 }
 
 /* Map an offset addr into a channel index.  */
-static inline int streamid_from_addr(target_phys_addr_t addr)
+static inline int streamid_from_addr(hwaddr addr)
 {
     int sid;
 
@@ -159,7 +158,7 @@ static void stream_desc_show(struct SDesc *d)
 }
 #endif
 
-static void stream_desc_load(struct Stream *s, target_phys_addr_t addr)
+static void stream_desc_load(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
     int i;
@@ -176,7 +175,7 @@ static void stream_desc_load(struct Stream *s, target_phys_addr_t addr)
     }
 }
 
-static void stream_desc_store(struct Stream *s, target_phys_addr_t addr)
+static void stream_desc_store(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
     int i;
@@ -364,7 +363,7 @@ axidma_push(StreamSlave *obj, unsigned char *buf, size_t len, uint32_t *app)
     stream_update_irq(s);
 }
 
-static uint64_t axidma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t axidma_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     struct XilinxAXIDMA *d = opaque;
@@ -399,7 +398,7 @@ static uint64_t axidma_read(void *opaque, target_phys_addr_t addr,
 
 }
 
-static void axidma_write(void *opaque, target_phys_addr_t addr,
+static void axidma_write(void *opaque, hwaddr addr,
                          uint64_t value, unsigned size)
 {
     struct XilinxAXIDMA *d = opaque;
@@ -504,7 +503,7 @@ static void axidma_class_init(ObjectClass *klass, void *data)
     ssc->push = axidma_push;
 }
 
-static TypeInfo axidma_info = {
+static const TypeInfo axidma_info = {
     .name          = "xlnx.axi-dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XilinxAXIDMA),
diff --git a/hw/xilinx_axienet.c b/hw/xilinx_axienet.c
index eec155d4404..51c2896e444 100644
--- a/hw/xilinx_axienet.c
+++ b/hw/xilinx_axienet.c
@@ -23,9 +23,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "net.h"
+#include "qemu/log.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #include "stream.h"
@@ -412,7 +411,7 @@ static void enet_update_irq(struct XilinxAXIEnet *s)
     qemu_set_irq(s->irq, !!s->regs[R_IP]);
 }
 
-static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t enet_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct XilinxAXIEnet *s = opaque;
     uint32_t r = 0;
@@ -503,7 +502,7 @@ static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void enet_write(void *opaque, target_phys_addr_t addr,
+static void enet_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     struct XilinxAXIEnet *s = opaque;
@@ -591,6 +590,10 @@ static void enet_write(void *opaque, target_phys_addr_t addr,
             s->maddr[s->fmi & 3][addr & 1] = value;
             break;
 
+        case R_IS:
+            s->regs[addr] &= ~value;
+            break;
+
         case 0x8000 ... 0x83ff:
             s->ext_mtable[addr - 0x8000] = value;
             break;
@@ -890,7 +893,7 @@ static void xilinx_enet_class_init(ObjectClass *klass, void *data)
     ssc->push = axienet_stream_push;
 }
 
-static TypeInfo xilinx_enet_info = {
+static const TypeInfo xilinx_enet_info = {
     .name          = "xlnx.axi-ethernet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XilinxAXIEnet),
diff --git a/hw/xilinx_ethlite.c b/hw/xilinx_ethlite.c
index 56ca620dd7a..11dfbc3ac18 100644
--- a/hw/xilinx_ethlite.c
+++ b/hw/xilinx_ethlite.c
@@ -24,7 +24,7 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 
 #define D(x)
 #define R_TX_BUF0     0
@@ -72,7 +72,7 @@ static inline void eth_pulse_irq(struct xlx_ethlite *s)
 }
 
 static uint64_t
-eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+eth_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_ethlite *s = opaque;
     uint32_t r = 0;
@@ -89,7 +89,7 @@ eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
         case R_RX_CTRL1:
         case R_RX_CTRL0:
             r = s->regs[addr];
-            D(qemu_log("%s %x=%x\n", __func__, addr * 4, r));
+            D(qemu_log("%s " TARGET_FMT_plx "=%x\n", __func__, addr * 4, r));
             break;
 
         default:
@@ -100,7 +100,7 @@ eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-eth_write(void *opaque, target_phys_addr_t addr,
+eth_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct xlx_ethlite *s = opaque;
@@ -115,7 +115,8 @@ eth_write(void *opaque, target_phys_addr_t addr,
             if (addr == R_TX_CTRL1)
                 base = 0x800 / 4;
 
-            D(qemu_log("%s addr=%x val=%x\n", __func__, addr * 4, value));
+            D(qemu_log("%s addr=" TARGET_FMT_plx " val=%x\n",
+                       __func__, addr * 4, value));
             if ((value & (CTRL_P | CTRL_S)) == CTRL_S) {
                 qemu_send_packet(&s->nic->nc,
                                  (void *) &s->regs[base],
@@ -135,12 +136,16 @@ eth_write(void *opaque, target_phys_addr_t addr,
             break;
 
         /* Keep these native.  */
+        case R_RX_CTRL0:
+        case R_RX_CTRL1:
+            if (!(value & CTRL_S)) {
+                qemu_flush_queued_packets(&s->nic->nc);
+            }
         case R_TX_LEN0:
         case R_TX_LEN1:
         case R_TX_GIE0:
-        case R_RX_CTRL0:
-        case R_RX_CTRL1:
-            D(qemu_log("%s addr=%x val=%x\n", __func__, addr * 4, value));
+            D(qemu_log("%s addr=" TARGET_FMT_plx " val=%x\n",
+                       __func__, addr * 4, value));
             s->regs[addr] = value;
             break;
 
@@ -163,9 +168,9 @@ static const MemoryRegionOps eth_ops = {
 static int eth_can_rx(NetClientState *nc)
 {
     struct xlx_ethlite *s = DO_UPCAST(NICState, nc, nc)->opaque;
-    int r;
-    r = !(s->regs[R_RX_CTRL0] & CTRL_S);
-    return r;
+    unsigned int rxbase = s->rxbuf * (0x800 / 4);
+
+    return !(s->regs[rxbase + R_RX_CTRL0] & CTRL_S);
 }
 
 static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
@@ -182,7 +187,7 @@ static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
         return -1;
     }
 
-    D(qemu_log("%s %d rxbase=%x\n", __func__, size, rxbase));
+    D(qemu_log("%s %zd rxbase=%x\n", __func__, size, rxbase));
     memcpy(&s->regs[rxbase + R_RX_BUF0], buf, size);
 
     s->regs[rxbase + R_RX_CTRL0] |= CTRL_S;
@@ -243,7 +248,7 @@ static void xilinx_ethlite_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_ethlite_properties;
 }
 
-static TypeInfo xilinx_ethlite_info = {
+static const TypeInfo xilinx_ethlite_info = {
     .name          = "xlnx.xps-ethernetlite",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct xlx_ethlite),
diff --git a/hw/xilinx_intc.c b/hw/xilinx_intc.c
index 386fd307436..0c34149c276 100644
--- a/hw/xilinx_intc.c
+++ b/hw/xilinx_intc.c
@@ -74,7 +74,7 @@ static void update_irq(struct xlx_pic *p)
 }
 
 static uint64_t
-pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+pic_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_pic *p = opaque;
     uint32_t r = 0;
@@ -93,7 +93,7 @@ pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-pic_write(void *opaque, target_phys_addr_t addr,
+pic_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct xlx_pic *p = opaque;
@@ -175,7 +175,7 @@ static void xilinx_intc_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_intc_properties;
 }
 
-static TypeInfo xilinx_intc_info = {
+static const TypeInfo xilinx_intc_info = {
     .name          = "xlnx.xps-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct xlx_pic),
diff --git a/hw/xilinx_spi.c b/hw/xilinx_spi.c
new file mode 100644
index 00000000000..be581c2ac5c
--- /dev/null
+++ b/hw/xilinx_spi.c
@@ -0,0 +1,385 @@
+/*
+ * QEMU model of the Xilinx SPI Controller
+ *
+ * Copyright (C) 2010 Edgar E. Iglesias.
+ * Copyright (C) 2012 Peter A. G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ * Copyright (C) 2012 PetaLogix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysbus.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+#include "fifo.h"
+
+#include "ssi.h"
+
+#ifdef XILINX_SPI_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+#define R_DGIER     (0x1c / 4)
+#define R_DGIER_IE  (1 << 31)
+
+#define R_IPISR     (0x20 / 4)
+#define IRQ_DRR_NOT_EMPTY    (1 << (31 - 23))
+#define IRQ_DRR_OVERRUN      (1 << (31 - 26))
+#define IRQ_DRR_FULL         (1 << (31 - 27))
+#define IRQ_TX_FF_HALF_EMPTY (1 << 6)
+#define IRQ_DTR_UNDERRUN     (1 << 3)
+#define IRQ_DTR_EMPTY        (1 << (31 - 29))
+
+#define R_IPIER     (0x28 / 4)
+#define R_SRR       (0x40 / 4)
+#define R_SPICR     (0x60 / 4)
+#define R_SPICR_TXFF_RST     (1 << 5)
+#define R_SPICR_RXFF_RST     (1 << 6)
+#define R_SPICR_MTI          (1 << 8)
+
+#define R_SPISR     (0x64 / 4)
+#define SR_TX_FULL    (1 << 3)
+#define SR_TX_EMPTY   (1 << 2)
+#define SR_RX_FULL    (1 << 1)
+#define SR_RX_EMPTY   (1 << 0)
+
+#define R_SPIDTR    (0x68 / 4)
+#define R_SPIDRR    (0x6C / 4)
+#define R_SPISSR    (0x70 / 4)
+#define R_TX_FF_OCY (0x74 / 4)
+#define R_RX_FF_OCY (0x78 / 4)
+#define R_MAX       (0x7C / 4)
+
+#define FIFO_CAPACITY 256
+
+typedef struct XilinxSPI {
+    SysBusDevice busdev;
+    MemoryRegion mmio;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    qemu_irq *cs_lines;
+
+    SSIBus *spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint32_t regs[R_MAX];
+} XilinxSPI;
+
+static void txfifo_reset(XilinxSPI *s)
+{
+    fifo8_reset(&s->tx_fifo);
+
+    s->regs[R_SPISR] &= ~SR_TX_FULL;
+    s->regs[R_SPISR] |= SR_TX_EMPTY;
+}
+
+static void rxfifo_reset(XilinxSPI *s)
+{
+    fifo8_reset(&s->rx_fifo);
+
+    s->regs[R_SPISR] |= SR_RX_EMPTY;
+    s->regs[R_SPISR] &= ~SR_RX_FULL;
+}
+
+static void xlx_spi_update_cs(XilinxSPI *s)
+{
+   int i;
+
+    for (i = 0; i < s->num_cs; ++i) {
+        qemu_set_irq(s->cs_lines[i], !(~s->regs[R_SPISSR] & 1 << i));
+    }
+}
+
+static void xlx_spi_update_irq(XilinxSPI *s)
+{
+    uint32_t pending;
+
+    s->regs[R_IPISR] |=
+            (!fifo8_is_empty(&s->rx_fifo) ? IRQ_DRR_NOT_EMPTY : 0) |
+            (fifo8_is_full(&s->rx_fifo) ? IRQ_DRR_FULL : 0);
+
+    pending = s->regs[R_IPISR] & s->regs[R_IPIER];
+
+    pending = pending && (s->regs[R_DGIER] & R_DGIER_IE);
+    pending = !!pending;
+
+    /* This call lies right in the data paths so don't call the
+       irq chain unless things really changed.  */
+    if (pending != s->irqline) {
+        s->irqline = pending;
+        DB_PRINT("irq_change of state %d ISR:%x IER:%X\n",
+                    pending, s->regs[R_IPISR], s->regs[R_IPIER]);
+        qemu_set_irq(s->irq, pending);
+    }
+
+}
+
+static void xlx_spi_do_reset(XilinxSPI *s)
+{
+    memset(s->regs, 0, sizeof s->regs);
+
+    rxfifo_reset(s);
+    txfifo_reset(s);
+
+    s->regs[R_SPISSR] = ~0;
+    xlx_spi_update_irq(s);
+    xlx_spi_update_cs(s);
+}
+
+static void xlx_spi_reset(DeviceState *d)
+{
+    xlx_spi_do_reset(DO_UPCAST(XilinxSPI, busdev.qdev, d));
+}
+
+static inline int spi_master_enabled(XilinxSPI *s)
+{
+    return !(s->regs[R_SPICR] & R_SPICR_MTI);
+}
+
+static void spi_flush_txfifo(XilinxSPI *s)
+{
+    uint32_t tx;
+    uint32_t rx;
+
+    while (!fifo8_is_empty(&s->tx_fifo)) {
+        tx = (uint32_t)fifo8_pop(&s->tx_fifo);
+        DB_PRINT("data tx:%x\n", tx);
+        rx = ssi_transfer(s->spi, tx);
+        DB_PRINT("data rx:%x\n", rx);
+        if (fifo8_is_full(&s->rx_fifo)) {
+            s->regs[R_IPISR] |= IRQ_DRR_OVERRUN;
+        } else {
+            fifo8_push(&s->rx_fifo, (uint8_t)rx);
+            if (fifo8_is_full(&s->rx_fifo)) {
+                s->regs[R_SPISR] |= SR_RX_FULL;
+                s->regs[R_IPISR] |= IRQ_DRR_FULL;
+            }
+        }
+
+        s->regs[R_SPISR] &= ~SR_RX_EMPTY;
+        s->regs[R_SPISR] &= ~SR_TX_FULL;
+        s->regs[R_SPISR] |= SR_TX_EMPTY;
+
+        s->regs[R_IPISR] |= IRQ_DTR_EMPTY;
+        s->regs[R_IPISR] |= IRQ_DRR_NOT_EMPTY;
+    }
+
+}
+
+static uint64_t
+spi_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    XilinxSPI *s = opaque;
+    uint32_t r = 0;
+
+    addr >>= 2;
+    switch (addr) {
+    case R_SPIDRR:
+        if (fifo8_is_empty(&s->rx_fifo)) {
+            DB_PRINT("Read from empty FIFO!\n");
+            return 0xdeadbeef;
+        }
+
+        s->regs[R_SPISR] &= ~SR_RX_FULL;
+        r = fifo8_pop(&s->rx_fifo);
+        if (fifo8_is_empty(&s->rx_fifo)) {
+            s->regs[R_SPISR] |= SR_RX_EMPTY;
+        }
+        break;
+
+    case R_SPISR:
+        r = s->regs[addr];
+        break;
+
+    default:
+        if (addr < ARRAY_SIZE(s->regs)) {
+            r = s->regs[addr];
+        }
+        break;
+
+    }
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, r);
+    xlx_spi_update_irq(s);
+    return r;
+}
+
+static void
+spi_write(void *opaque, hwaddr addr,
+            uint64_t val64, unsigned int size)
+{
+    XilinxSPI *s = opaque;
+    uint32_t value = val64;
+
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr, value);
+    addr >>= 2;
+    switch (addr) {
+    case R_SRR:
+        if (value != 0xa) {
+            DB_PRINT("Invalid write to SRR %x\n", value);
+        } else {
+            xlx_spi_do_reset(s);
+        }
+        break;
+
+    case R_SPIDTR:
+        s->regs[R_SPISR] &= ~SR_TX_EMPTY;
+        fifo8_push(&s->tx_fifo, (uint8_t)value);
+        if (fifo8_is_full(&s->tx_fifo)) {
+            s->regs[R_SPISR] |= SR_TX_FULL;
+        }
+        if (!spi_master_enabled(s)) {
+            goto done;
+        } else {
+            DB_PRINT("DTR and master enabled\n");
+        }
+        spi_flush_txfifo(s);
+        break;
+
+    case R_SPISR:
+        DB_PRINT("Invalid write to SPISR %x\n", value);
+        break;
+
+    case R_IPISR:
+        /* Toggle the bits.  */
+        s->regs[addr] ^= value;
+        break;
+
+    /* Slave Select Register.  */
+    case R_SPISSR:
+        s->regs[addr] = value;
+        xlx_spi_update_cs(s);
+        break;
+
+    case R_SPICR:
+        /* FIXME: reset irq and sr state to empty queues.  */
+        if (value & R_SPICR_RXFF_RST) {
+            rxfifo_reset(s);
+        }
+
+        if (value & R_SPICR_TXFF_RST) {
+            txfifo_reset(s);
+        }
+        value &= ~(R_SPICR_RXFF_RST | R_SPICR_TXFF_RST);
+        s->regs[addr] = value;
+
+        if (!(value & R_SPICR_MTI)) {
+            spi_flush_txfifo(s);
+        }
+        break;
+
+    default:
+        if (addr < ARRAY_SIZE(s->regs)) {
+            s->regs[addr] = value;
+        }
+        break;
+    }
+
+done:
+    xlx_spi_update_irq(s);
+}
+
+static const MemoryRegionOps spi_ops = {
+    .read = spi_read,
+    .write = spi_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
+};
+
+static int xilinx_spi_init(SysBusDevice *dev)
+{
+    int i;
+    XilinxSPI *s = FROM_SYSBUS(typeof(*s), dev);
+
+    DB_PRINT("\n");
+
+    s->spi = ssi_create_bus(&dev->qdev, "spi");
+
+    sysbus_init_irq(dev, &s->irq);
+    s->cs_lines = g_new(qemu_irq, s->num_cs);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi);
+    for (i = 0; i < s->num_cs; ++i) {
+        sysbus_init_irq(dev, &s->cs_lines[i]);
+    }
+
+    memory_region_init_io(&s->mmio, &spi_ops, s, "xilinx-spi", R_MAX * 4);
+    sysbus_init_mmio(dev, &s->mmio);
+
+    s->irqline = -1;
+
+    fifo8_create(&s->tx_fifo, FIFO_CAPACITY);
+    fifo8_create(&s->rx_fifo, FIFO_CAPACITY);
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_xilinx_spi = {
+    .name = "xilinx_spi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_FIFO8(tx_fifo, XilinxSPI),
+        VMSTATE_FIFO8(rx_fifo, XilinxSPI),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPI, R_MAX),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property xilinx_spi_properties[] = {
+    DEFINE_PROP_UINT8("num-ss-bits", XilinxSPI, num_cs, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void xilinx_spi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = xilinx_spi_init;
+    dc->reset = xlx_spi_reset;
+    dc->props = xilinx_spi_properties;
+    dc->vmsd = &vmstate_xilinx_spi;
+}
+
+static const TypeInfo xilinx_spi_info = {
+    .name           = "xlnx.xps-spi",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(XilinxSPI),
+    .class_init     = xilinx_spi_class_init,
+};
+
+static void xilinx_spi_register_types(void)
+{
+    type_register_static(&xilinx_spi_info);
+}
+
+type_init(xilinx_spi_register_types)
diff --git a/hw/xilinx_spips.c b/hw/xilinx_spips.c
new file mode 100644
index 00000000000..42e019dc057
--- /dev/null
+++ b/hw/xilinx_spips.c
@@ -0,0 +1,575 @@
+/*
+ * QEMU model of the Xilinx Zynq SPI controller
+ *
+ * Copyright (c) 2012 Peter A. G. Crosthwaite
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysbus.h"
+#include "sysemu/sysemu.h"
+#include "ptimer.h"
+#include "qemu/log.h"
+#include "fifo.h"
+#include "ssi.h"
+#include "qemu/bitops.h"
+
+#ifdef XILINX_SPIPS_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+/* config register */
+#define R_CONFIG            (0x00 / 4)
+#define IFMODE              (1 << 31)
+#define ENDIAN              (1 << 26)
+#define MODEFAIL_GEN_EN     (1 << 17)
+#define MAN_START_COM       (1 << 16)
+#define MAN_START_EN        (1 << 15)
+#define MANUAL_CS           (1 << 14)
+#define CS                  (0xF << 10)
+#define CS_SHIFT            (10)
+#define PERI_SEL            (1 << 9)
+#define REF_CLK             (1 << 8)
+#define FIFO_WIDTH          (3 << 6)
+#define BAUD_RATE_DIV       (7 << 3)
+#define CLK_PH              (1 << 2)
+#define CLK_POL             (1 << 1)
+#define MODE_SEL            (1 << 0)
+
+/* interrupt mechanism */
+#define R_INTR_STATUS       (0x04 / 4)
+#define R_INTR_EN           (0x08 / 4)
+#define R_INTR_DIS          (0x0C / 4)
+#define R_INTR_MASK         (0x10 / 4)
+#define IXR_TX_FIFO_UNDERFLOW   (1 << 6)
+#define IXR_RX_FIFO_FULL        (1 << 5)
+#define IXR_RX_FIFO_NOT_EMPTY   (1 << 4)
+#define IXR_TX_FIFO_FULL        (1 << 3)
+#define IXR_TX_FIFO_NOT_FULL    (1 << 2)
+#define IXR_TX_FIFO_MODE_FAIL   (1 << 1)
+#define IXR_RX_FIFO_OVERFLOW    (1 << 0)
+#define IXR_ALL                 ((IXR_TX_FIFO_UNDERFLOW<<1)-1)
+
+#define R_EN                (0x14 / 4)
+#define R_DELAY             (0x18 / 4)
+#define R_TX_DATA           (0x1C / 4)
+#define R_RX_DATA           (0x20 / 4)
+#define R_SLAVE_IDLE_COUNT  (0x24 / 4)
+#define R_TX_THRES          (0x28 / 4)
+#define R_RX_THRES          (0x2C / 4)
+#define R_TXD1              (0x80 / 4)
+#define R_TXD2              (0x84 / 4)
+#define R_TXD3              (0x88 / 4)
+
+#define R_LQSPI_CFG         (0xa0 / 4)
+#define R_LQSPI_CFG_RESET       0x03A002EB
+#define LQSPI_CFG_LQ_MODE       (1 << 31)
+#define LQSPI_CFG_TWO_MEM       (1 << 30)
+#define LQSPI_CFG_SEP_BUS       (1 << 30)
+#define LQSPI_CFG_U_PAGE        (1 << 28)
+#define LQSPI_CFG_MODE_EN       (1 << 25)
+#define LQSPI_CFG_MODE_WIDTH    8
+#define LQSPI_CFG_MODE_SHIFT    16
+#define LQSPI_CFG_DUMMY_WIDTH   3
+#define LQSPI_CFG_DUMMY_SHIFT   8
+#define LQSPI_CFG_INST_CODE     0xFF
+
+#define R_LQSPI_STS         (0xA4 / 4)
+#define LQSPI_STS_WR_RECVD      (1 << 1)
+
+#define R_MOD_ID            (0xFC / 4)
+
+#define R_MAX (R_MOD_ID+1)
+
+/* size of TXRX FIFOs */
+#define RXFF_A          32
+#define TXFF_A          32
+
+/* 16MB per linear region */
+#define LQSPI_ADDRESS_BITS 24
+/* Bite off 4k chunks at a time */
+#define LQSPI_CACHE_SIZE 1024
+
+#define SNOOP_CHECKING 0xFF
+#define SNOOP_NONE 0xFE
+#define SNOOP_STRIPING 0
+
+typedef struct {
+    SysBusDevice busdev;
+    MemoryRegion iomem;
+    MemoryRegion mmlqspi;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    uint8_t num_busses;
+
+    uint8_t snoop_state;
+    qemu_irq *cs_lines;
+    SSIBus **spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint8_t num_txrx_bytes;
+
+    uint32_t regs[R_MAX];
+
+    uint32_t lqspi_buf[LQSPI_CACHE_SIZE];
+    hwaddr lqspi_cached_addr;
+} XilinxSPIPS;
+
+static inline int num_effective_busses(XilinxSPIPS *s)
+{
+    return (s->regs[R_LQSPI_STS] & LQSPI_CFG_SEP_BUS &&
+            s->regs[R_LQSPI_STS] & LQSPI_CFG_TWO_MEM) ? s->num_busses : 1;
+}
+
+static void xilinx_spips_update_cs_lines(XilinxSPIPS *s)
+{
+    int i, j;
+    bool found = false;
+    int field = s->regs[R_CONFIG] >> CS_SHIFT;
+
+    for (i = 0; i < s->num_cs; i++) {
+        for (j = 0; j < num_effective_busses(s); j++) {
+            int upage = !!(s->regs[R_LQSPI_STS] & LQSPI_CFG_U_PAGE);
+            int cs_to_set = (j * s->num_cs + i + upage) %
+                                (s->num_cs * s->num_busses);
+
+            if (~field & (1 << i) && !found) {
+                DB_PRINT("selecting slave %d\n", i);
+                qemu_set_irq(s->cs_lines[cs_to_set], 0);
+            } else {
+                qemu_set_irq(s->cs_lines[cs_to_set], 1);
+            }
+        }
+        if (~field & (1 << i)) {
+            found = true;
+        }
+    }
+    if (!found) {
+        s->snoop_state = SNOOP_CHECKING;
+    }
+}
+
+static void xilinx_spips_update_ixr(XilinxSPIPS *s)
+{
+    /* These are set/cleared as they occur */
+    s->regs[R_INTR_STATUS] &= (IXR_TX_FIFO_UNDERFLOW | IXR_RX_FIFO_OVERFLOW |
+                                IXR_TX_FIFO_MODE_FAIL);
+    /* these are pure functions of fifo state, set them here */
+    s->regs[R_INTR_STATUS] |=
+        (fifo8_is_full(&s->rx_fifo) ? IXR_RX_FIFO_FULL : 0) |
+        (s->rx_fifo.num >= s->regs[R_RX_THRES] ? IXR_RX_FIFO_NOT_EMPTY : 0) |
+        (fifo8_is_full(&s->tx_fifo) ? IXR_TX_FIFO_FULL : 0) |
+        (s->tx_fifo.num < s->regs[R_TX_THRES] ? IXR_TX_FIFO_NOT_FULL : 0);
+    /* drive external interrupt pin */
+    int new_irqline = !!(s->regs[R_INTR_MASK] & s->regs[R_INTR_STATUS] &
+                                                                IXR_ALL);
+    if (new_irqline != s->irqline) {
+        s->irqline = new_irqline;
+        qemu_set_irq(s->irq, s->irqline);
+    }
+}
+
+static void xilinx_spips_reset(DeviceState *d)
+{
+    XilinxSPIPS *s = DO_UPCAST(XilinxSPIPS, busdev.qdev, d);
+
+    int i;
+    for (i = 0; i < R_MAX; i++) {
+        s->regs[i] = 0;
+    }
+
+    fifo8_reset(&s->rx_fifo);
+    fifo8_reset(&s->rx_fifo);
+    /* non zero resets */
+    s->regs[R_CONFIG] |= MODEFAIL_GEN_EN;
+    s->regs[R_SLAVE_IDLE_COUNT] = 0xFF;
+    s->regs[R_TX_THRES] = 1;
+    s->regs[R_RX_THRES] = 1;
+    /* FIXME: move magic number definition somewhere sensible */
+    s->regs[R_MOD_ID] = 0x01090106;
+    s->regs[R_LQSPI_CFG] = R_LQSPI_CFG_RESET;
+    s->snoop_state = SNOOP_CHECKING;
+    xilinx_spips_update_ixr(s);
+    xilinx_spips_update_cs_lines(s);
+}
+
+static void xilinx_spips_flush_txfifo(XilinxSPIPS *s)
+{
+    for (;;) {
+        int i;
+        uint8_t rx;
+        uint8_t tx = 0;
+
+        for (i = 0; i < num_effective_busses(s); ++i) {
+            if (!i || s->snoop_state == SNOOP_STRIPING) {
+                if (fifo8_is_empty(&s->tx_fifo)) {
+                    s->regs[R_INTR_STATUS] |= IXR_TX_FIFO_UNDERFLOW;
+                    xilinx_spips_update_ixr(s);
+                    return;
+                } else {
+                    tx = fifo8_pop(&s->tx_fifo);
+                }
+            }
+            rx = ssi_transfer(s->spi[i], (uint32_t)tx);
+            DB_PRINT("tx = %02x rx = %02x\n", tx, rx);
+            if (!i || s->snoop_state == SNOOP_STRIPING) {
+                if (fifo8_is_full(&s->rx_fifo)) {
+                    s->regs[R_INTR_STATUS] |= IXR_RX_FIFO_OVERFLOW;
+                    DB_PRINT("rx FIFO overflow");
+                } else {
+                    fifo8_push(&s->rx_fifo, (uint8_t)rx);
+                }
+            }
+        }
+
+        switch (s->snoop_state) {
+        case (SNOOP_CHECKING):
+            switch (tx) { /* new instruction code */
+            case 0x0b: /* dual/quad output read DOR/QOR */
+            case 0x6b:
+                s->snoop_state = 4;
+                break;
+            /* FIXME: these vary between vendor - set to spansion */
+            case 0xbb: /* high performance dual read DIOR */
+                s->snoop_state = 4;
+                break;
+            case 0xeb: /* high performance quad read QIOR */
+                s->snoop_state = 6;
+                break;
+            default:
+                s->snoop_state = SNOOP_NONE;
+            }
+            break;
+        case (SNOOP_STRIPING):
+        case (SNOOP_NONE):
+            break;
+        default:
+            s->snoop_state--;
+        }
+    }
+}
+
+static inline void rx_data_bytes(XilinxSPIPS *s, uint32_t *value, int max)
+{
+    int i;
+
+    *value = 0;
+    for (i = 0; i < max && !fifo8_is_empty(&s->rx_fifo); ++i) {
+        uint32_t next = fifo8_pop(&s->rx_fifo) & 0xFF;
+        *value |= next << 8 * (s->regs[R_CONFIG] & ENDIAN ? 3-i : i);
+    }
+}
+
+static uint64_t xilinx_spips_read(void *opaque, hwaddr addr,
+                                                        unsigned size)
+{
+    XilinxSPIPS *s = opaque;
+    uint32_t mask = ~0;
+    uint32_t ret;
+
+    addr >>= 2;
+    switch (addr) {
+    case R_CONFIG:
+        mask = 0x0002FFFF;
+        break;
+    case R_INTR_STATUS:
+    case R_INTR_MASK:
+        mask = IXR_ALL;
+        break;
+    case  R_EN:
+        mask = 0x1;
+        break;
+    case R_SLAVE_IDLE_COUNT:
+        mask = 0xFF;
+        break;
+    case R_MOD_ID:
+        mask = 0x01FFFFFF;
+        break;
+    case R_INTR_EN:
+    case R_INTR_DIS:
+    case R_TX_DATA:
+        mask = 0;
+        break;
+    case R_RX_DATA:
+        rx_data_bytes(s, &ret, s->num_txrx_bytes);
+        DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, ret);
+        xilinx_spips_update_ixr(s);
+        return ret;
+    }
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, s->regs[addr] & mask);
+    return s->regs[addr] & mask;
+
+}
+
+static inline void tx_data_bytes(XilinxSPIPS *s, uint32_t value, int num)
+{
+    int i;
+    for (i = 0; i < num && !fifo8_is_full(&s->tx_fifo); ++i) {
+        if (s->regs[R_CONFIG] & ENDIAN) {
+            fifo8_push(&s->tx_fifo, (uint8_t)(value >> 24));
+            value <<= 8;
+        } else {
+            fifo8_push(&s->tx_fifo, (uint8_t)value);
+            value >>= 8;
+        }
+    }
+}
+
+static void xilinx_spips_write(void *opaque, hwaddr addr,
+                                        uint64_t value, unsigned size)
+{
+    int mask = ~0;
+    int man_start_com = 0;
+    XilinxSPIPS *s = opaque;
+
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr, (unsigned)value);
+    addr >>= 2;
+    switch (addr) {
+    case R_CONFIG:
+        mask = 0x0002FFFF;
+        if (value & MAN_START_COM) {
+            man_start_com = 1;
+        }
+        break;
+    case R_INTR_STATUS:
+        mask = IXR_ALL;
+        s->regs[R_INTR_STATUS] &= ~(mask & value);
+        goto no_reg_update;
+    case R_INTR_DIS:
+        mask = IXR_ALL;
+        s->regs[R_INTR_MASK] &= ~(mask & value);
+        goto no_reg_update;
+    case R_INTR_EN:
+        mask = IXR_ALL;
+        s->regs[R_INTR_MASK] |= mask & value;
+        goto no_reg_update;
+    case R_EN:
+        mask = 0x1;
+        break;
+    case R_SLAVE_IDLE_COUNT:
+        mask = 0xFF;
+        break;
+    case R_RX_DATA:
+    case R_INTR_MASK:
+    case R_MOD_ID:
+        mask = 0;
+        break;
+    case R_TX_DATA:
+        tx_data_bytes(s, (uint32_t)value, s->num_txrx_bytes);
+        goto no_reg_update;
+    case R_TXD1:
+        tx_data_bytes(s, (uint32_t)value, 1);
+        goto no_reg_update;
+    case R_TXD2:
+        tx_data_bytes(s, (uint32_t)value, 2);
+        goto no_reg_update;
+    case R_TXD3:
+        tx_data_bytes(s, (uint32_t)value, 3);
+        goto no_reg_update;
+    }
+    s->regs[addr] = (s->regs[addr] & ~mask) | (value & mask);
+no_reg_update:
+    if (man_start_com) {
+        xilinx_spips_flush_txfifo(s);
+    }
+    xilinx_spips_update_ixr(s);
+    xilinx_spips_update_cs_lines(s);
+}
+
+static const MemoryRegionOps spips_ops = {
+    .read = xilinx_spips_read,
+    .write = xilinx_spips_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+#define LQSPI_CACHE_SIZE 1024
+
+static uint64_t
+lqspi_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    int i;
+    XilinxSPIPS *s = opaque;
+
+    if (addr >= s->lqspi_cached_addr &&
+            addr <= s->lqspi_cached_addr + LQSPI_CACHE_SIZE - 4) {
+        return s->lqspi_buf[(addr - s->lqspi_cached_addr) >> 2];
+    } else {
+        int flash_addr = (addr / num_effective_busses(s));
+        int slave = flash_addr >> LQSPI_ADDRESS_BITS;
+        int cache_entry = 0;
+
+        DB_PRINT("config reg status: %08x\n", s->regs[R_LQSPI_CFG]);
+
+        fifo8_reset(&s->tx_fifo);
+        fifo8_reset(&s->rx_fifo);
+
+        s->regs[R_CONFIG] &= ~CS;
+        s->regs[R_CONFIG] |= (~(1 << slave) << CS_SHIFT) & CS;
+        xilinx_spips_update_cs_lines(s);
+
+        /* instruction */
+        DB_PRINT("pushing read instruction: %02x\n",
+                 (uint8_t)(s->regs[R_LQSPI_CFG] & LQSPI_CFG_INST_CODE));
+        fifo8_push(&s->tx_fifo, s->regs[R_LQSPI_CFG] & LQSPI_CFG_INST_CODE);
+        /* read address */
+        DB_PRINT("pushing read address %06x\n", flash_addr);
+        fifo8_push(&s->tx_fifo, (uint8_t)(flash_addr >> 16));
+        fifo8_push(&s->tx_fifo, (uint8_t)(flash_addr >> 8));
+        fifo8_push(&s->tx_fifo, (uint8_t)flash_addr);
+        /* mode bits */
+        if (s->regs[R_LQSPI_CFG] & LQSPI_CFG_MODE_EN) {
+            fifo8_push(&s->tx_fifo, extract32(s->regs[R_LQSPI_CFG],
+                                              LQSPI_CFG_MODE_SHIFT,
+                                              LQSPI_CFG_MODE_WIDTH));
+        }
+        /* dummy bytes */
+        for (i = 0; i < (extract32(s->regs[R_LQSPI_CFG], LQSPI_CFG_DUMMY_SHIFT,
+                                   LQSPI_CFG_DUMMY_WIDTH)); ++i) {
+            DB_PRINT("pushing dummy byte\n");
+            fifo8_push(&s->tx_fifo, 0);
+        }
+        xilinx_spips_flush_txfifo(s);
+        fifo8_reset(&s->rx_fifo);
+
+        DB_PRINT("starting QSPI data read\n");
+
+        for (i = 0; i < LQSPI_CACHE_SIZE / 4; ++i) {
+            tx_data_bytes(s, 0, 4);
+            xilinx_spips_flush_txfifo(s);
+            rx_data_bytes(s, &s->lqspi_buf[cache_entry], 4);
+            cache_entry++;
+        }
+
+        s->regs[R_CONFIG] |= CS;
+        xilinx_spips_update_cs_lines(s);
+
+        s->lqspi_cached_addr = addr;
+        return lqspi_read(opaque, addr, size);
+    }
+}
+
+static const MemoryRegionOps lqspi_ops = {
+    .read = lqspi_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
+};
+
+static int xilinx_spips_init(SysBusDevice *dev)
+{
+    XilinxSPIPS *s = FROM_SYSBUS(typeof(*s), dev);
+    int i;
+
+    DB_PRINT("inited device model\n");
+
+    s->spi = g_new(SSIBus *, s->num_busses);
+    for (i = 0; i < s->num_busses; ++i) {
+        char bus_name[16];
+        snprintf(bus_name, 16, "spi%d", i);
+        s->spi[i] = ssi_create_bus(&dev->qdev, bus_name);
+    }
+
+    s->cs_lines = g_new(qemu_irq, s->num_cs * s->num_busses);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi[0]);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi[1]);
+    sysbus_init_irq(dev, &s->irq);
+    for (i = 0; i < s->num_cs * s->num_busses; ++i) {
+        sysbus_init_irq(dev, &s->cs_lines[i]);
+    }
+
+    memory_region_init_io(&s->iomem, &spips_ops, s, "spi", R_MAX*4);
+    sysbus_init_mmio(dev, &s->iomem);
+
+    memory_region_init_io(&s->mmlqspi, &lqspi_ops, s, "lqspi",
+                          (1 << LQSPI_ADDRESS_BITS) * 2);
+    sysbus_init_mmio(dev, &s->mmlqspi);
+
+    s->irqline = -1;
+    s->lqspi_cached_addr = ~0ULL;
+
+    fifo8_create(&s->rx_fifo, RXFF_A);
+    fifo8_create(&s->tx_fifo, TXFF_A);
+
+    return 0;
+}
+
+static int xilinx_spips_post_load(void *opaque, int version_id)
+{
+    xilinx_spips_update_ixr((XilinxSPIPS *)opaque);
+    xilinx_spips_update_cs_lines((XilinxSPIPS *)opaque);
+    return 0;
+}
+
+static const VMStateDescription vmstate_xilinx_spips = {
+    .name = "xilinx_spips",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
+    .post_load = xilinx_spips_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_FIFO8(tx_fifo, XilinxSPIPS),
+        VMSTATE_FIFO8(rx_fifo, XilinxSPIPS),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, R_MAX),
+        VMSTATE_UINT8(snoop_state, XilinxSPIPS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property xilinx_spips_properties[] = {
+    DEFINE_PROP_UINT8("num-busses", XilinxSPIPS, num_busses, 1),
+    DEFINE_PROP_UINT8("num-ss-bits", XilinxSPIPS, num_cs, 4),
+    DEFINE_PROP_UINT8("num-txrx-bytes", XilinxSPIPS, num_txrx_bytes, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+static void xilinx_spips_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+
+    sdc->init = xilinx_spips_init;
+    dc->reset = xilinx_spips_reset;
+    dc->props = xilinx_spips_properties;
+    dc->vmsd = &vmstate_xilinx_spips;
+}
+
+static const TypeInfo xilinx_spips_info = {
+    .name  = "xilinx,spips",
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(XilinxSPIPS),
+    .class_init = xilinx_spips_class_init,
+};
+
+static void xilinx_spips_register_types(void)
+{
+    type_register_static(&xilinx_spips_info);
+}
+
+type_init(xilinx_spips_register_types)
diff --git a/hw/xilinx_timer.c b/hw/xilinx_timer.c
index b562bd065e2..aa162efaadb 100644
--- a/hw/xilinx_timer.c
+++ b/hw/xilinx_timer.c
@@ -24,6 +24,7 @@
 
 #include "sysbus.h"
 #include "ptimer.h"
+#include "qemu/log.h"
 
 #define D(x)
 
@@ -71,7 +72,7 @@ static inline unsigned int num_timers(struct timerblock *t)
     return 2 - t->one_timer_only;
 }
 
-static inline unsigned int timer_from_addr(target_phys_addr_t addr)
+static inline unsigned int timer_from_addr(hwaddr addr)
 {
     /* Timers get a 4x32bit control reg area each.  */
     return addr >> 2;
@@ -92,7 +93,7 @@ static void timer_update_irq(struct timerblock *t)
 }
 
 static uint64_t
-timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+timer_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct timerblock *t = opaque;
     struct xlx_timer *xt;
@@ -119,7 +120,7 @@ timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
             break;
 
     }
-    D(printf("%s timer=%d %x=%x\n", __func__, timer, addr * 4, r));
+    D(fprintf(stderr, "%s timer=%d %x=%x\n", __func__, timer, addr * 4, r));
     return r;
 }
 
@@ -127,7 +128,7 @@ static void timer_enable(struct xlx_timer *xt)
 {
     uint64_t count;
 
-    D(printf("%s timer=%d down=%d\n", __func__,
+    D(fprintf(stderr, "%s timer=%d down=%d\n", __func__,
               xt->nr, xt->regs[R_TCSR] & TCSR_UDT));
 
     ptimer_stop(xt->ptimer);
@@ -141,7 +142,7 @@ static void timer_enable(struct xlx_timer *xt)
 }
 
 static void
-timer_write(void *opaque, target_phys_addr_t addr,
+timer_write(void *opaque, hwaddr addr,
             uint64_t val64, unsigned int size)
 {
     struct timerblock *t = opaque;
@@ -152,7 +153,7 @@ timer_write(void *opaque, target_phys_addr_t addr,
     addr >>= 2;
     timer = timer_from_addr(addr);
     xt = &t->timers[timer];
-    D(printf("%s addr=%x val=%x (timer=%d off=%d)\n",
+    D(fprintf(stderr, "%s addr=%x val=%x (timer=%d off=%d)\n",
              __func__, addr * 4, value, timer, addr & 3));
     /* Further decoding to address a specific timers reg.  */
     addr &= 3;
@@ -189,7 +190,7 @@ static void timer_hit(void *opaque)
 {
     struct xlx_timer *xt = opaque;
     struct timerblock *t = xt->parent;
-    D(printf("%s %d\n", __func__, timer));
+    D(fprintf(stderr, "%s %d\n", __func__, xt->nr));
     xt->regs[R_TCSR] |= TCSR_TINT;
 
     if (xt->regs[R_TCSR] & TCSR_ARHT)
@@ -217,14 +218,15 @@ static int xilinx_timer_init(SysBusDevice *dev)
         ptimer_set_freq(xt->ptimer, t->freq_hz);
     }
 
-    memory_region_init_io(&t->mmio, &timer_ops, t, "xlnx,xps-timer",
+    memory_region_init_io(&t->mmio, &timer_ops, t, "xlnx.xps-timer",
                           R_MAX * 4 * num_timers(t));
     sysbus_init_mmio(dev, &t->mmio);
     return 0;
 }
 
 static Property xilinx_timer_properties[] = {
-    DEFINE_PROP_UINT32("frequency", struct timerblock, freq_hz,   62 * 1000000),
+    DEFINE_PROP_UINT32("clock-frequency", struct timerblock, freq_hz,
+                                                                62 * 1000000),
     DEFINE_PROP_UINT8("one-timer-only", struct timerblock, one_timer_only, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -238,8 +240,8 @@ static void xilinx_timer_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_timer_properties;
 }
 
-static TypeInfo xilinx_timer_info = {
-    .name          = "xlnx,xps-timer",
+static const TypeInfo xilinx_timer_info = {
+    .name          = "xlnx.xps-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct timerblock),
     .class_init    = xilinx_timer_class_init,
diff --git a/hw/xilinx_uartlite.c b/hw/xilinx_uartlite.c
index d0f32db2c6c..9963982ef66 100644
--- a/hw/xilinx_uartlite.c
+++ b/hw/xilinx_uartlite.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #define DUART(x)
 
@@ -84,7 +84,7 @@ static void uart_update_status(struct xlx_uartlite *s)
 }
 
 static uint64_t
-uart_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+uart_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_uartlite *s = opaque;
     uint32_t r = 0;
@@ -97,6 +97,7 @@ uart_read(void *opaque, target_phys_addr_t addr, unsigned int size)
                 s->rx_fifo_len--;
             uart_update_status(s);
             uart_update_irq(s);
+            qemu_chr_accept_input(s->chr);
             break;
 
         default:
@@ -109,7 +110,7 @@ uart_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-uart_write(void *opaque, target_phys_addr_t addr,
+uart_write(void *opaque, hwaddr addr,
            uint64_t val64, unsigned int size)
 {
     struct xlx_uartlite *s = opaque;
@@ -182,12 +183,8 @@ static void uart_rx(void *opaque, const uint8_t *buf, int size)
 static int uart_can_rx(void *opaque)
 {
     struct xlx_uartlite *s = opaque;
-    int r;
 
-    r = s->rx_fifo_len < sizeof(s->rx_fifo);
-    if (!r)
-        printf("cannot receive!\n");
-    return r;
+    return s->rx_fifo_len < sizeof(s->rx_fifo);
 }
 
 static void uart_event(void *opaque, int event)
@@ -219,7 +216,7 @@ static void xilinx_uartlite_class_init(ObjectClass *klass, void *data)
     sdc->init = xilinx_uartlite_init;
 }
 
-static TypeInfo xilinx_uartlite_info = {
+static const TypeInfo xilinx_uartlite_info = {
     .name          = "xlnx.xps-uartlite",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (struct xlx_uartlite),
diff --git a/hw/xilinx_zynq.c b/hw/xilinx_zynq.c
index 7e6c27359ec..0ac33b5dab6 100644
--- a/hw/xilinx_zynq.c
+++ b/hw/xilinx_zynq.c
@@ -17,13 +17,18 @@
 
 #include "sysbus.h"
 #include "arm-misc.h"
-#include "net.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "loader.h"
+#include "ssi.h"
+
+#define NUM_SPI_FLASHES 4
+#define NUM_QSPI_FLASHES 2
+#define NUM_QSPI_BUSSES 2
 
 #define FLASH_SIZE (64 * 1024 * 1024)
 #define FLASH_SECTOR_SIZE (128 * 1024)
@@ -41,15 +46,60 @@ static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "cadence_gem");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
 
-static void zynq_init(ram_addr_t ram_size, const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static inline void zynq_init_spi_flashes(uint32_t base_addr, qemu_irq irq,
+                                         bool is_qspi)
 {
+    DeviceState *dev;
+    SysBusDevice *busdev;
+    SSIBus *spi;
+    DeviceState *flash_dev;
+    int i, j;
+    int num_busses =  is_qspi ? NUM_QSPI_BUSSES : 1;
+    int num_ss = is_qspi ? NUM_QSPI_FLASHES : NUM_SPI_FLASHES;
+
+    dev = qdev_create(NULL, "xilinx,spips");
+    qdev_prop_set_uint8(dev, "num-txrx-bytes", is_qspi ? 4 : 1);
+    qdev_prop_set_uint8(dev, "num-ss-bits", num_ss);
+    qdev_prop_set_uint8(dev, "num-busses", num_busses);
+    qdev_init_nofail(dev);
+    busdev = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(busdev, 0, base_addr);
+    if (is_qspi) {
+        sysbus_mmio_map(busdev, 1, 0xFC000000);
+    }
+    sysbus_connect_irq(busdev, 0, irq);
+
+    for (i = 0; i < num_busses; ++i) {
+        char bus_name[16];
+        qemu_irq cs_line;
+
+        snprintf(bus_name, 16, "spi%d", i);
+        spi = (SSIBus *)qdev_get_child_bus(dev, bus_name);
+
+        for (j = 0; j < num_ss; ++j) {
+            flash_dev = ssi_create_slave_no_init(spi, "m25p80");
+            qdev_prop_set_string(flash_dev, "partname", "n25q128");
+            qdev_init_nofail(flash_dev);
+
+            cs_line = qdev_get_gpio_in(flash_dev, 0);
+            sysbus_connect_irq(busdev, i * num_ss + j + 1, cs_line);
+        }
+    }
+
+}
+
+static void zynq_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ext_ram = g_new(MemoryRegion, 1);
@@ -100,12 +150,12 @@ static void zynq_init(ram_addr_t ram_size, const char *boot_device,
 
     dev = qdev_create(NULL, "xilinx,zynq_slcr");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xF8000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xF8000000);
 
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", 1);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xF8F00000);
     sysbus_connect_irq(busdev, 0, cpu_irq);
 
@@ -113,6 +163,13 @@ static void zynq_init(ram_addr_t ram_size, const char *boot_device,
         pic[n] = qdev_get_gpio_in(dev, n);
     }
 
+    zynq_init_spi_flashes(0xE0006000, pic[58-IRQ_OFFSET], false);
+    zynq_init_spi_flashes(0xE0007000, pic[81-IRQ_OFFSET], false);
+    zynq_init_spi_flashes(0xE000D000, pic[51-IRQ_OFFSET], true);
+
+    sysbus_create_simple("xlnx,ps7-usb", 0xE0002000, pic[53-IRQ_OFFSET]);
+    sysbus_create_simple("xlnx,ps7-usb", 0xE0003000, pic[75-IRQ_OFFSET]);
+
     sysbus_create_simple("cadence_uart", 0xE0000000, pic[59-IRQ_OFFSET]);
     sysbus_create_simple("cadence_uart", 0xE0001000, pic[82-IRQ_OFFSET]);
 
@@ -144,9 +201,10 @@ static QEMUMachine zynq_machine = {
     .name = "xilinx-zynq-a9",
     .desc = "Xilinx Zynq Platform Baseboard for Cortex-A9",
     .init = zynq_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 1,
-    .no_sdcard = 1
+    .no_sdcard = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void zynq_machine_init(void)
diff --git a/hw/xio3130_downstream.c b/hw/xio3130_downstream.c
index 0d8a5e7020c..7f00bc82564 100644
--- a/hw/xio3130_downstream.c
+++ b/hw/xio3130_downstream.c
@@ -19,9 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "xio3130_downstream.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130D       0x8233  /* downstream port */
@@ -193,7 +193,7 @@ static void xio3130_downstream_class_init(ObjectClass *klass, void *data)
     dc->props = xio3130_downstream_properties;
 }
 
-static TypeInfo xio3130_downstream_info = {
+static const TypeInfo xio3130_downstream_info = {
     .name          = "xio3130-downstream",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIESlot),
diff --git a/hw/xio3130_downstream.h b/hw/xio3130_downstream.h
index 010487f2d9f..559dff65652 100644
--- a/hw/xio3130_downstream.h
+++ b/hw/xio3130_downstream.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_XIO3130_DOWNSTREAM_H
 #define QEMU_XIO3130_DOWNSTREAM_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIESlot *xio3130_downstream_init(PCIBus *bus, int devfn, bool multifunction,
                                   const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/xio3130_upstream.c b/hw/xio3130_upstream.c
index d46b86c74d8..70b15d37c8f 100644
--- a/hw/xio3130_upstream.c
+++ b/hw/xio3130_upstream.c
@@ -19,9 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "xio3130_upstream.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130U       0x8232  /* upstream port */
@@ -167,7 +167,7 @@ static void xio3130_upstream_class_init(ObjectClass *klass, void *data)
     dc->props = xio3130_upstream_properties;
 }
 
-static TypeInfo xio3130_upstream_info = {
+static const TypeInfo xio3130_upstream_info = {
     .name          = "x3130-upstream",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIEPort),
diff --git a/hw/xio3130_upstream.h b/hw/xio3130_upstream.h
index e9969975ff4..fa09656b352 100644
--- a/hw/xio3130_upstream.h
+++ b/hw/xio3130_upstream.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_XIO3130_UPSTREAM_H
 #define QEMU_XIO3130_UPSTREAM_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,
                                 const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/xtensa_lx60.c b/hw/xtensa_lx60.c
index 3653f65b1e5..a810b9eae08 100644
--- a/hw/xtensa_lx60.c
+++ b/hw/xtensa_lx60.c
@@ -25,16 +25,18 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "memory.h"
-#include "exec-memory.h"
-#include "pc.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "serial.h"
+#include "net/net.h"
 #include "sysbus.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
+#include "char/char.h"
 #include "xtensa_bootparam.h"
 
 typedef struct LxBoardDesc {
@@ -57,7 +59,7 @@ static void lx60_fpga_reset(void *opaque)
     s->switches = 0;
 }
 
-static uint64_t lx60_fpga_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lx60_fpga_read(void *opaque, hwaddr addr,
         unsigned size)
 {
     Lx60FpgaState *s = opaque;
@@ -78,7 +80,7 @@ static uint64_t lx60_fpga_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void lx60_fpga_write(void *opaque, target_phys_addr_t addr,
+static void lx60_fpga_write(void *opaque, hwaddr addr,
         uint64_t val, unsigned size)
 {
     Lx60FpgaState *s = opaque;
@@ -103,7 +105,7 @@ static const MemoryRegionOps lx60_fpga_ops = {
 };
 
 static Lx60FpgaState *lx60_fpga_init(MemoryRegion *address_space,
-        target_phys_addr_t base)
+        hwaddr base)
 {
     Lx60FpgaState *s = g_malloc(sizeof(Lx60FpgaState));
 
@@ -116,9 +118,9 @@ static Lx60FpgaState *lx60_fpga_init(MemoryRegion *address_space,
 }
 
 static void lx60_net_init(MemoryRegion *address_space,
-        target_phys_addr_t base,
-        target_phys_addr_t descriptors,
-        target_phys_addr_t buffers,
+        hwaddr base,
+        hwaddr descriptors,
+        hwaddr buffers,
         qemu_irq irq, NICInfo *nd)
 {
     DeviceState *dev;
@@ -129,7 +131,7 @@ static void lx60_net_init(MemoryRegion *address_space,
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(address_space, base,
             sysbus_mmio_get_region(s, 0));
@@ -154,10 +156,7 @@ static void lx60_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void lx_init(const LxBoardDesc *board,
-        ram_addr_t ram_size, const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void lx_init(const LxBoardDesc *board, QEMUMachineInitArgs *args)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     int be = 1;
@@ -170,6 +169,9 @@ static void lx_init(const LxBoardDesc *board,
     MemoryRegion *ram, *rom, *system_io;
     DriveInfo *dinfo;
     pflash_t *flash = NULL;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     int n;
 
     if (!cpu_model) {
@@ -193,7 +195,7 @@ static void lx_init(const LxBoardDesc *board,
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, "lx60.dram", ram_size);
+    memory_region_init_ram(ram, "lx60.dram", args->ram_size);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -268,34 +270,24 @@ static void lx_init(const LxBoardDesc *board,
     }
 }
 
-static void xtensa_lx60_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void xtensa_lx60_init(QEMUMachineInitArgs *args)
 {
     static const LxBoardDesc lx60_board = {
         .flash_size = 0x400000,
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
     };
-    lx_init(&lx60_board, ram_size, boot_device,
-            kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
+    lx_init(&lx60_board, args);
 }
 
-static void xtensa_lx200_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void xtensa_lx200_init(QEMUMachineInitArgs *args)
 {
     static const LxBoardDesc lx200_board = {
         .flash_size = 0x1000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
-    lx_init(&lx200_board, ram_size, boot_device,
-            kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
+    lx_init(&lx200_board, args);
 }
 
 static QEMUMachine xtensa_lx60_machine = {
@@ -303,6 +295,7 @@ static QEMUMachine xtensa_lx60_machine = {
     .desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
     .init = xtensa_lx60_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine xtensa_lx200_machine = {
@@ -310,6 +303,7 @@ static QEMUMachine xtensa_lx200_machine = {
     .desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
     .init = xtensa_lx200_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xtensa_lx_machines_init(void)
diff --git a/hw/xtensa_pic.c b/hw/xtensa_pic.c
index 653ded68201..97d36be2725 100644
--- a/hw/xtensa_pic.c
+++ b/hw/xtensa_pic.c
@@ -26,8 +26,8 @@
  */
 
 #include "hw.h"
-#include "qemu-log.h"
-#include "qemu-timer.h"
+#include "qemu/log.h"
+#include "qemu/timer.h"
 
 void xtensa_advance_ccount(CPUXtensaState *env, uint32_t d)
 {
@@ -125,12 +125,13 @@ void xtensa_rearm_ccompare_timer(CPUXtensaState *env)
 
 static void xtensa_ccompare_cb(void *opaque)
 {
-    CPUXtensaState *env = opaque;
+    XtensaCPU *cpu = opaque;
+    CPUXtensaState *env = &cpu->env;
 
     if (env->halted) {
         env->halt_clock = qemu_get_clock_ns(vm_clock);
         xtensa_advance_ccount(env, env->wake_ccount - env->sregs[CCOUNT]);
-        if (!cpu_has_work(env)) {
+        if (!cpu_has_work(CPU(cpu))) {
             env->sregs[CCOUNT] = env->wake_ccount + 1;
             xtensa_rearm_ccompare_timer(env);
         }
@@ -139,12 +140,14 @@ static void xtensa_ccompare_cb(void *opaque)
 
 void xtensa_irq_init(CPUXtensaState *env)
 {
+    XtensaCPU *cpu = xtensa_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(
             xtensa_set_irq, env, env->config->ninterrupt);
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_TIMER_INTERRUPT) &&
             env->config->nccompare > 0) {
         env->ccompare_timer =
-            qemu_new_timer_ns(vm_clock, &xtensa_ccompare_cb, env);
+            qemu_new_timer_ns(vm_clock, &xtensa_ccompare_cb, cpu);
     }
 }
 
diff --git a/hw/xtensa_sim.c b/hw/xtensa_sim.c
index 831460b7c48..864e57c52ca 100644
--- a/hw/xtensa_sim.c
+++ b/hw/xtensa_sim.c
@@ -25,12 +25,12 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 static uint64_t translate_phys_addr(void *env, uint64_t addr)
 {
@@ -44,16 +44,20 @@ static void sim_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void sim_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void xtensa_sim_init(QEMUMachineInitArgs *args)
 {
     XtensaCPU *cpu = NULL;
     CPUXtensaState *env = NULL;
     MemoryRegion *ram, *rom;
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     int n;
 
+    if (!cpu_model) {
+        cpu_model = XTENSA_DEFAULT_CPU_MODEL;
+    }
+
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_xtensa_init(cpu_model);
         if (cpu == NULL) {
@@ -96,24 +100,13 @@ static void sim_init(ram_addr_t ram_size,
     }
 }
 
-static void xtensa_sim_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
-{
-    if (!cpu_model) {
-        cpu_model = XTENSA_DEFAULT_CPU_MODEL;
-    }
-    sim_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
-}
-
 static QEMUMachine xtensa_sim_machine = {
     .name = "sim",
     .desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")",
     .is_default = true,
     .init = xtensa_sim_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xtensa_sim_machine_init(void)
diff --git a/hw/z2.c b/hw/z2.c
index 289cee90c41..731550f2d86 100644
--- a/hw/z2.c
+++ b/hw/z2.c
@@ -18,12 +18,12 @@
 #include "i2c.h"
 #include "ssi.h"
 #include "boards.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "console.h"
+#include "sysemu/blockdev.h"
+#include "ui/console.h"
 #include "audio/audio.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #ifdef DEBUG_Z2
 #define DPRINTF(fmt, ...) \
@@ -161,10 +161,11 @@ static int zipit_lcd_init(SSISlave *dev)
 
 static VMStateDescription vmstate_zipit_lcd_state = {
     .name = "zipit-lcd",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
     .fields = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, ZipitLCD),
         VMSTATE_INT32(selected, ZipitLCD),
         VMSTATE_INT32(enabled, ZipitLCD),
         VMSTATE_BUFFER(buf, ZipitLCD),
@@ -184,7 +185,7 @@ static void zipit_lcd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_zipit_lcd_state;
 }
 
-static TypeInfo zipit_lcd_info = {
+static const TypeInfo zipit_lcd_info = {
     .name          = "zipit-lcd",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ZipitLCD),
@@ -287,18 +288,19 @@ static void aer915_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_aer915_state;
 }
 
-static TypeInfo aer915_info = {
+static const TypeInfo aer915_info = {
     .name          = "aer915",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(AER915State),
     .class_init    = aer915_class_init,
 };
 
-static void z2_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void z2_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     uint32_t sector_len = 0x10000;
     PXA2xxState *mpu;
@@ -371,6 +373,7 @@ static QEMUMachine z2_machine = {
     .name = "z2",
     .desc = "Zipit Z2 (PXA27x)",
     .init = z2_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void z2_machine_init(void)
diff --git a/hw/zaurus.c b/hw/zaurus.c
index 72838ec4408..2defe3b48dd 100644
--- a/hw/zaurus.c
+++ b/hw/zaurus.c
@@ -68,7 +68,7 @@ static inline void scoop_gpio_handler_update(ScoopInfo *s) {
     s->prev_level = level;
 }
 
-static uint64_t scoop_read(void *opaque, target_phys_addr_t addr,
+static uint64_t scoop_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     ScoopInfo *s = (ScoopInfo *) opaque;
@@ -102,7 +102,7 @@ static uint64_t scoop_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void scoop_write(void *opaque, target_phys_addr_t addr,
+static void scoop_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     ScoopInfo *s = (ScoopInfo *) opaque;
@@ -236,7 +236,7 @@ static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = scoop_sysbus_properties;
 }
 
-static TypeInfo scoop_sysbus_info = {
+static const TypeInfo scoop_sysbus_info = {
     .name          = "scoop",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ScoopInfo),
@@ -285,7 +285,7 @@ static struct QEMU_PACKED sl_param_info {
     .phadadj		= 0x01,
 };
 
-void sl_bootparam_write(target_phys_addr_t ptr)
+void sl_bootparam_write(hwaddr ptr)
 {
     cpu_physical_memory_write(ptr, (void *)&zaurus_bootparam,
                               sizeof(struct sl_param_info));
diff --git a/hw/zynq_slcr.c b/hw/zynq_slcr.c
index 4f97575770a..27b00f07d4a 100644
--- a/hw/zynq_slcr.c
+++ b/hw/zynq_slcr.c
@@ -15,9 +15,9 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #ifdef ZYNQ_ARM_SLCR_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -91,7 +91,7 @@ typedef enum {
 typedef enum {
   PSS,
   DDDR,
-  DMAC,
+  DMAC = 3,
   USB,
   GEM,
   SDIO,
@@ -160,7 +160,7 @@ static void zynq_slcr_reset(DeviceState *d)
 {
     int i;
     ZynqSLCRState *s =
-            FROM_SYSBUS(ZynqSLCRState, sysbus_from_qdev(d));
+            FROM_SYSBUS(ZynqSLCRState, SYS_BUS_DEVICE(d));
 
     DB_PRINT("RESET\n");
 
@@ -246,7 +246,7 @@ static void zynq_slcr_reset(DeviceState *d)
 }
 
 static inline uint32_t zynq_slcr_read_imp(void *opaque,
-    target_phys_addr_t offset)
+    hwaddr offset)
 {
     ZynqSLCRState *s = (ZynqSLCRState *)opaque;
 
@@ -329,21 +329,21 @@ static inline uint32_t zynq_slcr_read_imp(void *opaque,
     }
 }
 
-static uint64_t zynq_slcr_read(void *opaque, target_phys_addr_t offset,
+static uint64_t zynq_slcr_read(void *opaque, hwaddr offset,
     unsigned size)
 {
     uint32_t ret = zynq_slcr_read_imp(opaque, offset);
 
-    DB_PRINT("addr: %08x data: %08x\n", offset, ret);
+    DB_PRINT("addr: %08x data: %08x\n", (unsigned)offset, (unsigned)ret);
     return ret;
 }
 
-static void zynq_slcr_write(void *opaque, target_phys_addr_t offset,
+static void zynq_slcr_write(void *opaque, hwaddr offset,
                           uint64_t val, unsigned size)
 {
     ZynqSLCRState *s = (ZynqSLCRState *)opaque;
 
-    DB_PRINT("offset: %08x data: %08x\n", offset, (unsigned)val);
+    DB_PRINT("offset: %08x data: %08x\n", (unsigned)offset, (unsigned)val);
 
     switch (offset) {
     case 0x00: /* SCL */
@@ -476,7 +476,8 @@ static void zynq_slcr_write(void *opaque, target_phys_addr_t offset,
             break;
         default:
         bad_reg:
-            DB_PRINT("Bad register write %x <= %08x\n", (int)offset, val);
+            DB_PRINT("Bad register write %x <= %08x\n", (int)offset,
+                     (unsigned)val);
         }
     } else {
         DB_PRINT("SCLR registers are locked. Unlock them first\n");
@@ -520,7 +521,7 @@ static void zynq_slcr_class_init(ObjectClass *klass, void *data)
     dc->reset = zynq_slcr_reset;
 }
 
-static TypeInfo zynq_slcr_info = {
+static const TypeInfo zynq_slcr_info = {
     .class_init = zynq_slcr_class_init,
     .name  = "xilinx,zynq_slcr",
     .parent = TYPE_SYS_BUS_DEVICE,
diff --git a/aes.h b/include/block/aes.h
index a0167eb7d5b..a0167eb7d5b 100644
--- a/aes.h
+++ b/include/block/aes.h
diff --git a/include/block/aio.h b/include/block/aio.h
new file mode 100644
index 00000000000..8eda9245990
--- /dev/null
+++ b/include/block/aio.h
@@ -0,0 +1,238 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_AIO_H
+#define QEMU_AIO_H
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/event_notifier.h"
+
+typedef struct BlockDriverAIOCB BlockDriverAIOCB;
+typedef void BlockDriverCompletionFunc(void *opaque, int ret);
+
+typedef struct AIOCBInfo {
+    void (*cancel)(BlockDriverAIOCB *acb);
+    size_t aiocb_size;
+} AIOCBInfo;
+
+struct BlockDriverAIOCB {
+    const AIOCBInfo *aiocb_info;
+    BlockDriverState *bs;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+};
+
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
+                   BlockDriverCompletionFunc *cb, void *opaque);
+void qemu_aio_release(void *p);
+
+typedef struct AioHandler AioHandler;
+typedef void QEMUBHFunc(void *opaque);
+typedef void IOHandler(void *opaque);
+
+typedef struct AioContext {
+    GSource source;
+
+    /* The list of registered AIO handlers */
+    QLIST_HEAD(, AioHandler) aio_handlers;
+
+    /* This is a simple lock used to protect the aio_handlers list.
+     * Specifically, it's used to ensure that no callbacks are removed while
+     * we're walking and dispatching callbacks.
+     */
+    int walking_handlers;
+
+    /* Anchor of the list of Bottom Halves belonging to the context */
+    struct QEMUBH *first_bh;
+
+    /* A simple lock used to protect the first_bh list, and ensure that
+     * no callbacks are removed while we're walking and dispatching callbacks.
+     */
+    int walking_bh;
+
+    /* Used for aio_notify.  */
+    EventNotifier notifier;
+} AioContext;
+
+/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
+typedef int (AioFlushEventNotifierHandler)(EventNotifier *e);
+
+/**
+ * aio_context_new: Allocate a new AioContext.
+ *
+ * AioContext provide a mini event-loop that can be waited on synchronously.
+ * They also provide bottom halves, a service to execute a piece of code
+ * as soon as possible.
+ */
+AioContext *aio_context_new(void);
+
+/**
+ * aio_context_ref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Add a reference to an AioContext.
+ */
+void aio_context_ref(AioContext *ctx);
+
+/**
+ * aio_context_unref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Drop a reference to an AioContext.
+ */
+void aio_context_unref(AioContext *ctx);
+
+/**
+ * aio_bh_new: Allocate a new bottom half structure.
+ *
+ * Bottom halves are lightweight callbacks whose invocation is guaranteed
+ * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
+ * is opaque and must be allocated prior to its use.
+ */
+QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque);
+
+/**
+ * aio_notify: Force processing of pending events.
+ *
+ * Similar to signaling a condition variable, aio_notify forces
+ * aio_wait to exit, so that the next call will re-examine pending events.
+ * The caller of aio_notify will usually call aio_wait again very soon,
+ * or go through another iteration of the GLib main loop.  Hence, aio_notify
+ * also has the side effect of recalculating the sets of file descriptors
+ * that the main loop waits for.
+ *
+ * Calling aio_notify is rarely necessary, because for example scheduling
+ * a bottom half calls it already.
+ */
+void aio_notify(AioContext *ctx);
+
+/**
+ * aio_bh_poll: Poll bottom halves for an AioContext.
+ *
+ * These are internal functions used by the QEMU main loop.
+ */
+int aio_bh_poll(AioContext *ctx);
+
+/**
+ * qemu_bh_schedule: Schedule a bottom half.
+ *
+ * Scheduling a bottom half interrupts the main loop and causes the
+ * execution of the callback that was passed to qemu_bh_new.
+ *
+ * Bottom halves that are scheduled from a bottom half handler are instantly
+ * invoked.  This can create an infinite loop if a bottom half handler
+ * schedules itself.
+ *
+ * @bh: The bottom half to be scheduled.
+ */
+void qemu_bh_schedule(QEMUBH *bh);
+
+/**
+ * qemu_bh_cancel: Cancel execution of a bottom half.
+ *
+ * Canceling execution of a bottom half undoes the effect of calls to
+ * qemu_bh_schedule without freeing its resources yet.  While cancellation
+ * itself is also wait-free and thread-safe, it can of course race with the
+ * loop that executes bottom halves unless you are holding the iothread
+ * mutex.  This makes it mostly useless if you are not holding the mutex.
+ *
+ * @bh: The bottom half to be canceled.
+ */
+void qemu_bh_cancel(QEMUBH *bh);
+
+/**
+ *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
+ *
+ * Deleting a bottom half frees the memory that was allocated for it by
+ * qemu_bh_new.  It also implies canceling the bottom half if it was
+ * scheduled.
+ *
+ * @bh: The bottom half to be deleted.
+ */
+void qemu_bh_delete(QEMUBH *bh);
+
+/* Return whether there are any pending callbacks from the GSource
+ * attached to the AioContext.
+ *
+ * This is used internally in the implementation of the GSource.
+ */
+bool aio_pending(AioContext *ctx);
+
+/* Progress in completing AIO work to occur.  This can issue new pending
+ * aio as a result of executing I/O completion or bh callbacks.
+ *
+ * If there is no pending AIO operation or completion (bottom half),
+ * return false.  If there are pending AIO operations of bottom halves,
+ * return true.
+ *
+ * If there are no pending bottom halves, but there are pending AIO
+ * operations, it may not be possible to make any progress without
+ * blocking.  If @blocking is true, this function will wait until one
+ * or more AIO events have completed, to ensure something has moved
+ * before returning.
+ */
+bool aio_poll(AioContext *ctx, bool blocking);
+
+#ifdef CONFIG_POSIX
+/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
+typedef int (AioFlushHandler)(void *opaque);
+
+/* Register a file descriptor and associated callbacks.  Behaves very similarly
+ * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
+ * be invoked when using qemu_aio_wait().
+ *
+ * Code that invokes AIO completion functions should rely on this function
+ * instead of qemu_set_fd_handler[2].
+ */
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        AioFlushHandler *io_flush,
+                        void *opaque);
+#endif
+
+/* Register an event notifier and associated callbacks.  Behaves very similarly
+ * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
+ * will be invoked when using qemu_aio_wait().
+ *
+ * Code that invokes AIO completion functions should rely on this function
+ * instead of event_notifier_set_handler.
+ */
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *notifier,
+                            EventNotifierHandler *io_read,
+                            AioFlushEventNotifierHandler *io_flush);
+
+/* Return a GSource that lets the main loop poll the file descriptors attached
+ * to this AioContext.
+ */
+GSource *aio_get_g_source(AioContext *ctx);
+
+/* Functions to operate on the main QEMU AioContext.  */
+
+bool qemu_aio_wait(void);
+void qemu_aio_set_event_notifier(EventNotifier *notifier,
+                                 EventNotifierHandler *io_read,
+                                 AioFlushEventNotifierHandler *io_flush);
+
+#ifdef CONFIG_POSIX
+void qemu_aio_set_fd_handler(int fd,
+                             IOHandler *io_read,
+                             IOHandler *io_write,
+                             AioFlushHandler *io_flush,
+                             void *opaque);
+#endif
+
+#endif
diff --git a/block.h b/include/block/block.h
index 2e2be110710..5c3b911c1b6 100644
--- a/block.h
+++ b/include/block/block.h
@@ -1,14 +1,16 @@
 #ifndef BLOCK_H
 #define BLOCK_H
 
-#include "qemu-aio.h"
+#include "block/aio.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-coroutine.h"
-#include "qobject.h"
+#include "qemu/option.h"
+#include "block/coroutine.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi-types.h"
 
 /* block.c */
 typedef struct BlockDriver BlockDriver;
+typedef struct BlockJob BlockJob;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
@@ -80,6 +82,7 @@ typedef struct BlockDevOps {
 #define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
 #define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
 #define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
+#define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
 
 #define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
 
@@ -88,21 +91,23 @@ typedef struct BlockDevOps {
 #define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
 
 typedef enum {
-    BLOCK_ERR_REPORT, BLOCK_ERR_IGNORE, BLOCK_ERR_STOP_ENOSPC,
-    BLOCK_ERR_STOP_ANY
+    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
 } BlockErrorAction;
 
-typedef enum {
-    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
-} BlockQMPEventAction;
+typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
+
+typedef struct BDRVReopenState {
+    BlockDriverState *bs;
+    int flags;
+    void *opaque;
+} BDRVReopenState;
+
 
 void bdrv_iostatus_enable(BlockDriverState *bs);
 void bdrv_iostatus_reset(BlockDriverState *bs);
 void bdrv_iostatus_disable(BlockDriverState *bs);
 bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
 void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
-void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               BlockQMPEventAction action, int is_read);
 void bdrv_info_print(Monitor *mon, const QObject *data);
 void bdrv_info(Monitor *mon, QObject **ret_data);
 void bdrv_stats_print(Monitor *mon, const QObject *data);
@@ -128,9 +133,19 @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
 void bdrv_delete(BlockDriverState *bs);
 int bdrv_parse_cache_flags(const char *mode, int *flags);
 int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags);
+int bdrv_open_backing_file(BlockDriverState *bs);
 int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
               BlockDriver *drv);
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags);
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
+                        BlockReopenQueue *queue, Error **errp);
+void bdrv_reopen_commit(BDRVReopenState *reopen_state);
+void bdrv_reopen_abort(BDRVReopenState *reopen_state);
 void bdrv_close(BlockDriverState *bs);
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
 int bdrv_attach_dev(BlockDriverState *bs, void *dev);
 void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev);
 void bdrv_detach_dev(BlockDriverState *bs, void *dev);
@@ -185,6 +200,11 @@ int bdrv_commit_all(void);
 int bdrv_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt);
 void bdrv_register(BlockDriver *bdrv);
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base);
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs);
+BlockDriverState *bdrv_find_base(BlockDriverState *bs);
 
 
 typedef struct BdrvCheckResult {
@@ -259,9 +279,12 @@ int bdrv_has_zero_init(BlockDriverState *bs);
 int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
                       int *pnum);
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockErrorAction on_read_error,
-                       BlockErrorAction on_write_error);
-BlockErrorAction bdrv_get_on_error(BlockDriverState *bs, int is_read);
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error);
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error);
 int bdrv_is_read_only(BlockDriverState *bs);
 int bdrv_is_sg(BlockDriverState *bs);
 int bdrv_enable_write_cache(BlockDriverState *bs);
@@ -286,12 +309,18 @@ int bdrv_get_flags(BlockDriverState *bs);
 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
                           const uint8_t *buf, int nb_sectors);
 int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors);
 
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs);
 void bdrv_get_backing_filename(BlockDriverState *bs,
                                char *filename, int filename_size);
 void bdrv_get_full_backing_filename(BlockDriverState *bs,
                                     char *dest, size_t sz);
+BlockInfo *bdrv_query_info(BlockDriverState *s);
+BlockStats *bdrv_query_stats(const BlockDriverState *bs);
 int bdrv_can_snapshot(BlockDriverState *bs);
 int bdrv_is_snapshot(BlockDriverState *bs);
 BlockDriverState *bdrv_snapshots(void);
@@ -318,19 +347,20 @@ int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
                       int64_t pos, int size);
 
-int bdrv_img_create(const char *filename, const char *fmt,
-                    const char *base_filename, const char *base_fmt,
-                    char *options, uint64_t img_size, int flags);
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags, Error **errp);
 
 void bdrv_set_buffer_alignment(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
 
-#define BDRV_SECTORS_PER_DIRTY_CHUNK 2048
-
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable);
+struct HBitmapIter;
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity);
 int bdrv_get_dirty(BlockDriverState *bs, int64_t sector);
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
-                      int nr_sectors);
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+void bdrv_dirty_iter_init(BlockDriverState *bs, struct HBitmapIter *hbi);
 int64_t bdrv_get_dirty_count(BlockDriverState *bs);
 
 void bdrv_enable_copy_on_read(BlockDriverState *bs);
@@ -339,6 +369,15 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs);
 void bdrv_set_in_use(BlockDriverState *bs, int in_use);
 int bdrv_in_use(BlockDriverState *bs);
 
+#ifdef CONFIG_LINUX_AIO
+int raw_get_aio_fd(BlockDriverState *bs);
+#else
+static inline int raw_get_aio_fd(BlockDriverState *bs)
+{
+    return -ENOTSUP;
+}
+#endif
+
 enum BlockAcctType {
     BDRV_ACCT_READ,
     BDRV_ACCT_WRITE,
@@ -405,4 +444,9 @@ typedef enum {
 #define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
 void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
 
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                           const char *tag);
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag);
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag);
+
 #endif
diff --git a/block_int.h b/include/block/block_int.h
index 4452f6f3981..f7279b978a8 100644
--- a/block_int.h
+++ b/include/block/block_int.h
@@ -24,13 +24,15 @@
 #ifndef BLOCK_INT_H
 #define BLOCK_INT_H
 
-#include "block.h"
-#include "qemu-option.h"
-#include "qemu-queue.h"
-#include "qemu-coroutine.h"
-#include "qemu-timer.h"
+#include "block/block.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "block/coroutine.h"
+#include "qemu/timer.h"
 #include "qapi-types.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
+#include "monitor/monitor.h"
+#include "qemu/hbitmap.h"
 
 #define BLOCK_FLAG_ENCRYPT          1
 #define BLOCK_FLAG_COMPAT6          4
@@ -67,78 +69,18 @@ typedef struct BlockIOBaseValue {
     uint64_t ios[2];
 } BlockIOBaseValue;
 
-typedef struct BlockJob BlockJob;
-
-/**
- * BlockJobType:
- *
- * A class type for block job objects.
- */
-typedef struct BlockJobType {
-    /** Derived BlockJob struct size */
-    size_t instance_size;
-
-    /** String describing the operation, part of query-block-jobs QMP API */
-    const char *job_type;
-
-    /** Optional callback for job types that support setting a speed limit */
-    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
-} BlockJobType;
-
-/**
- * BlockJob:
- *
- * Long-running operation on a BlockDriverState.
- */
-struct BlockJob {
-    /** The job type, including the job vtable.  */
-    const BlockJobType *job_type;
-
-    /** The block device on which the job is operating.  */
-    BlockDriverState *bs;
-
-    /**
-     * The coroutine that executes the job.  If not NULL, it is
-     * reentered when busy is false and the job is cancelled.
-     */
-    Coroutine *co;
-
-    /**
-     * Set to true if the job should cancel itself.  The flag must
-     * always be tested just before toggling the busy flag from false
-     * to true.  After a job has been cancelled, it should only yield
-     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
-     */
-    bool cancelled;
-
-    /**
-     * Set to false by the job while it is in a quiescent state, where
-     * no I/O is pending and the job has yielded on any condition
-     * that is not detected by #qemu_aio_wait, such as a timer.
-     */
-    bool busy;
-
-    /** Offset that is published by the query-block-jobs QMP API */
-    int64_t offset;
-
-    /** Length that is published by the query-block-jobs QMP API */
-    int64_t len;
-
-    /** Speed that was set with @block_job_set_speed.  */
-    int64_t speed;
-
-    /** The completion function that will be called when the job completes.  */
-    BlockDriverCompletionFunc *cb;
-
-    /** The opaque value that is passed to the completion function.  */
-    void *opaque;
-};
-
 struct BlockDriver {
     const char *format_name;
     int instance_size;
     int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename);
     int (*bdrv_probe_device)(const char *filename);
+
+    /* For handling image reopen for split or non-split files */
+    int (*bdrv_reopen_prepare)(BDRVReopenState *reopen_state,
+                               BlockReopenQueue *queue, Error **errp);
+    void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
+    void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
+
     int (*bdrv_open)(BlockDriverState *bs, int flags);
     int (*bdrv_file_open)(BlockDriverState *bs, const char *filename, int flags);
     int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
@@ -249,6 +191,12 @@ struct BlockDriver {
 
     void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
 
+    /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
+    int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
+        const char *tag);
+    int (*bdrv_debug_resume)(BlockDriverState *bs, const char *tag);
+    bool (*bdrv_debug_is_suspended)(BlockDriverState *bs, const char *tag);
+
     /*
      * Returns 1 if newly created images are guaranteed to contain only
      * zeros, 0 otherwise.
@@ -268,7 +216,6 @@ struct BlockDriverState {
     int64_t total_sectors; /* if we are reading a disk image, give its
                               size in sectors */
     int read_only; /* if true, the media is read only */
-    int keep_read_only; /* if true, the media was requested to stay read only */
     int open_flags; /* flags used to open the file, re-used for re-open */
     int encrypted; /* if true, the media is encrypted */
     int valid_key; /* if true, a valid encryption key has been set */
@@ -293,6 +240,8 @@ struct BlockDriverState {
     BlockDriverState *backing_hd;
     BlockDriverState *file;
 
+    NotifierList close_notifiers;
+
     /* number of in-flight copy-on-read requests */
     unsigned int copy_on_read_in_flight;
 
@@ -323,12 +272,11 @@ struct BlockDriverState {
 
     /* NOTE: the following infos are only hints for real hardware
        drivers. They are not used by the block driver */
-    BlockErrorAction on_read_error, on_write_error;
+    BlockdevOnError on_read_error, on_write_error;
     bool iostatus_enabled;
     BlockDeviceIoStatus iostatus;
     char device_name[32];
-    unsigned long *dirty_bitmap;
-    int64_t dirty_count;
+    HBitmap *dirty_bitmap;
     int in_use; /* users other than guest access, eg. block migration */
     QTAILQ_ENTRY(BlockDriverState) list;
 
@@ -336,6 +284,7 @@ struct BlockDriverState {
 
     /* long-running background operation */
     BlockJob *job;
+
 };
 
 int get_tmp_filename(char *filename, int size);
@@ -346,113 +295,73 @@ void bdrv_set_io_limits(BlockDriverState *bs,
 #ifdef _WIN32
 int is_windows_drive(const char *filename);
 #endif
+void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read);
 
 /**
- * block_job_create:
- * @job_type: The class object for the newly-created job.
- * @bs: The block
+ * stream_start:
+ * @bs: Block device to operate on.
+ * @base: Block device that will become the new base, or %NULL to
+ * flatten the whole backing file chain onto @bs.
+ * @base_id: The file name that will be written to @bs as the new
+ * backing file if the job completes.  Ignored if @base is %NULL.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Create a new long-running block device job and return it.  The job
- * will call @cb asynchronously when the job completes.  Note that
- * @bs may have been closed at the time the @cb it is called.  If
- * this is the case, the job may be reported as either cancelled or
- * completed.
- *
- * This function is not part of the public job interface; it should be
- * called from a wrapper that is specific to the job type.
- */
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
-                       void *opaque, Error **errp);
-
-/**
- * block_job_sleep_ns:
- * @job: The job that calls the function.
- * @clock: The clock to sleep on.
- * @ns: How many nanoseconds to stop for.
- *
- * Put the job to sleep (assuming that it wasn't canceled) for @ns
- * nanoseconds.  Canceling the job will interrupt the wait immediately.
- */
-void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
-
-/**
- * block_job_complete:
- * @job: The job being completed.
- * @ret: The status code.
- *
- * Call the completion function that was registered at creation time, and
- * free @job.
+ * Start a streaming operation on @bs.  Clusters that are unallocated
+ * in @bs, but allocated in any image between @base and @bs (both
+ * exclusive) will be written to @bs.  At the end of a successful
+ * streaming job, the backing file of @bs will be changed to
+ * @base_id in the written image and to @base in the live BlockDriverState.
  */
-void block_job_complete(BlockJob *job, int ret);
+void stream_start(BlockDriverState *bs, BlockDriverState *base,
+                  const char *base_id, int64_t speed, BlockdevOnError on_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp);
 
 /**
- * block_job_set_speed:
- * @job: The job to set the speed for.
- * @speed: The new value
+ * commit_start:
+ * @bs: Top Block device
+ * @base: Block device that will be written into, and become the new top
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Set a rate-limiting parameter for the job; the actual meaning may
- * vary depending on the job type.
- */
-void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
-
-/**
- * block_job_cancel:
- * @job: The job to be canceled.
- *
- * Asynchronously cancel the specified job.
  */
-void block_job_cancel(BlockJob *job);
+void commit_start(BlockDriverState *bs, BlockDriverState *base,
+                 BlockDriverState *top, int64_t speed,
+                 BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                 void *opaque, Error **errp);
 
-/**
- * block_job_is_cancelled:
- * @job: The job being queried.
- *
- * Returns whether the job is scheduled for cancellation.
- */
-bool block_job_is_cancelled(BlockJob *job);
-
-/**
- * block_job_cancel:
- * @job: The job to be canceled.
- *
- * Asynchronously cancel the job and wait for it to reach a quiescent
- * state.  Note that the completion callback will still be called
- * asynchronously, hence it is *not* valid to call #bdrv_delete
- * immediately after #block_job_cancel_sync.  Users of block jobs
- * will usually protect the BlockDriverState objects with a reference
- * count, should this be a concern.
- *
- * Returns the return value from the job if the job actually completed
- * during the call, or -ECANCELED if it was canceled.
- */
-int block_job_cancel_sync(BlockJob *job);
-
-/**
- * stream_start:
+/*
+ * mirror_start:
  * @bs: Block device to operate on.
- * @base: Block device that will become the new base, or %NULL to
- * flatten the whole backing file chain onto @bs.
- * @base_id: The file name that will be written to @bs as the new
- * backing file if the job completes.  Ignored if @base is %NULL.
+ * @target: Block device to write to.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @granularity: The chosen granularity for the dirty bitmap.
+ * @buf_size: The amount of data that can be in flight at one time.
+ * @mode: Whether to collapse all images in the chain to the target.
+ * @on_source_error: The action to take upon error reading from the source.
+ * @on_target_error: The action to take upon error writing to the target.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Start a streaming operation on @bs.  Clusters that are unallocated
- * in @bs, but allocated in any image between @base and @bs (both
- * exclusive) will be written to @bs.  At the end of a successful
- * streaming job, the backing file of @bs will be changed to
- * @base_id in the written image and to @base in the live BlockDriverState.
+ * Start a mirroring operation on @bs.  Clusters that are allocated
+ * in @bs will be written to @bs until the job is cancelled or
+ * manually completed.  At the end of a successful mirroring job,
+ * @bs will be switched to read from @target.
  */
-void stream_start(BlockDriverState *bs, BlockDriverState *base,
-                  const char *base_id, int64_t speed,
+void mirror_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, int64_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp);
 
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
new file mode 100644
index 00000000000..c290d07bba0
--- /dev/null
+++ b/include/block/blockjob.h
@@ -0,0 +1,278 @@
+/*
+ * Declarations for long-running block device operations
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef BLOCKJOB_H
+#define BLOCKJOB_H 1
+
+#include "block/block.h"
+
+/**
+ * BlockJobType:
+ *
+ * A class type for block job objects.
+ */
+typedef struct BlockJobType {
+    /** Derived BlockJob struct size */
+    size_t instance_size;
+
+    /** String describing the operation, part of query-block-jobs QMP API */
+    const char *job_type;
+
+    /** Optional callback for job types that support setting a speed limit */
+    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
+
+    /** Optional callback for job types that need to forward I/O status reset */
+    void (*iostatus_reset)(BlockJob *job);
+
+    /**
+     * Optional callback for job types whose completion must be triggered
+     * manually.
+     */
+    void (*complete)(BlockJob *job, Error **errp);
+} BlockJobType;
+
+/**
+ * BlockJob:
+ *
+ * Long-running operation on a BlockDriverState.
+ */
+struct BlockJob {
+    /** The job type, including the job vtable.  */
+    const BlockJobType *job_type;
+
+    /** The block device on which the job is operating.  */
+    BlockDriverState *bs;
+
+    /**
+     * The coroutine that executes the job.  If not NULL, it is
+     * reentered when busy is false and the job is cancelled.
+     */
+    Coroutine *co;
+
+    /**
+     * Set to true if the job should cancel itself.  The flag must
+     * always be tested just before toggling the busy flag from false
+     * to true.  After a job has been cancelled, it should only yield
+     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
+     */
+    bool cancelled;
+
+    /**
+     * Set to true if the job is either paused, or will pause itself
+     * as soon as possible (if busy == true).
+     */
+    bool paused;
+
+    /**
+     * Set to false by the job while it is in a quiescent state, where
+     * no I/O is pending and the job has yielded on any condition
+     * that is not detected by #qemu_aio_wait, such as a timer.
+     */
+    bool busy;
+
+    /** Status that is published by the query-block-jobs QMP API */
+    BlockDeviceIoStatus iostatus;
+
+    /** Offset that is published by the query-block-jobs QMP API */
+    int64_t offset;
+
+    /** Length that is published by the query-block-jobs QMP API */
+    int64_t len;
+
+    /** Speed that was set with @block_job_set_speed.  */
+    int64_t speed;
+
+    /** The completion function that will be called when the job completes.  */
+    BlockDriverCompletionFunc *cb;
+
+    /** The opaque value that is passed to the completion function.  */
+    void *opaque;
+};
+
+/**
+ * block_job_create:
+ * @job_type: The class object for the newly-created job.
+ * @bs: The block
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ * Create a new long-running block device job and return it.  The job
+ * will call @cb asynchronously when the job completes.  Note that
+ * @bs may have been closed at the time the @cb it is called.  If
+ * this is the case, the job may be reported as either cancelled or
+ * completed.
+ *
+ * This function is not part of the public job interface; it should be
+ * called from a wrapper that is specific to the job type.
+ */
+void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       void *opaque, Error **errp);
+
+/**
+ * block_job_sleep_ns:
+ * @job: The job that calls the function.
+ * @clock: The clock to sleep on.
+ * @ns: How many nanoseconds to stop for.
+ *
+ * Put the job to sleep (assuming that it wasn't canceled) for @ns
+ * nanoseconds.  Canceling the job will interrupt the wait immediately.
+ */
+void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
+
+/**
+ * block_job_completed:
+ * @job: The job being completed.
+ * @ret: The status code.
+ *
+ * Call the completion function that was registered at creation time, and
+ * free @job.
+ */
+void block_job_completed(BlockJob *job, int ret);
+
+/**
+ * block_job_set_speed:
+ * @job: The job to set the speed for.
+ * @speed: The new value
+ * @errp: Error object.
+ *
+ * Set a rate-limiting parameter for the job; the actual meaning may
+ * vary depending on the job type.
+ */
+void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
+
+/**
+ * block_job_cancel:
+ * @job: The job to be canceled.
+ *
+ * Asynchronously cancel the specified job.
+ */
+void block_job_cancel(BlockJob *job);
+
+/**
+ * block_job_complete:
+ * @job: The job to be completed.
+ * @errp: Error object.
+ *
+ * Asynchronously complete the specified job.
+ */
+void block_job_complete(BlockJob *job, Error **errp);
+
+/**
+ * block_job_is_cancelled:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is scheduled for cancellation.
+ */
+bool block_job_is_cancelled(BlockJob *job);
+
+/**
+ * block_job_query:
+ * @job: The job to get information about.
+ *
+ * Return information about a job.
+ */
+BlockJobInfo *block_job_query(BlockJob *job);
+
+/**
+ * block_job_pause:
+ * @job: The job to be paused.
+ *
+ * Asynchronously pause the specified job.
+ */
+void block_job_pause(BlockJob *job);
+
+/**
+ * block_job_resume:
+ * @job: The job to be resumed.
+ *
+ * Resume the specified job.
+ */
+void block_job_resume(BlockJob *job);
+
+/**
+ * qobject_from_block_job:
+ * @job: The job whose information is requested.
+ *
+ * Return a QDict corresponding to @job's query-block-jobs entry.
+ */
+QObject *qobject_from_block_job(BlockJob *job);
+
+/**
+ * block_job_ready:
+ * @job: The job which is now ready to complete.
+ *
+ * Send a BLOCK_JOB_READY event for the specified job.
+ */
+void block_job_ready(BlockJob *job);
+
+/**
+ * block_job_is_paused:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is currently paused, or will pause
+ * as soon as it reaches a sleeping point.
+ */
+bool block_job_is_paused(BlockJob *job);
+
+/**
+ * block_job_cancel_sync:
+ * @job: The job to be canceled.
+ *
+ * Synchronously cancel the job.  The completion callback is called
+ * before the function returns.  The job may actually complete
+ * instead of canceling itself; the circumstances under which this
+ * happens depend on the kind of job that is active.
+ *
+ * Returns the return value from the job if the job actually completed
+ * during the call, or -ECANCELED if it was canceled.
+ */
+int block_job_cancel_sync(BlockJob *job);
+
+/**
+ * block_job_iostatus_reset:
+ * @job: The job whose I/O status should be reset.
+ *
+ * Reset I/O status on @job and on BlockDriverState objects it uses,
+ * other than job->bs.
+ */
+void block_job_iostatus_reset(BlockJob *job);
+
+/**
+ * block_job_error_action:
+ * @job: The job to signal an error for.
+ * @bs: The block device on which to set an I/O error.
+ * @on_err: The error action setting.
+ * @is_read: Whether the operation was a read.
+ * @error: The error that was reported.
+ *
+ * Report an I/O error for a block job and possibly stop the VM.  Return the
+ * action that was selected based on @on_err and @error.
+ */
+BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
+                                        BlockdevOnError on_err,
+                                        int is_read, int error);
+#endif
diff --git a/qemu-coroutine.h b/include/block/coroutine.h
index 34c15d41169..c31fae3f3cf 100644
--- a/qemu-coroutine.h
+++ b/include/block/coroutine.h
@@ -16,8 +16,8 @@
 #define QEMU_COROUTINE_H
 
 #include <stdbool.h>
-#include "qemu-queue.h"
-#include "qemu-timer.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
 
 /**
  * Coroutines are a mechanism for stack switching and can be used for
diff --git a/qemu-coroutine-int.h b/include/block/coroutine_int.h
index 0f1bd80a8da..17eb71e7238 100644
--- a/qemu-coroutine-int.h
+++ b/include/block/coroutine_int.h
@@ -25,8 +25,8 @@
 #ifndef QEMU_COROUTINE_INT_H
 #define QEMU_COROUTINE_INT_H
 
-#include "qemu-queue.h"
-#include "qemu-coroutine.h"
+#include "qemu/queue.h"
+#include "block/coroutine.h"
 
 typedef enum {
     COROUTINE_YIELD = 1,
diff --git a/nbd.h b/include/block/nbd.h
index 40d58d359f3..344f05b794d 100644
--- a/nbd.h
+++ b/include/block/nbd.h
@@ -79,9 +79,22 @@ typedef struct NBDExport NBDExport;
 typedef struct NBDClient NBDClient;
 
 NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags);
+                          off_t size, uint32_t nbdflags,
+                          void (*close)(NBDExport *));
 void nbd_export_close(NBDExport *exp);
+void nbd_export_get(NBDExport *exp);
+void nbd_export_put(NBDExport *exp);
+
+BlockDriverState *nbd_export_get_blockdev(NBDExport *exp);
+
+NBDExport *nbd_export_find(const char *name);
+void nbd_export_set_name(NBDExport *exp, const char *name);
+void nbd_export_close_all(void);
+
 NBDClient *nbd_client_new(NBDExport *exp, int csock,
                           void (*close)(NBDClient *));
+void nbd_client_close(NBDClient *client);
+void nbd_client_get(NBDClient *client);
+void nbd_client_put(NBDClient *client);
 
 #endif
diff --git a/include/block/thread-pool.h b/include/block/thread-pool.h
new file mode 100644
index 00000000000..200703e35f8
--- /dev/null
+++ b/include/block/thread-pool.h
@@ -0,0 +1,34 @@
+/*
+ * QEMU block layer thread pool
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#ifndef QEMU_THREAD_POOL_H
+#define QEMU_THREAD_POOL_H 1
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
+#include "block/block_int.h"
+
+typedef int ThreadPoolFunc(void *opaque);
+
+BlockDriverAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg,
+     BlockDriverCompletionFunc *cb, void *opaque);
+int coroutine_fn thread_pool_submit_co(ThreadPoolFunc *func, void *arg);
+void thread_pool_submit(ThreadPoolFunc *func, void *arg);
+
+#endif
diff --git a/include/bt/bt.h b/include/bt/bt.h
new file mode 100644
index 00000000000..2bc6d53ccad
--- /dev/null
+++ b/include/bt/bt.h
@@ -0,0 +1,20 @@
+#ifndef BT_HOST_H
+#define BT_HOST_H
+
+/* BT HCI info */
+
+struct HCIInfo {
+    int (*bdaddr_set)(struct HCIInfo *hci, const uint8_t *bd_addr);
+    void (*cmd_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void (*sco_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void (*acl_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void *opaque;
+    void (*evt_recv)(void *opaque, const uint8_t *data, int len);
+    void (*acl_recv)(void *opaque, const uint8_t *data, int len);
+};
+
+/* bt-host.c */
+struct HCIInfo *bt_host_hci(const char *id);
+struct HCIInfo *qemu_next_hci(void);
+
+#endif
diff --git a/qemu-char.h b/include/char/char.h
index 486644b3bdf..c91ce3c98a7 100644
--- a/qemu-char.h
+++ b/include/char/char.h
@@ -2,12 +2,13 @@
 #define QEMU_CHAR_H
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qobject.h"
-#include "qstring.h"
-#include "main-loop.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "block/aio.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/main-loop.h"
 
 /* character device */
 
@@ -69,11 +70,12 @@ struct CharDriverState {
     void (*chr_guest_open)(struct CharDriverState *chr);
     void (*chr_guest_close)(struct CharDriverState *chr);
     void *opaque;
-    QEMUBH *bh;
+    QEMUTimer *open_timer;
     char *label;
     char *filename;
     int opened;
     int avail_connections;
+    QemuOpts *opts;
     QTAILQ_ENTRY(CharDriverState) next;
 };
 
@@ -88,7 +90,8 @@ struct CharDriverState {
  * Returns: a new character backend
  */
 CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
-                                    void (*init)(struct CharDriverState *s));
+                                    void (*init)(struct CharDriverState *s),
+                                    Error **errp);
 
 /**
  * @qemu_chr_new:
diff --git a/config.h b/include/config.h
index e20f78696a1..e20f78696a1 100644
--- a/config.h
+++ b/include/config.h
diff --git a/dis-asm.h b/include/disas/bfd.h
index 3944b3cf7ee..3944b3cf7ee 100644
--- a/dis-asm.h
+++ b/include/disas/bfd.h
diff --git a/disas.h b/include/disas/disas.h
index 3ab42af0b5a..c13ca9a3a4d 100644
--- a/disas.h
+++ b/include/disas/disas.h
@@ -6,7 +6,8 @@
 #ifdef NEED_CPU_H
 /* Disassemble this for me please... (debugging). */
 void disas(FILE *out, void *code, unsigned long size);
-void target_disas(FILE *out, target_ulong code, target_ulong size, int flags);
+void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+                  target_ulong size, int flags);
 
 void monitor_disas(Monitor *mon, CPUArchState *env,
                    target_ulong pc, int nb_insn, int is_physical, int flags);
@@ -22,7 +23,7 @@ struct elf64_sym;
 #if defined(CONFIG_USER_ONLY)
 typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_ulong orig_addr);
 #else
-typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_phys_addr_t orig_addr);
+typedef const char *(*lookup_symbol_t)(struct syminfo *s, hwaddr orig_addr);
 #endif
 
 struct syminfo {
diff --git a/elf.h b/include/elf.h
index a21ea535bd1..a21ea535bd1 100644
--- a/elf.h
+++ b/include/elf.h
diff --git a/exec-memory.h b/include/exec/address-spaces.h
index 1cd92eec711..3d12cddeec1 100644
--- a/exec-memory.h
+++ b/include/exec/address-spaces.h
@@ -19,7 +19,7 @@
  * you're one of them.
  */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 #ifndef CONFIG_USER_ONLY
 
@@ -33,11 +33,8 @@ MemoryRegion *get_system_memory(void);
  */
 MemoryRegion *get_system_io(void);
 
-/* Set the root memory region.  This region is the system memory map. */
-void set_system_memory_map(MemoryRegion *mr);
-
-/* Set the I/O memory region.  This region is the I/O memory map. */
-void set_system_io_map(MemoryRegion *mr);
+extern AddressSpace address_space_memory;
+extern AddressSpace address_space_io;
 
 #endif
 
diff --git a/cpu-all.h b/include/exec/cpu-all.h
index 5e07d285087..249e0464f2f 100644
--- a/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -20,8 +20,9 @@
 #define CPU_ALL_H
 
 #include "qemu-common.h"
-#include "qemu-tls.h"
-#include "cpu-common.h"
+#include "qemu/tls.h"
+#include "exec/cpu-common.h"
+#include "qemu/thread.h"
 
 /* some important defines:
  *
@@ -180,7 +181,7 @@ static inline void tswap64s(uint64_t *s)
 
 #if defined(CONFIG_USER_ONLY)
 #include <assert.h>
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
@@ -260,14 +261,6 @@ extern unsigned long reserved_va;
 #define stfl(p, v) stfl_raw(p, v)
 #define stfq(p, v) stfq_raw(p, v)
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ldub_code(p) ldub_raw(p)
-#define ldsb_code(p) ldsb_raw(p)
-#define lduw_code(p) lduw_raw(p)
-#define ldsw_code(p) ldsw_raw(p)
-#define ldl_code(p) ldl_raw(p)
-#define ldq_code(p) ldq_raw(p)
-#else
 #define cpu_ldub_code(env1, p) ldub_raw(p)
 #define cpu_ldsb_code(env1, p) ldsb_raw(p)
 #define cpu_lduw_code(env1, p) lduw_raw(p)
@@ -296,7 +289,6 @@ extern unsigned long reserved_va;
 #define cpu_stw_kernel(env, addr, data) stw_raw(addr, data)
 #define cpu_stl_kernel(env, addr, data) stl_raw(addr, data)
 #define cpu_stq_kernel(env, addr, data) stq_raw(addr, data)
-#endif
 
 #define ldub_kernel(p) ldub_raw(p)
 #define ldsb_kernel(p) ldsb_raw(p)
@@ -313,7 +305,6 @@ extern unsigned long reserved_va;
 #define stfl_kernel(p, v) stfl_raw(p, v)
 #define stfq_kernel(p, vt) stfq_raw(p, v)
 
-#ifdef CONFIG_TCG_PASS_AREG0
 #define cpu_ldub_data(env, addr) ldub_raw(addr)
 #define cpu_lduw_data(env, addr) lduw_raw(addr)
 #define cpu_ldl_data(env, addr) ldl_raw(addr)
@@ -321,7 +312,6 @@ extern unsigned long reserved_va;
 #define cpu_stb_data(env, addr, data) stb_raw(addr, data)
 #define cpu_stw_data(env, addr, data) stw_raw(addr, data)
 #define cpu_stl_data(env, addr, data) stl_raw(addr, data)
-#endif
 #endif /* defined(CONFIG_USER_ONLY) */
 
 /* page related stuff */
@@ -364,9 +354,11 @@ int page_check_range(target_ulong start, target_ulong len, int flags);
 #endif
 
 CPUArchState *cpu_copy(CPUArchState *env);
-CPUArchState *qemu_get_cpu(int cpu);
 
 #define CPU_DUMP_CODE 0x00010000
+#define CPU_DUMP_FPU 0x00020000 /* dump FPU register state, not just integer */
+/* dump info about TCG QEMU's condition code optimization state */
+#define CPU_DUMP_CCOP 0x00040000
 
 void cpu_dump_state(CPUArchState *env, FILE *f, fprintf_function cpu_fprintf,
                     int flags);
@@ -446,8 +438,6 @@ void cpu_reset_interrupt(CPUArchState *env, int mask);
 
 void cpu_exit(CPUArchState *s);
 
-bool qemu_cpu_has_work(CPUArchState *env);
-
 /* Breakpoint/watchpoint flags */
 #define BP_MEM_READ           0x01
 #define BP_MEM_WRITE          0x02
@@ -474,15 +464,13 @@ void cpu_watchpoint_remove_all(CPUArchState *env, int mask);
 #define SSTEP_NOTIMER 0x4  /* Do not Timers while single stepping */
 
 void cpu_single_step(CPUArchState *env, int enabled);
-int cpu_is_stopped(CPUArchState *env);
-void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data);
 
 #if !defined(CONFIG_USER_ONLY)
 
 /* Return the physical page corresponding to a virtual one. Use it
    only for debugging because no protection checks are done. Return -1
    if no page found. */
-target_phys_addr_t cpu_get_phys_page_debug(CPUArchState *env, target_ulong addr);
+hwaddr cpu_get_phys_page_debug(CPUArchState *env, target_ulong addr);
 
 /* memory API */
 
@@ -499,16 +487,23 @@ typedef struct RAMBlock {
     ram_addr_t length;
     uint32_t flags;
     char idstr[256];
-    QLIST_ENTRY(RAMBlock) next;
+    /* Reads can take either the iothread or the ramlist lock.
+     * Writes must take both locks.
+     */
+    QTAILQ_ENTRY(RAMBlock) next;
 #if defined(__linux__) && !defined(TARGET_S390X)
     int fd;
 #endif
 } RAMBlock;
 
 typedef struct RAMList {
+    QemuMutex mutex;
+    /* Protected by the iothread lock.  */
     uint8_t *phys_dirty;
-    QLIST_HEAD(, RAMBlock) blocks;
-    uint64_t dirty_pages;
+    RAMBlock *mru_block;
+    /* Protected by the ramlist lock.  */
+    QTAILQ_HEAD(, RAMBlock) blocks;
+    uint32_t version;
 } RAMList;
 extern RAMList ram_list;
 
@@ -526,6 +521,9 @@ extern int mem_prealloc;
 #define TLB_MMIO        (1 << 5)
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
+ram_addr_t last_ram_offset(void);
+void qemu_mutex_lock_ramlist(void);
+void qemu_mutex_unlock_ramlist(void);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
new file mode 100644
index 00000000000..2e5f11f47f5
--- /dev/null
+++ b/include/exec/cpu-common.h
@@ -0,0 +1,124 @@
+#ifndef CPU_COMMON_H
+#define CPU_COMMON_H 1
+
+/* CPU interfaces that are target independent.  */
+
+#include "exec/hwaddr.h"
+
+#ifndef NEED_CPU_H
+#include "exec/poison.h"
+#endif
+
+#include "qemu/bswap.h"
+#include "qemu/queue.h"
+
+/**
+ * CPUListState:
+ * @cpu_fprintf: Print function.
+ * @file: File to print to using @cpu_fprint.
+ *
+ * State commonly used for iterating over CPU models.
+ */
+typedef struct CPUListState {
+    fprintf_function cpu_fprintf;
+    FILE *file;
+} CPUListState;
+
+#if !defined(CONFIG_USER_ONLY)
+
+enum device_endian {
+    DEVICE_NATIVE_ENDIAN,
+    DEVICE_BIG_ENDIAN,
+    DEVICE_LITTLE_ENDIAN,
+};
+
+/* address in the RAM (different from a physical address) */
+#if defined(CONFIG_XEN_BACKEND)
+typedef uint64_t ram_addr_t;
+#  define RAM_ADDR_MAX UINT64_MAX
+#  define RAM_ADDR_FMT "%" PRIx64
+#else
+typedef uintptr_t ram_addr_t;
+#  define RAM_ADDR_MAX UINTPTR_MAX
+#  define RAM_ADDR_FMT "%" PRIxPTR
+#endif
+
+/* memory API */
+
+typedef void CPUWriteMemoryFunc(void *opaque, hwaddr addr, uint32_t value);
+typedef uint32_t CPUReadMemoryFunc(void *opaque, hwaddr addr);
+
+void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
+/* This should only be used for ram local to a device.  */
+void *qemu_get_ram_ptr(ram_addr_t addr);
+void qemu_put_ram_ptr(void *addr);
+/* This should not be used by devices.  */
+int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
+ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr);
+void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
+
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
+                            int len, int is_write);
+static inline void cpu_physical_memory_read(hwaddr addr,
+                                            void *buf, int len)
+{
+    cpu_physical_memory_rw(addr, buf, len, 0);
+}
+static inline void cpu_physical_memory_write(hwaddr addr,
+                                             const void *buf, int len)
+{
+    cpu_physical_memory_rw(addr, (void *)buf, len, 1);
+}
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
+                              int is_write);
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len);
+void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
+
+bool cpu_physical_memory_is_io(hwaddr phys_addr);
+
+/* Coalesced MMIO regions are areas where write operations can be reordered.
+ * This usually implies that write operations are side-effect free.  This allows
+ * batching which can make a major impact on performance when using
+ * virtualization.
+ */
+void qemu_flush_coalesced_mmio_buffer(void);
+
+uint32_t ldub_phys(hwaddr addr);
+uint32_t lduw_le_phys(hwaddr addr);
+uint32_t lduw_be_phys(hwaddr addr);
+uint32_t ldl_le_phys(hwaddr addr);
+uint32_t ldl_be_phys(hwaddr addr);
+uint64_t ldq_le_phys(hwaddr addr);
+uint64_t ldq_be_phys(hwaddr addr);
+void stb_phys(hwaddr addr, uint32_t val);
+void stw_le_phys(hwaddr addr, uint32_t val);
+void stw_be_phys(hwaddr addr, uint32_t val);
+void stl_le_phys(hwaddr addr, uint32_t val);
+void stl_be_phys(hwaddr addr, uint32_t val);
+void stq_le_phys(hwaddr addr, uint64_t val);
+void stq_be_phys(hwaddr addr, uint64_t val);
+
+#ifdef NEED_CPU_H
+uint32_t lduw_phys(hwaddr addr);
+uint32_t ldl_phys(hwaddr addr);
+uint64_t ldq_phys(hwaddr addr);
+void stl_phys_notdirty(hwaddr addr, uint32_t val);
+void stq_phys_notdirty(hwaddr addr, uint64_t val);
+void stw_phys(hwaddr addr, uint32_t val);
+void stl_phys(hwaddr addr, uint32_t val);
+void stq_phys(hwaddr addr, uint64_t val);
+#endif
+
+void cpu_physical_memory_write_rom(hwaddr addr,
+                                   const uint8_t *buf, int len);
+
+extern struct MemoryRegion io_mem_ram;
+extern struct MemoryRegion io_mem_rom;
+extern struct MemoryRegion io_mem_unassigned;
+extern struct MemoryRegion io_mem_notdirty;
+
+#endif
+
+#endif /* !CPU_COMMON_H */
diff --git a/cpu-defs.h b/include/exec/cpu-defs.h
index 4018b88a1ad..2911b9fc908 100644
--- a/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -27,9 +27,9 @@
 #include <setjmp.h>
 #include <inttypes.h>
 #include <signal.h>
-#include "osdep.h"
-#include "qemu-queue.h"
-#include "targphys.h"
+#include "qemu/osdep.h"
+#include "qemu/queue.h"
+#include "exec/hwaddr.h"
 
 #ifndef TARGET_LONG_BITS
 #error TARGET_LONG_BITS must be defined before including this header
@@ -111,7 +111,7 @@ extern int CPUTLBEntry_wrong_size[sizeof(CPUTLBEntry) == (1 << CPU_TLB_ENTRY_BIT
 #define CPU_COMMON_TLB \
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
-    target_phys_addr_t iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
+    hwaddr iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
     target_ulong tlb_flush_addr;                                        \
     target_ulong tlb_flush_mask;
 
@@ -134,10 +134,6 @@ typedef struct icount_decr_u16 {
 } icount_decr_u16;
 #endif
 
-struct kvm_run;
-struct KVMState;
-struct qemu_work_item;
-
 typedef struct CPUBreakpoint {
     target_ulong pc;
     int flags; /* BP_* */
@@ -195,25 +191,11 @@ typedef struct CPUWatchpoint {
     int exception_index;                                                \
                                                                         \
     CPUArchState *next_cpu; /* next CPU sharing TB cache */                 \
-    int cpu_index; /* CPU index (informative) */                        \
     uint32_t host_tid; /* host thread ID */                             \
-    int numa_node; /* NUMA node this cpu is belonging to  */            \
-    int nr_cores;  /* number of cores within this CPU package */        \
-    int nr_threads;/* number of threads within this CPU */              \
     int running; /* Nonzero if cpu is currently running(usermode).  */  \
-    int thread_id;                                                      \
     /* user data */                                                     \
     void *opaque;                                                       \
                                                                         \
-    uint32_t created;                                                   \
-    uint32_t stop;   /* Stop request */                                 \
-    uint32_t stopped; /* Artificially stopped */                        \
-    struct QemuCond *halt_cond;                                         \
-    struct qemu_work_item *queued_work_first, *queued_work_last;        \
-    const char *cpu_model_str;                                          \
-    struct KVMState *kvm_state;                                         \
-    struct kvm_run *kvm_run;                                            \
-    int kvm_fd;                                                         \
-    int kvm_vcpu_dirty;
+    const char *cpu_model_str;
 
 #endif
diff --git a/cputlb.h b/include/exec/cputlb.h
index 2dc2c96cd80..733c885a1f8 100644
--- a/cputlb.h
+++ b/include/exec/cputlb.h
@@ -26,17 +26,18 @@ void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr,
                              target_ulong vaddr);
 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                            uintptr_t length);
-MemoryRegionSection *phys_page_find(target_phys_addr_t index);
+MemoryRegionSection *phys_page_find(struct AddressSpaceDispatch *d,
+                                    hwaddr index);
 void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
 void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
 extern int tlb_flush_count;
 
 /* exec.c */
 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr);
-target_phys_addr_t memory_region_section_get_iotlb(CPUArchState *env,
+hwaddr memory_region_section_get_iotlb(CPUArchState *env,
                                                    MemoryRegionSection *section,
                                                    target_ulong vaddr,
-                                                   target_phys_addr_t paddr,
+                                                   hwaddr paddr,
                                                    int prot,
                                                    target_ulong *address);
 bool memory_region_is_unassigned(MemoryRegion *mr);
diff --git a/def-helper.h b/include/exec/def-helper.h
index b98ff69e4e9..022a9ceb6a2 100644
--- a/def-helper.h
+++ b/include/exec/def-helper.h
@@ -128,6 +128,8 @@
 #define DEF_HELPER_5(name, ret, t1, t2, t3, t4, t5) \
     DEF_HELPER_FLAGS_5(name, 0, ret, t1, t2, t3, t4, t5)
 
+/* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */
+
 #endif /* DEF_HELPER_H */
 
 #ifndef GEN_HELPER
diff --git a/exec-all.h b/include/exec/exec-all.h
index c5ec8e11580..d235ef8b2e1 100644
--- a/exec-all.h
+++ b/include/exec/exec-all.h
@@ -51,7 +51,7 @@ typedef struct TranslationBlock TranslationBlock;
 #else
 #define MAX_OPC_PARAM_PER_ARG 1
 #endif
-#define MAX_OPC_PARAM_IARGS 4
+#define MAX_OPC_PARAM_IARGS 5
 #define MAX_OPC_PARAM_OARGS 1
 #define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
 
@@ -70,11 +70,7 @@ typedef struct TranslationBlock TranslationBlock;
 
 #define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
 
-extern target_ulong gen_opc_pc[OPC_BUF_SIZE];
-extern uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
-extern uint16_t gen_opc_icount[OPC_BUF_SIZE];
-
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 void gen_intermediate_code(CPUArchState *env, struct TranslationBlock *tb);
 void gen_intermediate_code_pc(CPUArchState *env, struct TranslationBlock *tb);
@@ -84,8 +80,8 @@ void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
 void cpu_gen_init(void);
 int cpu_gen_code(CPUArchState *env, struct TranslationBlock *tb,
                  int *gen_code_size_ptr);
-int cpu_restore_state(struct TranslationBlock *tb,
-                      CPUArchState *env, uintptr_t searched_pc);
+bool cpu_restore_state(CPUArchState *env, uintptr_t searched_pc);
+
 void QEMU_NORETURN cpu_resume_from_signal(CPUArchState *env1, void *puc);
 void QEMU_NORETURN cpu_io_recompile(CPUArchState *env, uintptr_t retaddr);
 TranslationBlock *tb_gen_code(CPUArchState *env, 
@@ -103,9 +99,9 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
 void tlb_flush_page(CPUArchState *env, target_ulong addr);
 void tlb_flush(CPUArchState *env, int flush_global);
 void tlb_set_page(CPUArchState *env, target_ulong vaddr,
-                  target_phys_addr_t paddr, int prot,
+                  hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size);
-void tb_invalidate_phys_addr(target_phys_addr_t addr);
+void tb_invalidate_phys_addr(hwaddr addr);
 #else
 static inline void tlb_flush_page(CPUArchState *env, target_ulong addr)
 {
@@ -121,8 +117,6 @@ static inline void tlb_flush(CPUArchState *env, int flush_global)
 #define CODE_GEN_PHYS_HASH_BITS     15
 #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
 
-#define MIN_CODE_GEN_BUFFER_SIZE     (1024 * 1024)
-
 /* estimated block size for TB allocation */
 /* XXX: use a per code average code fragment size and modulate it
    according to the host CPU */
@@ -132,9 +126,10 @@ static inline void tlb_flush(CPUArchState *env, int flush_global)
 #define CODE_GEN_AVG_BLOCK_SIZE 64
 #endif
 
-#if defined(_ARCH_PPC) || defined(__x86_64__) || defined(__arm__) || defined(__i386__)
-#define USE_DIRECT_JUMP
-#elif defined(CONFIG_TCG_INTERPRETER)
+#if defined(__arm__) || defined(_ARCH_PPC) \
+    || defined(__x86_64__) || defined(__i386__) \
+    || defined(__sparc__) \
+    || defined(CONFIG_TCG_INTERPRETER)
 #define USE_DIRECT_JUMP
 #endif
 
@@ -195,8 +190,6 @@ static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
 
 void tb_free(TranslationBlock *tb);
 void tb_flush(CPUArchState *env);
-void tb_link_page(TranslationBlock *tb,
-                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2);
 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
 
 extern TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
@@ -244,6 +237,8 @@ static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
+#elif defined(__sparc__)
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
 #else
 #error tb_set_jmp_target1 is missing
 #endif
@@ -280,9 +275,7 @@ static inline void tb_add_jump(TranslationBlock *tb, int n,
     }
 }
 
-TranslationBlock *tb_find_pc(uintptr_t pc_ptr);
-
-#include "qemu-lock.h"
+#include "exec/spinlock.h"
 
 extern spinlock_t tb_lock;
 
@@ -291,12 +284,8 @@ extern int tb_invalidated_flag;
 /* The return address may point to the start of the next instruction.
    Subtracting one gets us the call instruction itself.  */
 #if defined(CONFIG_TCG_INTERPRETER)
-/* Alpha and SH4 user mode emulations and Softmmu call GETPC().
-   For all others, GETPC remains undefined (which makes TCI a little faster. */
-# if defined(CONFIG_SOFTMMU) || defined(TARGET_ALPHA) || defined(TARGET_SH4)
 extern uintptr_t tci_tb_ptr;
-#  define GETPC() tci_tb_ptr
-# endif
+# define GETPC() tci_tb_ptr
 #elif defined(__s390__) && !defined(__s390x__)
 # define GETPC() \
     (((uintptr_t)__builtin_return_address(0) & 0x7fffffffUL) - 1)
@@ -308,40 +297,75 @@ extern uintptr_t tci_tb_ptr;
 # define GETPC() ((uintptr_t)__builtin_return_address(0) - 1)
 #endif
 
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* qemu_ld/st optimization split code generation to fast and slow path, thus,
+   it needs special handling for an MMU helper which is called from the slow
+   path, to get the fast path's pc without any additional argument.
+   It uses a tricky solution which embeds the fast path pc into the slow path.
+
+   Code flow in slow path:
+   (1) pre-process
+   (2) call MMU helper
+   (3) jump to (5)
+   (4) fast path information (implementation specific)
+   (5) post-process (e.g. stack adjust)
+   (6) jump to corresponding code of the next of fast path
+ */
+# if defined(__i386__) || defined(__x86_64__)
+/* To avoid broken disassembling, long jmp is used for embedding fast path pc,
+   so that the destination is the next code of fast path, though this jmp is
+   never executed.
+
+   call MMU helper
+   jmp POST_PROC (2byte)    <- GETRA()
+   jmp NEXT_CODE (5byte)
+   POST_PROCESS ...         <- GETRA() + 7
+ */
+#  define GETRA() ((uintptr_t)__builtin_return_address(0))
+#  define GETPC_LDST() ((uintptr_t)(GETRA() + 7 + \
+                                    *(int32_t *)((void *)GETRA() + 3) - 1))
+# elif defined (_ARCH_PPC) && !defined (_ARCH_PPC64)
+#  define GETRA() ((uintptr_t)__builtin_return_address(0))
+#  define GETPC_LDST() ((uintptr_t) ((*(int32_t *)(GETRA() - 4)) - 1))
+# else
+#  error "CONFIG_QEMU_LDST_OPTIMIZATION needs GETPC_LDST() implementation!"
+# endif
+bool is_tcg_gen_code(uintptr_t pc_ptr);
+# define GETPC_EXT() (is_tcg_gen_code(GETRA()) ? GETPC_LDST() : GETPC())
+#else
+# define GETPC_EXT() GETPC()
+#endif
+
 #if !defined(CONFIG_USER_ONLY)
 
-struct MemoryRegion *iotlb_to_region(target_phys_addr_t index);
-uint64_t io_mem_read(struct MemoryRegion *mr, target_phys_addr_t addr,
+struct MemoryRegion *iotlb_to_region(hwaddr index);
+uint64_t io_mem_read(struct MemoryRegion *mr, hwaddr addr,
                      unsigned size);
-void io_mem_write(struct MemoryRegion *mr, target_phys_addr_t addr,
+void io_mem_write(struct MemoryRegion *mr, hwaddr addr,
                   uint64_t value, unsigned size);
 
 void tlb_fill(CPUArchState *env1, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr);
 
-#include "softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 #define ACCESS_TYPE (NB_MMU_MODES + 1)
 #define MEMSUFFIX _code
-#ifndef CONFIG_TCG_PASS_AREG0
-#define env cpu_single_env
-#endif
 
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
-#undef env
 
 #endif
 
diff --git a/gdbstub.h b/include/exec/gdbstub.h
index 668de660004..49231feb29d 100644
--- a/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -35,7 +35,8 @@ static inline int cpu_index(CPUArchState *env)
 #if defined(CONFIG_USER_ONLY) && defined(CONFIG_USE_NPTL)
     return env->host_tid;
 #else
-    return env->cpu_index + 1;
+    CPUState *cpu = ENV_GET_CPU(env);
+    return cpu->cpu_index + 1;
 #endif
 }
 
diff --git a/gen-icount.h b/include/exec/gen-icount.h
index 430cb446d0e..8043b3ba268 100644
--- a/gen-icount.h
+++ b/include/exec/gen-icount.h
@@ -1,4 +1,7 @@
-#include "qemu-timer.h"
+#ifndef GEN_ICOUNT_H
+#define GEN_ICOUNT_H 1
+
+#include "qemu/timer.h"
 
 /* Helpers for instruction counting code generation.  */
 
@@ -16,7 +19,7 @@ static inline void gen_icount_start(void)
     count = tcg_temp_local_new_i32();
     tcg_gen_ld_i32(count, cpu_env, offsetof(CPUArchState, icount_decr.u32));
     /* This is a horrid hack to allow fixing up the value later.  */
-    icount_arg = gen_opparam_ptr + 1;
+    icount_arg = tcg_ctx.gen_opparam_ptr + 1;
     tcg_gen_subi_i32(count, count, 0xdeadbeef);
 
     tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, icount_label);
@@ -46,3 +49,5 @@ static inline void gen_io_end(void)
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUArchState, can_do_io));
     tcg_temp_free_i32(tmp);
 }
+
+#endif
diff --git a/include/exec/hwaddr.h b/include/exec/hwaddr.h
new file mode 100644
index 00000000000..251cf9216f6
--- /dev/null
+++ b/include/exec/hwaddr.h
@@ -0,0 +1,24 @@
+/* Define hwaddr if it exists.  */
+
+#ifndef HWADDR_H
+#define HWADDR_H
+
+#ifndef CONFIG_USER_ONLY
+
+#define HWADDR_BITS 64
+/* hwaddr is the type of a physical address (its size can
+   be different from 'target_ulong').  */
+
+typedef uint64_t hwaddr;
+#define HWADDR_MAX UINT64_MAX
+#define TARGET_FMT_plx "%016" PRIx64
+#define HWADDR_PRId PRId64
+#define HWADDR_PRIi PRIi64
+#define HWADDR_PRIo PRIo64
+#define HWADDR_PRIu PRIu64
+#define HWADDR_PRIx PRIx64
+#define HWADDR_PRIX PRIX64
+
+#endif
+
+#endif
diff --git a/ioport.h b/include/exec/ioport.h
index 23441cba088..fc28350a3cc 100644
--- a/ioport.h
+++ b/include/exec/ioport.h
@@ -25,7 +25,7 @@
 #define IOPORT_H
 
 #include "qemu-common.h"
-#include "iorange.h"
+#include "exec/iorange.h"
 
 typedef uint32_t pio_addr_t;
 #define FMT_pioaddr     PRIx32
diff --git a/iorange.h b/include/exec/iorange.h
index cd980a8312a..cd980a8312a 100644
--- a/iorange.h
+++ b/include/exec/iorange.h
diff --git a/exec-obsolete.h b/include/exec/memory-internal.h
index c09925610d0..1da240039dd 100644
--- a/exec-obsolete.h
+++ b/include/exec/memory-internal.h
@@ -16,14 +16,32 @@
  * The functions declared here will be removed soon.
  */
 
-#ifndef EXEC_OBSOLETE_H
-#define EXEC_OBSOLETE_H
-
-#ifndef WANT_EXEC_OBSOLETE
-#error Do not include exec-obsolete.h
-#endif
+#ifndef MEMORY_INTERNAL_H
+#define MEMORY_INTERNAL_H
 
 #ifndef CONFIG_USER_ONLY
+#include "hw/xen.h"
+
+typedef struct PhysPageEntry PhysPageEntry;
+
+struct PhysPageEntry {
+    uint16_t is_leaf : 1;
+     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
+    uint16_t ptr : 15;
+};
+
+typedef struct AddressSpaceDispatch AddressSpaceDispatch;
+
+struct AddressSpaceDispatch {
+    /* This is a multi-level map on the physical address space.
+     * The bottom level has pointers to MemoryRegionSections.
+     */
+    PhysPageEntry phys_map;
+    MemoryListener listener;
+};
+
+void address_space_init_dispatch(AddressSpace *as);
+void address_space_destroy_dispatch(AddressSpace *as);
 
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr);
@@ -33,13 +51,9 @@ void qemu_ram_free_from_ptr(ram_addr_t addr);
 
 struct MemoryRegion;
 struct MemoryRegionSection;
-void cpu_register_physical_memory_log(struct MemoryRegionSection *section,
-                                      bool readonly);
-
-void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
-void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
 
-int cpu_physical_memory_set_dirty_tracking(int enable);
+void qemu_register_coalesced_mmio(hwaddr addr, ram_addr_t size);
+void qemu_unregister_coalesced_mmio(hwaddr addr, ram_addr_t size);
 
 #define VGA_DIRTY_FLAG       0x01
 #define CODE_DIRTY_FLAG      0x02
@@ -74,11 +88,6 @@ static inline int cpu_physical_memory_get_dirty(ram_addr_t start,
 static inline int cpu_physical_memory_set_dirty_flags(ram_addr_t addr,
                                                       int dirty_flags)
 {
-    if ((dirty_flags & MIGRATION_DIRTY_FLAG) &&
-        !cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
-                                       MIGRATION_DIRTY_FLAG)) {
-        ram_list.dirty_pages++;
-    }
     return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] |= dirty_flags;
 }
 
@@ -92,11 +101,6 @@ static inline int cpu_physical_memory_clear_dirty_flags(ram_addr_t addr,
 {
     int mask = ~dirty_flags;
 
-    if ((dirty_flags & MIGRATION_DIRTY_FLAG) &&
-        cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
-                                      MIGRATION_DIRTY_FLAG)) {
-        ram_list.dirty_pages--;
-    }
     return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] &= mask;
 }
 
@@ -111,6 +115,7 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
     for (addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
         cpu_physical_memory_set_dirty_flags(addr, dirty_flags);
     }
+    xen_modified_memory(addr, length);
 }
 
 static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
diff --git a/memory.h b/include/exec/memory.h
index bd1bbaeabed..2322732dce2 100644
--- a/memory.h
+++ b/include/exec/memory.h
@@ -19,12 +19,12 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include "qemu-common.h"
-#include "cpu-common.h"
-#include "targphys.h"
-#include "qemu-queue.h"
-#include "iorange.h"
-#include "ioport.h"
-#include "int128.h"
+#include "exec/cpu-common.h"
+#include "exec/hwaddr.h"
+#include "qemu/queue.h"
+#include "exec/iorange.h"
+#include "exec/ioport.h"
+#include "qemu/int128.h"
 
 typedef struct MemoryRegionOps MemoryRegionOps;
 typedef struct MemoryRegion MemoryRegion;
@@ -48,7 +48,7 @@ typedef struct MemoryRegionIORange MemoryRegionIORange;
 struct MemoryRegionIORange {
     IORange iorange;
     MemoryRegion *mr;
-    target_phys_addr_t offset;
+    hwaddr offset;
 };
 
 /*
@@ -58,12 +58,12 @@ struct MemoryRegionOps {
     /* Read from the memory region. @addr is relative to @mr; @size is
      * in bytes. */
     uint64_t (*read)(void *opaque,
-                     target_phys_addr_t addr,
+                     hwaddr addr,
                      unsigned size);
     /* Write to the memory region. @addr is relative to @mr; @size is
      * in bytes. */
     void (*write)(void *opaque,
-                  target_phys_addr_t addr,
+                  hwaddr addr,
                   uint64_t data,
                   unsigned size);
 
@@ -84,7 +84,7 @@ struct MemoryRegionOps {
          * by the device (and results in machine dependent behaviour such
          * as a machine check exception).
          */
-        bool (*accepts)(void *opaque, target_phys_addr_t addr,
+        bool (*accepts)(void *opaque, hwaddr addr,
                         unsigned size, bool is_write);
     } valid;
     /* Internal implementation constraints: */
@@ -122,7 +122,7 @@ struct MemoryRegion {
     void *opaque;
     MemoryRegion *parent;
     Int128 size;
-    target_phys_addr_t addr;
+    hwaddr addr;
     void (*destructor)(MemoryRegion *mr);
     ram_addr_t ram_addr;
     bool subpage;
@@ -133,8 +133,9 @@ struct MemoryRegion {
     bool enabled;
     bool rom_device;
     bool warning_printed; /* For reservations */
+    bool flush_coalesced_mmio;
     MemoryRegion *alias;
-    target_phys_addr_t alias_offset;
+    hwaddr alias_offset;
     unsigned priority;
     bool may_overlap;
     QTAILQ_HEAD(subregions, MemoryRegion) subregions;
@@ -156,6 +157,22 @@ struct MemoryRegionPortio {
 
 #define PORTIO_END_OF_LIST() { }
 
+typedef struct AddressSpace AddressSpace;
+
+/**
+ * AddressSpace: describes a mapping of addresses to #MemoryRegion objects
+ */
+struct AddressSpace {
+    /* All fields are private. */
+    const char *name;
+    MemoryRegion *root;
+    struct FlatView *current_map;
+    int ioeventfd_nb;
+    struct MemoryRegionIoeventfd *ioeventfds;
+    struct AddressSpaceDispatch *dispatch;
+    QTAILQ_ENTRY(AddressSpace) address_spaces_link;
+};
+
 typedef struct MemoryRegionSection MemoryRegionSection;
 
 /**
@@ -171,10 +188,10 @@ typedef struct MemoryRegionSection MemoryRegionSection;
  */
 struct MemoryRegionSection {
     MemoryRegion *mr;
-    MemoryRegion *address_space;
-    target_phys_addr_t offset_within_region;
+    AddressSpace *address_space;
+    hwaddr offset_within_region;
     uint64_t size;
-    target_phys_addr_t offset_within_address_space;
+    hwaddr offset_within_address_space;
     bool readonly;
 };
 
@@ -201,9 +218,13 @@ struct MemoryListener {
                         bool match_data, uint64_t data, EventNotifier *e);
     void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
                         bool match_data, uint64_t data, EventNotifier *e);
+    void (*coalesced_mmio_add)(MemoryListener *listener, MemoryRegionSection *section,
+                               hwaddr addr, hwaddr len);
+    void (*coalesced_mmio_del)(MemoryListener *listener, MemoryRegionSection *section,
+                               hwaddr addr, hwaddr len);
     /* Lower = earlier (during add), later (during del) */
     unsigned priority;
-    MemoryRegion *address_space_filter;
+    AddressSpace *address_space_filter;
     QTAILQ_ENTRY(MemoryListener) link;
 };
 
@@ -252,9 +273,9 @@ void memory_region_init_ram(MemoryRegion *mr,
                             uint64_t size);
 
 /**
- * memory_region_init_ram:  Initialize RAM memory region from a user-provided.
- *                          pointer.  Accesses into the region will modify
- *                          memory directly.
+ * memory_region_init_ram_ptr:  Initialize RAM memory region from a
+ *                              user-provided pointer.  Accesses into the
+ *                              region will modify memory directly.
  *
  * @mr: the #MemoryRegion to be initialized.
  * @name: the name of the region.
@@ -280,7 +301,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
 void memory_region_init_alias(MemoryRegion *mr,
                               const char *name,
                               MemoryRegion *orig,
-                              target_phys_addr_t offset,
+                              hwaddr offset,
                               uint64_t size);
 
 /**
@@ -416,8 +437,8 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client);
  * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
  *          %DIRTY_MEMORY_VGA.
  */
-bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size, unsigned client);
+bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size, unsigned client);
 
 /**
  * memory_region_set_dirty: Mark a range of bytes as dirty in a memory region.
@@ -429,10 +450,26 @@ bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
  * @addr: the address (relative to the start of the region) being dirtied.
  * @size: size of the range being dirtied.
  */
-void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size);
+void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size);
 
 /**
+ * memory_region_test_and_clear_dirty: Check whether a range of bytes is dirty
+ *                                     for a specified client. It clears them.
+ *
+ * Checks whether a range of bytes has been written to since the last
+ * call to memory_region_reset_dirty() with the same @client.  Dirty logging
+ * must be enabled.
+ *
+ * @mr: the memory region being queried.
+ * @addr: the address (relative to the start of the region) being queried.
+ * @size: the size of the range being queried.
+ * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
+ *          %DIRTY_MEMORY_VGA.
+ */
+bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
+                                        hwaddr size, unsigned client);
+/**
  * memory_region_sync_dirty_bitmap: Synchronize a region's dirty bitmap with
  *                                  any external TLBs (e.g. kvm)
  *
@@ -455,8 +492,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr);
  * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
  *          %DIRTY_MEMORY_VGA.
  */
-void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                               target_phys_addr_t size, unsigned client);
+void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
+                               hwaddr size, unsigned client);
 
 /**
  * memory_region_set_readonly: Turn a memory region read-only (or read-write)
@@ -506,7 +543,7 @@ void memory_region_set_coalescing(MemoryRegion *mr);
  * @size: the size of the subrange to be coalesced.
  */
 void memory_region_add_coalescing(MemoryRegion *mr,
-                                  target_phys_addr_t offset,
+                                  hwaddr offset,
                                   uint64_t size);
 
 /**
@@ -521,6 +558,31 @@ void memory_region_add_coalescing(MemoryRegion *mr,
 void memory_region_clear_coalescing(MemoryRegion *mr);
 
 /**
+ * memory_region_set_flush_coalesced: Enforce memory coalescing flush before
+ *                                    accesses.
+ *
+ * Ensure that pending coalesced MMIO request are flushed before the memory
+ * region is accessed. This property is automatically enabled for all regions
+ * passed to memory_region_set_coalescing() and memory_region_add_coalescing().
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_set_flush_coalesced(MemoryRegion *mr);
+
+/**
+ * memory_region_clear_flush_coalesced: Disable memory coalescing flush before
+ *                                      accesses.
+ *
+ * Clear the automatic coalesced MMIO flushing enabled via
+ * memory_region_set_flush_coalesced. Note that this service has no effect on
+ * memory regions that have MMIO coalescing enabled for themselves. For them,
+ * automatic flushing will stop once coalescing is disabled.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_clear_flush_coalesced(MemoryRegion *mr);
+
+/**
  * memory_region_add_eventfd: Request an eventfd to be triggered when a word
  *                            is written to a location.
  *
@@ -537,7 +599,7 @@ void memory_region_clear_coalescing(MemoryRegion *mr);
  * @fd: the eventfd to be triggered when @addr, @size, and @data all match.
  **/
 void memory_region_add_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -557,7 +619,7 @@ void memory_region_add_eventfd(MemoryRegion *mr,
  * @fd: the eventfd to be triggered when @addr, @size, and @data all match.
  */
 void memory_region_del_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -578,10 +640,11 @@ void memory_region_del_eventfd(MemoryRegion *mr,
  * @subregion: the subregion to be added.
  */
 void memory_region_add_subregion(MemoryRegion *mr,
-                                 target_phys_addr_t offset,
+                                 hwaddr offset,
                                  MemoryRegion *subregion);
 /**
- * memory_region_add_subregion: Add a subregion to a container, with overlap.
+ * memory_region_add_subregion_overlap: Add a subregion to a container
+ *                                      with overlap.
  *
  * Adds a subregion at @offset.  The subregion may overlap with other
  * subregions.  Conflicts are resolved by having a higher @priority hide a
@@ -597,7 +660,7 @@ void memory_region_add_subregion(MemoryRegion *mr,
  * @priority: used for resolving overlaps; highest priority wins.
  */
 void memory_region_add_subregion_overlap(MemoryRegion *mr,
-                                         target_phys_addr_t offset,
+                                         hwaddr offset,
                                          MemoryRegion *subregion,
                                          unsigned priority);
 
@@ -645,7 +708,7 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled);
  * @mr: the region to be updated
  * @addr: new address, relative to parent region
  */
-void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr);
+void memory_region_set_address(MemoryRegion *mr, hwaddr addr);
 
 /*
  * memory_region_set_alias_offset: dynamically update a memory alias's offset
@@ -657,7 +720,7 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr);
  * @offset: the new offset into the target memory region
  */
 void memory_region_set_alias_offset(MemoryRegion *mr,
-                                    target_phys_addr_t offset);
+                                    hwaddr offset);
 
 /**
  * memory_region_find: locate a MemoryRegion in an address space
@@ -678,7 +741,7 @@ void memory_region_set_alias_offset(MemoryRegion *mr,
  * @size: size of the area to be searched
  */
 MemoryRegionSection memory_region_find(MemoryRegion *address_space,
-                                       target_phys_addr_t addr, uint64_t size);
+                                       hwaddr addr, uint64_t size);
 
 /**
  * memory_region_section_addr: get offset within MemoryRegionSection
@@ -688,9 +751,9 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
  * @section: the memory region section being queried
  * @addr: address in address space
  */
-static inline target_phys_addr_t
+static inline hwaddr
 memory_region_section_addr(MemoryRegionSection *section,
-                           target_phys_addr_t addr)
+                           hwaddr addr)
 {
     addr -= section->offset_within_address_space;
     addr += section->offset_within_region;
@@ -728,7 +791,7 @@ void memory_region_transaction_commit(void);
  * @listener: an object containing the callbacks to be called
  * @filter: if non-%NULL, only regions in this address space will be observed
  */
-void memory_listener_register(MemoryListener *listener, MemoryRegion *filter);
+void memory_listener_register(MemoryListener *listener, AddressSpace *filter);
 
 /**
  * memory_listener_unregister: undo the effect of memory_listener_register()
@@ -743,12 +806,93 @@ void memory_listener_unregister(MemoryListener *listener);
 void memory_global_dirty_log_start(void);
 
 /**
- * memory_global_dirty_log_stop: begin dirty logging for all regions
+ * memory_global_dirty_log_stop: end dirty logging for all regions
  */
 void memory_global_dirty_log_stop(void);
 
 void mtree_info(fprintf_function mon_printf, void *f);
 
+/**
+ * address_space_init: initializes an address space
+ *
+ * @as: an uninitialized #AddressSpace
+ * @root: a #MemoryRegion that routes addesses for the address space
+ */
+void address_space_init(AddressSpace *as, MemoryRegion *root);
+
+
+/**
+ * address_space_destroy: destroy an address space
+ *
+ * Releases all resources associated with an address space.  After an address space
+ * is destroyed, its root memory region (given by address_space_init()) may be destroyed
+ * as well.
+ *
+ * @as: address space to be destroyed
+ */
+void address_space_destroy(AddressSpace *as);
+
+/**
+ * address_space_rw: read from or write to an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ * @is_write: indicates the transfer direction
+ */
+void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
+                      int len, bool is_write);
+
+/**
+ * address_space_write: write to address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_write(AddressSpace *as, hwaddr addr,
+                         const uint8_t *buf, int len);
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len);
+
+/* address_space_map: map a physical memory region into a host virtual address
+ *
+ * May map a subset of the requested range, given by and returned in @plen.
+ * May return %NULL if resources needed to perform the mapping are exhausted.
+ * Use only for reads OR writes - not for read-modify-write operations.
+ * Use cpu_register_map_client() to know when retrying the map operation is
+ * likely to succeed.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @plen: pointer to length of buffer; updated on return
+ * @is_write: indicates the transfer direction
+ */
+void *address_space_map(AddressSpace *as, hwaddr addr,
+                        hwaddr *plen, bool is_write);
+
+/* address_space_unmap: Unmaps a memory region previously mapped by address_space_map()
+ *
+ * Will also mark the memory as dirty if @is_write == %true.  @access_len gives
+ * the amount of memory that was actually read or written by the caller.
+ *
+ * @as: #AddressSpace used
+ * @addr: address within that address space
+ * @len: buffer length as returned by address_space_map()
+ * @access_len: amount of data actually transferred
+ * @is_write: indicates the transfer direction
+ */
+void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
+                         int is_write, hwaddr access_len);
+
+
 #endif
 
 #endif
diff --git a/poison.h b/include/exec/poison.h
index 7d7b23b1fc8..7d7b23b1fc8 100644
--- a/poison.h
+++ b/include/exec/poison.h
diff --git a/softmmu-semi.h b/include/exec/softmmu-semi.h
index bcb979a5b03..93798b96142 100644
--- a/softmmu-semi.h
+++ b/include/exec/softmmu-semi.h
@@ -6,6 +6,8 @@
  *
  * This code is licensed under the GPL
  */
+#ifndef SOFTMMU_SEMI_H
+#define SOFTMMU_SEMI_H 1
 
 static inline uint32_t softmmu_tget32(CPUArchState *env, uint32_t addr)
 {
@@ -71,3 +73,5 @@ static void softmmu_unlock_user(CPUArchState *env, void *p, target_ulong addr,
     free(p);
 }
 #define unlock_user(s, args, len) softmmu_unlock_user(env, s, args, len)
+
+#endif
diff --git a/softmmu_defs.h b/include/exec/softmmu_defs.h
index 8d59f9d4f8d..1f25e33ce40 100644
--- a/softmmu_defs.h
+++ b/include/exec/softmmu_defs.h
@@ -9,25 +9,6 @@
 #ifndef SOFTMMU_DEFS_H
 #define SOFTMMU_DEFS_H
 
-#ifndef CONFIG_TCG_PASS_AREG0
-uint8_t __ldb_mmu(target_ulong addr, int mmu_idx);
-void __stb_mmu(target_ulong addr, uint8_t val, int mmu_idx);
-uint16_t __ldw_mmu(target_ulong addr, int mmu_idx);
-void __stw_mmu(target_ulong addr, uint16_t val, int mmu_idx);
-uint32_t __ldl_mmu(target_ulong addr, int mmu_idx);
-void __stl_mmu(target_ulong addr, uint32_t val, int mmu_idx);
-uint64_t __ldq_mmu(target_ulong addr, int mmu_idx);
-void __stq_mmu(target_ulong addr, uint64_t val, int mmu_idx);
-
-uint8_t __ldb_cmmu(target_ulong addr, int mmu_idx);
-void __stb_cmmu(target_ulong addr, uint8_t val, int mmu_idx);
-uint16_t __ldw_cmmu(target_ulong addr, int mmu_idx);
-void __stw_cmmu(target_ulong addr, uint16_t val, int mmu_idx);
-uint32_t __ldl_cmmu(target_ulong addr, int mmu_idx);
-void __stl_cmmu(target_ulong addr, uint32_t val, int mmu_idx);
-uint64_t __ldq_cmmu(target_ulong addr, int mmu_idx);
-void __stq_cmmu(target_ulong addr, uint64_t val, int mmu_idx);
-#else
 uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 void helper_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
                     int mmu_idx);
@@ -54,5 +35,3 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 void helper_stq_cmmu(CPUArchState *env, target_ulong addr, uint64_t val,
                      int mmu_idx);
 #endif
-
-#endif
diff --git a/softmmu_exec.h b/include/exec/softmmu_exec.h
index 8c73985599f..3e4e886a30d 100644
--- a/softmmu_exec.h
+++ b/include/exec/softmmu_exec.h
@@ -19,37 +19,37 @@
 #define ldul_executive  ldl_executive
 #define ldul_supervisor ldl_supervisor
 
-#include "softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 #define ACCESS_TYPE 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
 #define ACCESS_TYPE 1
 #define MEMSUFFIX MMU_MODE1_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
@@ -58,16 +58,16 @@
 #define ACCESS_TYPE 2
 #define MEMSUFFIX MMU_MODE2_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 3) */
@@ -77,16 +77,16 @@
 #define ACCESS_TYPE 3
 #define MEMSUFFIX MMU_MODE3_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 4) */
@@ -96,16 +96,16 @@
 #define ACCESS_TYPE 4
 #define MEMSUFFIX MMU_MODE4_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 5) */
@@ -115,16 +115,16 @@
 #define ACCESS_TYPE 5
 #define MEMSUFFIX MMU_MODE5_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 6) */
@@ -137,16 +137,16 @@
 #define ACCESS_TYPE (NB_MMU_MODES)
 #define MEMSUFFIX _data
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
diff --git a/softmmu_header.h b/include/exec/softmmu_header.h
index cf1aa38fc7b..d8d9c81b054 100644
--- a/softmmu_header.h
+++ b/include/exec/softmmu_header.h
@@ -78,23 +78,10 @@
 #define ADDR_READ addr_read
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ENV_PARAM
-#define ENV_VAR
-#define CPU_PREFIX
-#define HELPER_PREFIX __
-#else
-#define ENV_PARAM CPUArchState *env,
-#define ENV_VAR env,
-#define CPU_PREFIX cpu_
-#define HELPER_PREFIX helper_
-#endif
-
 /* generic load/store macros */
 
 static inline RES_TYPE
-glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
-                                                     target_ulong ptr)
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     int page_index;
     RES_TYPE res;
@@ -106,9 +93,7 @@ glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_VAR
-                                                                     addr,
-                                                                     mmu_idx);
+        res = glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(ld, USUFFIX), _raw)(hostaddr);
@@ -118,8 +103,7 @@ glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
 
 #if DATA_SIZE <= 2
 static inline int
-glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
-                                                     target_ulong ptr)
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     int res, page_index;
     target_ulong addr;
@@ -130,8 +114,8 @@ glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = (DATA_STYPE)glue(glue(glue(HELPER_PREFIX, ld), SUFFIX),
-                               MMUSUFFIX)(ENV_VAR addr, mmu_idx);
+        res = (DATA_STYPE)glue(glue(helper_ld, SUFFIX),
+                               MMUSUFFIX)(env, addr, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(lds, SUFFIX), _raw)(hostaddr);
@@ -145,8 +129,8 @@ glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
 /* generic store macro */
 
 static inline void
-glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
-                                                    RES_TYPE v)
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
 {
     int page_index;
     target_ulong addr;
@@ -157,8 +141,7 @@ glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_VAR addr, v,
-                                                               mmu_idx);
+        glue(glue(helper_st, SUFFIX), MMUSUFFIX)(env, addr, v, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         glue(glue(st, SUFFIX), _raw)(hostaddr, v);
@@ -170,52 +153,50 @@ glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
 #if ACCESS_TYPE != (NB_MMU_MODES + 1)
 
 #if DATA_SIZE == 8
-static inline float64 glue(glue(CPU_PREFIX, ldfq), MEMSUFFIX)(ENV_PARAM
-                                                              target_ulong ptr)
+static inline float64 glue(cpu_ldfq, MEMSUFFIX)(CPUArchState *env,
+                                                target_ulong ptr)
 {
     union {
         float64 d;
         uint64_t i;
     } u;
-    u.i = glue(glue(CPU_PREFIX, ldq), MEMSUFFIX)(ENV_VAR ptr);
+    u.i = glue(cpu_ldq, MEMSUFFIX)(env, ptr);
     return u.d;
 }
 
-static inline void glue(glue(CPU_PREFIX, stfq), MEMSUFFIX)(ENV_PARAM
-                                                           target_ulong ptr,
-                                                           float64 v)
+static inline void glue(cpu_stfq, MEMSUFFIX)(CPUArchState *env,
+                                             target_ulong ptr, float64 v)
 {
     union {
         float64 d;
         uint64_t i;
     } u;
     u.d = v;
-    glue(glue(CPU_PREFIX, stq), MEMSUFFIX)(ENV_VAR ptr, u.i);
+    glue(cpu_stq, MEMSUFFIX)(env, ptr, u.i);
 }
 #endif /* DATA_SIZE == 8 */
 
 #if DATA_SIZE == 4
-static inline float32 glue(glue(CPU_PREFIX, ldfl), MEMSUFFIX)(ENV_PARAM
-                                                              target_ulong ptr)
+static inline float32 glue(cpu_ldfl, MEMSUFFIX)(CPUArchState *env,
+                                                target_ulong ptr)
 {
     union {
         float32 f;
         uint32_t i;
     } u;
-    u.i = glue(glue(CPU_PREFIX, ldl), MEMSUFFIX)(ENV_VAR ptr);
+    u.i = glue(cpu_ldl, MEMSUFFIX)(env, ptr);
     return u.f;
 }
 
-static inline void glue(glue(CPU_PREFIX, stfl), MEMSUFFIX)(ENV_PARAM
-                                                           target_ulong ptr,
-                                                           float32 v)
+static inline void glue(cpu_stfl, MEMSUFFIX)(CPUArchState *env,
+                                             target_ulong ptr, float32 v)
 {
     union {
         float32 f;
         uint32_t i;
     } u;
     u.f = v;
-    glue(glue(CPU_PREFIX, stl), MEMSUFFIX)(ENV_VAR ptr, u.i);
+    glue(cpu_stl, MEMSUFFIX)(env, ptr, u.i);
 }
 #endif /* DATA_SIZE == 4 */
 
@@ -230,7 +211,3 @@ static inline void glue(glue(CPU_PREFIX, stfl), MEMSUFFIX)(ENV_PARAM
 #undef CPU_MMU_INDEX
 #undef MMUSUFFIX
 #undef ADDR_READ
-#undef ENV_PARAM
-#undef ENV_VAR
-#undef CPU_PREFIX
-#undef HELPER_PREFIX
diff --git a/softmmu_template.h b/include/exec/softmmu_template.h
index b8bd7003250..b219191abde 100644
--- a/softmmu_template.h
+++ b/include/exec/softmmu_template.h
@@ -21,8 +21,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu-timer.h"
-#include "memory.h"
+#include "qemu/timer.h"
+#include "exec/memory.h"
 
 #define DATA_SIZE (1 << SHIFT)
 
@@ -54,24 +54,12 @@
 #define ADDR_READ addr_read
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ENV_PARAM
-#define ENV_VAR
-#define CPU_PREFIX
-#define HELPER_PREFIX __
-#else
-#define ENV_PARAM CPUArchState *env,
-#define ENV_VAR env,
-#define CPU_PREFIX cpu_
-#define HELPER_PREFIX helper_
-#endif
-
-static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                         target_ulong addr,
                                                         int mmu_idx,
                                                         uintptr_t retaddr);
-static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
-                                              target_phys_addr_t physaddr,
+static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
+                                              hwaddr physaddr,
                                               target_ulong addr,
                                               uintptr_t retaddr)
 {
@@ -104,14 +92,13 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
 
 /* handle all cases except unaligned access which span two pages */
 DATA_TYPE
-glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
-                                                       target_ulong addr,
-                                                       int mmu_idx)
+glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
+                                         int mmu_idx)
 {
     DATA_TYPE res;
     int index;
     target_ulong tlb_addr;
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     uintptr_t retaddr;
 
     /* test if there is match for unaligned or IO access */
@@ -124,25 +111,25 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
             ioaddr = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
+            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
             /* slow unaligned access (it spans two pages or IO) */
         do_unaligned_access:
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
 #endif
-            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr,
+            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr,
                                                          mmu_idx, retaddr);
         } else {
             /* unaligned/aligned access in the same page */
             uintptr_t addend;
 #ifdef ALIGNED_ONLY
             if ((addr & (DATA_SIZE - 1)) != 0) {
-                retaddr = GETPC();
-                do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+                retaddr = GETPC_EXT();
+                do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
             }
 #endif
             addend = env->tlb_table[mmu_idx][index].addend;
@@ -151,10 +138,10 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
         }
     } else {
         /* the page is not in the TLB : fill it */
-        retaddr = GETPC();
+        retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
 #endif
         tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
         goto redo;
@@ -164,14 +151,14 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
 
 /* handle all unaligned cases */
 static DATA_TYPE
-glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
+glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                        target_ulong addr,
                                        int mmu_idx,
                                        uintptr_t retaddr)
 {
     DATA_TYPE res, res1, res2;
     int index, shift;
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr, addr1, addr2;
 
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
@@ -183,15 +170,15 @@ glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             ioaddr = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
+            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             /* slow unaligned access (it spans two pages) */
             addr1 = addr & ~(DATA_SIZE - 1);
             addr2 = addr1 + DATA_SIZE;
-            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr1,
+            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr1,
                                                           mmu_idx, retaddr);
-            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr2,
+            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr2,
                                                           mmu_idx, retaddr);
             shift = (addr & (DATA_SIZE - 1)) * 8;
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -216,14 +203,14 @@ glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
 
 #ifndef SOFTMMU_CODE_ACCESS
 
-static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                    target_ulong addr,
                                                    DATA_TYPE val,
                                                    int mmu_idx,
                                                    uintptr_t retaddr);
 
-static inline void glue(io_write, SUFFIX)(ENV_PARAM
-                                          target_phys_addr_t physaddr,
+static inline void glue(io_write, SUFFIX)(CPUArchState *env,
+                                          hwaddr physaddr,
                                           DATA_TYPE val,
                                           target_ulong addr,
                                           uintptr_t retaddr)
@@ -253,12 +240,11 @@ static inline void glue(io_write, SUFFIX)(ENV_PARAM
 #endif /* SHIFT > 2 */
 }
 
-void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
-                                                            target_ulong addr,
-                                                            DATA_TYPE val,
-                                                            int mmu_idx)
+void glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
+                                              target_ulong addr, DATA_TYPE val,
+                                              int mmu_idx)
 {
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr;
     uintptr_t retaddr;
     int index;
@@ -271,24 +257,24 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
             ioaddr = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
+            glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
 #endif
-            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_VAR addr, val,
+            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(env, addr, val,
                                                    mmu_idx, retaddr);
         } else {
             /* aligned/unaligned access in the same page */
             uintptr_t addend;
 #ifdef ALIGNED_ONLY
             if ((addr & (DATA_SIZE - 1)) != 0) {
-                retaddr = GETPC();
-                do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+                retaddr = GETPC_EXT();
+                do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
             }
 #endif
             addend = env->tlb_table[mmu_idx][index].addend;
@@ -297,10 +283,10 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
         }
     } else {
         /* the page is not in the TLB : fill it */
-        retaddr = GETPC();
+        retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
 #endif
         tlb_fill(env, addr, 1, mmu_idx, retaddr);
         goto redo;
@@ -308,13 +294,13 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
 }
 
 /* handles all unaligned cases */
-static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                    target_ulong addr,
                                                    DATA_TYPE val,
                                                    int mmu_idx,
                                                    uintptr_t retaddr)
 {
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr;
     int index, i;
 
@@ -327,7 +313,7 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             ioaddr = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
+            glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             /* XXX: not efficient, but simple */
@@ -335,11 +321,11 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
              * previous page from the TLB cache.  */
             for(i = DATA_SIZE - 1; i >= 0; i--) {
 #ifdef TARGET_WORDS_BIGENDIAN
-                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
+                glue(slow_stb, MMUSUFFIX)(env, addr + i,
                                           val >> (((DATA_SIZE - 1) * 8) - (i * 8)),
                                           mmu_idx, retaddr);
 #else
-                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
+                glue(slow_stb, MMUSUFFIX)(env, addr + i,
                                           val >> (i * 8),
                                           mmu_idx, retaddr);
 #endif
@@ -366,7 +352,3 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
 #undef USUFFIX
 #undef DATA_SIZE
 #undef ADDR_READ
-#undef ENV_PARAM
-#undef ENV_VAR
-#undef CPU_PREFIX
-#undef HELPER_PREFIX
diff --git a/qemu-lock.h b/include/exec/spinlock.h
index a72edda1d22..a72edda1d22 100644
--- a/qemu-lock.h
+++ b/include/exec/spinlock.h
diff --git a/linux-user/qemu-types.h b/include/exec/user/abitypes.h
index fe7f6624f9f..fe7f6624f9f 100644
--- a/linux-user/qemu-types.h
+++ b/include/exec/user/abitypes.h
diff --git a/thunk.h b/include/exec/user/thunk.h
index 87025c3b04d..87025c3b04d 100644
--- a/thunk.h
+++ b/include/exec/user/thunk.h
diff --git a/fpu/softfloat.h b/include/fpu/softfloat.h
index feec3a180e0..f3927e2419f 100644
--- a/fpu/softfloat.h
+++ b/include/fpu/softfloat.h
@@ -44,7 +44,7 @@ these four paragraphs for those parts of this code that are retained.
 
 #include <inttypes.h>
 #include "config-host.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 /*----------------------------------------------------------------------------
 | Each of the following `typedef's defines the most convenient type that holds
@@ -219,7 +219,7 @@ void float_raise( int8 flags STATUS_PARAM);
 enum {
     float_muladd_negate_c = 1,
     float_muladd_negate_product = 2,
-    float_muladd_negate_result = 3,
+    float_muladd_negate_result = 4,
 };
 
 /*----------------------------------------------------------------------------
@@ -237,6 +237,7 @@ float64 int64_to_float64( int64 STATUS_PARAM );
 float64 uint64_to_float64( uint64 STATUS_PARAM );
 floatx80 int64_to_floatx80( int64 STATUS_PARAM );
 float128 int64_to_float128( int64 STATUS_PARAM );
+float128 uint64_to_float128( uint64 STATUS_PARAM );
 
 /*----------------------------------------------------------------------------
 | Software half-precision conversion routines.
@@ -251,6 +252,11 @@ int float16_is_quiet_nan( float16 );
 int float16_is_signaling_nan( float16 );
 float16 float16_maybe_silence_nan( float16 );
 
+INLINE int float16_is_any_nan(float16 a)
+{
+    return ((float16_val(a) & ~0x8000) > 0x7c00);
+}
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
@@ -625,6 +631,8 @@ INLINE int float128_is_any_nan(float128 a)
         ((a.low != 0) || ((a.high & 0xffffffffffffLL) != 0));
 }
 
+#define float128_zero make_float128(0, 0)
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated quadruple-precision NaN.
 *----------------------------------------------------------------------------*/
diff --git a/libfdt_env.h b/include/libfdt_env.h
index 90d7f3b162c..3667d4cb3a1 100644
--- a/libfdt_env.h
+++ b/include/libfdt_env.h
@@ -19,18 +19,18 @@
 #ifndef _LIBFDT_ENV_H
 #define _LIBFDT_ENV_H
 
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 #ifdef HOST_WORDS_BIGENDIAN
-#define fdt32_to_cpu(x)		(x)
-#define cpu_to_fdt32(x)		(x)
-#define fdt64_to_cpu(x)		(x)
-#define cpu_to_fdt64(x)		(x)
+#define fdt32_to_cpu(x)  (x)
+#define cpu_to_fdt32(x)  (x)
+#define fdt64_to_cpu(x)  (x)
+#define cpu_to_fdt64(x)  (x)
 #else
-#define fdt32_to_cpu(x)		(bswap_32((x)))
-#define cpu_to_fdt32(x)		(bswap_32((x)))
-#define fdt64_to_cpu(x)		(bswap_64((x)))
-#define cpu_to_fdt64(x)		(bswap_64((x)))
+#define fdt32_to_cpu(x)  bswap32(x)
+#define cpu_to_fdt32(x)  bswap32(x)
+#define fdt64_to_cpu(x)  bswap64(x)
+#define cpu_to_fdt64(x)  bswap64(x)
 #endif
 
 #endif /* _LIBFDT_ENV_H */
diff --git a/block-migration.h b/include/migration/block.h
index ffa8ac0bdd6..ffa8ac0bdd6 100644
--- a/block-migration.h
+++ b/include/migration/block.h
diff --git a/migration.h b/include/migration/migration.h
index a9852fcae06..a8c96397320 100644
--- a/migration.h
+++ b/include/migration/migration.h
@@ -14,11 +14,12 @@
 #ifndef QEMU_MIGRATION_H
 #define QEMU_MIGRATION_H
 
-#include "qdict.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu-common.h"
-#include "notify.h"
-#include "error.h"
-#include "vmstate.h"
+#include "qemu/thread.h"
+#include "qemu/notify.h"
+#include "qapi/error.h"
+#include "migration/vmstate.h"
 #include "qapi-types.h"
 
 struct MigrationParams {
@@ -31,6 +32,13 @@ typedef struct MigrationState MigrationState;
 struct MigrationState
 {
     int64_t bandwidth_limit;
+    size_t bytes_xfer;
+    size_t xfer_limit;
+    uint8_t *buffer;
+    size_t buffer_size;
+    size_t buffer_capacity;
+    QemuThread thread;
+
     QEMUFile *file;
     int fd;
     int state;
@@ -40,13 +48,17 @@ struct MigrationState
     void *opaque;
     MigrationParams params;
     int64_t total_time;
+    int64_t downtime;
+    int64_t expected_downtime;
+    int64_t dirty_pages_rate;
     bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
     int64_t xbzrle_cache_size;
+    bool complete;
 };
 
 void process_incoming_migration(QEMUFile *f);
 
-int qemu_start_incoming_migration(const char *uri, Error **errp);
+void qemu_start_incoming_migration(const char *uri, Error **errp);
 
 uint64_t migrate_max_downtime(void);
 
@@ -54,32 +66,34 @@ void do_info_migrate_print(Monitor *mon, const QObject *data);
 
 void do_info_migrate(Monitor *mon, QObject **ret_data);
 
-int exec_start_incoming_migration(const char *host_port);
+void exec_start_incoming_migration(const char *host_port, Error **errp);
 
-int exec_start_outgoing_migration(MigrationState *s, const char *host_port);
+void exec_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
 
-int tcp_start_incoming_migration(const char *host_port, Error **errp);
+void tcp_start_incoming_migration(const char *host_port, Error **errp);
 
-int tcp_start_outgoing_migration(MigrationState *s, const char *host_port,
-                                 Error **errp);
+void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
 
-int unix_start_incoming_migration(const char *path);
+void unix_start_incoming_migration(const char *path, Error **errp);
 
-int unix_start_outgoing_migration(MigrationState *s, const char *path);
+void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp);
 
-int fd_start_incoming_migration(const char *path);
+void fd_start_incoming_migration(const char *path, Error **errp);
 
-int fd_start_outgoing_migration(MigrationState *s, const char *fdname);
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp);
 
 void migrate_fd_error(MigrationState *s);
 
 void migrate_fd_connect(MigrationState *s);
 
+int migrate_fd_close(MigrationState *s);
+
 void add_migration_state_change_notifier(Notifier *notify);
 void remove_migration_state_change_notifier(Notifier *notify);
 bool migration_is_active(MigrationState *);
 bool migration_has_finished(MigrationState *);
 bool migration_has_failed(MigrationState *);
+MigrationState *migrate_get_current(void);
 
 uint64_t ram_bytes_remaining(void);
 uint64_t ram_bytes_transferred(void);
@@ -118,5 +132,4 @@ int migrate_use_xbzrle(void);
 int64_t migrate_xbzrle_cache_size(void);
 
 int64_t xbzrle_cache_resize(int64_t new_size);
-
 #endif
diff --git a/include/qemu/page_cache.h b/include/migration/page_cache.h
index 3839ac7726f..3839ac7726f 100644
--- a/include/qemu/page_cache.h
+++ b/include/migration/page_cache.h
diff --git a/qemu-file.h b/include/migration/qemu-file.h
index 31b83f6bb32..68deefbcfba 100644
--- a/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -47,6 +47,10 @@ typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
  */
 typedef int (QEMUFileCloseFunc)(void *opaque);
 
+/* Called to return the OS file descriptor associated to the QEMUFile.
+ */
+typedef int (QEMUFileGetFD)(void *opaque);
+
 /* Called to determine if the file has exceeded its bandwidth allocation.  The
  * bandwidth capping is a soft limit, not a hard limit.
  */
@@ -59,19 +63,23 @@ typedef int (QEMUFileRateLimit)(void *opaque);
 typedef int64_t (QEMUFileSetRateLimit)(void *opaque, int64_t new_rate);
 typedef int64_t (QEMUFileGetRateLimit)(void *opaque);
 
-QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
-                         QEMUFileGetBufferFunc *get_buffer,
-                         QEMUFileCloseFunc *close,
-                         QEMUFileRateLimit *rate_limit,
-                         QEMUFileSetRateLimit *set_rate_limit,
-                         QEMUFileGetRateLimit *get_rate_limit);
+typedef struct QEMUFileOps {
+    QEMUFilePutBufferFunc *put_buffer;
+    QEMUFileGetBufferFunc *get_buffer;
+    QEMUFileCloseFunc *close;
+    QEMUFileGetFD *get_fd;
+    QEMUFileRateLimit *rate_limit;
+    QEMUFileSetRateLimit *set_rate_limit;
+    QEMUFileGetRateLimit *get_rate_limit;
+} QEMUFileOps;
+
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd);
 QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
-int qemu_stdio_fd(QEMUFile *f);
-void qemu_fflush(QEMUFile *f);
+int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
 void qemu_put_byte(QEMUFile *f, int v);
@@ -104,12 +112,6 @@ int qemu_file_rate_limit(QEMUFile *f);
 int64_t qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
-void qemu_file_set_error(QEMUFile *f, int error);
-
-/* Try to send any outstanding data.  This function is useful when output is
- * halted due to rate limiting or EAGAIN errors occur as it can be used to
- * resume output. */
-void qemu_file_put_notify(QEMUFile *f);
 
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
@@ -231,8 +233,4 @@ static inline void qemu_get_sbe64s(QEMUFile *f, int64_t *pv)
 {
     qemu_get_be64s(f, (uint64_t *)pv);
 }
-
-int64_t qemu_ftell(QEMUFile *f);
-int64_t qemu_fseek(QEMUFile *f, int64_t pos, int whence);
-
 #endif
diff --git a/vmstate.h b/include/migration/vmstate.h
index 5bd2b762ab0..f27276c2d83 100644
--- a/vmstate.h
+++ b/include/migration/vmstate.h
@@ -35,6 +35,7 @@ typedef struct SaveVMHandlers {
     int (*save_live_setup)(QEMUFile *f, void *opaque);
     int (*save_live_iterate)(QEMUFile *f, void *opaque);
     int (*save_live_complete)(QEMUFile *f, void *opaque);
+    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
     void (*cancel)(void *opaque);
     LoadStateHandler *load_state;
     bool (*is_active)(void *opaque);
@@ -139,6 +140,7 @@ extern const VMStateInfo vmstate_info_uint64;
 extern const VMStateInfo vmstate_info_timer;
 extern const VMStateInfo vmstate_info_buffer;
 extern const VMStateInfo vmstate_info_unused_buffer;
+extern const VMStateInfo vmstate_info_bitmap;
 
 #define type_check_array(t1,t2,n) ((t1(*)[n])0 - (t2*)0)
 #define type_check_pointer(t1,t2) ((t1**)0 - (t2*)0)
@@ -411,6 +413,18 @@ extern const VMStateInfo vmstate_info_unused_buffer;
     .flags        = VMS_BUFFER,                                      \
 }
 
+/* _field_size should be a int32_t field in the _state struct giving the
+ * size of the bitmap _field in bits.
+ */
+#define VMSTATE_BITMAP(_field, _state, _version, _field_size) {      \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .size_offset  = vmstate_offset_value(_state, _field_size, int32_t),\
+    .info         = &vmstate_info_bitmap,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER,                         \
+    .offset       = offsetof(_state, _field),                        \
+}
+
 /* _f : field name
    _f_n : num of elements field_name
    _n : num of elements
@@ -503,8 +517,11 @@ extern const VMStateInfo vmstate_info_unused_buffer;
 #define VMSTATE_TIMER_TEST(_f, _s, _test)                             \
     VMSTATE_POINTER_TEST(_f, _s, _test, vmstate_info_timer, QEMUTimer *)
 
+#define VMSTATE_TIMER_V(_f, _s, _v)                                   \
+    VMSTATE_POINTER(_f, _s, _v, vmstate_info_timer, QEMUTimer *)
+
 #define VMSTATE_TIMER(_f, _s)                                         \
-    VMSTATE_TIMER_TEST(_f, _s, NULL)
+    VMSTATE_TIMER_V(_f, _s, 0)
 
 #define VMSTATE_TIMER_ARRAY(_f, _s, _n)                              \
     VMSTATE_ARRAY_OF_POINTER(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer *)
diff --git a/monitor.h b/include/monitor/monitor.h
index 47d556b9d19..87fb49c0926 100644
--- a/monitor.h
+++ b/include/monitor/monitor.h
@@ -2,11 +2,10 @@
 #define MONITOR_H
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qerror.h"
-#include "qdict.h"
-#include "block.h"
-#include "readline.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qdict.h"
+#include "block/block.h"
+#include "monitor/readline.h"
 
 extern Monitor *cur_mon;
 extern Monitor *default_mon;
@@ -38,11 +37,14 @@ typedef enum MonitorEvent {
     QEVENT_SPICE_DISCONNECTED,
     QEVENT_BLOCK_JOB_COMPLETED,
     QEVENT_BLOCK_JOB_CANCELLED,
+    QEVENT_BLOCK_JOB_ERROR,
+    QEVENT_BLOCK_JOB_READY,
     QEVENT_DEVICE_TRAY_MOVED,
     QEVENT_SUSPEND,
     QEVENT_SUSPEND_DISK,
     QEVENT_WAKEUP,
     QEVENT_BALLOON_CHANGE,
+    QEVENT_SPICE_MIGRATE_COMPLETED,
 
     /* Add to 'monitor_event_names' array in monitor.c when
      * defining new events here */
@@ -65,7 +67,8 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
                                   BlockDriverCompletionFunc *completion_cb,
                                   void *opaque);
 
-int monitor_get_fd(Monitor *mon, const char *fdname);
+int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp);
+int monitor_handle_fd_param(Monitor *mon, const char *fdname);
 
 void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
     GCC_FMT_ATTR(2, 0);
@@ -87,6 +90,9 @@ int qmp_qom_set(Monitor *mon, const QDict *qdict, QObject **ret);
 
 int qmp_qom_get(Monitor *mon, const QDict *qdict, QObject **ret);
 
+AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
+                                bool has_opaque, const char *opaque,
+                                Error **errp);
 int monitor_fdset_get_fd(int64_t fdset_id, int flags);
 int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd);
 int monitor_fdset_dup_fd_remove(int dup_fd);
diff --git a/readline.h b/include/monitor/readline.h
index fc9806ecf1a..fc9806ecf1a 100644
--- a/readline.h
+++ b/include/monitor/readline.h
diff --git a/net/checksum.h b/include/net/checksum.h
index 1f052986e64..1f052986e64 100644
--- a/net/checksum.h
+++ b/include/net/checksum.h
diff --git a/net.h b/include/net/net.h
index 29750567ed9..4a92b6c3d2b 100644
--- a/net.h
+++ b/include/net/net.h
@@ -1,12 +1,12 @@
 #ifndef QEMU_NET_H
 #define QEMU_NET_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qdict.h"
-#include "qemu-option.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/option.h"
 #include "net/queue.h"
-#include "vmstate.h"
+#include "migration/vmstate.h"
 #include "qapi-types.h"
 
 struct MACAddr {
@@ -112,7 +112,7 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                             void *opaque);
 
 void print_net_client(Monitor *mon, NetClientState *nc);
-void do_info_network(Monitor *mon);
+void do_info_network(Monitor *mon, const QDict *qdict);
 
 /* NIC info */
 
@@ -133,20 +133,6 @@ extern int nb_nics;
 extern NICInfo nd_table[MAX_NICS];
 extern int default_net;
 
-/* BT HCI info */
-
-struct HCIInfo {
-    int (*bdaddr_set)(struct HCIInfo *hci, const uint8_t *bd_addr);
-    void (*cmd_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void (*sco_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void (*acl_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void *opaque;
-    void (*evt_recv)(void *opaque, const uint8_t *data, int len);
-    void (*acl_recv)(void *opaque, const uint8_t *data, int len);
-};
-
-struct HCIInfo *qemu_next_hci(void);
-
 /* from net.c */
 extern const char *legacy_tftp_prefix;
 extern const char *legacy_bootp_filename;
@@ -161,6 +147,9 @@ void net_host_device_remove(Monitor *mon, const QDict *qdict);
 void netdev_add(QemuOpts *opts, Error **errp);
 int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret);
 
+int net_hub_id_for_client(NetClientState *nc, int *id);
+NetClientState *net_hub_port_find(int hub_id);
+
 #define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
 #define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
 #define DEFAULT_BRIDGE_HELPER CONFIG_QEMU_HELPERDIR "/qemu-bridge-helper"
@@ -168,8 +157,6 @@ int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret);
 
 void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd);
 
-int net_handle_fd_param(Monitor *mon, const char *param);
-
 #define POLYNOMIAL 0x04c11db6
 unsigned compute_mcast_idx(const uint8_t *ep);
 
diff --git a/net/queue.h b/include/net/queue.h
index 9d44a9b3b87..fc02b33915f 100644
--- a/net/queue.h
+++ b/include/net/queue.h
@@ -53,6 +53,6 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue,
                                 NetPacketSent *sent_cb);
 
 void qemu_net_queue_purge(NetQueue *queue, NetClientState *from);
-void qemu_net_queue_flush(NetQueue *queue);
+bool qemu_net_queue_flush(NetQueue *queue);
 
 #endif /* QEMU_NET_QUEUE_H */
diff --git a/net/slirp.h b/include/net/slirp.h
index 5f685c4fb11..0502389c680 100644
--- a/net/slirp.h
+++ b/include/net/slirp.h
@@ -25,15 +25,12 @@
 #define QEMU_NET_SLIRP_H
 
 #include "qemu-common.h"
-#include "qdict.h"
-#include "qemu-option.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/option.h"
 #include "qapi-types.h"
 
 #ifdef CONFIG_SLIRP
 
-int net_init_slirp(const NetClientOptions *opts, const char *name,
-                   NetClientState *peer);
-
 void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict);
 void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict);
 
@@ -43,7 +40,7 @@ int net_slirp_parse_legacy(QemuOptsList *opts_list, const char *optarg, int *ret
 
 int net_slirp_smb(const char *exported_dir);
 
-void do_info_usernet(Monitor *mon);
+void do_info_usernet(Monitor *mon, const QDict *qdict);
 
 #endif
 
diff --git a/include/net/tap.h b/include/net/tap.h
new file mode 100644
index 00000000000..bb7efb5439a
--- /dev/null
+++ b/include/net/tap.h
@@ -0,0 +1,67 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_NET_TAP_H
+#define QEMU_NET_TAP_H
+
+#include "qemu-common.h"
+#include "qapi-types.h"
+
+int tap_has_ufo(NetClientState *nc);
+int tap_has_vnet_hdr(NetClientState *nc);
+int tap_has_vnet_hdr_len(NetClientState *nc, int len);
+void tap_using_vnet_hdr(NetClientState *nc, int using_vnet_hdr);
+void tap_set_offload(NetClientState *nc, int csum, int tso4, int tso6, int ecn, int ufo);
+void tap_set_vnet_hdr_len(NetClientState *nc, int len);
+
+int tap_get_fd(NetClientState *nc);
+
+struct vhost_net;
+struct vhost_net *tap_get_vhost_net(NetClientState *nc);
+
+struct virtio_net_hdr
+{
+#define VIRTIO_NET_HDR_F_NEEDS_CSUM     1       // Use csum_start, csum_offset
+#define VIRTIO_NET_HDR_F_DATA_VALID    2       // Csum is valid
+    uint8_t flags;
+#define VIRTIO_NET_HDR_GSO_NONE         0       // Not a GSO frame
+#define VIRTIO_NET_HDR_GSO_TCPV4        1       // GSO frame, IPv4 TCP (TSO)
+#define VIRTIO_NET_HDR_GSO_UDP          3       // GSO frame, IPv4 UDP (UFO)
+#define VIRTIO_NET_HDR_GSO_TCPV6        4       // GSO frame, IPv6 TCP
+#define VIRTIO_NET_HDR_GSO_ECN          0x80    // TCP has ECN set
+    uint8_t gso_type;
+    uint16_t hdr_len;
+    uint16_t gso_size;
+    uint16_t csum_start;
+    uint16_t csum_offset;
+};
+
+struct virtio_net_hdr_mrg_rxbuf
+{
+    struct virtio_net_hdr hdr;
+    uint16_t num_buffers;   /* Number of merged rx buffers */
+};
+
+#endif /* QEMU_NET_TAP_H */
diff --git a/qapi/qapi-dealloc-visitor.h b/include/qapi/dealloc-visitor.h
index 5842bc79bde..cf4c36d2d34 100644
--- a/qapi/qapi-dealloc-visitor.h
+++ b/include/qapi/dealloc-visitor.h
@@ -14,7 +14,7 @@
 #ifndef QAPI_DEALLOC_VISITOR_H
 #define QAPI_DEALLOC_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct QapiDeallocVisitor QapiDeallocVisitor;
 
diff --git a/error.h b/include/qapi/error.h
index 96fc20328f9..5cd2f0c3025 100644
--- a/error.h
+++ b/include/qapi/error.h
@@ -12,7 +12,7 @@
 #ifndef ERROR_H
 #define ERROR_H
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "qapi-types.h"
 #include <stdbool.h>
 
@@ -30,6 +30,21 @@ typedef struct Error Error;
 void error_set(Error **err, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(3, 4);
 
 /**
+ * Set an indirect pointer to an error given a ErrorClass value and a
+ * printf-style human message, followed by a strerror() string if
+ * @os_error is not zero.
+ */
+void error_set_errno(Error **err, int os_error, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+
+/**
+ * Same as error_set(), but sets a generic error
+ */
+#define error_setg(err, fmt, ...) \
+    error_set(err, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+#define error_setg_errno(err, os_error, fmt, ...) \
+    error_set_errno(err, os_error, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+
+/**
  * Returns true if an indirect pointer to an error is pointing to a valid
  * error object.
  */
diff --git a/qapi/opts-visitor.h b/include/qapi/opts-visitor.h
index ea1a395573b..5939eeebc7f 100644
--- a/qapi/opts-visitor.h
+++ b/include/qapi/opts-visitor.h
@@ -13,8 +13,8 @@
 #ifndef OPTS_VISITOR_H
 #define OPTS_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qemu-option.h"
+#include "qapi/visitor.h"
+#include "qemu/option.h"
 
 typedef struct OptsVisitor OptsVisitor;
 
diff --git a/qapi/qmp-input-visitor.h b/include/qapi/qmp-input-visitor.h
index e0a48a5f3b1..3ed499cc42d 100644
--- a/qapi/qmp-input-visitor.h
+++ b/include/qapi/qmp-input-visitor.h
@@ -14,8 +14,8 @@
 #ifndef QMP_INPUT_VISITOR_H
 #define QMP_INPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qobject.h"
+#include "qapi/visitor.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QmpInputVisitor QmpInputVisitor;
 
diff --git a/qapi/qmp-output-visitor.h b/include/qapi/qmp-output-visitor.h
index 4a649c2504a..22667706ab9 100644
--- a/qapi/qmp-output-visitor.h
+++ b/include/qapi/qmp-output-visitor.h
@@ -14,8 +14,8 @@
 #ifndef QMP_OUTPUT_VISITOR_H
 #define QMP_OUTPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qobject.h"
+#include "qapi/visitor.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QmpOutputVisitor QmpOutputVisitor;
 
diff --git a/qapi/qmp-core.h b/include/qapi/qmp/dispatch.h
index 00446cff9be..1ce11f5df00 100644
--- a/qapi/qmp-core.h
+++ b/include/qapi/qmp/dispatch.h
@@ -14,9 +14,9 @@
 #ifndef QMP_CORE_H
 #define QMP_CORE_H
 
-#include "qobject.h"
-#include "qdict.h"
-#include "error.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
 
 typedef void (QmpCommandFunc)(QDict *, QObject **, Error **);
 
diff --git a/json-lexer.h b/include/qapi/qmp/json-lexer.h
index 10bc0a77984..cdff0460a83 100644
--- a/json-lexer.h
+++ b/include/qapi/qmp/json-lexer.h
@@ -14,8 +14,8 @@
 #ifndef QEMU_JSON_LEXER_H
 #define QEMU_JSON_LEXER_H
 
-#include "qstring.h"
-#include "qlist.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
 
 typedef enum json_token_type {
     JSON_OPERATOR = 100,
diff --git a/json-parser.h b/include/qapi/qmp/json-parser.h
index 8f2b5ec4bcb..44d88f34685 100644
--- a/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -15,8 +15,8 @@
 #define QEMU_JSON_PARSER_H
 
 #include "qemu-common.h"
-#include "qlist.h"
-#include "error.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/error.h"
 
 QObject *json_parser_parse(QList *tokens, va_list *ap);
 QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
diff --git a/json-streamer.h b/include/qapi/qmp/json-streamer.h
index f09bc4daec9..823f7d7fa42 100644
--- a/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -14,8 +14,8 @@
 #ifndef QEMU_JSON_STREAMER_H
 #define QEMU_JSON_STREAMER_H
 
-#include "qlist.h"
-#include "json-lexer.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/json-lexer.h"
 
 typedef struct JSONMessageParser
 {
diff --git a/qbool.h b/include/qapi/qmp/qbool.h
index fe66fcd4551..c4eaab9bb9f 100644
--- a/qbool.h
+++ b/include/qapi/qmp/qbool.h
@@ -15,7 +15,7 @@
 #define QBOOL_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QBool {
     QObject_HEAD;
diff --git a/qdict.h b/include/qapi/qmp/qdict.h
index 929d8d22f50..6d9a4be3a5f 100644
--- a/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -13,9 +13,9 @@
 #ifndef QDICT_H
 #define QDICT_H
 
-#include "qobject.h"
-#include "qlist.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qlist.h"
+#include "qemu/queue.h"
 #include <stdint.h>
 
 #define QDICT_BUCKET_MAX 512
diff --git a/qerror.h b/include/qapi/qmp/qerror.h
index d0a76a4f71b..6c0a18dfc40 100644
--- a/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -12,10 +12,10 @@
 #ifndef QERROR_H
 #define QERROR_H
 
-#include "qdict.h"
-#include "qstring.h"
-#include "qemu-error.h"
-#include "error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "qapi-types.h"
 #include <stdarg.h>
 
@@ -48,6 +48,15 @@ void assert_no_error(Error *err);
 #define QERR_BASE_NOT_FOUND \
     ERROR_CLASS_GENERIC_ERROR, "Base '%s' not found"
 
+#define QERR_BLOCK_JOB_NOT_ACTIVE \
+    ERROR_CLASS_DEVICE_NOT_ACTIVE, "No active block job on device '%s'"
+
+#define QERR_BLOCK_JOB_PAUSED \
+    ERROR_CLASS_GENERIC_ERROR, "The block job for device '%s' is currently paused"
+
+#define QERR_BLOCK_JOB_NOT_READY \
+    ERROR_CLASS_GENERIC_ERROR, "The active block job for device '%s' cannot be completed"
+
 #define QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED \
     ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by device '%s' does not support feature '%s'"
 
@@ -159,9 +168,6 @@ void assert_no_error(Error *err);
 #define QERR_MIGRATION_NOT_SUPPORTED \
     ERROR_CLASS_GENERIC_ERROR, "State blocked by non-migratable device '%s'"
 
-#define QERR_MIGRATION_EXPECTED \
-    ERROR_CLASS_MIGRATION_EXPECTED, "An incoming migration is expected before this command can be executed"
-
 #define QERR_MISSING_PARAMETER \
     ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' is missing"
 
@@ -231,9 +237,6 @@ void assert_no_error(Error *err);
 #define QERR_VIRTFS_FEATURE_BLOCKS_MIGRATION \
     ERROR_CLASS_GENERIC_ERROR, "Migration is disabled when VirtFS export path '%s' is mounted in the guest using mount_tag '%s'"
 
-#define QERR_VNC_SERVER_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Could not start VNC server on %s"
-
 #define QERR_SOCKET_CONNECT_FAILED \
     ERROR_CLASS_GENERIC_ERROR, "Failed to connect to socket"
 
diff --git a/qfloat.h b/include/qapi/qmp/qfloat.h
index 9d6787659c5..a8658443dc2 100644
--- a/qfloat.h
+++ b/include/qapi/qmp/qfloat.h
@@ -15,7 +15,7 @@
 #define QFLOAT_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QFloat {
     QObject_HEAD;
diff --git a/qint.h b/include/qapi/qmp/qint.h
index 6b1a15c088b..48a41b0f2ae 100644
--- a/qint.h
+++ b/include/qapi/qmp/qint.h
@@ -14,7 +14,7 @@
 #define QINT_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QInt {
     QObject_HEAD;
diff --git a/qjson.h b/include/qapi/qmp/qjson.h
index 1190d8a86c2..73351ed6d68 100644
--- a/qjson.h
+++ b/include/qapi/qmp/qjson.h
@@ -15,9 +15,9 @@
 #define QJSON_H
 
 #include <stdarg.h>
-#include "compiler.h"
-#include "qobject.h"
-#include "qstring.h"
+#include "qemu/compiler.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
 
 QObject *qobject_from_json(const char *string) GCC_FMT_ATTR(1, 0);
 QObject *qobject_from_jsonf(const char *string, ...) GCC_FMT_ATTR(1, 2);
diff --git a/qlist.h b/include/qapi/qmp/qlist.h
index ae776f99c38..382f04c3c4a 100644
--- a/qlist.h
+++ b/include/qapi/qmp/qlist.h
@@ -13,10 +13,9 @@
 #ifndef QLIST_H
 #define QLIST_H
 
-#include "qobject.h"
-#include "qemu-queue.h"
-#include "qemu-common.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
+#include "qemu/queue.h"
 
 typedef struct QListEntry {
     QObject *value;
diff --git a/qobject.h b/include/qapi/qmp/qobject.h
index d42386dde13..9124649ed20 100644
--- a/qobject.h
+++ b/include/qapi/qmp/qobject.h
@@ -71,7 +71,7 @@ typedef struct QObject {
 
 /* High-level interface for qobject_decref() */
 #define QDECREF(obj)              \
-    qobject_decref(QOBJECT(obj))
+    qobject_decref(obj ? QOBJECT(obj) : NULL)
 
 /* Initialize an object to default values */
 #define QOBJECT_INIT(obj, qtype_type)   \
diff --git a/qstring.h b/include/qapi/qmp/qstring.h
index 84ccd96d61f..0e690f48498 100644
--- a/qstring.h
+++ b/include/qapi/qmp/qstring.h
@@ -14,7 +14,7 @@
 #define QSTRING_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QString {
     QObject_HEAD;
diff --git a/qemu-objects.h b/include/qapi/qmp/types.h
index c53fbaa2175..7782ec5a602 100644
--- a/qemu-objects.h
+++ b/include/qapi/qmp/types.h
@@ -13,13 +13,13 @@
 #ifndef QEMU_OBJECTS_H
 #define QEMU_OBJECTS_H
 
-#include "qobject.h"
-#include "qint.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qjson.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qjson.h"
 
 #endif /* QEMU_OBJECTS_H */
diff --git a/qapi/string-input-visitor.h b/include/qapi/string-input-visitor.h
index d269d42cef9..089243c09eb 100644
--- a/qapi/string-input-visitor.h
+++ b/include/qapi/string-input-visitor.h
@@ -13,7 +13,7 @@
 #ifndef STRING_INPUT_VISITOR_H
 #define STRING_INPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct StringInputVisitor StringInputVisitor;
 
diff --git a/qapi/string-output-visitor.h b/include/qapi/string-output-visitor.h
index 88684541105..ec81e42b60a 100644
--- a/qapi/string-output-visitor.h
+++ b/include/qapi/string-output-visitor.h
@@ -13,7 +13,7 @@
 #ifndef STRING_OUTPUT_VISITOR_H
 #define STRING_OUTPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct StringOutputVisitor StringOutputVisitor;
 
diff --git a/include/qapi/visitor-impl.h b/include/qapi/visitor-impl.h
new file mode 100644
index 00000000000..5159964863e
--- /dev/null
+++ b/include/qapi/visitor-impl.h
@@ -0,0 +1,63 @@
+/*
+ * Core Definitions for QAPI Visitor implementations
+ *
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * Author: Paolo Bonizni <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+#ifndef QAPI_VISITOR_IMPL_H
+#define QAPI_VISITOR_IMPL_H
+
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+
+struct Visitor
+{
+    /* Must be set */
+    void (*start_struct)(Visitor *v, void **obj, const char *kind,
+                         const char *name, size_t size, Error **errp);
+    void (*end_struct)(Visitor *v, Error **errp);
+
+    void (*start_list)(Visitor *v, const char *name, Error **errp);
+    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
+    void (*end_list)(Visitor *v, Error **errp);
+
+    void (*type_enum)(Visitor *v, int *obj, const char *strings[],
+                      const char *kind, const char *name, Error **errp);
+
+    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
+    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
+    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
+    void (*type_number)(Visitor *v, double *obj, const char *name,
+                        Error **errp);
+
+    /* May be NULL */
+    void (*start_optional)(Visitor *v, bool *present, const char *name,
+                           Error **errp);
+    void (*end_optional)(Visitor *v, Error **errp);
+
+    void (*start_handle)(Visitor *v, void **obj, const char *kind,
+                         const char *name, Error **errp);
+    void (*end_handle)(Visitor *v, Error **errp);
+    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
+    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
+    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
+    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
+    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
+    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
+    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
+    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
+    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
+    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
+};
+
+void input_type_enum(Visitor *v, int *obj, const char *strings[],
+                     const char *kind, const char *name, Error **errp);
+void output_type_enum(Visitor *v, int *obj, const char *strings[],
+                      const char *kind, const char *name, Error **errp);
+
+#endif
diff --git a/qapi/qapi-visit-core.h b/include/qapi/visitor.h
index 60acedac77e..1fef18c08fa 100644
--- a/qapi/qapi-visit-core.h
+++ b/include/qapi/visitor.h
@@ -13,7 +13,7 @@
 #ifndef QAPI_VISITOR_CORE_H
 #define QAPI_VISITOR_CORE_H
 
-#include "qapi/qapi-types-core.h"
+#include "qapi/error.h"
 #include <stdlib.h>
 
 typedef struct GenericList
@@ -24,46 +24,6 @@ typedef struct GenericList
 
 typedef struct Visitor Visitor;
 
-struct Visitor
-{
-    /* Must be set */
-    void (*start_struct)(Visitor *v, void **obj, const char *kind,
-                         const char *name, size_t size, Error **errp);
-    void (*end_struct)(Visitor *v, Error **errp);
-
-    void (*start_list)(Visitor *v, const char *name, Error **errp);
-    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
-    void (*end_list)(Visitor *v, Error **errp);
-
-    void (*type_enum)(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
-
-    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
-    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
-    void (*type_number)(Visitor *v, double *obj, const char *name,
-                        Error **errp);
-
-    /* May be NULL */
-    void (*start_optional)(Visitor *v, bool *present, const char *name,
-                           Error **errp);
-    void (*end_optional)(Visitor *v, Error **errp);
-
-    void (*start_handle)(Visitor *v, void **obj, const char *kind,
-                         const char *name, Error **errp);
-    void (*end_handle)(Visitor *v, Error **errp);
-    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
-    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
-    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
-    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
-    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-};
-
 void visit_start_handle(Visitor *v, void **obj, const char *kind,
                         const char *name, Error **errp);
 void visit_end_handle(Visitor *v, Error **errp);
diff --git a/qemu-common.h b/include/qemu-common.h
index e5c2bcd204a..af2379ff384 100644
--- a/qemu-common.h
+++ b/include/qemu-common.h
@@ -1,10 +1,20 @@
 
-/* Common header file that is included by all of qemu.  */
+/* Common header file that is included by all of QEMU.
+ *
+ * This file is supposed to be included only by .c files. No header file should
+ * depend on qemu-common.h, as this would easily lead to circular header
+ * dependencies.
+ *
+ * If a header file uses a definition from qemu-common.h, that definition
+ * must be moved to a separate header file, and the header that uses it
+ * must include that header.
+ */
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "config-host.h"
+#include "qemu/typedefs.h"
 
 #if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
 #define WORDS_ALIGNED
@@ -12,14 +22,6 @@
 
 #define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
 
-typedef struct QEMUTimer QEMUTimer;
-typedef struct QEMUFile QEMUFile;
-typedef struct DeviceState DeviceState;
-
-struct Monitor;
-typedef struct Monitor Monitor;
-typedef struct MigrationParams MigrationParams;
-
 /* we put basic includes here to avoid repeating them in device drivers */
 #include <stdlib.h>
 #include <stdio.h>
@@ -41,11 +43,11 @@ typedef struct MigrationParams MigrationParams;
 #include <glib.h>
 
 #ifdef _WIN32
-#include "qemu-os-win32.h"
+#include "sysemu/os-win32.h"
 #endif
 
 #ifdef CONFIG_POSIX
-#include "qemu-os-posix.h"
+#include "sysemu/os-posix.h"
 #endif
 
 #ifndef O_LARGEFILE
@@ -66,6 +68,9 @@ typedef struct MigrationParams MigrationParams;
 #if !defined(ECANCELED)
 #define ECANCELED 4097
 #endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
 #ifndef TIME_MAX
 #define TIME_MAX LONG_MAX
 #endif
@@ -120,8 +125,8 @@ extern int use_icount;
 /* FIXME: Remove NEED_CPU_H.  */
 #ifndef NEED_CPU_H
 
-#include "osdep.h"
-#include "bswap.h"
+#include "qemu/osdep.h"
+#include "qemu/bswap.h"
 
 #else
 
@@ -167,7 +172,6 @@ int qemu_fls(int i);
 int qemu_fdatasync(int fd);
 int fcntl_setfl(int fd, int flag);
 int qemu_parse_fd(const char *param);
-int qemu_parse_fdset(const char *param);
 
 /*
  * strtosz() suffixes used to specify the default treatment of an
@@ -208,8 +212,6 @@ const char *path(const char *pathname);
 
 void *qemu_oom_check(void *ptr);
 
-int qemu_open(const char *name, int flags, ...);
-int qemu_close(int fd);
 ssize_t qemu_write_full(int fd, const void *buf, size_t count)
     QEMU_WARN_UNUSED_RESULT;
 ssize_t qemu_send_full(int fd, const void *buf, size_t count, int flags)
@@ -218,14 +220,26 @@ ssize_t qemu_recv_full(int fd, void *buf, size_t count, int flags)
     QEMU_WARN_UNUSED_RESULT;
 
 #ifndef _WIN32
-int qemu_eventfd(int pipefd[2]);
 int qemu_pipe(int pipefd[2]);
 #endif
 
 #ifdef _WIN32
+/* MinGW needs type casts for the 'buf' and 'optval' arguments. */
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, (void *)optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, (const void *)optval, optlen)
 #define qemu_recv(sockfd, buf, len, flags) recv(sockfd, (void *)buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, (const void *)buf, len, flags, destaddr, addrlen)
 #else
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, optval, optlen)
 #define qemu_recv(sockfd, buf, len, flags) recv(sockfd, buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, buf, len, flags, destaddr, addrlen)
 #endif
 
 /* Error handling.  */
@@ -239,51 +253,6 @@ struct ParallelIOArg {
 
 typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size);
 
-/* A load of opaque types so that device init declarations don't have to
-   pull in all the real definitions.  */
-typedef struct NICInfo NICInfo;
-typedef struct HCIInfo HCIInfo;
-typedef struct AudioState AudioState;
-typedef struct BlockDriverState BlockDriverState;
-typedef struct DriveInfo DriveInfo;
-typedef struct DisplayState DisplayState;
-typedef struct DisplayChangeListener DisplayChangeListener;
-typedef struct DisplaySurface DisplaySurface;
-typedef struct DisplayAllocator DisplayAllocator;
-typedef struct PixelFormat PixelFormat;
-typedef struct TextConsole TextConsole;
-typedef TextConsole QEMUConsole;
-typedef struct CharDriverState CharDriverState;
-typedef struct MACAddr MACAddr;
-typedef struct NetClientState NetClientState;
-typedef struct i2c_bus i2c_bus;
-typedef struct ISABus ISABus;
-typedef struct ISADevice ISADevice;
-typedef struct SMBusDevice SMBusDevice;
-typedef struct PCIHostState PCIHostState;
-typedef struct PCIExpressHost PCIExpressHost;
-typedef struct PCIBus PCIBus;
-typedef struct PCIDevice PCIDevice;
-typedef struct PCIExpressDevice PCIExpressDevice;
-typedef struct PCIBridge PCIBridge;
-typedef struct PCIEAERMsg PCIEAERMsg;
-typedef struct PCIEAERLog PCIEAERLog;
-typedef struct PCIEAERErr PCIEAERErr;
-typedef struct PCIEPort PCIEPort;
-typedef struct PCIESlot PCIESlot;
-typedef struct MSIMessage MSIMessage;
-typedef struct SerialState SerialState;
-typedef struct IRQState *qemu_irq;
-typedef struct PCMCIACardState PCMCIACardState;
-typedef struct MouseTransformInfo MouseTransformInfo;
-typedef struct uWireSlave uWireSlave;
-typedef struct I2SCodec I2SCodec;
-typedef struct SSIBus SSIBus;
-typedef struct EventNotifier EventNotifier;
-typedef struct VirtIODevice VirtIODevice;
-typedef struct QEMUSGList QEMUSGList;
-typedef struct SHPCDevice SHPCDevice;
-
 typedef uint64_t pcibus_t;
 
 typedef enum LostTickPolicy {
@@ -311,9 +280,7 @@ void cpu_save(QEMUFile *f, void *opaque);
 int cpu_load(QEMUFile *f, void *opaque, int version_id);
 
 /* Unblock cpu */
-void qemu_cpu_kick(void *env);
 void qemu_cpu_kick_self(void);
-int qemu_cpu_is_self(void *env);
 
 /* work queue */
 struct qemu_work_item {
@@ -324,7 +291,9 @@ struct qemu_work_item {
 };
 
 #ifdef CONFIG_USER_ONLY
-#define qemu_init_vcpu(env) do { } while (0)
+static inline void qemu_init_vcpu(void *env)
+{
+}
 #else
 void qemu_init_vcpu(void *env);
 #endif
@@ -365,6 +334,9 @@ void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
 void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
 void qemu_iovec_concat(QEMUIOVector *dst,
                        QEMUIOVector *src, size_t soffset, size_t sbytes);
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes);
 void qemu_iovec_destroy(QEMUIOVector *qiov);
 void qemu_iovec_reset(QEMUIOVector *qiov);
 size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
@@ -444,7 +416,7 @@ static inline bool is_power_of_2(uint64_t value)
 /* round down to the nearest power of 2*/
 int64_t pow2floor(int64_t value);
 
-#include "module.h"
+#include "qemu/module.h"
 
 /*
  * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
diff --git a/acl.h b/include/qemu/acl.h
index 0ef780401f9..116487e2823 100644
--- a/acl.h
+++ b/include/qemu/acl.h
@@ -25,7 +25,7 @@
 #ifndef __QEMU_ACL_H__
 #define __QEMU_ACL_H__
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef struct qemu_acl_entry qemu_acl_entry;
 typedef struct qemu_acl qemu_acl;
diff --git a/qemu-barrier.h b/include/qemu/atomic.h
index 7e11197814e..96a194bbee6 100644
--- a/qemu-barrier.h
+++ b/include/qemu/atomic.h
@@ -6,6 +6,8 @@
 
 #if defined(__i386__)
 
+#include "qemu/compiler.h"        /* QEMU_GNUC_PREREQ */
+
 /*
  * Because of the strongly ordered x86 storage model, wmb() and rmb() are nops
  * on x86(well, a compiler barrier only).  Well, at least as long as
@@ -19,7 +21,7 @@
  * mfence on 32 bit as well, e.g. if built with -march=pentium-m.
  * However, on i386, there seem to be known bugs as recently as 4.3.
  * */
-#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 4
+#if QEMU_GNUC_PREREQ(4, 4)
 #define smp_mb() __sync_synchronize()
 #else
 #define smp_mb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
diff --git a/bitmap.h b/include/qemu/bitmap.h
index 08755eba11b..308bbb71e9b 100644
--- a/bitmap.h
+++ b/include/qemu/bitmap.h
@@ -13,7 +13,7 @@
 #define BITMAP_H
 
 #include "qemu-common.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 /*
  * The available bitmap operations and their rough meaning in the
diff --git a/bitops.h b/include/qemu/bitops.h
index 74e14e57247..74e14e57247 100644
--- a/bitops.h
+++ b/include/qemu/bitops.h
diff --git a/include/qemu/bswap.h b/include/qemu/bswap.h
new file mode 100644
index 00000000000..e6d47981420
--- /dev/null
+++ b/include/qemu/bswap.h
@@ -0,0 +1,469 @@
+#ifndef BSWAP_H
+#define BSWAP_H
+
+#include "config-host.h"
+
+#include <inttypes.h>
+#include "fpu/softfloat.h"
+
+#ifdef CONFIG_MACHINE_BSWAP_H
+# include <sys/endian.h>
+# include <sys/types.h>
+# include <machine/bswap.h>
+#elif defined(CONFIG_BYTESWAP_H)
+# include <byteswap.h>
+
+static inline uint16_t bswap16(uint16_t x)
+{
+    return bswap_16(x);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return bswap_32(x);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return bswap_64(x);
+}
+# else
+static inline uint16_t bswap16(uint16_t x)
+{
+    return (((x & 0x00ff) << 8) |
+            ((x & 0xff00) >> 8));
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return (((x & 0x000000ffU) << 24) |
+            ((x & 0x0000ff00U) <<  8) |
+            ((x & 0x00ff0000U) >>  8) |
+            ((x & 0xff000000U) >> 24));
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return (((x & 0x00000000000000ffULL) << 56) |
+            ((x & 0x000000000000ff00ULL) << 40) |
+            ((x & 0x0000000000ff0000ULL) << 24) |
+            ((x & 0x00000000ff000000ULL) <<  8) |
+            ((x & 0x000000ff00000000ULL) >>  8) |
+            ((x & 0x0000ff0000000000ULL) >> 24) |
+            ((x & 0x00ff000000000000ULL) >> 40) |
+            ((x & 0xff00000000000000ULL) >> 56));
+}
+#endif /* ! CONFIG_MACHINE_BSWAP_H */
+
+static inline void bswap16s(uint16_t *s)
+{
+    *s = bswap16(*s);
+}
+
+static inline void bswap32s(uint32_t *s)
+{
+    *s = bswap32(*s);
+}
+
+static inline void bswap64s(uint64_t *s)
+{
+    *s = bswap64(*s);
+}
+
+#if defined(HOST_WORDS_BIGENDIAN)
+#define be_bswap(v, size) (v)
+#define le_bswap(v, size) glue(bswap, size)(v)
+#define be_bswaps(v, size)
+#define le_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#else
+#define le_bswap(v, size) (v)
+#define be_bswap(v, size) glue(bswap, size)(v)
+#define le_bswaps(v, size)
+#define be_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#endif
+
+#define CPU_CONVERT(endian, size, type)\
+static inline type endian ## size ## _to_cpu(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline type cpu_to_ ## endian ## size(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline void endian ## size ## _to_cpus(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## s(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline type endian ## size ## _to_cpup(const type *p)\
+{\
+    return glue(glue(endian, size), _to_cpu)(*p);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
+{\
+    *p = glue(glue(cpu_to_, endian), size)(v);\
+}
+
+CPU_CONVERT(be, 16, uint16_t)
+CPU_CONVERT(be, 32, uint32_t)
+CPU_CONVERT(be, 64, uint64_t)
+
+CPU_CONVERT(le, 16, uint16_t)
+CPU_CONVERT(le, 32, uint32_t)
+CPU_CONVERT(le, 64, uint64_t)
+
+/* len must be one of 1, 2, 4 */
+static inline uint32_t qemu_bswap_len(uint32_t value, int len)
+{
+    return bswap32(value) >> (32 - 8 * len);
+}
+
+/* Unions for reinterpreting between floats and integers.  */
+
+typedef union {
+    float32 f;
+    uint32_t l;
+} CPU_FloatU;
+
+typedef union {
+    float64 d;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upper;
+        uint32_t lower;
+    } l;
+#else
+    struct {
+        uint32_t lower;
+        uint32_t upper;
+    } l;
+#endif
+    uint64_t ll;
+} CPU_DoubleU;
+
+typedef union {
+     floatx80 d;
+     struct {
+         uint64_t lower;
+         uint16_t upper;
+     } l;
+} CPU_LDoubleU;
+
+typedef union {
+    float128 q;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upmost;
+        uint32_t upper;
+        uint32_t lower;
+        uint32_t lowest;
+    } l;
+    struct {
+        uint64_t upper;
+        uint64_t lower;
+    } ll;
+#else
+    struct {
+        uint32_t lowest;
+        uint32_t lower;
+        uint32_t upper;
+        uint32_t upmost;
+    } l;
+    struct {
+        uint64_t lower;
+        uint64_t upper;
+    } ll;
+#endif
+} CPU_QuadU;
+
+/* unaligned/endian-independent pointer access */
+
+/*
+ * the generic syntax is:
+ *
+ * load: ld{type}{sign}{size}{endian}_p(ptr)
+ *
+ * store: st{type}{size}{endian}_p(ptr, val)
+ *
+ * Note there are small differences with the softmmu access API!
+ *
+ * type is:
+ * (empty): integer access
+ *   f    : float access
+ *
+ * sign is:
+ * (empty): for floats or 32 bit size
+ *   u    : unsigned
+ *   s    : signed
+ *
+ * size is:
+ *   b: 8 bits
+ *   w: 16 bits
+ *   l: 32 bits
+ *   q: 64 bits
+ *
+ * endian is:
+ * (empty): host endian
+ *   be   : big endian
+ *   le   : little endian
+ */
+
+static inline int ldub_p(const void *ptr)
+{
+    return *(uint8_t *)ptr;
+}
+
+static inline int ldsb_p(const void *ptr)
+{
+    return *(int8_t *)ptr;
+}
+
+static inline void stb_p(void *ptr, int v)
+{
+    *(uint8_t *)ptr = v;
+}
+
+/* Any compiler worth its salt will turn these memcpy into native unaligned
+   operations.  Thus we don't need to play games with packed attributes, or
+   inline byte-by-byte stores.  */
+
+static inline int lduw_p(const void *ptr)
+{
+    uint16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline int ldsw_p(const void *ptr)
+{
+    int16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stw_p(void *ptr, uint16_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int ldl_p(const void *ptr)
+{
+    int32_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stl_p(void *ptr, uint32_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline uint64_t ldq_p(const void *ptr)
+{
+    uint64_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stq_p(void *ptr, uint64_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int lduw_le_p(const void *ptr)
+{
+    return (uint16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_le_p(const void *ptr)
+{
+    return (int16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_le_p(const void *ptr)
+{
+    return le_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_le_p(const void *ptr)
+{
+    return le_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_le_p(void *ptr, int v)
+{
+    stw_p(ptr, le_bswap(v, 16));
+}
+
+static inline void stl_le_p(void *ptr, int v)
+{
+    stl_p(ptr, le_bswap(v, 32));
+}
+
+static inline void stq_le_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, le_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_le_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_le_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_le_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_le_p(ptr, u.l);
+}
+
+static inline float64 ldfq_le_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_le_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_le_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_le_p(ptr, u.ll);
+}
+
+static inline int lduw_be_p(const void *ptr)
+{
+    return (uint16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_be_p(const void *ptr)
+{
+    return (int16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_be_p(const void *ptr)
+{
+    return be_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_be_p(const void *ptr)
+{
+    return be_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_be_p(void *ptr, int v)
+{
+    stw_p(ptr, be_bswap(v, 16));
+}
+
+static inline void stl_be_p(void *ptr, int v)
+{
+    stl_p(ptr, be_bswap(v, 32));
+}
+
+static inline void stq_be_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, be_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_be_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_be_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_be_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_be_p(ptr, u.l);
+}
+
+static inline float64 ldfq_be_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_be_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_be_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_be_p(ptr, u.ll);
+}
+
+/* Legacy unaligned versions.  Note that we never had a complete set.  */
+
+static inline void cpu_to_le16wu(uint16_t *p, uint16_t v)
+{
+    stw_le_p(p, v);
+}
+
+static inline void cpu_to_le32wu(uint32_t *p, uint32_t v)
+{
+    stl_le_p(p, v);
+}
+
+static inline uint16_t le16_to_cpupu(const uint16_t *p)
+{
+    return lduw_le_p(p);
+}
+
+static inline uint32_t le32_to_cpupu(const uint32_t *p)
+{
+    return ldl_le_p(p);
+}
+
+static inline uint32_t be32_to_cpupu(const uint32_t *p)
+{
+    return ldl_be_p(p);
+}
+
+static inline void cpu_to_be16wu(uint16_t *p, uint16_t v)
+{
+    stw_be_p(p, v);
+}
+
+static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
+{
+    stl_be_p(p, v);
+}
+
+static inline void cpu_to_be64wu(uint64_t *p, uint64_t v)
+{
+    stq_be_p(p, v);
+}
+
+static inline void cpu_to_32wu(uint32_t *p, uint32_t v)
+{
+    stl_p(p, v);
+}
+
+static inline unsigned long leul_to_cpu(unsigned long v)
+{
+    return le_bswap(v, HOST_LONG_BITS);
+}
+
+#undef le_bswap
+#undef be_bswap
+#undef le_bswaps
+#undef be_bswaps
+
+#endif /* BSWAP_H */
diff --git a/cache-utils.h b/include/qemu/cache-utils.h
index 2c57f78fc19..2c57f78fc19 100644
--- a/cache-utils.h
+++ b/include/qemu/cache-utils.h
diff --git a/compatfd.h b/include/qemu/compatfd.h
index 6b04877b970..6b04877b970 100644
--- a/compatfd.h
+++ b/include/qemu/compatfd.h
diff --git a/compiler.h b/include/qemu/compiler.h
index 07ba1f81139..2f7998b6c12 100644
--- a/compiler.h
+++ b/include/qemu/compiler.h
@@ -44,14 +44,12 @@
    /* Use gnu_printf when supported (qemu uses standard format strings). */
 #  define GCC_ATTR __attribute__((__unused__, format(gnu_printf, 1, 2)))
 #  define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+#  if defined(_WIN32)
+    /* Map __printf__ to __gnu_printf__ because we want standard format strings
+     * even when MinGW or GLib include files use __printf__. */
+#   define __printf__ __gnu_printf__
+#  endif
 # endif
-#if defined(_WIN32)
-#define GCC_WEAK __attribute__((weak))
-#define GCC_WEAK_DECL GCC_WEAK
-#else
-#define GCC_WEAK __attribute__((weak))
-#define GCC_WEAK_DECL
-#endif
 #else
 #define GCC_ATTR /**/
 #define GCC_FMT_ATTR(n, m)
diff --git a/qemu-config.h b/include/qemu/config-file.h
index 5557562c337..ccfccae2b4f 100644
--- a/qemu-config.h
+++ b/include/qemu/config-file.h
@@ -1,12 +1,10 @@
 #ifndef QEMU_CONFIG_H
 #define QEMU_CONFIG_H
 
-#include "error.h"
-
-extern QemuOptsList qemu_fsdev_opts;
-extern QemuOptsList qemu_virtfs_opts;
-extern QemuOptsList qemu_spice_opts;
-extern QemuOptsList qemu_sandbox_opts;
+#include <stdio.h>
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "qemu/option.h"
 
 QemuOptsList *qemu_find_opts(const char *group);
 QemuOptsList *qemu_find_opts_err(const char *group, Error **errp);
diff --git a/include/qemu/cpu.h b/include/qemu/cpu.h
deleted file mode 100644
index ad706a6dbd7..00000000000
--- a/include/qemu/cpu.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * QEMU CPU model
- *
- * Copyright (c) 2012 SUSE LINUX Products GmbH
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- */
-#ifndef QEMU_CPU_H
-#define QEMU_CPU_H
-
-#include "qemu/object.h"
-#include "qemu-thread.h"
-
-/**
- * SECTION:cpu
- * @section_id: QEMU-cpu
- * @title: CPU Class
- * @short_description: Base class for all CPUs
- */
-
-#define TYPE_CPU "cpu"
-
-#define CPU(obj) OBJECT_CHECK(CPUState, (obj), TYPE_CPU)
-#define CPU_CLASS(class) OBJECT_CLASS_CHECK(CPUClass, (class), TYPE_CPU)
-#define CPU_GET_CLASS(obj) OBJECT_GET_CLASS(CPUClass, (obj), TYPE_CPU)
-
-typedef struct CPUState CPUState;
-
-/**
- * CPUClass:
- * @reset: Callback to reset the #CPUState to its initial state.
- *
- * Represents a CPU family or model.
- */
-typedef struct CPUClass {
-    /*< private >*/
-    ObjectClass parent_class;
-    /*< public >*/
-
-    void (*reset)(CPUState *cpu);
-} CPUClass;
-
-/**
- * CPUState:
- *
- * State of one CPU core or thread.
- */
-struct CPUState {
-    /*< private >*/
-    Object parent_obj;
-    /*< public >*/
-
-    struct QemuThread *thread;
-#ifdef _WIN32
-    HANDLE hThread;
-#endif
-    bool thread_kicked;
-
-    /* TODO Move common fields from CPUArchState here. */
-};
-
-
-/**
- * cpu_reset:
- * @cpu: The CPU whose state is to be reset.
- */
-void cpu_reset(CPUState *cpu);
-
-
-#endif
diff --git a/envlist.h b/include/qemu/envlist.h
index b9addcc11f7..b9addcc11f7 100644
--- a/envlist.h
+++ b/include/qemu/envlist.h
diff --git a/qemu-error.h b/include/qemu/error-report.h
index 93d74b4a305..c902cc10de9 100644
--- a/qemu-error.h
+++ b/include/qemu/error-report.h
@@ -13,6 +13,8 @@
 #ifndef QEMU_ERROR_H
 #define QEMU_ERROR_H
 
+#include <stdarg.h>
+
 typedef struct Location {
     /* all members are private to qemu-error.c */
     enum { LOC_NONE, LOC_CMDLINE, LOC_FILE } kind;
diff --git a/event_notifier.h b/include/qemu/event_notifier.h
index f0ec2f2171b..88b57af7ce8 100644
--- a/event_notifier.h
+++ b/include/qemu/event_notifier.h
@@ -15,18 +15,32 @@
 
 #include "qemu-common.h"
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 struct EventNotifier {
-    int fd;
+#ifdef _WIN32
+    HANDLE event;
+#else
+    int rfd;
+    int wfd;
+#endif
 };
 
 typedef void EventNotifierHandler(EventNotifier *);
 
-void event_notifier_init_fd(EventNotifier *, int fd);
 int event_notifier_init(EventNotifier *, int active);
 void event_notifier_cleanup(EventNotifier *);
-int event_notifier_get_fd(EventNotifier *);
 int event_notifier_set(EventNotifier *);
 int event_notifier_test_and_clear(EventNotifier *);
 int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
 
+#ifdef CONFIG_POSIX
+void event_notifier_init_fd(EventNotifier *, int fd);
+int event_notifier_get_fd(EventNotifier *);
+#else
+HANDLE event_notifier_get_handle(EventNotifier *);
+#endif
+
 #endif
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
new file mode 100644
index 00000000000..73f5d1d8d3e
--- /dev/null
+++ b/include/qemu/hbitmap.h
@@ -0,0 +1,208 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef HBITMAP_H
+#define HBITMAP_H 1
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include "bitops.h"
+
+typedef struct HBitmap HBitmap;
+typedef struct HBitmapIter HBitmapIter;
+
+#define BITS_PER_LEVEL         (BITS_PER_LONG == 32 ? 5 : 6)
+
+/* For 32-bit, the largest that fits in a 4 GiB address space.
+ * For 64-bit, the number of sectors in 1 PiB.  Good luck, in
+ * either case... :)
+ */
+#define HBITMAP_LOG_MAX_SIZE   (BITS_PER_LONG == 32 ? 34 : 41)
+
+/* We need to place a sentinel in level 0 to speed up iteration.  Thus,
+ * we do this instead of HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL.  The
+ * difference is that it allocates an extra level when HBITMAP_LOG_MAX_SIZE
+ * is an exact multiple of BITS_PER_LEVEL.
+ */
+#define HBITMAP_LEVELS         ((HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL) + 1)
+
+struct HBitmapIter {
+    const HBitmap *hb;
+
+    /* Copied from hb for access in the inline functions (hb is opaque).  */
+    int granularity;
+
+    /* Entry offset into the last-level array of longs.  */
+    size_t pos;
+
+    /* The currently-active path in the tree.  Each item of cur[i] stores
+     * the bits (i.e. the subtrees) yet to be processed under that node.
+     */
+    unsigned long cur[HBITMAP_LEVELS];
+};
+
+/**
+ * hbitmap_alloc:
+ * @size: Number of bits in the bitmap.
+ * @granularity: Granularity of the bitmap.  Aligned groups of 2^@granularity
+ * bits will be represented by a single bit.  Each operation on a
+ * range of bits first rounds the bits to determine which group they land
+ * in, and then affect the entire set; iteration will only visit the first
+ * bit of each group.
+ *
+ * Allocate a new HBitmap.
+ */
+HBitmap *hbitmap_alloc(uint64_t size, int granularity);
+
+/**
+ * hbitmap_empty:
+ * @hb: HBitmap to operate on.
+ *
+ * Return whether the bitmap is empty.
+ */
+bool hbitmap_empty(const HBitmap *hb);
+
+/**
+ * hbitmap_granularity:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the granularity of the HBitmap.
+ */
+int hbitmap_granularity(const HBitmap *hb);
+
+/**
+ * hbitmap_count:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the number of bits set in the HBitmap.
+ */
+uint64_t hbitmap_count(const HBitmap *hb);
+
+/**
+ * hbitmap_set:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to set (0-based).
+ * @count: Number of bits to set.
+ *
+ * Set a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_reset:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to reset (0-based).
+ * @count: Number of bits to reset.
+ *
+ * Reset a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_get:
+ * @hb: HBitmap to operate on.
+ * @item: Bit to query (0-based).
+ *
+ * Return whether the @item-th bit in an HBitmap is set.
+ */
+bool hbitmap_get(const HBitmap *hb, uint64_t item);
+
+/**
+ * hbitmap_free:
+ * @hb: HBitmap to operate on.
+ *
+ * Free an HBitmap and all of its associated memory.
+ */
+void hbitmap_free(HBitmap *hb);
+
+/**
+ * hbitmap_iter_init:
+ * @hbi: HBitmapIter to initialize.
+ * @hb: HBitmap to iterate on.
+ * @first: First bit to visit (0-based, must be strictly less than the
+ * size of the bitmap).
+ *
+ * Set up @hbi to iterate on the HBitmap @hb.  hbitmap_iter_next will return
+ * the lowest-numbered bit that is set in @hb, starting at @first.
+ *
+ * Concurrent setting of bits is acceptable, and will at worst cause the
+ * iteration to miss some of those bits.  Resetting bits before the current
+ * position of the iterator is also okay.  However, concurrent resetting of
+ * bits can lead to unexpected behavior if the iterator has not yet reached
+ * those bits.
+ */
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first);
+
+/* hbitmap_iter_skip_words:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Internal function used by hbitmap_iter_next and hbitmap_iter_next_word.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi);
+
+/**
+ * hbitmap_iter_next:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Return the next bit that is set in @hbi's associated HBitmap,
+ * or -1 if all remaining bits are zero.
+ */
+static inline int64_t hbitmap_iter_next(HBitmapIter *hbi)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+    int64_t item;
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next bit.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1);
+    item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ffsl(cur) - 1;
+
+    return item << hbi->granularity;
+}
+
+/**
+ * hbitmap_iter_next_word:
+ * @hbi: HBitmapIter to operate on.
+ * @p_cur: Location where to store the next non-zero word.
+ *
+ * Return the index of the next nonzero word that is set in @hbi's
+ * associated HBitmap, and set *p_cur to the content of that word
+ * (bits before the index that was passed to hbitmap_iter_init are
+ * trimmed on the first call).  Return -1, and set *p_cur to zero,
+ * if all remaining words are zero.
+ */
+static inline size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long *p_cur)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            *p_cur = 0;
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next word.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = 0;
+    *p_cur = cur;
+    return hbi->pos;
+}
+
+
+#endif
diff --git a/host-utils.h b/include/qemu/host-utils.h
index 821db936717..2a32be4cc06 100644
--- a/host-utils.h
+++ b/include/qemu/host-utils.h
@@ -22,8 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HOST_UTILS_H
+#define HOST_UTILS_H 1
 
-#include "compiler.h"   /* QEMU_GNUC_PREREQ */
+#include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
+#include <string.h>     /* ffsl */
 
 #if defined(__x86_64__)
 #define __HAVE_FAST_MULU64__
@@ -234,3 +237,30 @@ static inline int ctpop64(uint64_t val)
     return val;
 #endif
 }
+
+/* glibc does not provide an inline version of ffsl, so always define
+ * ours.  We need to give it a different name, however.
+ */
+#ifdef __GLIBC__
+#define ffsl qemu_ffsl
+#endif
+static inline int ffsl(long val)
+{
+    if (!val) {
+        return 0;
+    }
+
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_ctzl(val) + 1;
+#else
+    if (sizeof(long) == 4) {
+        return ctz32(val) + 1;
+    } else if (sizeof(long) == 8) {
+        return ctz64(val) + 1;
+    } else {
+        abort();
+    }
+#endif
+}
+
+#endif
diff --git a/int128.h b/include/qemu/int128.h
index b3864b6cd46..b3864b6cd46 100644
--- a/int128.h
+++ b/include/qemu/int128.h
diff --git a/iov.h b/include/qemu/iov.h
index 381f37a5463..68d25f29b76 100644
--- a/iov.h
+++ b/include/qemu/iov.h
@@ -11,6 +11,9 @@
  * the COPYING file in the top-level directory.
  */
 
+#ifndef IOV_H
+#define IOV_H
+
 #include "qemu-common.h"
 
 /**
@@ -36,7 +39,7 @@ size_t iov_size(const struct iovec *iov, const unsigned int iov_cnt);
  * such "large" value is -1 (sinice size_t is unsigned),
  * so specifying `-1' as `bytes' means 'up to the end of iovec'.
  */
-size_t iov_from_buf(struct iovec *iov, unsigned int iov_cnt,
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
                     size_t offset, const void *buf, size_t bytes);
 size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
                   size_t offset, void *buf, size_t bytes);
@@ -86,3 +89,27 @@ ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
  */
 void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
                  FILE *fp, const char *prefix, size_t limit);
+
+/*
+ * Partial copy of vector from iov to dst_iov (data is not copied).
+ * dst_iov overlaps iov at a specified offset.
+ * size of dst_iov is at most bytes. dst vector count is returned.
+ */
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes);
+
+/*
+ * Remove a given number of bytes from the front or back of a vector.
+ * This may update iov and/or iov_cnt to exclude iovec elements that are
+ * no longer required.
+ *
+ * The number of bytes actually discarded is returned.  This number may be
+ * smaller than requested if the vector is too small.
+ */
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes);
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes);
+
+#endif
diff --git a/qemu-log.h b/include/qemu/log.h
index 5ccecf30aff..58f69cb494d 100644
--- a/qemu-log.h
+++ b/include/qemu/log.h
@@ -3,7 +3,7 @@
 
 #include <stdarg.h>
 #ifdef NEED_CPU_H
-#include "disas.h"
+#include "disas/disas.h"
 #endif
 
 /* Private global variables, don't use */
@@ -35,6 +35,7 @@ static inline bool qemu_log_enabled(void)
 #define CPU_LOG_TB_CPU     (1 << 8)
 #define CPU_LOG_RESET      (1 << 9)
 #define LOG_UNIMP          (1 << 10)
+#define LOG_GUEST_ERROR    (1 << 11)
 
 /* Returns true if a bit is set in the current loglevel mask
  */
@@ -83,10 +84,10 @@ static inline void log_cpu_state_mask(int mask, CPUArchState *env1, int flags)
 }
 
 /* disas() and target_disas() to qemu_logfile: */
-static inline void log_target_disas(target_ulong start, target_ulong len,
-                                    int flags)
+static inline void log_target_disas(CPUArchState *env, target_ulong start,
+                                    target_ulong len, int flags)
 {
-    target_disas(qemu_logfile, start, len, flags);
+    target_disas(qemu_logfile, env, start, len, flags);
 }
 
 static inline void log_disas(void *code, unsigned long size)
diff --git a/main-loop.h b/include/qemu/main-loop.h
index dce1cd9d7ca..e8059c3d0a7 100644
--- a/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -25,6 +25,8 @@
 #ifndef QEMU_MAIN_LOOP_H
 #define QEMU_MAIN_LOOP_H 1
 
+#include "block/aio.h"
+
 #define SIG_IPI SIGUSR1
 
 /**
@@ -43,16 +45,6 @@
 int qemu_init_main_loop(void);
 
 /**
- * main_loop_init: Initializes main loop
- *
- * Internal (but shared for compatibility reasons) initialization routine
- * for the main loop. This should not be used by applications directly,
- * use qemu_init_main_loop() instead.
- *
- */
-int main_loop_init(void);
-
-/**
  * main_loop_wait: Run one iteration of the main loop.
  *
  * If @nonblocking is true, poll for events, otherwise suspend until
@@ -173,7 +165,6 @@ void qemu_del_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque);
 
 typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size);
 typedef int IOCanReadHandler(void *opaque);
-typedef void IOHandler(void *opaque);
 
 /**
  * qemu_set_fd_handler2: Register a file descriptor with the main loop
@@ -254,56 +245,6 @@ int qemu_set_fd_handler(int fd,
                         IOHandler *fd_write,
                         void *opaque);
 
-typedef struct QEMUBH QEMUBH;
-typedef void QEMUBHFunc(void *opaque);
-
-/**
- * qemu_bh_new: Allocate a new bottom half structure.
- *
- * Bottom halves are lightweight callbacks whose invocation is guaranteed
- * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
- * is opaque and must be allocated prior to its use.
- */
-QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
-
-/**
- * qemu_bh_schedule: Schedule a bottom half.
- *
- * Scheduling a bottom half interrupts the main loop and causes the
- * execution of the callback that was passed to qemu_bh_new.
- *
- * Bottom halves that are scheduled from a bottom half handler are instantly
- * invoked.  This can create an infinite loop if a bottom half handler
- * schedules itself.
- *
- * @bh: The bottom half to be scheduled.
- */
-void qemu_bh_schedule(QEMUBH *bh);
-
-/**
- * qemu_bh_cancel: Cancel execution of a bottom half.
- *
- * Canceling execution of a bottom half undoes the effect of calls to
- * qemu_bh_schedule without freeing its resources yet.  While cancellation
- * itself is also wait-free and thread-safe, it can of course race with the
- * loop that executes bottom halves unless you are holding the iothread
- * mutex.  This makes it mostly useless if you are not holding the mutex.
- *
- * @bh: The bottom half to be canceled.
- */
-void qemu_bh_cancel(QEMUBH *bh);
-
-/**
- *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
- *
- * Deleting a bottom half frees the memory that was allocated for it by
- * qemu_bh_new.  It also implies canceling the bottom half if it was
- * scheduled.
- *
- * @bh: The bottom half to be deleted.
- */
-void qemu_bh_delete(QEMUBH *bh);
-
 #ifdef CONFIG_POSIX
 /**
  * qemu_add_child_watch: Register a child process for reaping.
@@ -359,8 +300,7 @@ void qemu_fd_register(int fd);
 void qemu_iohandler_fill(int *pnfds, fd_set *readfds, fd_set *writefds, fd_set *xfds);
 void qemu_iohandler_poll(fd_set *readfds, fd_set *writefds, fd_set *xfds, int rc);
 
+QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule_idle(QEMUBH *bh);
-int qemu_bh_poll(void);
-void qemu_bh_update_timeout(uint32_t *timeout);
 
 #endif
diff --git a/module.h b/include/qemu/module.h
index c4ccd571664..c4ccd571664 100644
--- a/module.h
+++ b/include/qemu/module.h
diff --git a/notify.h b/include/qemu/notify.h
index 03cf26c0b62..4e2e7f0ec4d 100644
--- a/notify.h
+++ b/include/qemu/notify.h
@@ -14,7 +14,7 @@
 #ifndef QEMU_NOTIFY_H
 #define QEMU_NOTIFY_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef struct Notifier Notifier;
 
diff --git a/qemu-option.h b/include/qemu/option.h
index ca729862d57..ba197cddcf2 100644
--- a/qemu-option.h
+++ b/include/qemu/option.h
@@ -27,9 +27,9 @@
 #define QEMU_OPTIONS_H
 
 #include <stdint.h>
-#include "qemu-queue.h"
-#include "error.h"
-#include "qdict.h"
+#include "qemu/queue.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
 
 enum QEMUOptionParType {
     OPT_FLAG,
@@ -126,6 +126,7 @@ int qemu_opt_set(QemuOpts *opts, const char *name, const char *value);
 void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
                       Error **errp);
 int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val);
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val);
 typedef int (*qemu_opt_loopfunc)(const char *name, const char *value, void *opaque);
 int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
                      int abort_on_failure);
@@ -133,6 +134,7 @@ int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
 QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id);
 QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
                            int fail_if_exists, Error **errp);
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list);
 void qemu_opts_reset(QemuOptsList *list);
 void qemu_opts_loc_restore(QemuOpts *opts);
 int qemu_opts_set(QemuOptsList *list, const char *id,
diff --git a/qemu-option-internal.h b/include/qemu/option_int.h
index 19fdc1ca855..8212fa4a485 100644
--- a/qemu-option-internal.h
+++ b/include/qemu/option_int.h
@@ -26,7 +26,8 @@
 #ifndef QEMU_OPTIONS_INTERNAL_H
 #define QEMU_OPTIONS_INTERNAL_H
 
-#include "qemu-option.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
 
 struct QemuOpt {
     const char   *name;
diff --git a/osdep.h b/include/qemu/osdep.h
index cb213e02953..24ff6d05034 100644
--- a/osdep.h
+++ b/include/qemu/osdep.h
@@ -60,6 +60,10 @@ typedef signed int              int_fast16_t;
 #define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
+#ifndef ROUND_UP
+#define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
+#endif
+
 #ifndef DIV_ROUND_UP
 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
 #endif
@@ -108,6 +112,11 @@ void qemu_vfree(void *ptr);
 #else
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #endif
+#ifdef MADV_HUGEPAGE
+#define QEMU_MADV_HUGEPAGE MADV_HUGEPAGE
+#else
+#define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
+#endif
 
 #elif defined(CONFIG_POSIX_MADVISE)
 
@@ -116,6 +125,7 @@ void qemu_vfree(void *ptr);
 #define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
 #define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
 
 #else /* no-op */
 
@@ -124,11 +134,15 @@ void qemu_vfree(void *ptr);
 #define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
 #define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
 
 #endif
 
 int qemu_madvise(void *addr, size_t len, int advice);
 
+int qemu_open(const char *name, int flags, ...);
+int qemu_close(int fd);
+
 #if defined(__HAIKU__) && defined(__i386__)
 #define FMT_pid "%ld"
 #elif defined(WIN64)
diff --git a/qemu-queue.h b/include/qemu/queue.h
index 9288cd8e826..d433b9017ce 100644
--- a/qemu-queue.h
+++ b/include/qemu/queue.h
@@ -78,7 +78,7 @@
  * For details on the use of these macros, see the queue(3) manual page.
  */
 
-#include "qemu-barrier.h" /* for smp_wmb() */
+#include "qemu/atomic.h" /* for smp_wmb() */
 
 /*
  * List definitions.
diff --git a/range.h b/include/qemu/range.h
index 350237212b6..350237212b6 100644
--- a/range.h
+++ b/include/qemu/range.h
diff --git a/include/qemu/ratelimit.h b/include/qemu/ratelimit.h
index c6ac2811418..d1610f135be 100644
--- a/include/qemu/ratelimit.h
+++ b/include/qemu/ratelimit.h
@@ -42,7 +42,7 @@ static inline void ratelimit_set_speed(RateLimit *limit, uint64_t speed,
                                        uint64_t slice_ns)
 {
     limit->slice_ns = slice_ns;
-    limit->slice_quota = ((double)speed * 1000000000ULL) / slice_ns;
+    limit->slice_quota = ((double)speed * slice_ns)/1000000000ULL;
 }
 
 #endif
diff --git a/include/qemu/rng-random.h b/include/qemu/rng-random.h
new file mode 100644
index 00000000000..4332772a24b
--- /dev/null
+++ b/include/qemu/rng-random.h
@@ -0,0 +1,22 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_RNG_RANDOM_H
+#define QEMU_RNG_RANDOM_H
+
+#include "qom/object.h"
+
+#define TYPE_RNG_RANDOM "rng-random"
+#define RNG_RANDOM(obj) OBJECT_CHECK(RndRandom, (obj), TYPE_RNG_RANDOM)
+
+typedef struct RndRandom RndRandom;
+
+#endif
diff --git a/include/qemu/rng.h b/include/qemu/rng.h
new file mode 100644
index 00000000000..509abd023d1
--- /dev/null
+++ b/include/qemu/rng.h
@@ -0,0 +1,93 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_RNG_H
+#define QEMU_RNG_H
+
+#include "qom/object.h"
+#include "qemu-common.h"
+#include "qapi/error.h"
+
+#define TYPE_RNG_BACKEND "rng-backend"
+#define RNG_BACKEND(obj) \
+    OBJECT_CHECK(RngBackend, (obj), TYPE_RNG_BACKEND)
+#define RNG_BACKEND_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(RngBackendClass, (obj), TYPE_RNG_BACKEND)
+#define RNG_BACKEND_CLASS(klass) \
+    OBJECT_CLASS_CHECK(RngBackendClass, (klass), TYPE_RNG_BACKEND)
+
+typedef struct RngBackendClass RngBackendClass;
+typedef struct RngBackend RngBackend;
+
+typedef void (EntropyReceiveFunc)(void *opaque,
+                                  const void *data,
+                                  size_t size);
+
+struct RngBackendClass
+{
+    ObjectClass parent_class;
+
+    void (*request_entropy)(RngBackend *s, size_t size,
+                            EntropyReceiveFunc *recieve_entropy, void *opaque);
+    void (*cancel_requests)(RngBackend *s);
+
+    void (*opened)(RngBackend *s, Error **errp);
+};
+
+struct RngBackend
+{
+    Object parent;
+
+    /*< protected >*/
+    bool opened;
+};
+
+/**
+ * rng_backend_request_entropy:
+ * @s: the backend to request entropy from
+ * @size: the number of bytes of data to request
+ * @receive_entropy: a function to be invoked when entropy is available
+ * @opaque: data that should be passed to @receive_entropy
+ *
+ * This function is used by the front-end to request entropy from an entropy
+ * source.  This function can be called multiple times before @receive_entropy
+ * is invoked with different values of @receive_entropy and @opaque.  The
+ * backend will queue each request and handle appropriately.
+ *
+ * The backend does not need to pass the full amount of data to @receive_entropy
+ * but will pass a value greater than 0.
+ */
+void rng_backend_request_entropy(RngBackend *s, size_t size,
+                                 EntropyReceiveFunc *receive_entropy,
+                                 void *opaque);
+
+/**
+ * rng_backend_cancel_requests:
+ * @s: the backend to cancel all pending requests in
+ *
+ * Cancels all pending requests submitted by @rng_backend_request_entropy.  This
+ * should be used by a device during reset or in preparation for live migration
+ * to stop tracking any request.
+ */
+void rng_backend_cancel_requests(RngBackend *s);
+
+/**
+ * rng_backend_open:
+ * @s: the backend to open
+ * @errp: a pointer to return the #Error object if an error occurs.
+ *
+ * This function will open the backend if it is not already open.  Calling this
+ * function on an already opened backend will not result in an error.
+ */
+void rng_backend_open(RngBackend *s, Error **errp);
+
+#endif
diff --git a/include/qemu/sockets.h b/include/qemu/sockets.h
new file mode 100644
index 00000000000..803ae1798c4
--- /dev/null
+++ b/include/qemu/sockets.h
@@ -0,0 +1,77 @@
+/* headers to use the BSD sockets */
+#ifndef QEMU_SOCKET_H
+#define QEMU_SOCKET_H
+
+#ifdef _WIN32
+#include <windows.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#define socket_error() WSAGetLastError()
+
+int inet_aton(const char *cp, struct in_addr *ia);
+
+#else
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/un.h>
+
+#define socket_error() errno
+#define closesocket(s) close(s)
+
+#endif /* !_WIN32 */
+
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+
+/* misc helpers */
+int qemu_socket(int domain, int type, int protocol);
+int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
+int socket_set_cork(int fd, int v);
+void socket_set_block(int fd);
+void socket_set_nonblock(int fd);
+int send_all(int fd, const void *buf, int len1);
+
+/* callback function for nonblocking connect
+ * valid fd on success, negative error code on failure
+ */
+typedef void NonBlockingConnectHandler(int fd, void *opaque);
+
+int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
+int inet_listen(const char *str, char *ostr, int olen,
+                int socktype, int port_offset, Error **errp);
+int inet_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int inet_connect(const char *str, Error **errp);
+int inet_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+int inet_dgram_opts(QemuOpts *opts, Error **errp);
+const char *inet_strfamily(int family);
+
+int unix_listen_opts(QemuOpts *opts, Error **errp);
+int unix_listen(const char *path, char *ostr, int olen, Error **errp);
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int unix_connect(const char *path, Error **errp);
+int unix_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+SocketAddress *socket_parse(const char *str, Error **errp);
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque);
+int socket_listen(SocketAddress *addr, Error **errp);
+
+/* Old, ipv4 only bits.  Don't use for new code. */
+int parse_host_port(struct sockaddr_in *saddr, const char *str);
+int socket_init(void);
+
+#endif /* QEMU_SOCKET_H */
diff --git a/qemu-thread-posix.h b/include/qemu/thread-posix.h
index ee4618e6209..0f30dccb53c 100644
--- a/qemu-thread-posix.h
+++ b/include/qemu/thread-posix.h
@@ -1,6 +1,7 @@
 #ifndef __QEMU_THREAD_POSIX_H
 #define __QEMU_THREAD_POSIX_H 1
 #include "pthread.h"
+#include <semaphore.h>
 
 struct QemuMutex {
     pthread_mutex_t lock;
@@ -10,6 +11,16 @@ struct QemuCond {
     pthread_cond_t cond;
 };
 
+struct QemuSemaphore {
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    int count;
+#else
+    sem_t sem;
+#endif
+};
+
 struct QemuThread {
     pthread_t thread;
 };
diff --git a/qemu-thread-win32.h b/include/qemu/thread-win32.h
index b9d1be84780..13adb958f04 100644
--- a/qemu-thread-win32.h
+++ b/include/qemu/thread-win32.h
@@ -13,6 +13,10 @@ struct QemuCond {
     HANDLE continue_event;
 };
 
+struct QemuSemaphore {
+    HANDLE sema;
+};
+
 typedef struct QemuThreadData QemuThreadData;
 struct QemuThread {
     QemuThreadData *data;
diff --git a/qemu-thread.h b/include/qemu/thread.h
index 05fdaaf50e4..c02404b9fbf 100644
--- a/qemu-thread.h
+++ b/include/qemu/thread.h
@@ -6,12 +6,13 @@
 
 typedef struct QemuMutex QemuMutex;
 typedef struct QemuCond QemuCond;
+typedef struct QemuSemaphore QemuSemaphore;
 typedef struct QemuThread QemuThread;
 
 #ifdef _WIN32
-#include "qemu-thread-win32.h"
+#include "qemu/thread-win32.h"
 #else
-#include "qemu-thread-posix.h"
+#include "qemu/thread-posix.h"
 #endif
 
 #define QEMU_THREAD_JOINABLE 0
@@ -38,6 +39,12 @@ void qemu_cond_signal(QemuCond *cond);
 void qemu_cond_broadcast(QemuCond *cond);
 void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex);
 
+void qemu_sem_init(QemuSemaphore *sem, int init);
+void qemu_sem_post(QemuSemaphore *sem);
+void qemu_sem_wait(QemuSemaphore *sem);
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms);
+void qemu_sem_destroy(QemuSemaphore *sem);
+
 void qemu_thread_create(QemuThread *thread,
                         void *(*start_routine)(void *),
                         void *arg, int mode);
diff --git a/qemu-timer.h b/include/qemu/timer.h
index f8af595f133..1766b2d6c76 100644
--- a/qemu-timer.h
+++ b/include/qemu/timer.h
@@ -2,8 +2,8 @@
 #define QEMU_TIMER_H
 
 #include "qemu-common.h"
-#include "main-loop.h"
-#include "notify.h"
+#include "qemu/main-loop.h"
+#include "qemu/notify.h"
 
 #ifdef __FreeBSD__
 #include <sys/param.h>
@@ -218,7 +218,7 @@ static inline int64_t cpu_get_real_ticks(void)
     return val;
 }
 
-#elif defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || defined(__sparc_v9__)
+#elif defined(__sparc__)
 
 static inline int64_t cpu_get_real_ticks (void)
 {
@@ -227,6 +227,8 @@ static inline int64_t cpu_get_real_ticks (void)
     asm volatile("rd %%tick,%0" : "=r"(rval));
     return rval;
 #else
+    /* We need an %o or %g register for this.  For recent enough gcc
+       there is an "h" constraint for that.  Don't bother with that.  */
     union {
         uint64_t i64;
         struct {
@@ -234,8 +236,8 @@ static inline int64_t cpu_get_real_ticks (void)
             uint32_t low;
         }       i32;
     } rval;
-    asm volatile("rd %%tick,%1; srlx %1,32,%0"
-                 : "=r"(rval.i32.high), "=r"(rval.i32.low));
+    asm volatile("rd %%tick,%%g1; srlx %%g1,32,%0; mov %%g1,%1"
+                 : "=r"(rval.i32.high), "=r"(rval.i32.low) : : "g1");
     return rval.i64;
 #endif
 }
diff --git a/qemu-tls.h b/include/qemu/tls.h
index b92ea9d7da3..b92ea9d7da3 100644
--- a/qemu-tls.h
+++ b/include/qemu/tls.h
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
new file mode 100644
index 00000000000..fd532a268d8
--- /dev/null
+++ b/include/qemu/typedefs.h
@@ -0,0 +1,61 @@
+#ifndef QEMU_TYPEDEFS_H
+#define QEMU_TYPEDEFS_H
+
+/* A load of opaque types so that device init declarations don't have to
+   pull in all the real definitions.  */
+typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUFile QEMUFile;
+typedef struct QEMUBH QEMUBH;
+
+struct Monitor;
+typedef struct Monitor Monitor;
+typedef struct MigrationParams MigrationParams;
+
+typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
+typedef struct CompatProperty CompatProperty;
+typedef struct DeviceState DeviceState;
+typedef struct BusState BusState;
+typedef struct BusClass BusClass;
+
+typedef struct NICInfo NICInfo;
+typedef struct HCIInfo HCIInfo;
+typedef struct AudioState AudioState;
+typedef struct BlockDriverState BlockDriverState;
+typedef struct DriveInfo DriveInfo;
+typedef struct DisplayState DisplayState;
+typedef struct DisplayChangeListener DisplayChangeListener;
+typedef struct DisplaySurface DisplaySurface;
+typedef struct PixelFormat PixelFormat;
+typedef struct QemuConsole QemuConsole;
+typedef struct CharDriverState CharDriverState;
+typedef struct MACAddr MACAddr;
+typedef struct NetClientState NetClientState;
+typedef struct i2c_bus i2c_bus;
+typedef struct ISABus ISABus;
+typedef struct ISADevice ISADevice;
+typedef struct SMBusDevice SMBusDevice;
+typedef struct PCIHostState PCIHostState;
+typedef struct PCIExpressHost PCIExpressHost;
+typedef struct PCIBus PCIBus;
+typedef struct PCIDevice PCIDevice;
+typedef struct PCIExpressDevice PCIExpressDevice;
+typedef struct PCIBridge PCIBridge;
+typedef struct PCIEAERMsg PCIEAERMsg;
+typedef struct PCIEAERLog PCIEAERLog;
+typedef struct PCIEAERErr PCIEAERErr;
+typedef struct PCIEPort PCIEPort;
+typedef struct PCIESlot PCIESlot;
+typedef struct MSIMessage MSIMessage;
+typedef struct SerialState SerialState;
+typedef struct PCMCIACardState PCMCIACardState;
+typedef struct MouseTransformInfo MouseTransformInfo;
+typedef struct uWireSlave uWireSlave;
+typedef struct I2SCodec I2SCodec;
+typedef struct SSIBus SSIBus;
+typedef struct EventNotifier EventNotifier;
+typedef struct VirtIODevice VirtIODevice;
+typedef struct QEMUSGList QEMUSGList;
+typedef struct SHPCDevice SHPCDevice;
+
+#endif /* QEMU_TYPEDEFS_H */
diff --git a/include/qemu/uri.h b/include/qemu/uri.h
new file mode 100644
index 00000000000..de99b3bd4b8
--- /dev/null
+++ b/include/qemu/uri.h
@@ -0,0 +1,113 @@
+/**
+ * Summary: library of generic URI related routines
+ * Description: library of generic URI related routines
+ *              Implements RFC 2396
+ *
+ * Copyright (C) 1998-2003 Daniel Veillard.  All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * DANIEL VEILLARD BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Except as contained in this notice, the name of Daniel Veillard shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization from him.
+ *
+ * Author: Daniel Veillard
+ **
+ * Copyright (C) 2007 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *    Richard W.M. Jones <rjones@redhat.com>
+ *
+ * Utility functions to help parse and assemble query strings.
+ */
+
+#ifndef QEMU_URI_H
+#define QEMU_URI_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * URI:
+ *
+ * A parsed URI reference. This is a struct containing the various fields
+ * as described in RFC 2396 but separated for further processing.
+ */
+typedef struct URI {
+    char *scheme;	/* the URI scheme */
+    char *opaque;	/* opaque part */
+    char *authority;	/* the authority part */
+    char *server;	/* the server part */
+    char *user;		/* the user part */
+    int port;		/* the port number */
+    char *path;		/* the path string */
+    char *fragment;	/* the fragment identifier */
+    int  cleanup;	/* parsing potentially unclean URI */
+    char *query;	/* the query string (as it appears in the URI) */
+} URI;
+
+URI *uri_new(void);
+char *uri_resolve(const char *URI, const char *base);
+char *uri_resolve_relative(const char *URI, const char *base);
+URI *uri_parse(const char *str);
+URI *uri_parse_raw(const char *str, int raw);
+int uri_parse_into(URI *uri, const char *str);
+char *uri_to_string(URI *uri);
+char *uri_string_escape(const char *str, const char *list);
+char *uri_string_unescape(const char *str, int len, char *target);
+void uri_free(URI *uri);
+
+/* Single web service query parameter 'name=value'. */
+typedef struct QueryParam {
+  char *name;			/* Name (unescaped). */
+  char *value;			/* Value (unescaped). */
+  int ignore;			/* Ignore this field in qparam_get_query */
+} QueryParam;
+
+/* Set of parameters. */
+typedef struct QueryParams {
+  int n;			/* number of parameters used */
+  int alloc;			/* allocated space */
+  QueryParam *p;		/* array of parameters */
+} QueryParams;
+
+struct QueryParams *query_params_new (int init_alloc);
+int query_param_append (QueryParams *ps, const char *name, const char *value);
+extern char *query_param_to_string (const QueryParams *ps);
+extern QueryParams *query_params_parse (const char *query);
+extern void query_params_free (QueryParams *ps);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* QEMU_URI_H */
diff --git a/qemu-xattr.h b/include/qemu/xattr.h
index f910d96eafb..f910d96eafb 100644
--- a/qemu-xattr.h
+++ b/include/qemu/xattr.h
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
new file mode 100644
index 00000000000..46f2247274a
--- /dev/null
+++ b/include/qom/cpu.h
@@ -0,0 +1,183 @@
+/*
+ * QEMU CPU model
+ *
+ * Copyright (c) 2012 SUSE LINUX Products GmbH
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see
+ * <http://www.gnu.org/licenses/gpl-2.0.html>
+ */
+#ifndef QEMU_CPU_H
+#define QEMU_CPU_H
+
+#include "hw/qdev-core.h"
+#include "qemu/thread.h"
+
+/**
+ * SECTION:cpu
+ * @section_id: QEMU-cpu
+ * @title: CPU Class
+ * @short_description: Base class for all CPUs
+ */
+
+#define TYPE_CPU "cpu"
+
+#define CPU(obj) OBJECT_CHECK(CPUState, (obj), TYPE_CPU)
+#define CPU_CLASS(class) OBJECT_CLASS_CHECK(CPUClass, (class), TYPE_CPU)
+#define CPU_GET_CLASS(obj) OBJECT_GET_CLASS(CPUClass, (obj), TYPE_CPU)
+
+typedef struct CPUState CPUState;
+
+/**
+ * CPUClass:
+ * @class_by_name: Callback to map -cpu command line model name to an
+ * instantiatable CPU type.
+ * @reset: Callback to reset the #CPUState to its initial state.
+ *
+ * Represents a CPU family or model.
+ */
+typedef struct CPUClass {
+    /*< private >*/
+    DeviceClass parent_class;
+    /*< public >*/
+
+    ObjectClass *(*class_by_name)(const char *cpu_model);
+
+    void (*reset)(CPUState *cpu);
+} CPUClass;
+
+struct KVMState;
+struct kvm_run;
+
+/**
+ * CPUState:
+ * @cpu_index: CPU index (informative).
+ * @nr_cores: Number of cores within this CPU package.
+ * @nr_threads: Number of threads within this CPU.
+ * @numa_node: NUMA node this CPU is belonging to.
+ * @created: Indicates whether the CPU thread has been successfully created.
+ * @stop: Indicates a pending stop request.
+ * @stopped: Indicates the CPU has been artificially stopped.
+ * @kvm_fd: vCPU file descriptor for KVM.
+ *
+ * State of one CPU core or thread.
+ */
+struct CPUState {
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
+
+    int nr_cores;
+    int nr_threads;
+    int numa_node;
+
+    struct QemuThread *thread;
+#ifdef _WIN32
+    HANDLE hThread;
+#endif
+    int thread_id;
+    struct QemuCond *halt_cond;
+    struct qemu_work_item *queued_work_first, *queued_work_last;
+    bool thread_kicked;
+    bool created;
+    bool stop;
+    bool stopped;
+
+    int kvm_fd;
+    bool kvm_vcpu_dirty;
+    struct KVMState *kvm_state;
+    struct kvm_run *kvm_run;
+
+    /* TODO Move common fields from CPUArchState here. */
+    int cpu_index; /* used by alpha TCG */
+};
+
+
+/**
+ * cpu_reset:
+ * @cpu: The CPU whose state is to be reset.
+ */
+void cpu_reset(CPUState *cpu);
+
+/**
+ * cpu_class_by_name:
+ * @typename: The CPU base type.
+ * @cpu_model: The model string without any parameters.
+ *
+ * Looks up a CPU #ObjectClass matching name @cpu_model.
+ *
+ * Returns: A #CPUClass or %NULL if not matching class is found.
+ */
+ObjectClass *cpu_class_by_name(const char *typename, const char *cpu_model);
+
+/**
+ * qemu_cpu_has_work:
+ * @cpu: The vCPU to check.
+ *
+ * Checks whether the CPU has work to do.
+ *
+ * Returns: %true if the CPU has work, %false otherwise.
+ */
+bool qemu_cpu_has_work(CPUState *cpu);
+
+/**
+ * qemu_cpu_is_self:
+ * @cpu: The vCPU to check against.
+ *
+ * Checks whether the caller is executing on the vCPU thread.
+ *
+ * Returns: %true if called from @cpu's thread, %false otherwise.
+ */
+bool qemu_cpu_is_self(CPUState *cpu);
+
+/**
+ * qemu_cpu_kick:
+ * @cpu: The vCPU to kick.
+ *
+ * Kicks @cpu's thread.
+ */
+void qemu_cpu_kick(CPUState *cpu);
+
+/**
+ * cpu_is_stopped:
+ * @cpu: The CPU to check.
+ *
+ * Checks whether the CPU is stopped.
+ *
+ * Returns: %true if run state is not running or if artificially stopped;
+ * %false otherwise.
+ */
+bool cpu_is_stopped(CPUState *cpu);
+
+/**
+ * run_on_cpu:
+ * @cpu: The vCPU to run on.
+ * @func: The function to be executed.
+ * @data: Data to pass to the function.
+ *
+ * Schedules the function @func for execution on the vCPU @cpu.
+ */
+void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data);
+
+/**
+ * qemu_get_cpu:
+ * @index: The CPUState@cpu_index value of the CPU to obtain.
+ *
+ * Gets a CPU matching @index.
+ *
+ * Returns: The CPU or %NULL if there is no matching CPU.
+ */
+CPUState *qemu_get_cpu(int index);
+
+
+#endif
diff --git a/include/qemu/object.h b/include/qom/object.h
index cc75feed661..48e80ba229c 100644
--- a/include/qemu/object.h
+++ b/include/qom/object.h
@@ -17,7 +17,7 @@
 #include <glib.h>
 #include <stdint.h>
 #include <stdbool.h>
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 struct Visitor;
 struct Error;
@@ -65,7 +65,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     int reg0, reg1, reg2;
  * } MyDevice;
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -138,7 +138,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     dc->reset = my_device_reset;
  * }
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -147,9 +147,9 @@ typedef struct InterfaceInfo InterfaceInfo;
  *   </programlisting>
  * </example>
  *
- * Introducing new virtual functions requires a class to define its own
- * struct and to add a .class_size member to the TypeInfo.  Each function
- * will also have a wrapper to call it easily:
+ * Introducing new virtual methods requires a class to define its own
+ * struct and to add a .class_size member to the #TypeInfo.  Each method
+ * will also have a wrapper function to call it easily:
  *
  * <example>
  *   <title>Defining an abstract class</title>
@@ -163,7 +163,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     void (*frobnicate) (MyDevice *obj);
  * } MyDeviceClass;
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -186,6 +186,104 @@ typedef struct InterfaceInfo InterfaceInfo;
  * similar to normal types except for the fact that are only defined by
  * their classes and never carry any state.  You can dynamically cast an object
  * to one of its #Interface types and vice versa.
+ *
+ * # Methods #
+ *
+ * A <emphasis>method</emphasis> is a function within the namespace scope of
+ * a class. It usually operates on the object instance by passing it as a
+ * strongly-typed first argument.
+ * If it does not operate on an object instance, it is dubbed
+ * <emphasis>class method</emphasis>.
+ *
+ * Methods cannot be overloaded. That is, the #ObjectClass and method name
+ * uniquely identity the function to be called; the signature does not vary
+ * except for trailing varargs.
+ *
+ * Methods are always <emphasis>virtual</emphasis>. Overriding a method in
+ * #TypeInfo.class_init of a subclass leads to any user of the class obtained
+ * via OBJECT_GET_CLASS() accessing the overridden function.
+ * The original function is not automatically invoked. It is the responsability
+ * of the overriding class to determine whether and when to invoke the method
+ * being overridden.
+ *
+ * To invoke the method being overridden, the preferred solution is to store
+ * the original value in the overriding class before overriding the method.
+ * This corresponds to |[ {super,base}.method(...) ]| in Java and C#
+ * respectively; this frees the overriding class from hardcoding its parent
+ * class, which someone might choose to change at some point.
+ *
+ * <example>
+ *   <title>Overriding a virtual method</title>
+ *   <programlisting>
+ * typedef struct MyState MyState;
+ *
+ * typedef void (*MyDoSomething)(MyState *obj);
+ *
+ * typedef struct MyClass {
+ *     ObjectClass parent_class;
+ *
+ *     MyDoSomething do_something;
+ * } MyClass;
+ *
+ * static void my_do_something(MyState *obj)
+ * {
+ *     // do something
+ * }
+ *
+ * static void my_class_init(ObjectClass *oc, void *data)
+ * {
+ *     MyClass *mc = MY_CLASS(oc);
+ *
+ *     mc->do_something = my_do_something;
+ * }
+ *
+ * static const TypeInfo my_type_info = {
+ *     .name = TYPE_MY,
+ *     .parent = TYPE_OBJECT,
+ *     .instance_size = sizeof(MyState),
+ *     .class_size = sizeof(MyClass),
+ *     .class_init = my_class_init,
+ * };
+ *
+ * typedef struct DerivedClass {
+ *     MyClass parent_class;
+ *
+ *     MyDoSomething parent_do_something;
+ * } MyClass;
+ *
+ * static void derived_do_something(MyState *obj)
+ * {
+ *     DerivedClass *dc = DERIVED_GET_CLASS(obj);
+ *
+ *     // do something here
+ *     dc->parent_do_something(obj);
+ *     // do something else here
+ * }
+ *
+ * static void derived_class_init(ObjectClass *oc, void *data)
+ * {
+ *     MyClass *mc = MY_CLASS(oc);
+ *     DerivedClass *dc = DERIVED_CLASS(oc);
+ *
+ *     dc->parent_do_something = mc->do_something;
+ *     mc->do_something = derived_do_something;
+ * }
+ *
+ * static const TypeInfo derived_type_info = {
+ *     .name = TYPE_DERIVED,
+ *     .parent = TYPE_MY,
+ *     .class_size = sizeof(DerivedClass),
+ *     .class_init = my_class_init,
+ * };
+ *   </programlisting>
+ * </example>
+ *
+ * Alternatively, object_class_by_name() can be used to obtain the class and
+ * its non-overridden methods for a specific type. This would correspond to
+ * |[ MyClass::method(...) ]| in C++.
+ *
+ * The first example of such a QOM method was #CPUClass.reset,
+ * another example is #DeviceClass.realize.
  */
 
 
@@ -230,6 +328,23 @@ typedef struct ObjectProperty
 } ObjectProperty;
 
 /**
+ * ObjectUnparent:
+ * @obj: the object that is being removed from the composition tree
+ *
+ * Called when an object is being removed from the QOM composition tree.
+ * The function should remove any backlinks from children objects to @obj.
+ */
+typedef void (ObjectUnparent)(Object *obj);
+
+/**
+ * ObjectFree:
+ * @obj: the object being freed
+ *
+ * Called when an object's last reference is removed.
+ */
+typedef void (ObjectFree)(void *obj);
+
+/**
  * ObjectClass:
  *
  * The base for all classes.  The only thing that #ObjectClass contains is an
@@ -240,6 +355,8 @@ struct ObjectClass
     /*< private >*/
     Type type;
     GSList *interfaces;
+
+    ObjectUnparent *unparent;
 };
 
 /**
@@ -261,6 +378,7 @@ struct Object
 {
     /*< private >*/
     ObjectClass *class;
+    ObjectFree *free;
     QTAILQ_HEAD(, ObjectProperty) properties;
     uint32_t ref;
     Object *parent;
@@ -485,15 +603,6 @@ void object_initialize_with_type(void *data, Type type);
 void object_initialize(void *obj, const char *typename);
 
 /**
- * object_finalize:
- * @obj: The object to finalize.
- *
- * This function destroys and object without freeing the memory associated with
- * it.
- */
-void object_finalize(void *obj);
-
-/**
  * object_dynamic_cast:
  * @obj: The object to cast.
  * @typename: The @typename to cast to.
@@ -582,6 +691,14 @@ ObjectClass *object_class_get_parent(ObjectClass *klass);
 const char *object_class_get_name(ObjectClass *klass);
 
 /**
+ * object_class_is_abstract:
+ * @klass: The class to obtain the abstractness for.
+ *
+ * Returns: %true if @klass is abstract, %false otherwise.
+ */
+bool object_class_is_abstract(ObjectClass *klass);
+
+/**
  * object_class_by_name:
  * @typename: The QOM typename to obtain the class for.
  *
@@ -889,7 +1006,7 @@ Object *object_resolve_path_type(const char *path, const char *typename,
  *
  * Returns: The resolved object or NULL on path lookup failure.
  */
-Object *object_resolve_path_component(Object *parent, gchar *part);
+Object *object_resolve_path_component(Object *parent, const gchar *part);
 
 /**
  * object_property_add_child:
@@ -947,6 +1064,22 @@ void object_property_add_str(Object *obj, const char *name,
                              struct Error **errp);
 
 /**
+ * object_property_add_bool:
+ * @obj: the object to add a property to
+ * @name: the name of the property
+ * @get: the getter or NULL if the property is write-only.
+ * @set: the setter or NULL if the property is read-only
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Add a bool property using getters/setters.  This function will add a
+ * property of type 'bool'.
+ */
+void object_property_add_bool(Object *obj, const char *name,
+                              bool (*get)(Object *, struct Error **),
+                              void (*set)(Object *, bool, struct Error **),
+                              struct Error **errp);
+
+/**
  * object_child_foreach:
  * @obj: the object whose children will be navigated
  * @fn: the iterator function to be called
diff --git a/include/qemu/qom-qobject.h b/include/qom/qom-qobject.h
index f9dff12f117..77cd717e3fa 100644
--- a/include/qemu/qom-qobject.h
+++ b/include/qom/qom-qobject.h
@@ -13,7 +13,7 @@
 #ifndef QEMU_QOM_QOBJECT_H
 #define QEMU_QOM_QOBJECT_H
 
-#include "qemu/object.h"
+#include "qom/object.h"
 
 /*
  * object_property_get_qobject:
diff --git a/arch_init.h b/include/sysemu/arch_init.h
index d9c572aee86..5fc780c63d0 100644
--- a/arch_init.h
+++ b/include/sysemu/arch_init.h
@@ -34,6 +34,6 @@ int tcg_available(void);
 int kvm_available(void);
 int xen_available(void);
 
-CpuDefinitionInfoList GCC_WEAK_DECL *arch_query_cpu_definitions(Error **errp);
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp);
 
 #endif
diff --git a/balloon.h b/include/sysemu/balloon.h
index b803a00741f..bd9d39505d4 100644
--- a/balloon.h
+++ b/include/sysemu/balloon.h
@@ -14,7 +14,7 @@
 #ifndef _QEMU_BALLOON_H
 #define _QEMU_BALLOON_H
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "qapi-types.h"
 
 typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
diff --git a/blockdev.h b/include/sysemu/blockdev.h
index 5f27b643be7..1fe533299ec 100644
--- a/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -10,17 +10,22 @@
 #ifndef BLOCKDEV_H
 #define BLOCKDEV_H
 
-#include "block.h"
-#include "error.h"
-#include "qemu-queue.h"
+#include "block/block.h"
+#include "qapi/error.h"
+#include "qemu/queue.h"
 
 void blockdev_mark_auto_del(BlockDriverState *bs);
 void blockdev_auto_del(BlockDriverState *bs);
 
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
+    /*
+     * IF_IDE must be zero, because we want QEMUMachine member
+     * block_default_type to default-initialize to IF_IDE
+     */
+    IF_IDE = 0,
     IF_NONE,
-    IF_IDE, IF_SCSI, IF_FLOPPY, IF_PFLASH, IF_MTD, IF_SD, IF_VIRTIO, IF_XEN,
+    IF_SCSI, IF_FLOPPY, IF_PFLASH, IF_MTD, IF_SD, IF_VIRTIO, IF_XEN,
     IF_COUNT
 } BlockInterfaceType;
 
@@ -51,7 +56,7 @@ DriveInfo *drive_get_by_blockdev(BlockDriverState *bs);
 QemuOpts *drive_def(const char *optstr);
 QemuOpts *drive_add(BlockInterfaceType type, int index, const char *file,
                     const char *optstr);
-DriveInfo *drive_init(QemuOpts *arg, int default_to_scsi);
+DriveInfo *drive_init(QemuOpts *arg, BlockInterfaceType block_default_type);
 
 /* device-hotplug */
 
diff --git a/cpus.h b/include/sysemu/cpus.h
index 81bd81773f8..f7f68542596 100644
--- a/cpus.h
+++ b/include/sysemu/cpus.h
@@ -13,9 +13,16 @@ void cpu_synchronize_all_post_init(void);
 
 void qtest_clock_warp(int64_t dest);
 
+#ifndef CONFIG_USER_ONLY
 /* vl.c */
 extern int smp_cores;
 extern int smp_threads;
+#else
+/* *-user doesn't have configurable SMP topology */
+#define smp_cores   1
+#define smp_threads 1
+#endif
+
 void set_numa_modes(void);
 void set_cpu_log(const char *optarg);
 void set_cpu_log_filename(const char *optarg);
diff --git a/device_tree.h b/include/sysemu/device_tree.h
index f7a3e6cfc5f..f0b3f35e03c 100644
--- a/device_tree.h
+++ b/include/sysemu/device_tree.h
@@ -49,4 +49,6 @@ int qemu_devtree_add_subnode(void *fdt, const char *name);
                              sizeof(qdt_tmp));                                \
     } while (0)
 
+void qemu_devtree_dumpdtb(void *fdt, int size);
+
 #endif /* __DEVICE_TREE_H__ */
diff --git a/dma.h b/include/sysemu/dma.h
index f35c4b6632b..a52c93a5536 100644
--- a/dma.h
+++ b/include/sysemu/dma.h
@@ -11,9 +11,10 @@
 #define DMA_H
 
 #include <stdio.h>
+#include "exec/memory.h"
 #include "hw/hw.h"
-#include "block.h"
-#include "kvm.h"
+#include "block/block.h"
+#include "sysemu/kvm.h"
 
 typedef struct DMAContext DMAContext;
 typedef struct ScatterGatherEntry ScatterGatherEntry;
@@ -31,7 +32,7 @@ struct QEMUSGList {
     DMAContext *dma;
 };
 
-#if defined(TARGET_PHYS_ADDR_BITS)
+#ifndef CONFIG_USER_ONLY
 
 /*
  * When an IOMMU is present, bus addresses become distinct from
@@ -47,8 +48,8 @@ typedef uint64_t dma_addr_t;
 
 typedef int DMATranslateFunc(DMAContext *dma,
                              dma_addr_t addr,
-                             target_phys_addr_t *paddr,
-                             target_phys_addr_t *len,
+                             hwaddr *paddr,
+                             hwaddr *len,
                              DMADirection dir);
 typedef void* DMAMapFunc(DMAContext *dma,
                          dma_addr_t addr,
@@ -61,11 +62,17 @@ typedef void DMAUnmapFunc(DMAContext *dma,
                           dma_addr_t access_len);
 
 struct DMAContext {
+    AddressSpace *as;
     DMATranslateFunc *translate;
     DMAMapFunc *map;
     DMAUnmapFunc *unmap;
 };
 
+/* A global DMA context corresponding to the address_space_memory
+ * AddressSpace, for sysbus devices which do DMA.
+ */
+extern DMAContext dma_context_memory;
+
 static inline void dma_barrier(DMAContext *dma, DMADirection dir)
 {
     /*
@@ -93,7 +100,7 @@ static inline void dma_barrier(DMAContext *dma, DMADirection dir)
 
 static inline bool dma_has_iommu(DMAContext *dma)
 {
-    return !!dma;
+    return dma && dma->translate;
 }
 
 /* Checks that the given range of addresses is valid for DMA.  This is
@@ -120,8 +127,7 @@ static inline int dma_memory_rw_relaxed(DMAContext *dma, dma_addr_t addr,
 {
     if (!dma_has_iommu(dma)) {
         /* Fast-path for no IOMMU */
-        cpu_physical_memory_rw(addr, buf, len,
-                               dir == DMA_DIRECTION_FROM_DEVICE);
+        address_space_rw(dma->as, addr, buf, len, dir == DMA_DIRECTION_FROM_DEVICE);
         return 0;
     } else {
         return iommu_dma_memory_rw(dma, addr, buf, len, dir);
@@ -176,11 +182,10 @@ static inline void *dma_memory_map(DMAContext *dma,
                                    DMADirection dir)
 {
     if (!dma_has_iommu(dma)) {
-        target_phys_addr_t xlen = *len;
+        hwaddr xlen = *len;
         void *p;
 
-        p = cpu_physical_memory_map(addr, &xlen,
-                                    dir == DMA_DIRECTION_FROM_DEVICE);
+        p = address_space_map(dma->as, addr, &xlen, dir == DMA_DIRECTION_FROM_DEVICE);
         *len = xlen;
         return p;
     } else {
@@ -196,9 +201,8 @@ static inline void dma_memory_unmap(DMAContext *dma,
                                     DMADirection dir, dma_addr_t access_len)
 {
     if (!dma_has_iommu(dma)) {
-        cpu_physical_memory_unmap(buffer, (target_phys_addr_t)len,
-                                  dir == DMA_DIRECTION_FROM_DEVICE,
-                                  access_len);
+        address_space_unmap(dma->as, buffer, (hwaddr)len,
+                            dir == DMA_DIRECTION_FROM_DEVICE, access_len);
     } else {
         iommu_dma_memory_unmap(dma, buffer, len, dir, access_len);
     }
@@ -242,7 +246,7 @@ DEFINE_LDST_DMA(q, q, 64, be);
 
 #undef DEFINE_LDST_DMA
 
-void dma_context_init(DMAContext *dma, DMATranslateFunc translate,
+void dma_context_init(DMAContext *dma, AddressSpace *as, DMATranslateFunc translate,
                       DMAMapFunc map, DMAUnmapFunc unmap);
 
 struct ScatterGatherEntry {
diff --git a/dump.h b/include/sysemu/dump.h
index e25b7cfb73c..e25b7cfb73c 100644
--- a/dump.h
+++ b/include/sysemu/dump.h
diff --git a/kvm.h b/include/sysemu/kvm.h
index 5b8f588813a..6e6dfb374a3 100644
--- a/kvm.h
+++ b/include/sysemu/kvm.h
@@ -16,10 +16,27 @@
 
 #include <errno.h>
 #include "config-host.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
+#include "qom/cpu.h"
 
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
+#include <linux/kvm_para.h>
+#else
+/* These constants must never be used at runtime if kvm_enabled() is false.
+ * They exist so we don't need #ifdefs around KVM-specific code that already
+ * checks kvm_enabled() properly.
+ */
+#define KVM_CPUID_SIGNATURE      0
+#define KVM_CPUID_FEATURES       0
+#define KVM_FEATURE_CLOCKSOURCE  0
+#define KVM_FEATURE_NOP_IO_DELAY 0
+#define KVM_FEATURE_MMU_OP       0
+#define KVM_FEATURE_CLOCKSOURCE2 0
+#define KVM_FEATURE_ASYNC_PF     0
+#define KVM_FEATURE_STEAL_TIME   0
+#define KVM_FEATURE_PV_EOI       0
+#define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 0
 #endif
 
 extern int kvm_allowed;
@@ -117,10 +134,11 @@ int kvm_has_xcrs(void);
 int kvm_has_pit_state2(void);
 int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
+int kvm_has_intx_set_mask(void);
 
-#ifdef NEED_CPU_H
-int kvm_init_vcpu(CPUArchState *env);
+int kvm_init_vcpu(CPUState *cpu);
 
+#ifdef NEED_CPU_H
 int kvm_cpu_exec(CPUArchState *env);
 
 #if !defined(CONFIG_USER_ONLY)
@@ -128,8 +146,6 @@ void *kvm_vmalloc(ram_addr_t size);
 void *kvm_arch_vmalloc(ram_addr_t size);
 void kvm_setup_guest_memory(void *start, size_t size);
 
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size);
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size);
 void kvm_flush_coalesced_mmio_buffer(void);
 #endif
 
@@ -143,7 +159,7 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap);
 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset);
 #endif
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr);
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_on_sigbus(int code, void *addr);
 
 /* internal API */
@@ -152,20 +168,20 @@ int kvm_ioctl(KVMState *s, int type, ...);
 
 int kvm_vm_ioctl(KVMState *s, int type, ...);
 
-int kvm_vcpu_ioctl(CPUArchState *env, int type, ...);
+int kvm_vcpu_ioctl(CPUState *cpu, int type, ...);
 
 /* Arch specific hooks */
 
 extern const KVMCapabilityInfo kvm_arch_required_capabilities[];
 
-void kvm_arch_pre_run(CPUArchState *env, struct kvm_run *run);
-void kvm_arch_post_run(CPUArchState *env, struct kvm_run *run);
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run);
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run);
 
-int kvm_arch_handle_exit(CPUArchState *env, struct kvm_run *run);
+int kvm_arch_handle_exit(CPUState *cpu, struct kvm_run *run);
 
-int kvm_arch_process_async_events(CPUArchState *env);
+int kvm_arch_process_async_events(CPUState *cpu);
 
-int kvm_arch_get_registers(CPUArchState *env);
+int kvm_arch_get_registers(CPUState *cpu);
 
 /* state subset only touched by the VCPU itself during runtime */
 #define KVM_PUT_RUNTIME_STATE   1
@@ -174,15 +190,18 @@ int kvm_arch_get_registers(CPUArchState *env);
 /* full state set, modified during initialization or on vmload */
 #define KVM_PUT_FULL_STATE      3
 
-int kvm_arch_put_registers(CPUArchState *env, int level);
+int kvm_arch_put_registers(CPUState *cpu, int level);
 
 int kvm_arch_init(KVMState *s);
 
-int kvm_arch_init_vcpu(CPUArchState *env);
+int kvm_arch_init_vcpu(CPUState *cpu);
+
+/* Returns VCPU ID to be used on KVM_CREATE_VCPU ioctl() */
+unsigned long kvm_arch_vcpu_id(CPUState *cpu);
 
-void kvm_arch_reset_vcpu(CPUArchState *env);
+void kvm_arch_reset_vcpu(CPUState *cpu);
 
-int kvm_arch_on_sigbus_vcpu(CPUArchState *env, int code, void *addr);
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_arch_on_sigbus(int code, void *addr);
 
 void kvm_arch_init_irq_routing(KVMState *s);
@@ -207,14 +226,14 @@ struct kvm_sw_breakpoint {
 
 QTAILQ_HEAD(kvm_sw_breakpoint_head, kvm_sw_breakpoint);
 
-struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
+struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
                                                  target_ulong pc);
 
-int kvm_sw_breakpoints_active(CPUArchState *env);
+int kvm_sw_breakpoints_active(CPUState *cpu);
 
-int kvm_arch_insert_sw_breakpoint(CPUArchState *current_env,
+int kvm_arch_insert_sw_breakpoint(CPUState *current_cpu,
                                   struct kvm_sw_breakpoint *bp);
-int kvm_arch_remove_sw_breakpoint(CPUArchState *current_env,
+int kvm_arch_remove_sw_breakpoint(CPUState *current_cpu,
                                   struct kvm_sw_breakpoint *bp);
 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type);
@@ -222,9 +241,9 @@ int kvm_arch_remove_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type);
 void kvm_arch_remove_all_hw_breakpoints(void);
 
-void kvm_arch_update_guest_debug(CPUArchState *env, struct kvm_guest_debug *dbg);
+void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg);
 
-bool kvm_arch_stop_on_emulation_error(CPUArchState *env);
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu);
 
 int kvm_check_extension(KVMState *s, unsigned int extension);
 
@@ -260,7 +279,7 @@ static inline void cpu_synchronize_post_init(CPUArchState *env)
 
 #if !defined(CONFIG_USER_ONLY)
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
-                                       target_phys_addr_t *phys_addr);
+                                       hwaddr *phys_addr);
 #endif
 
 #endif
@@ -270,10 +289,11 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign,
 int kvm_set_ioeventfd_pio_word(int fd, uint16_t adr, uint16_t val, bool assign);
 
 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq);
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq);
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq);
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq);
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
+void kvm_pc_gsi_handler(void *opaque, int n, int level);
+void kvm_pc_setup_irq_routing(bool pci_enabled);
 #endif
diff --git a/memory_mapping.h b/include/sysemu/memory_mapping.h
index ef72b0abadf..12561259635 100644
--- a/memory_mapping.h
+++ b/include/sysemu/memory_mapping.h
@@ -14,11 +14,11 @@
 #ifndef MEMORY_MAPPING_H
 #define MEMORY_MAPPING_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /* The physical and virtual address in the memory mapping are contiguous. */
 typedef struct MemoryMapping {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong virt_addr;
     ram_addr_t length;
     QTAILQ_ENTRY(MemoryMapping) next;
@@ -39,8 +39,8 @@ bool cpu_paging_enabled(CPUArchState *env);
  * and is contiguous. The list is sorted by phys_addr.
  */
 void memory_mapping_list_add_merge_sorted(MemoryMappingList *list,
-                                          target_phys_addr_t phys_addr,
-                                          target_phys_addr_t virt_addr,
+                                          hwaddr phys_addr,
+                                          hwaddr virt_addr,
                                           ram_addr_t length);
 
 void memory_mapping_list_free(MemoryMappingList *list);
diff --git a/qemu-os-posix.h b/include/sysemu/os-posix.h
index 8e1149d9640..7f198e475cd 100644
--- a/qemu-os-posix.h
+++ b/include/sysemu/os-posix.h
@@ -46,4 +46,6 @@ typedef struct timeval qemu_timeval;
 typedef struct timespec qemu_timespec;
 int qemu_utimens(const char *path, const qemu_timespec *times);
 
+bool is_daemonized(void);
+
 #endif
diff --git a/qemu-os-win32.h b/include/sysemu/os-win32.h
index 753679b1946..d0e9234d24c 100644
--- a/qemu-os-win32.h
+++ b/include/sysemu/os-win32.h
@@ -28,7 +28,6 @@
 
 #include <windows.h>
 #include <winsock2.h>
-#include "main-loop.h"
 
 /* Workaround for older versions of MinGW. */
 #ifndef ECONNREFUSED
@@ -68,6 +67,12 @@
 /* Declaration of ffs() is missing in MinGW's strings.h. */
 int ffs(int i);
 
+/* Missing POSIX functions. Don't use MinGW-w64 macros. */
+#undef gmtime_r
+struct tm *gmtime_r(const time_t *timep, struct tm *result);
+#undef localtime_r
+struct tm *localtime_r(const time_t *timep, struct tm *result);
+
 static inline void os_setup_signal_handling(void) {}
 static inline void os_daemonize(void) {}
 static inline void os_setup_post(void) {}
@@ -86,4 +91,9 @@ typedef struct {
 } qemu_timeval;
 int qemu_gettimeofday(qemu_timeval *tp);
 
+static inline bool is_daemonized(void)
+{
+    return false;
+}
+
 #endif
diff --git a/qtest.h b/include/sysemu/qtest.h
index 723a4f9536d..723a4f9536d 100644
--- a/qtest.h
+++ b/include/sysemu/qtest.h
diff --git a/qemu-seccomp.h b/include/sysemu/seccomp.h
index b2fc3f8c3c4..1189fa241da 100644
--- a/qemu-seccomp.h
+++ b/include/sysemu/seccomp.h
@@ -16,7 +16,7 @@
 #define QEMU_SECCOMP_H
 
 #include <seccomp.h>
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 int seccomp_start(void);
 #endif
diff --git a/sysemu.h b/include/sysemu/sysemu.h
index 65552acee56..1d9599e5f49 100644
--- a/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -2,13 +2,13 @@
 #define SYSEMU_H
 /* Misc. things related to the system emulator.  */
 
-#include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
+#include "qemu/typedefs.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
 #include "qapi-types.h"
-#include "notify.h"
-#include "main-loop.h"
+#include "qemu/notify.h"
+#include "qemu/main-loop.h"
 
 /* vl.c */
 
@@ -19,7 +19,6 @@ extern uint8_t qemu_uuid[];
 int qemu_uuid_parse(const char *str, uint8_t *uuid);
 #define UUID_FMT "%02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx"
 
-void runstate_init(void);
 bool runstate_check(RunState state);
 void runstate_set(RunState new_state);
 int runstate_is_running(void);
@@ -52,16 +51,12 @@ void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
+void qemu_register_powerdown_notifier(Notifier *notifier);
 void qemu_system_debug_request(void);
 void qemu_system_vmstop_request(RunState reason);
 int qemu_shutdown_requested_get(void);
 int qemu_reset_requested_get(void);
-int qemu_shutdown_requested(void);
-int qemu_reset_requested(void);
-int qemu_powerdown_requested(void);
 void qemu_system_killed(int signal, pid_t pid);
-void qemu_kill_report(void);
-extern qemu_irq qemu_system_powerdown;
 void qemu_devices_reset(void);
 void qemu_system_reset(bool report);
 
@@ -73,7 +68,7 @@ void qemu_add_machine_init_done_notifier(Notifier *notify);
 void do_savevm(Monitor *mon, const QDict *qdict);
 int load_vmstate(const char *name);
 void do_delvm(Monitor *mon, const QDict *qdict);
-void do_info_snapshots(Monitor *mon);
+void do_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
@@ -82,7 +77,8 @@ int qemu_savevm_state_begin(QEMUFile *f,
                             const MigrationParams *params);
 int qemu_savevm_state_iterate(QEMUFile *f);
 int qemu_savevm_state_complete(QEMUFile *f);
-void qemu_savevm_state_cancel(QEMUFile *f);
+void qemu_savevm_state_cancel(void);
+uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
 int qemu_loadvm_state(QEMUFile *f);
 
 /* SLIRP */
@@ -105,10 +101,7 @@ typedef enum {
 } VGAInterfaceType;
 
 extern int vga_interface_type;
-#define cirrus_vga_enabled (vga_interface_type == VGA_CIRRUS)
-#define std_vga_enabled (vga_interface_type == VGA_STD)
 #define xenfb_enabled (vga_interface_type == VGA_XENFB)
-#define vmsvga_enabled (vga_interface_type == VGA_VMWARE)
 #define qxl_enabled (vga_interface_type == VGA_QXL)
 
 extern int graphic_width;
@@ -119,7 +112,6 @@ extern const char *keyboard_layout;
 extern int win2k_install_hack;
 extern int alt_grab;
 extern int ctrl_grab;
-extern int usb_enabled;
 extern int smp_cpus;
 extern int max_cpus;
 extern int cursor_hide;
@@ -130,7 +122,7 @@ extern int semihosting_enabled;
 extern int old_param;
 extern int boot_menu;
 extern uint8_t *boot_splash_filedata;
-extern int boot_splash_filedata_size;
+extern size_t boot_splash_filedata_size;
 extern uint8_t qemu_extra_params_fw[2];
 extern QEMUClock *rtc_clock;
 
@@ -154,8 +146,7 @@ extern unsigned int nb_prom_envs;
 
 /* pci-hotplug */
 void pci_device_hot_add(Monitor *mon, const QDict *qdict);
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type);
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo);
 void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
 
 /* generic hotplug */
@@ -180,7 +171,7 @@ extern CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 
 void do_usb_add(Monitor *mon, const QDict *qdict);
 void do_usb_del(Monitor *mon, const QDict *qdict);
-void usb_info(Monitor *mon);
+void usb_info(Monitor *mon, const QDict *qdict);
 
 void rtc_change_mon_event(struct tm *tm);
 
@@ -188,5 +179,16 @@ void register_devices(void);
 
 void add_boot_device_path(int32_t bootindex, DeviceState *dev,
                           const char *suffix);
-char *get_boot_devices_list(uint32_t *size);
+char *get_boot_devices_list(size_t *size);
+
+bool usb_enabled(bool default_usb);
+
+extern QemuOptsList qemu_drive_opts;
+extern QemuOptsList qemu_chardev_opts;
+extern QemuOptsList qemu_device_opts;
+extern QemuOptsList qemu_netdev_opts;
+extern QemuOptsList qemu_net_opts;
+extern QemuOptsList qemu_global_opts;
+extern QemuOptsList qemu_mon_opts;
+
 #endif
diff --git a/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index 70301a57e4e..c59804060b9 100644
--- a/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -11,14 +11,14 @@
 
 #include <stdlib.h>
 
-typedef target_phys_addr_t (*phys_offset_to_gaddr_t)(target_phys_addr_t start_addr,
+typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr start_addr,
                                                      ram_addr_t size,
                                                      void *opaque);
 #ifdef CONFIG_XEN
 
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
                         void *opaque);
-uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
+uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
                        uint8_t lock);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
 void xen_invalidate_map_cache_entry(uint8_t *buffer);
@@ -31,8 +31,8 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t f,
 {
 }
 
-static inline uint8_t *xen_map_cache(target_phys_addr_t phys_addr,
-                                     target_phys_addr_t size,
+static inline uint8_t *xen_map_cache(hwaddr phys_addr,
+                                     hwaddr size,
                                      uint8_t lock)
 {
     abort();
diff --git a/include/trace.h b/include/trace.h
new file mode 100644
index 00000000000..c15f4981280
--- /dev/null
+++ b/include/trace.h
@@ -0,0 +1,6 @@
+#ifndef TRACE_H
+#define TRACE_H
+
+#include "trace/generated-tracers.h"
+
+#endif  /* TRACE_H */
diff --git a/console.h b/include/ui/console.h
index 4334db5ca5f..fc23baa06b3 100644
--- a/console.h
+++ b/include/ui/console.h
@@ -1,11 +1,13 @@
 #ifndef CONSOLE_H
 #define CONSOLE_H
 
-#include "qemu-char.h"
-#include "qdict.h"
-#include "notify.h"
-#include "monitor.h"
+#include "ui/qemu-pixman.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/notify.h"
+#include "monitor/monitor.h"
 #include "trace.h"
+#include "qapi-types.h"
+#include "qapi/error.h"
 
 /* keyboard/mouse support */
 
@@ -105,7 +107,6 @@ void kbd_put_keysym(int keysym);
 
 #define QEMU_BIG_ENDIAN_FLAG    0x01
 #define QEMU_ALLOCATED_FLAG     0x02
-#define QEMU_REALPIXELS_FLAG    0x04
 
 struct PixelFormat {
     uint8_t bits_per_pixel;
@@ -118,11 +119,9 @@ struct PixelFormat {
 };
 
 struct DisplaySurface {
+    pixman_format_code_t format;
+    pixman_image_t *image;
     uint8_t flags;
-    int width;
-    int height;
-    int linesize;        /* bytes per line */
-    uint8_t *data;
 
     struct PixelFormat pf;
 };
@@ -152,35 +151,32 @@ struct DisplayChangeListener {
     int idle;
     uint64_t gui_timer_interval;
 
-    void (*dpy_update)(struct DisplayState *s, int x, int y, int w, int h);
-    void (*dpy_resize)(struct DisplayState *s);
-    void (*dpy_setdata)(struct DisplayState *s);
     void (*dpy_refresh)(struct DisplayState *s);
-    void (*dpy_copy)(struct DisplayState *s, int src_x, int src_y,
-                     int dst_x, int dst_y, int w, int h);
-    void (*dpy_fill)(struct DisplayState *s, int x, int y,
-                     int w, int h, uint32_t c);
+
+    void (*dpy_gfx_update)(struct DisplayState *s, int x, int y, int w, int h);
+    void (*dpy_gfx_resize)(struct DisplayState *s);
+    void (*dpy_gfx_setdata)(struct DisplayState *s);
+    void (*dpy_gfx_copy)(struct DisplayState *s, int src_x, int src_y,
+                         int dst_x, int dst_y, int w, int h);
+
     void (*dpy_text_cursor)(struct DisplayState *s, int x, int y);
+    void (*dpy_text_resize)(struct DisplayState *s, int w, int h);
+    void (*dpy_text_update)(struct DisplayState *s, int x, int y, int w, int h);
 
-    struct DisplayChangeListener *next;
-};
+    void (*dpy_mouse_set)(struct DisplayState *s, int x, int y, int on);
+    void (*dpy_cursor_define)(struct DisplayState *s, QEMUCursor *cursor);
 
-struct DisplayAllocator {
-    DisplaySurface* (*create_displaysurface)(int width, int height);
-    DisplaySurface* (*resize_displaysurface)(DisplaySurface *surface, int width, int height);
-    void (*free_displaysurface)(DisplaySurface *surface);
+    QLIST_ENTRY(DisplayChangeListener) next;
 };
 
 struct DisplayState {
     struct DisplaySurface *surface;
     void *opaque;
     struct QEMUTimer *gui_timer;
+    bool have_gfx;
+    bool have_text;
 
-    struct DisplayAllocator* allocator;
-    struct DisplayChangeListener* listeners;
-
-    void (*mouse_set)(int x, int y, int on);
-    void (*cursor_define)(QEMUCursor *cursor);
+    QLIST_HEAD(, DisplayChangeListener) listeners;
 
     struct DisplayState *next;
 };
@@ -189,143 +185,230 @@ void register_displaystate(DisplayState *ds);
 DisplayState *get_displaystate(void);
 DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
                                                 int linesize, uint8_t *data);
-void qemu_alloc_display(DisplaySurface *surface, int width, int height,
-                        int linesize, PixelFormat pf, int newflags);
 PixelFormat qemu_different_endianness_pixelformat(int bpp);
 PixelFormat qemu_default_pixelformat(int bpp);
 
-DisplayAllocator *register_displayallocator(DisplayState *ds, DisplayAllocator *da);
+DisplaySurface *qemu_create_displaysurface(DisplayState *ds,
+                                           int width, int height);
+DisplaySurface *qemu_resize_displaysurface(DisplayState *ds,
+                                           int width, int height);
+void qemu_free_displaysurface(DisplayState *ds);
 
-static inline DisplaySurface* qemu_create_displaysurface(DisplayState *ds, int width, int height)
+static inline int is_surface_bgr(DisplaySurface *surface)
 {
-    return ds->allocator->create_displaysurface(width, height);    
+    if (surface->pf.bits_per_pixel == 32 && surface->pf.rshift == 0)
+        return 1;
+    else
+        return 0;
 }
 
-static inline DisplaySurface* qemu_resize_displaysurface(DisplayState *ds, int width, int height)
+static inline int is_buffer_shared(DisplaySurface *surface)
 {
-    trace_displaysurface_resize(ds, ds->surface, width, height);
-    return ds->allocator->resize_displaysurface(ds->surface, width, height);
+    return !(surface->flags & QEMU_ALLOCATED_FLAG);
 }
 
-static inline void qemu_free_displaysurface(DisplayState *ds)
+void gui_setup_refresh(DisplayState *ds);
+
+static inline void register_displaychangelistener(DisplayState *ds, DisplayChangeListener *dcl)
 {
-    trace_displaysurface_free(ds, ds->surface);
-    ds->allocator->free_displaysurface(ds->surface);
+    QLIST_INSERT_HEAD(&ds->listeners, dcl, next);
+    gui_setup_refresh(ds);
+    if (dcl->dpy_gfx_resize) {
+        dcl->dpy_gfx_resize(ds);
+    }
 }
 
-static inline int is_surface_bgr(DisplaySurface *surface)
+static inline void unregister_displaychangelistener(DisplayState *ds,
+                                                    DisplayChangeListener *dcl)
 {
-    if (surface->pf.bits_per_pixel == 32 && surface->pf.rshift == 0)
-        return 1;
-    else
-        return 0;
+    QLIST_REMOVE(dcl, next);
+    gui_setup_refresh(ds);
 }
 
-static inline int is_buffer_shared(DisplaySurface *surface)
+static inline void dpy_gfx_update(DisplayState *s, int x, int y, int w, int h)
 {
-    return (!(surface->flags & QEMU_ALLOCATED_FLAG) &&
-            !(surface->flags & QEMU_REALPIXELS_FLAG));
+    struct DisplayChangeListener *dcl;
+    int width = pixman_image_get_width(s->surface->image);
+    int height = pixman_image_get_height(s->surface->image);
+
+    x = MAX(x, 0);
+    y = MAX(y, 0);
+    x = MIN(x, width);
+    y = MIN(y, height);
+    w = MIN(w, width - x);
+    h = MIN(h, height - y);
+
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_update) {
+            dcl->dpy_gfx_update(s, x, y, w, h);
+        }
+    }
 }
 
-static inline void register_displaychangelistener(DisplayState *ds, DisplayChangeListener *dcl)
+static inline void dpy_gfx_resize(DisplayState *s)
 {
-    dcl->next = ds->listeners;
-    ds->listeners = dcl;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_resize) {
+            dcl->dpy_gfx_resize(s);
+        }
+    }
 }
 
-static inline void dpy_update(DisplayState *s, int x, int y, int w, int h)
+static inline void dpy_gfx_setdata(DisplayState *s)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        dcl->dpy_update(s, x, y, w, h);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_setdata) {
+            dcl->dpy_gfx_setdata(s);
+        }
     }
 }
 
-static inline void dpy_resize(DisplayState *s)
+static inline void dpy_refresh(DisplayState *s)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        dcl->dpy_resize(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_refresh) {
+            dcl->dpy_refresh(s);
+        }
     }
 }
 
-static inline void dpy_setdata(DisplayState *s)
+static inline void dpy_gfx_copy(struct DisplayState *s, int src_x, int src_y,
+                             int dst_x, int dst_y, int w, int h)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_setdata) dcl->dpy_setdata(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_copy) {
+            dcl->dpy_gfx_copy(s, src_x, src_y, dst_x, dst_y, w, h);
+        } else { /* TODO */
+            dcl->dpy_gfx_update(s, dst_x, dst_y, w, h);
+        }
     }
 }
 
-static inline void dpy_refresh(DisplayState *s)
+static inline void dpy_text_cursor(struct DisplayState *s, int x, int y)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_cursor) {
+            dcl->dpy_text_cursor(s, x, y);
+        }
+    }
+}
+
+static inline void dpy_text_update(DisplayState *s, int x, int y, int w, int h)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_refresh) dcl->dpy_refresh(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_update) {
+            dcl->dpy_text_update(s, x, y, w, h);
+        }
     }
 }
 
-static inline void dpy_copy(struct DisplayState *s, int src_x, int src_y,
-                             int dst_x, int dst_y, int w, int h) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_copy)
-            dcl->dpy_copy(s, src_x, src_y, dst_x, dst_y, w, h);
-        else /* TODO */
-            dcl->dpy_update(s, dst_x, dst_y, w, h);
-        dcl = dcl->next;
+static inline void dpy_text_resize(DisplayState *s, int w, int h)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_resize) {
+            dcl->dpy_text_resize(s, w, h);
+        }
     }
 }
 
-static inline void dpy_fill(struct DisplayState *s, int x, int y,
-                             int w, int h, uint32_t c) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_fill) dcl->dpy_fill(s, x, y, w, h, c);
-        dcl = dcl->next;
+static inline void dpy_mouse_set(struct DisplayState *s, int x, int y, int on)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_mouse_set) {
+            dcl->dpy_mouse_set(s, x, y, on);
+        }
+    }
+}
+
+static inline void dpy_cursor_define(struct DisplayState *s, QEMUCursor *cursor)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_cursor_define) {
+            dcl->dpy_cursor_define(s, cursor);
+        }
     }
 }
 
-static inline void dpy_cursor(struct DisplayState *s, int x, int y) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_text_cursor) dcl->dpy_text_cursor(s, x, y);
-        dcl = dcl->next;
+static inline bool dpy_cursor_define_supported(struct DisplayState *s)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_cursor_define) {
+            return true;
+        }
     }
+    return false;
 }
 
 static inline int ds_get_linesize(DisplayState *ds)
 {
-    return ds->surface->linesize;
+    return pixman_image_get_stride(ds->surface->image);
 }
 
 static inline uint8_t* ds_get_data(DisplayState *ds)
 {
-    return ds->surface->data;
+    return (void *)pixman_image_get_data(ds->surface->image);
 }
 
 static inline int ds_get_width(DisplayState *ds)
 {
-    return ds->surface->width;
+    return pixman_image_get_width(ds->surface->image);
 }
 
 static inline int ds_get_height(DisplayState *ds)
 {
-    return ds->surface->height;
+    return pixman_image_get_height(ds->surface->image);
 }
 
 static inline int ds_get_bits_per_pixel(DisplayState *ds)
 {
-    return ds->surface->pf.bits_per_pixel;
+    int bits = PIXMAN_FORMAT_BPP(ds->surface->format);
+    return bits;
 }
 
 static inline int ds_get_bytes_per_pixel(DisplayState *ds)
 {
-    return ds->surface->pf.bytes_per_pixel;
+    int bits = PIXMAN_FORMAT_BPP(ds->surface->format);
+    return (bits + 7) / 8;
+}
+
+static inline pixman_format_code_t ds_get_format(DisplayState *ds)
+{
+    return ds->surface->format;
+}
+
+static inline pixman_image_t *ds_get_image(DisplayState *ds)
+{
+    return ds->surface->image;
+}
+
+static inline int ds_get_depth(DisplayState *ds)
+{
+    return ds->surface->pf.depth;
+}
+
+static inline int ds_get_rmask(DisplayState *ds)
+{
+    return ds->surface->pf.rmask;
+}
+
+static inline int ds_get_gmask(DisplayState *ds)
+{
+    return ds->surface->pf.gmask;
+}
+
+static inline int ds_get_bmask(DisplayState *ds)
+{
+    return ds->surface->pf.bmask;
 }
 
 #ifdef CONFIG_CURSES
@@ -343,7 +426,8 @@ static inline void console_write_ch(console_ch_t *dest, uint32_t ch)
 
 typedef void (*vga_hw_update_ptr)(void *);
 typedef void (*vga_hw_invalidate_ptr)(void *);
-typedef void (*vga_hw_screen_dump_ptr)(void *, const char *, bool cswitch);
+typedef void (*vga_hw_screen_dump_ptr)(void *, const char *, bool cswitch,
+                                       Error **errp);
 typedef void (*vga_hw_text_update_ptr)(void *, console_ch_t *);
 
 DisplayState *graphic_console_init(vga_hw_update_ptr update,
@@ -354,7 +438,6 @@ DisplayState *graphic_console_init(vga_hw_update_ptr update,
 
 void vga_hw_update(void);
 void vga_hw_invalidate(void);
-void vga_hw_screen_dump(const char *filename);
 void vga_hw_text_update(console_ch_t *chardata);
 
 int is_graphic_console(void);
@@ -375,10 +458,8 @@ void cocoa_display_init(DisplayState *ds, int full_screen);
 
 /* vnc.c */
 void vnc_display_init(DisplayState *ds);
-void vnc_display_close(DisplayState *ds);
-int vnc_display_open(DisplayState *ds, const char *display);
+void vnc_display_open(DisplayState *ds, const char *display, Error **errp);
 void vnc_display_add_client(DisplayState *ds, int csock, int skipauth);
-int vnc_display_disable_login(DisplayState *ds);
 char *vnc_display_local_addr(DisplayState *ds);
 #ifdef CONFIG_VNC
 int vnc_display_password(DisplayState *ds, const char *password);
@@ -397,4 +478,8 @@ static inline int vnc_display_pw_expire(DisplayState *ds, time_t expires)
 /* curses.c */
 void curses_display_init(DisplayState *ds, int full_screen);
 
+/* input.c */
+int index_from_key(const char *key);
+int index_from_keycode(int code);
+
 #endif
diff --git a/hw/pixel_ops.h b/include/ui/pixel_ops.h
index d390adfd1b3..d390adfd1b3 100644
--- a/hw/pixel_ops.h
+++ b/include/ui/pixel_ops.h
diff --git a/include/ui/qemu-pixman.h b/include/ui/qemu-pixman.h
new file mode 100644
index 00000000000..016fd877268
--- /dev/null
+++ b/include/ui/qemu-pixman.h
@@ -0,0 +1,46 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_PIXMAN_H
+#define QEMU_PIXMAN_H
+
+/* pixman-0.16.0 headers have a redundant declaration */
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#endif
+#include <pixman.h>
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic error "-Wredundant-decls"
+#endif
+
+#include "console.h"
+
+/*
+ * pixman image formats are defined to be native endian,
+ * that means host byte order on qemu.  So we go define
+ * fixed formats here for cases where it is needed, like
+ * feeding libjpeg / libpng and writing screenshots.
+ */
+
+#ifdef HOST_WORDS_BIGENDIAN
+# define PIXMAN_BE_r8g8b8     PIXMAN_r8g8b8
+#else
+# define PIXMAN_BE_r8g8b8     PIXMAN_b8g8r8
+#endif
+
+/* -------------------------------------------------------------------- */
+
+int qemu_pixman_get_type(int rshift, int gshift, int bshift);
+pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf);
+
+pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
+                                           int width);
+void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
+                              int width, int x, int y);
+pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
+                                          pixman_image_t *image);
+void qemu_pixman_image_unref(pixman_image_t *image);
+
+#endif /* QEMU_PIXMAN_H */
diff --git a/ui/qemu-spice.h b/include/ui/qemu-spice.h
index 3299da87d60..5a78fd764d7 100644
--- a/ui/qemu-spice.h
+++ b/include/ui/qemu-spice.h
@@ -22,10 +22,9 @@
 
 #include <spice.h>
 
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-char.h"
-#include "monitor.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "monitor/monitor.h"
 
 extern int using_spice;
 
@@ -46,9 +45,13 @@ void do_info_spice_print(Monitor *mon, const QObject *data);
 void do_info_spice(Monitor *mon, QObject **ret_data);
 
 CharDriverState *qemu_chr_open_spice(QemuOpts *opts);
+#if SPICE_SERVER_VERSION >= 0x000c02
+CharDriverState *qemu_chr_open_spice_port(QemuOpts *opts);
+void qemu_spice_register_ports(void);
+#endif
 
 #else  /* CONFIG_SPICE */
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #define using_spice 0
 static inline int qemu_spice_set_passwd(const char *passwd,
diff --git a/ui/spice-display.h b/include/ui/spice-display.h
index 12e50b6efc3..8b192e96138 100644
--- a/ui/spice-display.h
+++ b/include/ui/spice-display.h
@@ -19,10 +19,9 @@
 #include <spice/enums.h>
 #include <spice/qxl_dev.h>
 
-#include "qemu-thread.h"
-#include "console.h"
-#include "pflib.h"
-#include "sysemu.h"
+#include "qemu/thread.h"
+#include "ui/qemu-pixman.h"
+#include "sysemu/sysemu.h"
 
 #define NUM_MEMSLOTS 8
 #define MEMSLOT_GENERATION_BITS 8
@@ -32,8 +31,6 @@
 #define MEMSLOT_GROUP_GUEST 1
 #define NUM_MEMSLOTS_GROUPS 2
 
-#define NUM_SURFACES 1024
-
 /*
  * Internal enum to differenciate between options for
  * io calls that have a sync (old) version and an _async (new)
@@ -51,6 +48,7 @@ typedef enum qxl_async_io {
 enum {
     QXL_COOKIE_TYPE_IO,
     QXL_COOKIE_TYPE_RENDER_UPDATE_AREA,
+    QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
 };
 
 typedef struct QXLCookie {
@@ -78,11 +76,12 @@ struct SimpleSpiceDisplay {
     QXLWorker *worker;
     QXLInstance qxl;
     uint32_t unique;
-    QemuPfConv *conv;
+    pixman_image_t *surface;
+    pixman_image_t *mirror;
+    int32_t num_surfaces;
 
     QXLRect dirty;
     int notify;
-    int running;
 
     /*
      * All struct members below this comment can be accessed from
@@ -90,7 +89,7 @@ struct SimpleSpiceDisplay {
      * to them must be protected by the lock.
      */
     QemuMutex lock;
-    SimpleSpiceUpdate *update;
+    QTAILQ_HEAD(, SimpleSpiceUpdate) updates;
     QEMUCursor *cursor;
     int mouse_x, mouse_y;
 };
@@ -100,6 +99,7 @@ struct SimpleSpiceUpdate {
     QXLImage image;
     QXLCommandExt ext;
     uint8_t *bitmap;
+    QTAILQ_ENTRY(SimpleSpiceUpdate) next;
 };
 
 int qemu_spice_rect_is_empty(const QXLRect* r);
@@ -129,5 +129,6 @@ void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
 void qemu_spice_destroy_primary_surface(SimpleSpiceDisplay *ssd,
                                         uint32_t id, qxl_async_io async);
 void qemu_spice_wakeup(SimpleSpiceDisplay *ssd);
-void qemu_spice_start(SimpleSpiceDisplay *ssd);
-void qemu_spice_stop(SimpleSpiceDisplay *ssd);
+void qemu_spice_display_start(void);
+void qemu_spice_display_stop(void);
+int qemu_spice_display_is_running(SimpleSpiceDisplay *ssd);
diff --git a/iohandler.c b/iohandler.c
index dea43552d2a..2523adc11d6 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -24,9 +24,9 @@
 
 #include "config-host.h"
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-queue.h"
-#include "main-loop.h"
+#include "qemu/queue.h"
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 #ifndef _WIN32
 #include <sys/wait.h>
@@ -56,6 +56,8 @@ int qemu_set_fd_handler2(int fd,
 {
     IOHandlerRecord *ioh;
 
+    assert(fd >= 0);
+
     if (!fd_read && !fd_write) {
         QLIST_FOREACH(ioh, &io_handlers, next) {
             if (ioh->fd == fd) {
diff --git a/ioport.c b/ioport.c
index 6e4ca0dd911..a0ac2a0a414 100644
--- a/ioport.c
+++ b/ioport.c
@@ -25,9 +25,9 @@
  * splitted out ioport related stuffs from vl.c.
  */
 
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "trace.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 /***********************************************************/
 /* IO Port */
diff --git a/kvm-all.c b/kvm-all.c
index 34b02c1fba4..04ec2d541a7 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -21,24 +21,28 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
-#include "qemu-barrier.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "sysemu.h"
+#include "qemu/atomic.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
 #include "hw/hw.h"
-#include "hw/msi.h"
-#include "gdbstub.h"
-#include "kvm.h"
-#include "bswap.h"
-#include "memory.h"
-#include "exec-memory.h"
-#include "event_notifier.h"
+#include "hw/pci/msi.h"
+#include "exec/gdbstub.h"
+#include "sysemu/kvm.h"
+#include "qemu/bswap.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "qemu/event_notifier.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
 #include <sys/eventfd.h>
 #endif
 
+#ifdef CONFIG_VALGRIND_H
+#include <valgrind/memcheck.h>
+#endif
+
 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
 #define PAGE_SIZE TARGET_PAGE_SIZE
 
@@ -56,7 +60,7 @@
 
 typedef struct KVMSlot
 {
-    target_phys_addr_t start_addr;
+    hwaddr start_addr;
     ram_addr_t memory_size;
     void *ram;
     int slot;
@@ -84,10 +88,11 @@ struct KVMState
     int pit_state2;
     int xsave, xcrs;
     int many_ioeventfds;
+    int intx_set_mask;
     /* The man page (and posix) say ioctl numbers are signed int, but
      * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
      * unsigned, and treating them as signed here can break things */
-    unsigned irqchip_inject_ioctl;
+    unsigned irq_set_ioctl;
 #ifdef KVM_CAP_IRQ_ROUTING
     struct kvm_irq_routing *irq_routes;
     int nr_allocated_irq_routes;
@@ -126,8 +131,8 @@ static KVMSlot *kvm_alloc_slot(KVMState *s)
 }
 
 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
-                                         target_phys_addr_t start_addr,
-                                         target_phys_addr_t end_addr)
+                                         hwaddr start_addr,
+                                         hwaddr end_addr)
 {
     int i;
 
@@ -147,8 +152,8 @@ static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  * Find overlapping slot with lowest start address
  */
 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
-                                            target_phys_addr_t start_addr,
-                                            target_phys_addr_t end_addr)
+                                            hwaddr start_addr,
+                                            hwaddr end_addr)
 {
     KVMSlot *found = NULL;
     int i;
@@ -171,7 +176,7 @@ static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 }
 
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
-                                       target_phys_addr_t *phys_addr)
+                                       hwaddr *phys_addr)
 {
     int i;
 
@@ -204,12 +209,12 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 
 static void kvm_reset_vcpu(void *opaque)
 {
-    CPUArchState *env = opaque;
+    CPUState *cpu = opaque;
 
-    kvm_arch_reset_vcpu(env);
+    kvm_arch_reset_vcpu(cpu);
 }
 
-int kvm_init_vcpu(CPUArchState *env)
+int kvm_init_vcpu(CPUState *cpu)
 {
     KVMState *s = kvm_state;
     long mmap_size;
@@ -217,15 +222,15 @@ int kvm_init_vcpu(CPUArchState *env)
 
     DPRINTF("kvm_init_vcpu\n");
 
-    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
+    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
     if (ret < 0) {
         DPRINTF("kvm_create_vcpu failed\n");
         goto err;
     }
 
-    env->kvm_fd = ret;
-    env->kvm_state = s;
-    env->kvm_vcpu_dirty = 1;
+    cpu->kvm_fd = ret;
+    cpu->kvm_state = s;
+    cpu->kvm_vcpu_dirty = true;
 
     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
     if (mmap_size < 0) {
@@ -234,9 +239,9 @@ int kvm_init_vcpu(CPUArchState *env)
         goto err;
     }
 
-    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
-                        env->kvm_fd, 0);
-    if (env->kvm_run == MAP_FAILED) {
+    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                        cpu->kvm_fd, 0);
+    if (cpu->kvm_run == MAP_FAILED) {
         ret = -errno;
         DPRINTF("mmap'ing vcpu state failed\n");
         goto err;
@@ -244,13 +249,13 @@ int kvm_init_vcpu(CPUArchState *env)
 
     if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
         s->coalesced_mmio_ring =
-            (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
+            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
     }
 
-    ret = kvm_arch_init_vcpu(env);
+    ret = kvm_arch_init_vcpu(cpu);
     if (ret == 0) {
-        qemu_register_reset(kvm_reset_vcpu, env);
-        kvm_arch_reset_vcpu(env);
+        qemu_register_reset(kvm_reset_vcpu, cpu);
+        kvm_arch_reset_vcpu(cpu);
     }
 err:
     return ret;
@@ -288,7 +293,7 @@ static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
     return kvm_set_user_memory_region(s, mem);
 }
 
-static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
+static int kvm_dirty_pages_log_change(hwaddr phys_addr,
                                       ram_addr_t size, bool log_dirty)
 {
     KVMState *s = kvm_state;
@@ -297,7 +302,7 @@ static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
     if (mem == NULL)  {
         fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
                 TARGET_FMT_plx "\n", __func__, phys_addr,
-                (target_phys_addr_t)(phys_addr + size - 1));
+                (hwaddr)(phys_addr + size - 1));
         return -EINVAL;
     }
     return kvm_slot_dirty_pages_log_change(mem, log_dirty);
@@ -358,8 +363,8 @@ static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 {
     unsigned int i, j;
     unsigned long page_number, c;
-    target_phys_addr_t addr, addr1;
-    unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
+    hwaddr addr, addr1;
+    unsigned int len = ((section->size / getpagesize()) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 
     /*
@@ -401,8 +406,8 @@ static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
     KVMDirtyLog d;
     KVMSlot *mem;
     int ret = 0;
-    target_phys_addr_t start_addr = section->offset_within_address_space;
-    target_phys_addr_t end_addr = start_addr + section->size;
+    hwaddr start_addr = section->offset_within_address_space;
+    hwaddr end_addr = start_addr + section->size;
 
     d.dirty_bitmap = NULL;
     while (start_addr < end_addr) {
@@ -449,9 +454,10 @@ static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
     return ret;
 }
 
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
+static void kvm_coalesce_mmio_region(MemoryListener *listener,
+                                     MemoryRegionSection *secion,
+                                     hwaddr start, hwaddr size)
 {
-    int ret = -ENOSYS;
     KVMState *s = kvm_state;
 
     if (s->coalesced_mmio) {
@@ -461,15 +467,14 @@ int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
         zone.size = size;
         zone.pad = 0;
 
-        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
+        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
     }
-
-    return ret;
 }
 
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
+static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
+                                       MemoryRegionSection *secion,
+                                       hwaddr start, hwaddr size)
 {
-    int ret = -ENOSYS;
     KVMState *s = kvm_state;
 
     if (s->coalesced_mmio) {
@@ -479,10 +484,8 @@ int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
         zone.size = size;
         zone.pad = 0;
 
-        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
+        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
     }
-
-    return ret;
 }
 
 int kvm_check_extension(KVMState *s, unsigned int extension)
@@ -553,7 +556,7 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     int err;
     MemoryRegion *mr = section->mr;
     bool log_dirty = memory_region_is_logging(mr);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     void *ram = NULL;
     unsigned delta;
@@ -698,14 +701,6 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     }
 }
 
-static void kvm_begin(MemoryListener *listener)
-{
-}
-
-static void kvm_commit(MemoryListener *listener)
-{
-}
-
 static void kvm_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -718,11 +713,6 @@ static void kvm_region_del(MemoryListener *listener,
     kvm_set_phys_mem(section, false);
 }
 
-static void kvm_region_nop(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
 static void kvm_log_sync(MemoryListener *listener,
                          MemoryRegionSection *section)
 {
@@ -750,9 +740,12 @@ static void kvm_log_global_stop(struct MemoryListener *listener)
     assert(r >= 0);
 }
 
-static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
-                                  bool match_data, uint64_t data, int fd)
+static void kvm_mem_ioeventfd_add(MemoryListener *listener,
+                                  MemoryRegionSection *section,
+                                  bool match_data, uint64_t data,
+                                  EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     assert(match_data && section->size <= 8);
@@ -764,9 +757,12 @@ static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
     }
 }
 
-static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
-                                  bool match_data, uint64_t data, int fd)
+static void kvm_mem_ioeventfd_del(MemoryListener *listener,
+                                  MemoryRegionSection *section,
+                                  bool match_data, uint64_t data,
+                                  EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
@@ -776,9 +772,12 @@ static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
     }
 }
 
-static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
-                                 bool match_data, uint64_t data, int fd)
+static void kvm_io_ioeventfd_add(MemoryListener *listener,
+                                 MemoryRegionSection *section,
+                                 bool match_data, uint64_t data,
+                                 EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     assert(match_data && section->size == 2);
@@ -790,10 +789,13 @@ static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
     }
 }
 
-static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
-                                 bool match_data, uint64_t data, int fd)
+static void kvm_io_ioeventfd_del(MemoryListener *listener,
+                                 MemoryRegionSection *section,
+                                 bool match_data, uint64_t data,
+                                 EventNotifier *e)
 
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
@@ -803,56 +805,35 @@ static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
     }
 }
 
-static void kvm_eventfd_add(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-    if (section->address_space == get_system_memory()) {
-        kvm_mem_ioeventfd_add(section, match_data, data,
-			      event_notifier_get_fd(e));
-    } else {
-        kvm_io_ioeventfd_add(section, match_data, data,
-			     event_notifier_get_fd(e));
-    }
-}
-
-static void kvm_eventfd_del(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-    if (section->address_space == get_system_memory()) {
-        kvm_mem_ioeventfd_del(section, match_data, data,
-			      event_notifier_get_fd(e));
-    } else {
-        kvm_io_ioeventfd_del(section, match_data, data,
-			     event_notifier_get_fd(e));
-    }
-}
-
 static MemoryListener kvm_memory_listener = {
-    .begin = kvm_begin,
-    .commit = kvm_commit,
     .region_add = kvm_region_add,
     .region_del = kvm_region_del,
-    .region_nop = kvm_region_nop,
     .log_start = kvm_log_start,
     .log_stop = kvm_log_stop,
     .log_sync = kvm_log_sync,
     .log_global_start = kvm_log_global_start,
     .log_global_stop = kvm_log_global_stop,
-    .eventfd_add = kvm_eventfd_add,
-    .eventfd_del = kvm_eventfd_del,
+    .eventfd_add = kvm_mem_ioeventfd_add,
+    .eventfd_del = kvm_mem_ioeventfd_del,
+    .coalesced_mmio_add = kvm_coalesce_mmio_region,
+    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
+    .priority = 10,
+};
+
+static MemoryListener kvm_io_listener = {
+    .eventfd_add = kvm_io_ioeventfd_add,
+    .eventfd_del = kvm_io_ioeventfd_del,
     .priority = 10,
 };
 
 static void kvm_handle_interrupt(CPUArchState *env, int mask)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     env->interrupt_request |= mask;
 
-    if (!qemu_cpu_is_self(env)) {
-        qemu_cpu_kick(env);
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
     }
 }
 
@@ -865,13 +846,13 @@ int kvm_set_irq(KVMState *s, int irq, int level)
 
     event.level = level;
     event.irq = irq;
-    ret = kvm_vm_ioctl(s, s->irqchip_inject_ioctl, &event);
+    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
     if (ret < 0) {
         perror("kvm_set_irq");
         abort();
     }
 
-    return (s->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
+    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
 }
 
 #ifdef KVM_CAP_IRQ_ROUTING
@@ -959,6 +940,30 @@ static void kvm_add_routing_entry(KVMState *s,
     kvm_irqchip_commit_routes(s);
 }
 
+static int kvm_update_routing_entry(KVMState *s,
+                                    struct kvm_irq_routing_entry *new_entry)
+{
+    struct kvm_irq_routing_entry *entry;
+    int n;
+
+    for (n = 0; n < s->irq_routes->nr; n++) {
+        entry = &s->irq_routes->entries[n];
+        if (entry->gsi != new_entry->gsi) {
+            continue;
+        }
+
+        entry->type = new_entry->type;
+        entry->flags = new_entry->flags;
+        entry->u = new_entry->u;
+
+        kvm_irqchip_commit_routes(s);
+
+        return 0;
+    }
+
+    return -ESRCH;
+}
+
 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
 {
     struct kvm_irq_routing_entry e;
@@ -986,8 +991,6 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
         }
     }
     clear_gsi(s, virq);
-
-    kvm_irqchip_commit_routes(s);
 }
 
 static unsigned int kvm_hash_msi(uint32_t data)
@@ -1121,6 +1124,24 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     return virq;
 }
 
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+{
+    struct kvm_irq_routing_entry kroute;
+
+    if (!kvm_irqchip_in_kernel()) {
+        return -ENOSYS;
+    }
+
+    kroute.gsi = virq;
+    kroute.type = KVM_IRQ_ROUTING_MSI;
+    kroute.flags = 0;
+    kroute.u.msi.address_lo = (uint32_t)msg.address;
+    kroute.u.msi.address_hi = msg.address >> 32;
+    kroute.u.msi.data = msg.data;
+
+    return kvm_update_routing_entry(s, &kroute);
+}
+
 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     struct kvm_irqfd irqfd = {
@@ -1160,26 +1181,21 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     abort();
 }
-#endif /* !KVM_CAP_IRQ_ROUTING */
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
 {
-    return kvm_irqchip_assign_irqfd(s, fd, virq, true);
-}
-
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq)
-{
-    return kvm_irqchip_add_irqfd(s, event_notifier_get_fd(n), virq);
+    return -ENOSYS;
 }
+#endif /* !KVM_CAP_IRQ_ROUTING */
 
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq)
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
-    return kvm_irqchip_assign_irqfd(s, fd, virq, false);
+    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
 }
 
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
-    return kvm_irqchip_remove_irqfd(s, event_notifier_get_fd(n), virq);
+    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
 }
 
 static int kvm_irqchip_create(KVMState *s)
@@ -1200,10 +1216,6 @@ static int kvm_irqchip_create(KVMState *s)
         return ret;
     }
 
-    s->irqchip_inject_ioctl = KVM_IRQ_LINE;
-    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
-        s->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
-    }
     kvm_kernel_irqchip = true;
     /* If we have an in-kernel IRQ chip then we must have asynchronous
      * interrupt delivery (though the reverse is not necessarily true)
@@ -1350,6 +1362,13 @@ int kvm_init(void)
     s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
 #endif
 
+    s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
+
+    s->irq_set_ioctl = KVM_IRQ_LINE;
+    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
+        s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
+    }
+
     ret = kvm_arch_init(s);
     if (ret < 0) {
         goto err;
@@ -1361,7 +1380,8 @@ int kvm_init(void)
     }
 
     kvm_state = s;
-    memory_listener_register(&kvm_memory_listener, NULL);
+    memory_listener_register(&kvm_memory_listener, &address_space_memory);
+    memory_listener_register(&kvm_io_listener, &address_space_io);
 
     s->many_ioeventfds = kvm_check_many_ioeventfds();
 
@@ -1370,13 +1390,11 @@ int kvm_init(void)
     return 0;
 
 err:
-    if (s) {
-        if (s->vmfd >= 0) {
-            close(s->vmfd);
-        }
-        if (s->fd != -1) {
-            close(s->fd);
-        }
+    if (s->vmfd >= 0) {
+        close(s->vmfd);
+    }
+    if (s->fd != -1) {
+        close(s->fd);
     }
     g_free(s);
 
@@ -1422,6 +1440,8 @@ static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
 
 static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     fprintf(stderr, "KVM internal error.");
     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
         int i;
@@ -1436,7 +1456,7 @@ static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
     }
     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
         fprintf(stderr, "emulation failure\n");
-        if (!kvm_arch_stop_on_emulation_error(env)) {
+        if (!kvm_arch_stop_on_emulation_error(cpu)) {
             cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
             return EXCP_INTERRUPT;
         }
@@ -1473,54 +1493,61 @@ void kvm_flush_coalesced_mmio_buffer(void)
     s->coalesced_flush_in_progress = false;
 }
 
-static void do_kvm_cpu_synchronize_state(void *_env)
+static void do_kvm_cpu_synchronize_state(void *arg)
 {
-    CPUArchState *env = _env;
+    CPUState *cpu = arg;
 
-    if (!env->kvm_vcpu_dirty) {
-        kvm_arch_get_registers(env);
-        env->kvm_vcpu_dirty = 1;
+    if (!cpu->kvm_vcpu_dirty) {
+        kvm_arch_get_registers(cpu);
+        cpu->kvm_vcpu_dirty = true;
     }
 }
 
 void kvm_cpu_synchronize_state(CPUArchState *env)
 {
-    if (!env->kvm_vcpu_dirty) {
-        run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    if (!cpu->kvm_vcpu_dirty) {
+        run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
     }
 }
 
 void kvm_cpu_synchronize_post_reset(CPUArchState *env)
 {
-    kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
-    env->kvm_vcpu_dirty = 0;
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
+    cpu->kvm_vcpu_dirty = false;
 }
 
 void kvm_cpu_synchronize_post_init(CPUArchState *env)
 {
-    kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
-    env->kvm_vcpu_dirty = 0;
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
+    cpu->kvm_vcpu_dirty = false;
 }
 
 int kvm_cpu_exec(CPUArchState *env)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUState *cpu = ENV_GET_CPU(env);
+    struct kvm_run *run = cpu->kvm_run;
     int ret, run_ret;
 
     DPRINTF("kvm_cpu_exec()\n");
 
-    if (kvm_arch_process_async_events(env)) {
+    if (kvm_arch_process_async_events(cpu)) {
         env->exit_request = 0;
         return EXCP_HLT;
     }
 
     do {
-        if (env->kvm_vcpu_dirty) {
-            kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
-            env->kvm_vcpu_dirty = 0;
+        if (cpu->kvm_vcpu_dirty) {
+            kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
+            cpu->kvm_vcpu_dirty = false;
         }
 
-        kvm_arch_pre_run(env, run);
+        kvm_arch_pre_run(cpu, run);
         if (env->exit_request) {
             DPRINTF("interrupt exit requested\n");
             /*
@@ -1532,12 +1559,10 @@ int kvm_cpu_exec(CPUArchState *env)
         }
         qemu_mutex_unlock_iothread();
 
-        run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
+        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
 
         qemu_mutex_lock_iothread();
-        kvm_arch_post_run(env, run);
-
-        kvm_flush_coalesced_mmio_buffer();
+        kvm_arch_post_run(cpu, run);
 
         if (run_ret < 0) {
             if (run_ret == -EINTR || run_ret == -EAGAIN) {
@@ -1587,7 +1612,7 @@ int kvm_cpu_exec(CPUArchState *env)
             break;
         default:
             DPRINTF("kvm_arch_handle_exit\n");
-            ret = kvm_arch_handle_exit(env, run);
+            ret = kvm_arch_handle_exit(cpu, run);
             break;
         }
     } while (ret == 0);
@@ -1635,7 +1660,7 @@ int kvm_vm_ioctl(KVMState *s, int type, ...)
     return ret;
 }
 
-int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
+int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
 {
     int ret;
     void *arg;
@@ -1645,7 +1670,7 @@ int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
-    ret = ioctl(env->kvm_fd, type, arg);
+    ret = ioctl(cpu->kvm_fd, type, arg);
     if (ret == -1) {
         ret = -errno;
     }
@@ -1704,6 +1729,11 @@ int kvm_has_gsi_routing(void)
 #endif
 }
 
+int kvm_has_intx_set_mask(void)
+{
+    return kvm_state->intx_set_mask;
+}
+
 void *kvm_vmalloc(ram_addr_t size)
 {
 #ifdef TARGET_S390X
@@ -1719,6 +1749,9 @@ void *kvm_vmalloc(ram_addr_t size)
 
 void kvm_setup_guest_memory(void *start, size_t size)
 {
+#ifdef CONFIG_VALGRIND_H
+    VALGRIND_MAKE_MEM_DEFINED(start, size);
+#endif
     if (!kvm_has_sync_mmu()) {
         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
 
@@ -1732,12 +1765,12 @@ void kvm_setup_guest_memory(void *start, size_t size)
 }
 
 #ifdef KVM_CAP_SET_GUEST_DEBUG
-struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
+struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
                                                  target_ulong pc)
 {
     struct kvm_sw_breakpoint *bp;
 
-    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
+    QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
         if (bp->pc == pc) {
             return bp;
         }
@@ -1745,27 +1778,28 @@ struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
     return NULL;
 }
 
-int kvm_sw_breakpoints_active(CPUArchState *env)
+int kvm_sw_breakpoints_active(CPUState *cpu)
 {
-    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
+    return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
 }
 
 struct kvm_set_guest_debug_data {
     struct kvm_guest_debug dbg;
-    CPUArchState *env;
+    CPUState *cpu;
     int err;
 };
 
 static void kvm_invoke_set_guest_debug(void *data)
 {
     struct kvm_set_guest_debug_data *dbg_data = data;
-    CPUArchState *env = dbg_data->env;
 
-    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
+    dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
+                                   &dbg_data->dbg);
 }
 
 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     struct kvm_set_guest_debug_data data;
 
     data.dbg.control = reinject_trap;
@@ -1773,22 +1807,23 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
     if (env->singlestep_enabled) {
         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
     }
-    kvm_arch_update_guest_debug(env, &data.dbg);
-    data.env = env;
+    kvm_arch_update_guest_debug(cpu, &data.dbg);
+    data.cpu = cpu;
 
-    run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
+    run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
     return data.err;
 }
 
 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
                           target_ulong len, int type)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp;
     CPUArchState *env;
     int err;
 
     if (type == GDB_BREAKPOINT_SW) {
-        bp = kvm_find_sw_breakpoint(current_env, addr);
+        bp = kvm_find_sw_breakpoint(current_cpu, addr);
         if (bp) {
             bp->use_count++;
             return 0;
@@ -1801,13 +1836,13 @@ int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
 
         bp->pc = addr;
         bp->use_count = 1;
-        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
+        err = kvm_arch_insert_sw_breakpoint(current_cpu, bp);
         if (err) {
             g_free(bp);
             return err;
         }
 
-        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
+        QTAILQ_INSERT_HEAD(&current_cpu->kvm_state->kvm_sw_breakpoints,
                           bp, entry);
     } else {
         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
@@ -1828,12 +1863,13 @@ int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
                           target_ulong len, int type)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp;
     CPUArchState *env;
     int err;
 
     if (type == GDB_BREAKPOINT_SW) {
-        bp = kvm_find_sw_breakpoint(current_env, addr);
+        bp = kvm_find_sw_breakpoint(current_cpu, addr);
         if (!bp) {
             return -ENOENT;
         }
@@ -1843,12 +1879,12 @@ int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
             return 0;
         }
 
-        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
+        err = kvm_arch_remove_sw_breakpoint(current_cpu, bp);
         if (err) {
             return err;
         }
 
-        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
+        QTAILQ_REMOVE(&current_cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
         g_free(bp);
     } else {
         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
@@ -1868,19 +1904,24 @@ int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
 
 void kvm_remove_all_breakpoints(CPUArchState *current_env)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp, *next;
-    KVMState *s = current_env->kvm_state;
+    KVMState *s = current_cpu->kvm_state;
     CPUArchState *env;
+    CPUState *cpu;
 
     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
-        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
+        if (kvm_arch_remove_sw_breakpoint(current_cpu, bp) != 0) {
             /* Try harder to find a CPU that currently sees the breakpoint. */
             for (env = first_cpu; env != NULL; env = env->next_cpu) {
-                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
+                cpu = ENV_GET_CPU(env);
+                if (kvm_arch_remove_sw_breakpoint(cpu, bp) == 0) {
                     break;
                 }
             }
         }
+        QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
+        g_free(bp);
     }
     kvm_arch_remove_all_hw_breakpoints();
 
@@ -1915,18 +1956,19 @@ void kvm_remove_all_breakpoints(CPUArchState *current_env)
 
 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     struct kvm_signal_mask *sigmask;
     int r;
 
     if (!sigset) {
-        return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
+        return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
     }
 
     sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
 
     sigmask->len = 8;
     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
-    r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
+    r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
     g_free(sigmask);
 
     return r;
@@ -1984,9 +2026,9 @@ int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
     return 0;
 }
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
-    return kvm_arch_on_sigbus_vcpu(env, code, addr);
+    return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
 }
 
 int kvm_on_sigbus(int code, void *addr)
diff --git a/kvm-stub.c b/kvm-stub.c
index 94c9ea15b03..760aadc8743 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -12,10 +12,10 @@
 
 #include "qemu-common.h"
 #include "hw/hw.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 #include "cpu.h"
-#include "gdbstub.h"
-#include "kvm.h"
+#include "exec/gdbstub.h"
+#include "sysemu/kvm.h"
 
 KVMState *kvm_state;
 bool kvm_kernel_irqchip;
@@ -24,17 +24,7 @@ bool kvm_irqfds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
 
-int kvm_init_vcpu(CPUArchState *env)
-{
-    return -ENOSYS;
-}
-
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
-{
-    return -ENOSYS;
-}
-
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
+int kvm_init_vcpu(CPUState *cpu)
 {
     return -ENOSYS;
 }
@@ -122,7 +112,7 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign, uint
     return -ENOSYS;
 }
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
@@ -141,22 +131,17 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
 {
 }
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq)
-{
-    return -ENOSYS;
-}
-
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
 {
     return -ENOSYS;
 }
 
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq)
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
 }
 
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
 }
diff --git a/alpha.ld b/ldscripts/alpha.ld
index 906d76bba11..906d76bba11 100644
--- a/alpha.ld
+++ b/ldscripts/alpha.ld
diff --git a/arm.ld b/ldscripts/arm.ld
index 7f13da9ebbc..7f13da9ebbc 100644
--- a/arm.ld
+++ b/ldscripts/arm.ld
diff --git a/hppa.ld b/ldscripts/hppa.ld
index 3555b3e8e80..3555b3e8e80 100644
--- a/hppa.ld
+++ b/ldscripts/hppa.ld
diff --git a/i386.ld b/ldscripts/i386.ld
index cc3f160af07..cc3f160af07 100644
--- a/i386.ld
+++ b/ldscripts/i386.ld
diff --git a/ia64.ld b/ldscripts/ia64.ld
index 0c377967cd4..0c377967cd4 100644
--- a/ia64.ld
+++ b/ldscripts/ia64.ld
diff --git a/m68k.ld b/ldscripts/m68k.ld
index 0e3d9deb5a6..0e3d9deb5a6 100644
--- a/m68k.ld
+++ b/ldscripts/m68k.ld
diff --git a/mips.ld b/ldscripts/mips.ld
index 7b610ceed47..7b610ceed47 100644
--- a/mips.ld
+++ b/ldscripts/mips.ld
diff --git a/ppc.ld b/ldscripts/ppc.ld
index 2a0dcad63da..2a0dcad63da 100644
--- a/ppc.ld
+++ b/ldscripts/ppc.ld
diff --git a/ppc64.ld b/ldscripts/ppc64.ld
index e2dafa0b533..e2dafa0b533 100644
--- a/ppc64.ld
+++ b/ldscripts/ppc64.ld
diff --git a/s390.ld b/ldscripts/s390.ld
index a9c53707f16..a9c53707f16 100644
--- a/s390.ld
+++ b/ldscripts/s390.ld
diff --git a/sparc.ld b/ldscripts/sparc.ld
index 56efe34e73e..56efe34e73e 100644
--- a/sparc.ld
+++ b/ldscripts/sparc.ld
diff --git a/sparc64.ld b/ldscripts/sparc64.ld
index 9ea41431e8c..9ea41431e8c 100644
--- a/sparc64.ld
+++ b/ldscripts/sparc64.ld
diff --git a/x86_64.ld b/ldscripts/x86_64.ld
index b7a9f4eb369..b7a9f4eb369 100644
--- a/x86_64.ld
+++ b/ldscripts/x86_64.ld
diff --git a/libcacard/Makefile b/libcacard/Makefile
index 63990b70036..47827a0eb89 100644
--- a/libcacard/Makefile
+++ b/libcacard/Makefile
@@ -1,60 +1,49 @@
--include ../config-host.mak
--include $(SRC_PATH)/Makefile.objs
--include $(SRC_PATH)/rules.mak
-
 libcacard_includedir=$(includedir)/cacard
 
-$(call set-vpath, $(SRC_PATH))
+TOOLS += vscclient$(EXESUF)
 
 # objects linked into a shared library, built with libtool with -fPIC if required
-QEMU_OBJS=$(oslib-obj-y) qemu-timer-common.o $(trace-obj-y)
-QEMU_OBJS_LIB=$(patsubst %.o,%.lo,$(QEMU_OBJS))
-
-QEMU_CFLAGS+=-I../
+libcacard-obj-y = $(stub-obj-y) $(libcacard-y)
+libcacard-obj-y += util/osdep.o util/cutils.o util/qemu-timer-common.o util/error.o
+libcacard-obj-$(CONFIG_WIN32) += util/oslib-win32.o util/qemu-thread-win32.o
+libcacard-obj-$(CONFIG_POSIX) += util/oslib-posix.o util/qemu-thread-posix.o
+libcacard-obj-y += $(filter trace/%, $(util-obj-y))
 
-libcacard.lib-y=$(patsubst %.o,%.lo,$(libcacard-y))
+libcacard-lobj-y=$(patsubst %.o,%.lo,$(libcacard-obj-y))
 
-clean:
-	rm -f *.o */*.o *.d */*.d *.a */*.a *~ */*~ vscclient *.lo */*.lo .libs/* */.libs/* *.la */*.la *.pc
-	rm -Rf .libs */.libs
+# libtool will build the .o files, too
+$(libcacard-obj-y): | $(libcacard-lobj-y)
 
 all: libcacard.la libcacard.pc
-# Dummy command so that make thinks it has done something
-	@true
+
+vscclient$(EXESUF): libcacard/vscclient.o libcacard.la
+	$(call LINK,$^)
 
 #########################################################################
 # Rules for building libcacard standalone library
 
-ifeq ($(LIBTOOL),)
-libcacard.la:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-
-install-libcacard:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-else
-libcacard.la: $(libcacard.lib-y) $(QEMU_OBJS_LIB)
-	$(call quiet-command,$(LIBTOOL) --mode=link --quiet --tag=CC $(CC) -rpath $(libdir) -o $@ $^ $(libcacard_libs),"  lt LINK $@")
+libcacard.la: LDFLAGS += -rpath $(libdir) -no-undefined \
+	-export-syms $(SRC_PATH)/libcacard/libcacard.syms
+libcacard.la: LIBS += $(libcacard_libs)
+libcacard.la: $(libcacard-lobj-y)
+	$(call LINK,$^)
 
-libcacard_srcpath=$(SRC_PATH)/libcacard
-libcacard.pc: $(libcacard_srcpath)/libcacard.pc.in
+libcacard.pc: $(SRC_PATH)/libcacard/libcacard.pc.in
 	$(call quiet-command,sed -e 's|@LIBDIR@|$(libdir)|' \
 		-e 's|@INCLUDEDIR@|$(libcacard_includedir)|' \
 	    -e 's|@VERSION@|$(shell cat $(SRC_PATH)/VERSION)|' \
-		-e 's|@PREFIX@|$(prefix)|' \
-		< $(libcacard_srcpath)/libcacard.pc.in > libcacard.pc,\
+		-e 's|@PREFIX@|$(prefix)|' $< > libcacard.pc,\
 	"  GEN   $@")
 
 .PHONY: install-libcacard
 
-install-libcacard: libcacard.pc libcacard.la vscclient
+install: install-libcacard
+install-libcacard: libcacard.pc libcacard.la
 	$(INSTALL_DIR) "$(DESTDIR)$(libdir)"
 	$(INSTALL_DIR) "$(DESTDIR)$(libdir)/pkgconfig"
 	$(INSTALL_DIR) "$(DESTDIR)$(libcacard_includedir)"
-	$(INSTALL_DIR) "$(DESTDIR)$(bindir)"
-	$(LIBTOOL) --mode=install $(INSTALL_PROG) vscclient "$(DESTDIR)$(bindir)"
-	$(LIBTOOL) --mode=install $(INSTALL_DATA) libcacard.la "$(DESTDIR)$(libdir)"
-	$(LIBTOOL) --mode=install $(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
-	for inc in *.h; do \
-		$(LIBTOOL) --mode=install $(INSTALL_DATA) $(libcacard_srcpath)/$$inc "$(DESTDIR)$(libcacard_includedir)"; \
+	$(INSTALL_LIB) libcacard.la "$(DESTDIR)$(libdir)"
+	$(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
+	for inc in $(SRC_PATH)/libcacard/*.h; do \
+		$(INSTALL_DATA) $$inc "$(DESTDIR)$(libcacard_includedir)"; \
 	done
-endif
diff --git a/libcacard/event.c b/libcacard/event.c
index 61923761c92..2d7500fac0a 100644
--- a/libcacard/event.c
+++ b/libcacard/event.c
@@ -6,7 +6,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "vcard.h"
 #include "vreader.h"
diff --git a/libcacard/libcacard.syms b/libcacard/libcacard.syms
new file mode 100644
index 00000000000..1697515a7f0
--- /dev/null
+++ b/libcacard/libcacard.syms
@@ -0,0 +1,77 @@
+cac_card_init
+cac_is_cac_card
+vcard_add_applet
+vcard_apdu_delete
+vcard_apdu_new
+vcard_applet_get_aid
+vcard_buffer_response_delete
+vcard_buffer_response_new
+vcard_delete_applet
+vcard_emul_delete_key
+vcard_emul_force_card_insert
+vcard_emul_force_card_remove
+vcard_emul_get_atr
+vcard_emul_get_login_count
+vcard_emul_init
+vcard_emul_login
+vcard_emul_options
+vcard_emul_replay_insertion_events
+vcard_emul_reset
+vcard_emul_rsa_op
+vcard_emul_type_from_string
+vcard_emul_type_select
+vcard_emul_usage
+vcard_find_applet
+vcard_free
+vcard_get_atr
+vcard_get_buffer_response
+vcard_get_current_applet_private
+vcard_get_private
+vcard_get_type
+vcard_init
+vcard_make_response
+vcard_new
+vcard_new_applet
+vcard_process_apdu
+vcard_process_applet_apdu
+vcard_reference
+vcard_reset
+vcard_response_delete
+vcard_response_new
+vcard_response_new_bytes
+vcard_response_new_data
+vcard_response_new_status_bytes
+vcard_select_applet
+vcard_set_applet_private
+vcard_set_atr_func
+vcard_set_buffer_response
+vcard_set_type
+vevent_delete
+vevent_get_next_vevent
+vevent_new
+vevent_queue_init
+vevent_queue_vevent
+vevent_wait_next_vevent
+vreader_add_reader
+vreader_card_is_present
+vreader_free
+vreader_get_id
+vreader_get_name
+vreader_get_private
+vreader_get_reader_by_id
+vreader_get_reader_by_name
+vreader_get_reader_list
+vreader_init
+vreader_insert_card
+vreader_list_delete
+vreader_list_get_first
+vreader_list_get_next
+vreader_list_get_reader
+vreader_new
+vreader_power_off
+vreader_power_on
+vreader_queue_card_event
+vreader_reference
+vreader_remove_reader
+vreader_set_id
+vreader_xfr_bytes
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
index b02556ee0c1..539177bb4c0 100644
--- a/libcacard/vcard.c
+++ b/libcacard/vcard.c
@@ -200,7 +200,6 @@ vcard_free(VCard *vcard)
     }
     vcard_buffer_response_delete(vcard->vcard_buffer_response);
     g_free(vcard);
-    return;
 }
 
 void
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
index 802cae3a291..5f565e0b4ab 100644
--- a/libcacard/vcard_emul_nss.c
+++ b/libcacard/vcard_emul_nss.c
@@ -168,7 +168,6 @@ vcard_emul_delete_key(VCardKey *key)
     if (key->slot) {
         PK11_FreeSlot(key->slot);
     }
-    return;
 }
 
 /*
@@ -418,7 +417,6 @@ vcard_emul_reset(VCard *card, VCardPower power)
     /* TODO: we may also need to send insertion/removal events? */
     slot = vcard_emul_card_get_slot(card);
     PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
-    return;
 }
 
 
@@ -535,7 +533,6 @@ vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len)
 
     memcpy(atr, nss_atr, len);
     *atr_len = len;
-    return;
 }
 
 /*
@@ -1169,8 +1166,7 @@ vcard_emul_options(const char *args)
             NEXT_TOKEN(vname)
             NEXT_TOKEN(type_params)
             type_params_length = MIN(type_params_length, sizeof(type_str)-1);
-            strncpy(type_str, type_params, type_params_length);
-            type_str[type_params_length] = 0;
+            pstrcpy(type_str, type_params_length, type_params);
             type = vcard_emul_type_from_string(type_str);
 
             NEXT_TOKEN(type_params)
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
index ec126dfa466..313349b6567 100644
--- a/libcacard/vreader.c
+++ b/libcacard/vreader.c
@@ -6,7 +6,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "vcard.h"
 #include "vcard_emul.h"
@@ -93,7 +93,6 @@ vreader_free(VReader *reader)
         reader->reader_private_free(reader->reader_private);
     }
     g_free(reader);
-    return;
 }
 
 static VCard *
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
index b64c93dd692..2fce52bed50 100644
--- a/libcacard/vscclient.c
+++ b/libcacard/vscclient.c
@@ -13,8 +13,8 @@
 #include <netdb.h>
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
-#include "qemu_socket.h"
+#include "qemu/thread.h"
+#include "qemu/sockets.h"
 
 #include "vscard_common.h"
 
diff --git a/linux-headers/asm-powerpc/epapr_hcalls.h b/linux-headers/asm-powerpc/epapr_hcalls.h
new file mode 100644
index 00000000000..06f724786a4
--- /dev/null
+++ b/linux-headers/asm-powerpc/epapr_hcalls.h
@@ -0,0 +1,98 @@
+/*
+ * ePAPR hcall interface
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Author: Timur Tabi <timur@freescale.com>
+ *
+ * This file is provided under a dual BSD/GPL license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ASM_POWERPC_EPAPR_HCALLS_H
+#define _ASM_POWERPC_EPAPR_HCALLS_H
+
+#define EV_BYTE_CHANNEL_SEND		1
+#define EV_BYTE_CHANNEL_RECEIVE		2
+#define EV_BYTE_CHANNEL_POLL		3
+#define EV_INT_SET_CONFIG		4
+#define EV_INT_GET_CONFIG		5
+#define EV_INT_SET_MASK			6
+#define EV_INT_GET_MASK			7
+#define EV_INT_IACK			9
+#define EV_INT_EOI			10
+#define EV_INT_SEND_IPI			11
+#define EV_INT_SET_TASK_PRIORITY	12
+#define EV_INT_GET_TASK_PRIORITY	13
+#define EV_DOORBELL_SEND		14
+#define EV_MSGSND			15
+#define EV_IDLE				16
+
+/* vendor ID: epapr */
+#define EV_LOCAL_VENDOR_ID		0	/* for private use */
+#define EV_EPAPR_VENDOR_ID		1
+#define EV_FSL_VENDOR_ID		2	/* Freescale Semiconductor */
+#define EV_IBM_VENDOR_ID		3	/* IBM */
+#define EV_GHS_VENDOR_ID		4	/* Green Hills Software */
+#define EV_ENEA_VENDOR_ID		5	/* Enea */
+#define EV_WR_VENDOR_ID			6	/* Wind River Systems */
+#define EV_AMCC_VENDOR_ID		7	/* Applied Micro Circuits */
+#define EV_KVM_VENDOR_ID		42	/* KVM */
+
+/* The max number of bytes that a byte channel can send or receive per call */
+#define EV_BYTE_CHANNEL_MAX_BYTES	16
+
+
+#define _EV_HCALL_TOKEN(id, num) (((id) << 16) | (num))
+#define EV_HCALL_TOKEN(hcall_num) _EV_HCALL_TOKEN(EV_EPAPR_VENDOR_ID, hcall_num)
+
+/* epapr return codes */
+#define EV_SUCCESS		0
+#define EV_EPERM		1	/* Operation not permitted */
+#define EV_ENOENT		2	/*  Entry Not Found */
+#define EV_EIO			3	/* I/O error occured */
+#define EV_EAGAIN		4	/* The operation had insufficient
+					 * resources to complete and should be
+					 * retried
+					 */
+#define EV_ENOMEM		5	/* There was insufficient memory to
+					 * complete the operation */
+#define EV_EFAULT		6	/* Bad guest address */
+#define EV_ENODEV		7	/* No such device */
+#define EV_EINVAL		8	/* An argument supplied to the hcall
+					   was out of range or invalid */
+#define EV_INTERNAL		9	/* An internal error occured */
+#define EV_CONFIG		10	/* A configuration error was detected */
+#define EV_INVALID_STATE	11	/* The object is in an invalid state */
+#define EV_UNIMPLEMENTED	12	/* Unimplemented hypercall */
+#define EV_BUFFER_OVERFLOW	13	/* Caller-supplied buffer too small */
+
+#endif /* _ASM_POWERPC_EPAPR_HCALLS_H */
diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index 1bea4d8ea6f..16064d00adb 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -114,7 +114,10 @@ struct kvm_regs {
 /* Embedded Floating Point (SPE) -- IVOR32-34 if KVM_SREGS_E_IVOR */
 #define KVM_SREGS_E_SPE			(1 << 9)
 
-/* External Proxy (EXP) -- EPR */
+/*
+ * DEPRECATED! USE ONE_REG FOR THIS ONE!
+ * External Proxy (EXP) -- EPR
+ */
 #define KVM_SREGS_EXP			(1 << 10)
 
 /* External PID (E.PD) -- EPSC/EPLC */
@@ -221,6 +224,12 @@ struct kvm_sregs {
 
 			__u32 dbsr;	/* KVM_SREGS_E_UPDATE_DBSR */
 			__u32 dbcr[3];
+			/*
+			 * iac/dac registers are 64bit wide, while this API
+			 * interface provides only lower 32 bits on 64 bit
+			 * processors. ONE_REG interface is added for 64bit
+			 * iac/dac registers.
+			 */
 			__u32 iac[4];
 			__u32 dac[2];
 			__u32 dvc[2];
@@ -325,6 +334,87 @@ struct kvm_book3e_206_tlb_params {
 	__u32 reserved[8];
 };
 
+/* For KVM_PPC_GET_HTAB_FD */
+struct kvm_get_htab_fd {
+	__u64	flags;
+	__u64	start_index;
+	__u64	reserved[2];
+};
+
+/* Values for kvm_get_htab_fd.flags */
+#define KVM_GET_HTAB_BOLTED_ONLY	((__u64)0x1)
+#define KVM_GET_HTAB_WRITE		((__u64)0x2)
+
+/*
+ * Data read on the file descriptor is formatted as a series of
+ * records, each consisting of a header followed by a series of
+ * `n_valid' HPTEs (16 bytes each), which are all valid.  Following
+ * those valid HPTEs there are `n_invalid' invalid HPTEs, which
+ * are not represented explicitly in the stream.  The same format
+ * is used for writing.
+ */
+struct kvm_get_htab_header {
+	__u32	index;
+	__u16	n_valid;
+	__u16	n_invalid;
+};
+
 #define KVM_REG_PPC_HIOR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x1)
+#define KVM_REG_PPC_IAC1	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x2)
+#define KVM_REG_PPC_IAC2	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x3)
+#define KVM_REG_PPC_IAC3	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x4)
+#define KVM_REG_PPC_IAC4	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x5)
+#define KVM_REG_PPC_DAC1	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x6)
+#define KVM_REG_PPC_DAC2	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x7)
+#define KVM_REG_PPC_DABR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x8)
+#define KVM_REG_PPC_DSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x9)
+#define KVM_REG_PPC_PURR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa)
+#define KVM_REG_PPC_SPURR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb)
+#define KVM_REG_PPC_DAR		(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc)
+#define KVM_REG_PPC_DSISR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xd)
+#define KVM_REG_PPC_AMR		(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xe)
+#define KVM_REG_PPC_UAMOR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xf)
+
+#define KVM_REG_PPC_MMCR0	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x10)
+#define KVM_REG_PPC_MMCR1	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x11)
+#define KVM_REG_PPC_MMCRA	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x12)
+
+#define KVM_REG_PPC_PMC1	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x18)
+#define KVM_REG_PPC_PMC2	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x19)
+#define KVM_REG_PPC_PMC3	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1a)
+#define KVM_REG_PPC_PMC4	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1b)
+#define KVM_REG_PPC_PMC5	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1c)
+#define KVM_REG_PPC_PMC6	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1d)
+#define KVM_REG_PPC_PMC7	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1e)
+#define KVM_REG_PPC_PMC8	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x1f)
+
+/* 32 floating-point registers */
+#define KVM_REG_PPC_FPR0	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x20)
+#define KVM_REG_PPC_FPR(n)	(KVM_REG_PPC_FPR0 + (n))
+#define KVM_REG_PPC_FPR31	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x3f)
+
+/* 32 VMX/Altivec vector registers */
+#define KVM_REG_PPC_VR0		(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x40)
+#define KVM_REG_PPC_VR(n)	(KVM_REG_PPC_VR0 + (n))
+#define KVM_REG_PPC_VR31	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x5f)
+
+/* 32 double-width FP registers for VSX */
+/* High-order halves overlap with FP regs */
+#define KVM_REG_PPC_VSR0	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x60)
+#define KVM_REG_PPC_VSR(n)	(KVM_REG_PPC_VSR0 + (n))
+#define KVM_REG_PPC_VSR31	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x7f)
+
+/* FP and vector status/control registers */
+#define KVM_REG_PPC_FPSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x80)
+#define KVM_REG_PPC_VSCR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x81)
+
+/* Virtual processor areas */
+/* For SLB & DTL, address in high (first) half, length in low half */
+#define KVM_REG_PPC_VPA_ADDR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x82)
+#define KVM_REG_PPC_VPA_SLB	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x83)
+#define KVM_REG_PPC_VPA_DTL	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x84)
+
+#define KVM_REG_PPC_EPCR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x85)
+#define KVM_REG_PPC_EPR		(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x86)
 
 #endif /* __LINUX_KVM_POWERPC_H */
diff --git a/linux-headers/asm-powerpc/kvm_para.h b/linux-headers/asm-powerpc/kvm_para.h
index c047a840009..7e64f575f67 100644
--- a/linux-headers/asm-powerpc/kvm_para.h
+++ b/linux-headers/asm-powerpc/kvm_para.h
@@ -75,9 +75,10 @@ struct kvm_vcpu_arch_shared {
 };
 
 #define KVM_SC_MAGIC_R0		0x4b564d21 /* "KVM!" */
-#define HC_VENDOR_KVM		(42 << 16)
-#define HC_EV_SUCCESS		0
-#define HC_EV_UNIMPLEMENTED	12
+
+#define KVM_HCALL_TOKEN(num)     _EV_HCALL_TOKEN(EV_KVM_VENDOR_ID, num)
+
+#include <asm/epapr_hcalls.h>
 
 #define KVM_FEATURE_MAGIC_PAGE	1
 
diff --git a/linux-headers/asm-s390/kvm_para.h b/linux-headers/asm-s390/kvm_para.h
index 870051f645a..ff1f4e7b301 100644
--- a/linux-headers/asm-s390/kvm_para.h
+++ b/linux-headers/asm-s390/kvm_para.h
@@ -1,5 +1,5 @@
 /*
- * definition for paravirtual devices on s390
+ * User API definitions for paravirtual devices on s390
  *
  * Copyright IBM Corp. 2008
  *
@@ -9,9 +9,3 @@
  *
  *    Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
  */
-
-#ifndef __S390_KVM_PARA_H
-#define __S390_KVM_PARA_H
-
-
-#endif /* __S390_KVM_PARA_H */
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index 246617efd67..a65ec29e6ff 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -9,6 +9,22 @@
 #include <linux/types.h>
 #include <linux/ioctl.h>
 
+#define DE_VECTOR 0
+#define DB_VECTOR 1
+#define BP_VECTOR 3
+#define OF_VECTOR 4
+#define BR_VECTOR 5
+#define UD_VECTOR 6
+#define NM_VECTOR 7
+#define DF_VECTOR 8
+#define TS_VECTOR 10
+#define NP_VECTOR 11
+#define SS_VECTOR 12
+#define GP_VECTOR 13
+#define PF_VECTOR 14
+#define MF_VECTOR 16
+#define MC_VECTOR 18
+
 /* Select x86 specific features in <linux/kvm.h> */
 #define __KVM_HAVE_PIT
 #define __KVM_HAVE_IOAPIC
@@ -25,6 +41,7 @@
 #define __KVM_HAVE_DEBUGREGS
 #define __KVM_HAVE_XSAVE
 #define __KVM_HAVE_XCRS
+#define __KVM_HAVE_READONLY_MEM
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 4b9e575dd08..5af935761cc 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -101,9 +101,13 @@ struct kvm_userspace_memory_region {
 	__u64 userspace_addr; /* start of the userspace allocated memory */
 };
 
-/* for kvm_memory_region::flags */
-#define KVM_MEM_LOG_DIRTY_PAGES  1UL
-#define KVM_MEMSLOT_INVALID      (1UL << 1)
+/*
+ * The bit 0 ~ bit 15 of kvm_memory_region::flags are visible for userspace,
+ * other bits are reserved for kvm internal use which are defined in
+ * include/linux/kvm_host.h.
+ */
+#define KVM_MEM_LOG_DIRTY_PAGES	(1UL << 0)
+#define KVM_MEM_READONLY	(1UL << 1)
 
 /* for KVM_IRQ_LINE */
 struct kvm_irq_level {
@@ -163,10 +167,17 @@ struct kvm_pit_config {
 #define KVM_EXIT_OSI              18
 #define KVM_EXIT_PAPR_HCALL	  19
 #define KVM_EXIT_S390_UCONTROL	  20
+#define KVM_EXIT_WATCHDOG         21
+#define KVM_EXIT_S390_TSCH        22
+#define KVM_EXIT_EPR              23
 
 /* For KVM_EXIT_INTERNAL_ERROR */
-#define KVM_INTERNAL_ERROR_EMULATION 1
-#define KVM_INTERNAL_ERROR_SIMUL_EX 2
+/* Emulate instruction failed. */
+#define KVM_INTERNAL_ERROR_EMULATION	1
+/* Encounter unexpected simultaneous exceptions. */
+#define KVM_INTERNAL_ERROR_SIMUL_EX	2
+/* Encounter unexpected vm-exit due to delivery event. */
+#define KVM_INTERNAL_ERROR_DELIVERY_EV	3
 
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
@@ -276,6 +287,19 @@ struct kvm_run {
 			__u64 ret;
 			__u64 args[9];
 		} papr_hcall;
+		/* KVM_EXIT_S390_TSCH */
+		struct {
+			__u16 subchannel_id;
+			__u16 subchannel_nr;
+			__u32 io_int_parm;
+			__u32 io_int_word;
+			__u32 ipb;
+			__u8 dequeued;
+		} s390_tsch;
+		/* KVM_EXIT_EPR */
+		struct {
+			__u32 epr;
+		} epr;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -388,10 +412,20 @@ struct kvm_s390_psw {
 #define KVM_S390_PROGRAM_INT		0xfffe0001u
 #define KVM_S390_SIGP_SET_PREFIX	0xfffe0002u
 #define KVM_S390_RESTART		0xfffe0003u
+#define KVM_S390_MCHK			0xfffe1000u
 #define KVM_S390_INT_VIRTIO		0xffff2603u
 #define KVM_S390_INT_SERVICE		0xffff2401u
 #define KVM_S390_INT_EMERGENCY		0xffff1201u
 #define KVM_S390_INT_EXTERNAL_CALL	0xffff1202u
+/* Anything below 0xfffe0000u is taken by INT_IO */
+#define KVM_S390_INT_IO(ai,cssid,ssid,schid)   \
+	(((schid)) |			       \
+	 ((ssid) << 16) |		       \
+	 ((cssid) << 18) |		       \
+	 ((ai) << 26))
+#define KVM_S390_INT_IO_MIN		0x00000000u
+#define KVM_S390_INT_IO_MAX		0xfffdffffu
+
 
 struct kvm_s390_interrupt {
 	__u32 type;
@@ -473,6 +507,8 @@ struct kvm_ppc_smmu_info {
 	struct kvm_ppc_one_seg_page_size sps[KVM_PPC_PAGE_SIZES_MAX_SZ];
 };
 
+#define KVM_PPC_PVINFO_FLAGS_EV_IDLE   (1<<0)
+
 #define KVMIO 0xAE
 
 /* machine type bits, to be used as argument to KVM_CREATE_VM */
@@ -618,6 +654,14 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_GET_SMMU_INFO 78
 #define KVM_CAP_S390_COW 79
 #define KVM_CAP_PPC_ALLOC_HTAB 80
+#ifdef __KVM_HAVE_READONLY_MEM
+#define KVM_CAP_READONLY_MEM 81
+#endif
+#define KVM_CAP_IRQFD_RESAMPLE 82
+#define KVM_CAP_PPC_BOOKE_WATCHDOG 83
+#define KVM_CAP_PPC_HTAB_FD 84
+#define KVM_CAP_S390_CSS_SUPPORT 85
+#define KVM_CAP_PPC_EPR 86
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -683,12 +727,21 @@ struct kvm_xen_hvm_config {
 #endif
 
 #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)
+/*
+ * Available with KVM_CAP_IRQFD_RESAMPLE
+ *
+ * KVM_IRQFD_FLAG_RESAMPLE indicates resamplefd is valid and specifies
+ * the irqfd to operate in resampling mode for level triggered interrupt
+ * emlation.  See Documentation/virtual/kvm/api.txt.
+ */
+#define KVM_IRQFD_FLAG_RESAMPLE (1 << 1)
 
 struct kvm_irqfd {
 	__u32 fd;
 	__u32 gsi;
 	__u32 flags;
-	__u8  pad[20];
+	__u32 resamplefd;
+	__u8  pad[16];
 };
 
 struct kvm_clock_data {
@@ -831,6 +884,11 @@ struct kvm_s390_ucas_mapping {
 #define KVM_PPC_GET_SMMU_INFO	  _IOR(KVMIO,  0xa6, struct kvm_ppc_smmu_info)
 /* Available with KVM_CAP_PPC_ALLOC_HTAB */
 #define KVM_PPC_ALLOCATE_HTAB	  _IOWR(KVMIO, 0xa7, __u32)
+#define KVM_CREATE_SPAPR_TCE	  _IOW(KVMIO,  0xa8, struct kvm_create_spapr_tce)
+/* Available with KVM_CAP_RMA */
+#define KVM_ALLOCATE_RMA	  _IOR(KVMIO,  0xa9, struct kvm_allocate_rma)
+/* Available with KVM_CAP_PPC_HTAB_FD */
+#define KVM_PPC_GET_HTAB_FD	  _IOW(KVMIO,  0xaa, struct kvm_get_htab_fd)
 
 /*
  * ioctls for vcpu fds
@@ -894,9 +952,6 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_XCRS */
 #define KVM_GET_XCRS		  _IOR(KVMIO,  0xa6, struct kvm_xcrs)
 #define KVM_SET_XCRS		  _IOW(KVMIO,  0xa7, struct kvm_xcrs)
-#define KVM_CREATE_SPAPR_TCE	  _IOW(KVMIO,  0xa8, struct kvm_create_spapr_tce)
-/* Available with KVM_CAP_RMA */
-#define KVM_ALLOCATE_RMA	  _IOR(KVMIO,  0xa9, struct kvm_allocate_rma)
 /* Available with KVM_CAP_SW_TLB */
 #define KVM_DIRTY_TLB		  _IOW(KVMIO,  0xaa, struct kvm_dirty_tlb)
 /* Available with KVM_CAP_ONE_REG */
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
new file mode 100644
index 00000000000..f787b727a92
--- /dev/null
+++ b/linux-headers/linux/vfio.h
@@ -0,0 +1,368 @@
+/*
+ * VFIO API definition
+ *
+ * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
+ *     Author: Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef VFIO_H
+#define VFIO_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#define VFIO_API_VERSION	0
+
+
+/* Kernel & User level defines for VFIO IOCTLs. */
+
+/* Extensions */
+
+#define VFIO_TYPE1_IOMMU		1
+
+/*
+ * The IOCTL interface is designed for extensibility by embedding the
+ * structure length (argsz) and flags into structures passed between
+ * kernel and userspace.  We therefore use the _IO() macro for these
+ * defines to avoid implicitly embedding a size into the ioctl request.
+ * As structure fields are added, argsz will increase to match and flag
+ * bits will be defined to indicate additional fields with valid data.
+ * It's *always* the caller's responsibility to indicate the size of
+ * the structure passed by setting argsz appropriately.
+ */
+
+#define VFIO_TYPE	(';')
+#define VFIO_BASE	100
+
+/* -------- IOCTLs for VFIO file descriptor (/dev/vfio/vfio) -------- */
+
+/**
+ * VFIO_GET_API_VERSION - _IO(VFIO_TYPE, VFIO_BASE + 0)
+ *
+ * Report the version of the VFIO API.  This allows us to bump the entire
+ * API version should we later need to add or change features in incompatible
+ * ways.
+ * Return: VFIO_API_VERSION
+ * Availability: Always
+ */
+#define VFIO_GET_API_VERSION		_IO(VFIO_TYPE, VFIO_BASE + 0)
+
+/**
+ * VFIO_CHECK_EXTENSION - _IOW(VFIO_TYPE, VFIO_BASE + 1, __u32)
+ *
+ * Check whether an extension is supported.
+ * Return: 0 if not supported, 1 (or some other positive integer) if supported.
+ * Availability: Always
+ */
+#define VFIO_CHECK_EXTENSION		_IO(VFIO_TYPE, VFIO_BASE + 1)
+
+/**
+ * VFIO_SET_IOMMU - _IOW(VFIO_TYPE, VFIO_BASE + 2, __s32)
+ *
+ * Set the iommu to the given type.  The type must be supported by an
+ * iommu driver as verified by calling CHECK_EXTENSION using the same
+ * type.  A group must be set to this file descriptor before this
+ * ioctl is available.  The IOMMU interfaces enabled by this call are
+ * specific to the value set.
+ * Return: 0 on success, -errno on failure
+ * Availability: When VFIO group attached
+ */
+#define VFIO_SET_IOMMU			_IO(VFIO_TYPE, VFIO_BASE + 2)
+
+/* -------- IOCTLs for GROUP file descriptors (/dev/vfio/$GROUP) -------- */
+
+/**
+ * VFIO_GROUP_GET_STATUS - _IOR(VFIO_TYPE, VFIO_BASE + 3,
+ *						struct vfio_group_status)
+ *
+ * Retrieve information about the group.  Fills in provided
+ * struct vfio_group_info.  Caller sets argsz.
+ * Return: 0 on succes, -errno on failure.
+ * Availability: Always
+ */
+struct vfio_group_status {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_GROUP_FLAGS_VIABLE		(1 << 0)
+#define VFIO_GROUP_FLAGS_CONTAINER_SET	(1 << 1)
+};
+#define VFIO_GROUP_GET_STATUS		_IO(VFIO_TYPE, VFIO_BASE + 3)
+
+/**
+ * VFIO_GROUP_SET_CONTAINER - _IOW(VFIO_TYPE, VFIO_BASE + 4, __s32)
+ *
+ * Set the container for the VFIO group to the open VFIO file
+ * descriptor provided.  Groups may only belong to a single
+ * container.  Containers may, at their discretion, support multiple
+ * groups.  Only when a container is set are all of the interfaces
+ * of the VFIO file descriptor and the VFIO group file descriptor
+ * available to the user.
+ * Return: 0 on success, -errno on failure.
+ * Availability: Always
+ */
+#define VFIO_GROUP_SET_CONTAINER	_IO(VFIO_TYPE, VFIO_BASE + 4)
+
+/**
+ * VFIO_GROUP_UNSET_CONTAINER - _IO(VFIO_TYPE, VFIO_BASE + 5)
+ *
+ * Remove the group from the attached container.  This is the
+ * opposite of the SET_CONTAINER call and returns the group to
+ * an initial state.  All device file descriptors must be released
+ * prior to calling this interface.  When removing the last group
+ * from a container, the IOMMU will be disabled and all state lost,
+ * effectively also returning the VFIO file descriptor to an initial
+ * state.
+ * Return: 0 on success, -errno on failure.
+ * Availability: When attached to container
+ */
+#define VFIO_GROUP_UNSET_CONTAINER	_IO(VFIO_TYPE, VFIO_BASE + 5)
+
+/**
+ * VFIO_GROUP_GET_DEVICE_FD - _IOW(VFIO_TYPE, VFIO_BASE + 6, char)
+ *
+ * Return a new file descriptor for the device object described by
+ * the provided string.  The string should match a device listed in
+ * the devices subdirectory of the IOMMU group sysfs entry.  The
+ * group containing the device must already be added to this context.
+ * Return: new file descriptor on success, -errno on failure.
+ * Availability: When attached to container
+ */
+#define VFIO_GROUP_GET_DEVICE_FD	_IO(VFIO_TYPE, VFIO_BASE + 6)
+
+/* --------------- IOCTLs for DEVICE file descriptors --------------- */
+
+/**
+ * VFIO_DEVICE_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 7,
+ *						struct vfio_device_info)
+ *
+ * Retrieve information about the device.  Fills in provided
+ * struct vfio_device_info.  Caller sets argsz.
+ * Return: 0 on success, -errno on failure.
+ */
+struct vfio_device_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
+#define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
+	__u32	num_regions;	/* Max region index + 1 */
+	__u32	num_irqs;	/* Max IRQ index + 1 */
+};
+#define VFIO_DEVICE_GET_INFO		_IO(VFIO_TYPE, VFIO_BASE + 7)
+
+/**
+ * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
+ *				       struct vfio_region_info)
+ *
+ * Retrieve information about a device region.  Caller provides
+ * struct vfio_region_info with index value set.  Caller sets argsz.
+ * Implementation of region mapping is bus driver specific.  This is
+ * intended to describe MMIO, I/O port, as well as bus specific
+ * regions (ex. PCI config space).  Zero sized regions may be used
+ * to describe unimplemented regions (ex. unimplemented PCI BARs).
+ * Return: 0 on success, -errno on failure.
+ */
+struct vfio_region_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_REGION_INFO_FLAG_READ	(1 << 0) /* Region supports read */
+#define VFIO_REGION_INFO_FLAG_WRITE	(1 << 1) /* Region supports write */
+#define VFIO_REGION_INFO_FLAG_MMAP	(1 << 2) /* Region supports mmap */
+	__u32	index;		/* Region index */
+	__u32	resv;		/* Reserved for alignment */
+	__u64	size;		/* Region size (bytes) */
+	__u64	offset;		/* Region offset from start of device fd */
+};
+#define VFIO_DEVICE_GET_REGION_INFO	_IO(VFIO_TYPE, VFIO_BASE + 8)
+
+/**
+ * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9,
+ *				    struct vfio_irq_info)
+ *
+ * Retrieve information about a device IRQ.  Caller provides
+ * struct vfio_irq_info with index value set.  Caller sets argsz.
+ * Implementation of IRQ mapping is bus driver specific.  Indexes
+ * using multiple IRQs are primarily intended to support MSI-like
+ * interrupt blocks.  Zero count irq blocks may be used to describe
+ * unimplemented interrupt types.
+ *
+ * The EVENTFD flag indicates the interrupt index supports eventfd based
+ * signaling.
+ *
+ * The MASKABLE flags indicates the index supports MASK and UNMASK
+ * actions described below.
+ *
+ * AUTOMASKED indicates that after signaling, the interrupt line is
+ * automatically masked by VFIO and the user needs to unmask the line
+ * to receive new interrupts.  This is primarily intended to distinguish
+ * level triggered interrupts.
+ *
+ * The NORESIZE flag indicates that the interrupt lines within the index
+ * are setup as a set and new subindexes cannot be enabled without first
+ * disabling the entire index.  This is used for interrupts like PCI MSI
+ * and MSI-X where the driver may only use a subset of the available
+ * indexes, but VFIO needs to enable a specific number of vectors
+ * upfront.  In the case of MSI-X, where the user can enable MSI-X and
+ * then add and unmask vectors, it's up to userspace to make the decision
+ * whether to allocate the maximum supported number of vectors or tear
+ * down setup and incrementally increase the vectors as each is enabled.
+ */
+struct vfio_irq_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IRQ_INFO_EVENTFD		(1 << 0)
+#define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
+#define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
+#define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+	__u32	index;		/* IRQ index */
+	__u32	count;		/* Number of IRQs within this index */
+};
+#define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
+
+/**
+ * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
+ *
+ * Set signaling, masking, and unmasking of interrupts.  Caller provides
+ * struct vfio_irq_set with all fields set.  'start' and 'count' indicate
+ * the range of subindexes being specified.
+ *
+ * The DATA flags specify the type of data provided.  If DATA_NONE, the
+ * operation performs the specified action immediately on the specified
+ * interrupt(s).  For example, to unmask AUTOMASKED interrupt [0,0]:
+ * flags = (DATA_NONE|ACTION_UNMASK), index = 0, start = 0, count = 1.
+ *
+ * DATA_BOOL allows sparse support for the same on arrays of interrupts.
+ * For example, to mask interrupts [0,1] and [0,3] (but not [0,2]):
+ * flags = (DATA_BOOL|ACTION_MASK), index = 0, start = 1, count = 3,
+ * data = {1,0,1}
+ *
+ * DATA_EVENTFD binds the specified ACTION to the provided __s32 eventfd.
+ * A value of -1 can be used to either de-assign interrupts if already
+ * assigned or skip un-assigned interrupts.  For example, to set an eventfd
+ * to be trigger for interrupts [0,0] and [0,2]:
+ * flags = (DATA_EVENTFD|ACTION_TRIGGER), index = 0, start = 0, count = 3,
+ * data = {fd1, -1, fd2}
+ * If index [0,1] is previously set, two count = 1 ioctls calls would be
+ * required to set [0,0] and [0,2] without changing [0,1].
+ *
+ * Once a signaling mechanism is set, DATA_BOOL or DATA_NONE can be used
+ * with ACTION_TRIGGER to perform kernel level interrupt loopback testing
+ * from userspace (ie. simulate hardware triggering).
+ *
+ * Setting of an event triggering mechanism to userspace for ACTION_TRIGGER
+ * enables the interrupt index for the device.  Individual subindex interrupts
+ * can be disabled using the -1 value for DATA_EVENTFD or the index can be
+ * disabled as a whole with: flags = (DATA_NONE|ACTION_TRIGGER), count = 0.
+ *
+ * Note that ACTION_[UN]MASK specify user->kernel signaling (irqfds) while
+ * ACTION_TRIGGER specifies kernel->user signaling.
+ */
+struct vfio_irq_set {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IRQ_SET_DATA_NONE		(1 << 0) /* Data not present */
+#define VFIO_IRQ_SET_DATA_BOOL		(1 << 1) /* Data is bool (u8) */
+#define VFIO_IRQ_SET_DATA_EVENTFD	(1 << 2) /* Data is eventfd (s32) */
+#define VFIO_IRQ_SET_ACTION_MASK	(1 << 3) /* Mask interrupt */
+#define VFIO_IRQ_SET_ACTION_UNMASK	(1 << 4) /* Unmask interrupt */
+#define VFIO_IRQ_SET_ACTION_TRIGGER	(1 << 5) /* Trigger interrupt */
+	__u32	index;
+	__u32	start;
+	__u32	count;
+	__u8	data[];
+};
+#define VFIO_DEVICE_SET_IRQS		_IO(VFIO_TYPE, VFIO_BASE + 10)
+
+#define VFIO_IRQ_SET_DATA_TYPE_MASK	(VFIO_IRQ_SET_DATA_NONE | \
+					 VFIO_IRQ_SET_DATA_BOOL | \
+					 VFIO_IRQ_SET_DATA_EVENTFD)
+#define VFIO_IRQ_SET_ACTION_TYPE_MASK	(VFIO_IRQ_SET_ACTION_MASK | \
+					 VFIO_IRQ_SET_ACTION_UNMASK | \
+					 VFIO_IRQ_SET_ACTION_TRIGGER)
+/**
+ * VFIO_DEVICE_RESET - _IO(VFIO_TYPE, VFIO_BASE + 11)
+ *
+ * Reset a device.
+ */
+#define VFIO_DEVICE_RESET		_IO(VFIO_TYPE, VFIO_BASE + 11)
+
+/*
+ * The VFIO-PCI bus driver makes use of the following fixed region and
+ * IRQ index mapping.  Unimplemented regions return a size of zero.
+ * Unimplemented IRQ types return a count of zero.
+ */
+
+enum {
+	VFIO_PCI_BAR0_REGION_INDEX,
+	VFIO_PCI_BAR1_REGION_INDEX,
+	VFIO_PCI_BAR2_REGION_INDEX,
+	VFIO_PCI_BAR3_REGION_INDEX,
+	VFIO_PCI_BAR4_REGION_INDEX,
+	VFIO_PCI_BAR5_REGION_INDEX,
+	VFIO_PCI_ROM_REGION_INDEX,
+	VFIO_PCI_CONFIG_REGION_INDEX,
+	VFIO_PCI_NUM_REGIONS
+};
+
+enum {
+	VFIO_PCI_INTX_IRQ_INDEX,
+	VFIO_PCI_MSI_IRQ_INDEX,
+	VFIO_PCI_MSIX_IRQ_INDEX,
+	VFIO_PCI_NUM_IRQS
+};
+
+/* -------- API for Type1 VFIO IOMMU -------- */
+
+/**
+ * VFIO_IOMMU_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 12, struct vfio_iommu_info)
+ *
+ * Retrieve information about the IOMMU object. Fills in provided
+ * struct vfio_iommu_info. Caller sets argsz.
+ *
+ * XXX Should we do these by CHECK_EXTENSION too?
+ */
+struct vfio_iommu_type1_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
+	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+};
+
+#define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
+
+/**
+ * VFIO_IOMMU_MAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 13, struct vfio_dma_map)
+ *
+ * Map process virtual addresses to IO virtual addresses using the
+ * provided struct vfio_dma_map. Caller sets argsz. READ &/ WRITE required.
+ */
+struct vfio_iommu_type1_dma_map {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_DMA_MAP_FLAG_READ (1 << 0)		/* readable from device */
+#define VFIO_DMA_MAP_FLAG_WRITE (1 << 1)	/* writable from device */
+	__u64	vaddr;				/* Process virtual address */
+	__u64	iova;				/* IO virtual address */
+	__u64	size;				/* Size of mapping (bytes) */
+};
+
+#define VFIO_IOMMU_MAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 13)
+
+/**
+ * VFIO_IOMMU_UNMAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 14, struct vfio_dma_unmap)
+ *
+ * Unmap IO virtual addresses using the provided struct vfio_dma_unmap.
+ * Caller sets argsz.
+ */
+struct vfio_iommu_type1_dma_unmap {
+	__u32	argsz;
+	__u32	flags;
+	__u64	iova;				/* IO virtual address */
+	__u64	size;				/* Size of mapping (bytes) */
+};
+
+#define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14)
+
+#endif /* VFIO_H */
diff --git a/linux-user/alpha/target_signal.h b/linux-user/alpha/target_signal.h
index 94f15f612fe..d3822da60e3 100644
--- a/linux-user/alpha/target_signal.h
+++ b/linux-user/alpha/target_signal.h
@@ -6,9 +6,10 @@
 /* this struct defines a stack used during syscall handling */
 
 typedef struct target_sigaltstack {
-	abi_ulong ss_sp;
-	abi_long ss_flags;
-	abi_ulong ss_size;
+    abi_ulong ss_sp;
+    int32_t ss_flags;
+    int32_t dummy;
+    abi_ulong ss_size;
 } target_stack_t;
 
 
diff --git a/linux-user/arm/nwfpe/double_cpdo.c b/linux-user/arm/nwfpe/double_cpdo.c
index 8e9b28fa687..41c28f30969 100644
--- a/linux-user/arm/nwfpe/double_cpdo.c
+++ b/linux-user/arm/nwfpe/double_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 float64 float64_exp(float64 Fm);
diff --git a/linux-user/arm/nwfpe/extended_cpdo.c b/linux-user/arm/nwfpe/extended_cpdo.c
index 880ce030a54..48eca3b4a65 100644
--- a/linux-user/arm/nwfpe/extended_cpdo.c
+++ b/linux-user/arm/nwfpe/extended_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 floatx80 floatx80_exp(floatx80 Fm);
diff --git a/linux-user/arm/nwfpe/fpa11.h b/linux-user/arm/nwfpe/fpa11.h
index 002b3cbb826..bb9ac6554ee 100644
--- a/linux-user/arm/nwfpe/fpa11.h
+++ b/linux-user/arm/nwfpe/fpa11.h
@@ -43,7 +43,7 @@ extern CPUARMState *user_registers;
 
 /* includes */
 #include "fpsr.h"		/* FP control and status register definitions */
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define		typeNone		0x00
 #define		typeSingle		0x01
diff --git a/linux-user/arm/nwfpe/fpa11_cpdt.c b/linux-user/arm/nwfpe/fpa11_cpdt.c
index 3e7a938253a..007a3d65052 100644
--- a/linux-user/arm/nwfpe/fpa11_cpdt.c
+++ b/linux-user/arm/nwfpe/fpa11_cpdt.c
@@ -20,7 +20,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 //#include "fpmodule.h"
 //#include "fpmodule.inl"
diff --git a/linux-user/arm/nwfpe/fpa11_cprt.c b/linux-user/arm/nwfpe/fpa11_cprt.c
index 801189798bb..7be93fa54fd 100644
--- a/linux-user/arm/nwfpe/fpa11_cprt.c
+++ b/linux-user/arm/nwfpe/fpa11_cprt.c
@@ -20,7 +20,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 #include "fpa11.inl"
 //#include "fpmodule.h"
diff --git a/linux-user/arm/nwfpe/fpopcode.c b/linux-user/arm/nwfpe/fpopcode.c
index 82ac92f0cea..0dc5c9cd5d2 100644
--- a/linux-user/arm/nwfpe/fpopcode.c
+++ b/linux-user/arm/nwfpe/fpopcode.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 #include "fpsr.h"
 //#include "fpmodule.h"
diff --git a/linux-user/arm/nwfpe/single_cpdo.c b/linux-user/arm/nwfpe/single_cpdo.c
index 26168e2201d..2bfb359eb6a 100644
--- a/linux-user/arm/nwfpe/single_cpdo.c
+++ b/linux-user/arm/nwfpe/single_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 float32 float32_exp(float32 Fm);
diff --git a/linux-user/arm/syscall_nr.h b/linux-user/arm/syscall_nr.h
index 5356395659a..42d68550dc0 100644
--- a/linux-user/arm/syscall_nr.h
+++ b/linux-user/arm/syscall_nr.h
@@ -182,8 +182,8 @@
 #define TARGET_NR_rt_sigtimedwait		(177)
 #define TARGET_NR_rt_sigqueueinfo		(178)
 #define TARGET_NR_rt_sigsuspend		(179)
-#define TARGET_NR_pread			(180)
-#define TARGET_NR_pwrite			(181)
+#define TARGET_NR_pread64                       (180)
+#define TARGET_NR_pwrite64                      (181)
 #define TARGET_NR_chown			(182)
 #define TARGET_NR_getcwd			(183)
 #define TARGET_NR_capget			(184)
diff --git a/linux-user/cris/syscall.h b/linux-user/cris/syscall.h
index 24f92ba4212..50e50b4f3f2 100644
--- a/linux-user/cris/syscall.h
+++ b/linux-user/cris/syscall.h
@@ -1,3 +1,6 @@
+#ifndef CRIS_SYSCALL_H
+#define CRIS_SYSCALL_H 1
+
 
 #define UNAME_MACHINE "cris"
 
@@ -34,3 +37,5 @@ struct target_pt_regs {
         unsigned long exs;
         unsigned long eda;
 };
+
+#endif
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 819fdd515a5..89db49ccaac 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -14,7 +14,7 @@
 #include <time.h>
 
 #include "qemu.h"
-#include "disas.h"
+#include "disas/disas.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
@@ -2442,7 +2442,7 @@ static void fill_prstatus(struct target_elf_prstatus *prstatus,
 
 static int fill_psinfo(struct target_elf_prpsinfo *psinfo, const TaskState *ts)
 {
-    char *filename, *base_filename;
+    char *base_filename;
     unsigned int i, len;
 
     (void) memset(psinfo, 0, sizeof (*psinfo));
@@ -2464,13 +2464,15 @@ static int fill_psinfo(struct target_elf_prpsinfo *psinfo, const TaskState *ts)
     psinfo->pr_uid = getuid();
     psinfo->pr_gid = getgid();
 
-    filename = strdup(ts->bprm->filename);
-    base_filename = strdup(basename(filename));
+    base_filename = g_path_get_basename(ts->bprm->filename);
+    /*
+     * Using strncpy here is fine: at max-length,
+     * this field is not NUL-terminated.
+     */
     (void) strncpy(psinfo->pr_fname, base_filename,
                    sizeof(psinfo->pr_fname));
-    free(base_filename);
-    free(filename);
 
+    g_free(base_filename);
     bswap_psinfo(psinfo);
     return (0);
 }
diff --git a/linux-user/i386/syscall_nr.h b/linux-user/i386/syscall_nr.h
index 74abfcacb41..f0803050d8d 100644
--- a/linux-user/i386/syscall_nr.h
+++ b/linux-user/i386/syscall_nr.h
@@ -182,8 +182,8 @@
 #define TARGET_NR_rt_sigtimedwait	177
 #define TARGET_NR_rt_sigqueueinfo	178
 #define TARGET_NR_rt_sigsuspend	179
-#define TARGET_NR_pread		180
-#define TARGET_NR_pwrite		181
+#define TARGET_NR_pread64               180
+#define TARGET_NR_pwrite64              181
 #define TARGET_NR_chown		182
 #define TARGET_NR_getcwd		183
 #define TARGET_NR_capget		184
diff --git a/linux-user/linuxload.c b/linux-user/linuxload.c
index b47025f08a7..381ab8914be 100644
--- a/linux-user/linuxload.c
+++ b/linux-user/linuxload.c
@@ -140,8 +140,9 @@ int loader_exec(const char * filename, char ** argv, char ** envp,
     bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
     memset(bprm->page, 0, sizeof(bprm->page));
     retval = open(filename, O_RDONLY);
-    if (retval < 0)
-        return retval;
+    if (retval < 0) {
+        return -errno;
+    }
     bprm->fd = retval;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
@@ -165,8 +166,7 @@ int loader_exec(const char * filename, char ** argv, char ** envp,
             retval = load_flt_binary(bprm,regs,infop);
 #endif
         } else {
-            fprintf(stderr, "Unknown binary format\n");
-            return -1;
+            return -ENOEXEC;
         }
     }
 
diff --git a/linux-user/main.c b/linux-user/main.c
index 1a1c661ee4f..0181bc21125 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -28,11 +28,11 @@
 
 #include "qemu.h"
 #include "qemu-common.h"
-#include "cache-utils.h"
+#include "qemu/cache-utils.h"
 #include "cpu.h"
 #include "tcg.h"
-#include "qemu-timer.h"
-#include "envlist.h"
+#include "qemu/timer.h"
+#include "qemu/envlist.h"
 #include "elf.h"
 
 #define DEBUG_LOGFILE "/tmp/qemu.log"
@@ -57,7 +57,12 @@ int have_guest_base;
  * This way we will never overlap with our own libraries or binaries or stack
  * or anything else that QEMU maps.
  */
+# ifdef TARGET_MIPS
+/* MIPS only supports 31 bits of virtual address space for user space */
+unsigned long reserved_va = 0x77000000;
+# else
 unsigned long reserved_va = 0xf7000000;
+# endif
 #else
 unsigned long reserved_va;
 #endif
@@ -89,19 +94,6 @@ int cpu_get_pic_interrupt(CPUX86State *env)
 }
 #endif
 
-/* timers for rdtsc */
-
-#if 0
-
-static uint64_t emu_time;
-
-int64_t cpu_get_real_ticks(void)
-{
-    return emu_time++;
-}
-
-#endif
-
 #if defined(CONFIG_USE_NPTL)
 /***********************************************************/
 /* Helper routines for implementing atomic operations.  */
@@ -1127,6 +1119,11 @@ void cpu_loop (CPUSPARCState *env)
     while (1) {
         trapnr = cpu_sparc_exec (env);
 
+        /* Compute PSR before exposing state.  */
+        if (env->cc_op != CC_OP_FLAGS) {
+            cpu_get_psr(env);
+        }
+
         switch (trapnr) {
 #ifndef TARGET_SPARC64
         case 0x88:
@@ -2294,6 +2291,12 @@ done_syscall:
                 queue_signal(env, info.si_signo, &info);
             }
             break;
+        case EXCP_DSPDIS:
+            info.si_signo = TARGET_SIGILL;
+            info.si_errno = 0;
+            info.si_code = TARGET_ILL_ILLOPC;
+            queue_signal(env, info.si_signo, &info);
+            break;
         default:
             //        error:
             fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
@@ -2535,6 +2538,7 @@ void cpu_loop(CPUMBState *env)
         case EXCP_BREAK:
             /* Return address is 4 bytes after the call.  */
             env->regs[14] += 4;
+            env->sregs[SR_PC] = env->regs[14];
             ret = do_syscall(env, 
                              env->regs[12], 
                              env->regs[5], 
@@ -2545,7 +2549,6 @@ void cpu_loop(CPUMBState *env)
                              env->regs[10],
                              0, 0);
             env->regs[3] = ret;
-            env->sregs[SR_PC] = env->regs[14];
             break;
         case EXCP_HW_EXCP:
             env->regs[17] = env->sregs[SR_PC] + 4;
@@ -2935,71 +2938,115 @@ void cpu_loop(CPUAlphaState *env)
 #ifdef TARGET_S390X
 void cpu_loop(CPUS390XState *env)
 {
-    int trapnr;
+    int trapnr, n, sig;
     target_siginfo_t info;
+    target_ulong addr;
 
     while (1) {
-        trapnr = cpu_s390x_exec (env);
-
+        trapnr = cpu_s390x_exec(env);
         switch (trapnr) {
         case EXCP_INTERRUPT:
-            /* just indicate that signals should be handled asap */
+            /* Just indicate that signals should be handled asap.  */
             break;
-        case EXCP_DEBUG:
-            {
-                int sig;
 
-                sig = gdb_handlesig (env, TARGET_SIGTRAP);
-                if (sig) {
-                    info.si_signo = sig;
-                    info.si_errno = 0;
-                    info.si_code = TARGET_TRAP_BRKPT;
-                    queue_signal(env, info.si_signo, &info);
-                }
+        case EXCP_SVC:
+            n = env->int_svc_code;
+            if (!n) {
+                /* syscalls > 255 */
+                n = env->regs[1];
             }
+            env->psw.addr += env->int_svc_ilen;
+            env->regs[2] = do_syscall(env, n, env->regs[2], env->regs[3],
+                                      env->regs[4], env->regs[5],
+                                      env->regs[6], env->regs[7], 0, 0);
             break;
-        case EXCP_SVC:
-            {
-                int n = env->int_svc_code;
-                if (!n) {
-                    /* syscalls > 255 */
-                    n = env->regs[1];
-                }
-                env->psw.addr += env->int_svc_ilc;
-                env->regs[2] = do_syscall(env, n,
-                           env->regs[2],
-                           env->regs[3],
-                           env->regs[4],
-                           env->regs[5],
-                           env->regs[6],
-                           env->regs[7],
-                           0, 0);
+
+        case EXCP_DEBUG:
+            sig = gdb_handlesig(env, TARGET_SIGTRAP);
+            if (sig) {
+                n = TARGET_TRAP_BRKPT;
+                goto do_signal_pc;
             }
             break;
-        case EXCP_ADDR:
-            {
-                info.si_signo = SIGSEGV;
-                info.si_errno = 0;
+        case EXCP_PGM:
+            n = env->int_pgm_code;
+            switch (n) {
+            case PGM_OPERATION:
+            case PGM_PRIVILEGED:
+                sig = SIGILL;
+                n = TARGET_ILL_ILLOPC;
+                goto do_signal_pc;
+            case PGM_PROTECTION:
+            case PGM_ADDRESSING:
+                sig = SIGSEGV;
                 /* XXX: check env->error_code */
-                info.si_code = TARGET_SEGV_MAPERR;
-                info._sifields._sigfault._addr = env->__excp_addr;
-                queue_signal(env, info.si_signo, &info);
+                n = TARGET_SEGV_MAPERR;
+                addr = env->__excp_addr;
+                goto do_signal;
+            case PGM_EXECUTE:
+            case PGM_SPECIFICATION:
+            case PGM_SPECIAL_OP:
+            case PGM_OPERAND:
+            do_sigill_opn:
+                sig = SIGILL;
+                n = TARGET_ILL_ILLOPN;
+                goto do_signal_pc;
+
+            case PGM_FIXPT_OVERFLOW:
+                sig = SIGFPE;
+                n = TARGET_FPE_INTOVF;
+                goto do_signal_pc;
+            case PGM_FIXPT_DIVIDE:
+                sig = SIGFPE;
+                n = TARGET_FPE_INTDIV;
+                goto do_signal_pc;
+
+            case PGM_DATA:
+                n = (env->fpc >> 8) & 0xff;
+                if (n == 0xff) {
+                    /* compare-and-trap */
+                    goto do_sigill_opn;
+                } else {
+                    /* An IEEE exception, simulated or otherwise.  */
+                    if (n & 0x80) {
+                        n = TARGET_FPE_FLTINV;
+                    } else if (n & 0x40) {
+                        n = TARGET_FPE_FLTDIV;
+                    } else if (n & 0x20) {
+                        n = TARGET_FPE_FLTOVF;
+                    } else if (n & 0x10) {
+                        n = TARGET_FPE_FLTUND;
+                    } else if (n & 0x08) {
+                        n = TARGET_FPE_FLTRES;
+                    } else {
+                        /* ??? Quantum exception; BFP, DFP error.  */
+                        goto do_sigill_opn;
+                    }
+                    sig = SIGFPE;
+                    goto do_signal_pc;
+                }
+
+            default:
+                fprintf(stderr, "Unhandled program exception: %#x\n", n);
+                cpu_dump_state(env, stderr, fprintf, 0);
+                exit(1);
             }
             break;
-        case EXCP_SPEC:
-            {
-                fprintf(stderr,"specification exception insn 0x%08x%04x\n", ldl(env->psw.addr), lduw(env->psw.addr + 4));
-                info.si_signo = SIGILL;
-                info.si_errno = 0;
-                info.si_code = TARGET_ILL_ILLOPC;
-                info._sifields._sigfault._addr = env->__excp_addr;
-                queue_signal(env, info.si_signo, &info);
-            }
+
+        do_signal_pc:
+            addr = env->psw.addr;
+        do_signal:
+            info.si_signo = sig;
+            info.si_errno = 0;
+            info.si_code = n;
+            info._sifields._sigfault._addr = addr;
+            queue_signal(env, info.si_signo, &info);
             break;
+
         default:
-            printf ("Unhandled trap: 0x%x\n", trapnr);
+            fprintf(stderr, "Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(env, stderr, fprintf, 0);
-            exit (1);
+            exit(1);
         }
         process_pending_signals (env);
     }
@@ -3143,10 +3190,8 @@ static void handle_arg_cpu(const char *arg)
     cpu_model = strdup(arg);
     if (cpu_model == NULL || is_help_option(cpu_model)) {
         /* XXX: implement xxx_cpu_list for targets that still miss it */
-#if defined(cpu_list_id)
-        cpu_list_id(stdout, &fprintf, "");
-#elif defined(cpu_list)
-        cpu_list(stdout, &fprintf); /* deprecated */
+#if defined(cpu_list)
+        cpu_list(stdout, &fprintf);
 #endif
         exit(1);
     }
@@ -3584,7 +3629,7 @@ int main(int argc, char **argv, char **envp)
     ret = loader_exec(filename, target_argv, target_environ, regs,
         info, &bprm);
     if (ret != 0) {
-        printf("Error %d while loading %s\n", ret, filename);
+        printf("Error while loading %s: %s\n", filename, strerror(-ret));
         _exit(1);
     }
 
diff --git a/linux-user/microblaze/syscall.h b/linux-user/microblaze/syscall.h
index db1f98ae69a..c3e5c55b3d4 100644
--- a/linux-user/microblaze/syscall.h
+++ b/linux-user/microblaze/syscall.h
@@ -1,3 +1,7 @@
+#ifndef MICROBLAZE_SYSCALLS_H
+#define MICROBLAZE_SYSCALLS_H 1
+
+
 #define UNAME_MACHINE "microblaze"
 
 /* We use microblaze_reg_t to keep things similar to the kernel sources.  */
@@ -43,3 +47,5 @@ struct target_pt_regs {
         microblaze_reg_t fsr;
         uint32_t kernel_mode;
 };
+
+#endif
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 69b27d71463..31a220af816 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -11,14 +11,14 @@
 #include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
-#include "thunk.h"
+#include "exec/user/thunk.h"
 #include "syscall_defs.h"
 #include "syscall.h"
 #include "target_signal.h"
-#include "gdbstub.h"
-#include "qemu-queue.h"
+#include "exec/gdbstub.h"
+#include "qemu/queue.h"
 
 #if defined(CONFIG_USE_NPTL)
 #define THREAD __thread
@@ -217,7 +217,10 @@ unsigned long init_guest_space(unsigned long host_start,
                                unsigned long guest_start,
                                bool fixed);
 
-#include "qemu-log.h"
+#include "qemu/log.h"
+
+/* syscall.c */
+int host_to_target_waitstatus(int status);
 
 /* strace.c */
 void print_syscall(int num,
@@ -284,53 +287,39 @@ static inline int access_ok(int type, abi_ulong addr, abi_ulong size)
                             (type == VERIFY_READ) ? PAGE_READ : (PAGE_READ | PAGE_WRITE)) == 0;
 }
 
-/* NOTE __get_user and __put_user use host pointers and don't check access. */
-/* These are usually used to access struct data members once the
- * struct has been locked - usually with lock_user_struct().
- */
-#define __put_user(x, hptr)\
-({\
-    switch(sizeof(*hptr)) {\
-    case 1:\
-        *(uint8_t *)(hptr) = (uint8_t)(typeof(*hptr))(x);\
-        break;\
-    case 2:\
-        *(uint16_t *)(hptr) = tswap16((uint16_t)(typeof(*hptr))(x));\
-        break;\
-    case 4:\
-        *(uint32_t *)(hptr) = tswap32((uint32_t)(typeof(*hptr))(x));\
-        break;\
-    case 8:\
-        *(uint64_t *)(hptr) = tswap64((typeof(*hptr))(x));\
-        break;\
-    default:\
-        abort();\
-    }\
-    0;\
-})
-
-#define __get_user(x, hptr) \
-({\
-    switch(sizeof(*hptr)) {\
-    case 1:\
-        x = (typeof(*hptr))*(uint8_t *)(hptr);\
-        break;\
-    case 2:\
-        x = (typeof(*hptr))tswap16(*(uint16_t *)(hptr));\
-        break;\
-    case 4:\
-        x = (typeof(*hptr))tswap32(*(uint32_t *)(hptr));\
-        break;\
-    case 8:\
-        x = (typeof(*hptr))tswap64(*(uint64_t *)(hptr));\
-        break;\
-    default:\
-        /* avoid warning */\
-        x = 0;\
-        abort();\
-    }\
-    0;\
-})
+/* NOTE __get_user and __put_user use host pointers and don't check access.
+   These are usually used to access struct data members once the struct has
+   been locked - usually with lock_user_struct.  */
+
+/* Tricky points:
+   - Use __builtin_choose_expr to avoid type promotion from ?:,
+   - Invalid sizes result in a compile time error stemming from
+     the fact that abort has no parameters.
+   - It's easier to use the endian-specific unaligned load/store
+     functions than host-endian unaligned load/store plus tswapN.  */
+
+#define __put_user_e(x, hptr, e)                                        \
+  (__builtin_choose_expr(sizeof(*(hptr)) == 1, stb_p,                   \
+   __builtin_choose_expr(sizeof(*(hptr)) == 2, stw_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 4, stl_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 8, stq_##e##_p, abort))))   \
+     ((hptr), (x)), 0)
+
+#define __get_user_e(x, hptr, e)                                        \
+  ((x) =                                                                \
+   __builtin_choose_expr(sizeof(*(hptr)) == 1, ldub_p,                  \
+   __builtin_choose_expr(sizeof(*(hptr)) == 2, lduw_##e##_p,            \
+   __builtin_choose_expr(sizeof(*(hptr)) == 4, ldl_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 8, ldq_##e##_p, abort))))   \
+     (hptr), 0)
+
+#ifdef TARGET_WORDS_BIGENDIAN
+# define __put_user(x, hptr)  __put_user_e(x, hptr, be)
+# define __get_user(x, hptr)  __get_user_e(x, hptr, be)
+#else
+# define __put_user(x, hptr)  __put_user_e(x, hptr, le)
+# define __get_user(x, hptr)  __get_user_e(x, hptr, le)
+#endif
 
 /* put_user()/get_user() take a guest address and check access */
 /* These are usually used to access an atomic data type, such as an int,
diff --git a/linux-user/s390x/syscall.h b/linux-user/s390x/syscall.h
index c2ea151ea52..e4603b79c3d 100644
--- a/linux-user/s390x/syscall.h
+++ b/linux-user/s390x/syscall.h
@@ -16,7 +16,7 @@ struct target_pt_regs {
     target_psw_t psw;
     abi_ulong gprs[TARGET_NUM_GPRS];
     abi_ulong orig_gpr2;
-    unsigned short ilc;
+    unsigned short ilen;
     unsigned short trap;
 };
 
diff --git a/linux-user/signal.c b/linux-user/signal.c
index 78691473fa9..67c23118a04 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -202,46 +202,67 @@ void target_to_host_old_sigset(sigset_t *sigset,
 static inline void host_to_target_siginfo_noswap(target_siginfo_t *tinfo,
                                                  const siginfo_t *info)
 {
-    int sig;
-    sig = host_to_target_signal(info->si_signo);
+    int sig = host_to_target_signal(info->si_signo);
     tinfo->si_signo = sig;
     tinfo->si_errno = 0;
     tinfo->si_code = info->si_code;
-    if (sig == SIGILL || sig == SIGFPE || sig == SIGSEGV ||
-        sig == SIGBUS || sig == SIGTRAP) {
-        /* should never come here, but who knows. The information for
-           the target is irrelevant */
+
+    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
+        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
+        /* Should never come here, but who knows. The information for
+           the target is irrelevant.  */
         tinfo->_sifields._sigfault._addr = 0;
-    } else if (sig == SIGIO) {
+    } else if (sig == TARGET_SIGIO) {
+        tinfo->_sifields._sigpoll._band = info->si_band;
 	tinfo->_sifields._sigpoll._fd = info->si_fd;
+    } else if (sig == TARGET_SIGCHLD) {
+        tinfo->_sifields._sigchld._pid = info->si_pid;
+        tinfo->_sifields._sigchld._uid = info->si_uid;
+        tinfo->_sifields._sigchld._status
+            = host_to_target_waitstatus(info->si_status);
+        tinfo->_sifields._sigchld._utime = info->si_utime;
+        tinfo->_sifields._sigchld._stime = info->si_stime;
     } else if (sig >= TARGET_SIGRTMIN) {
         tinfo->_sifields._rt._pid = info->si_pid;
         tinfo->_sifields._rt._uid = info->si_uid;
         /* XXX: potential problem if 64 bit */
-        tinfo->_sifields._rt._sigval.sival_ptr =
-            (abi_ulong)(unsigned long)info->si_value.sival_ptr;
+        tinfo->_sifields._rt._sigval.sival_ptr
+            = (abi_ulong)(unsigned long)info->si_value.sival_ptr;
     }
 }
 
 static void tswap_siginfo(target_siginfo_t *tinfo,
                           const target_siginfo_t *info)
 {
-    int sig;
-    sig = info->si_signo;
+    int sig = info->si_signo;
     tinfo->si_signo = tswap32(sig);
     tinfo->si_errno = tswap32(info->si_errno);
     tinfo->si_code = tswap32(info->si_code);
-    if (sig == SIGILL || sig == SIGFPE || sig == SIGSEGV ||
-        sig == SIGBUS || sig == SIGTRAP) {
-        tinfo->_sifields._sigfault._addr =
-            tswapal(info->_sifields._sigfault._addr);
-    } else if (sig == SIGIO) {
-	tinfo->_sifields._sigpoll._fd = tswap32(info->_sifields._sigpoll._fd);
+
+    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
+        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
+        tinfo->_sifields._sigfault._addr
+            = tswapal(info->_sifields._sigfault._addr);
+    } else if (sig == TARGET_SIGIO) {
+        tinfo->_sifields._sigpoll._band
+            = tswap32(info->_sifields._sigpoll._band);
+        tinfo->_sifields._sigpoll._fd = tswap32(info->_sifields._sigpoll._fd);
+    } else if (sig == TARGET_SIGCHLD) {
+        tinfo->_sifields._sigchld._pid
+            = tswap32(info->_sifields._sigchld._pid);
+        tinfo->_sifields._sigchld._uid
+            = tswap32(info->_sifields._sigchld._uid);
+        tinfo->_sifields._sigchld._status
+            = tswap32(info->_sifields._sigchld._status);
+        tinfo->_sifields._sigchld._utime
+            = tswapal(info->_sifields._sigchld._utime);
+        tinfo->_sifields._sigchld._stime
+            = tswapal(info->_sifields._sigchld._stime);
     } else if (sig >= TARGET_SIGRTMIN) {
         tinfo->_sifields._rt._pid = tswap32(info->_sifields._rt._pid);
         tinfo->_sifields._rt._uid = tswap32(info->_sifields._rt._uid);
-        tinfo->_sifields._rt._sigval.sival_ptr =
-            tswapal(info->_sifields._rt._sigval.sival_ptr);
+        tinfo->_sifields._rt._sigval.sival_ptr
+            = tswapal(info->_sifields._rt._sigval.sival_ptr);
     }
 }
 
@@ -586,28 +607,22 @@ int do_sigaction(int sig, const struct target_sigaction *act,
             sig, act, oact);
 #endif
     if (oact) {
-        oact->_sa_handler = tswapal(k->_sa_handler);
-#if defined(TARGET_MIPS) || defined (TARGET_ALPHA)
-        oact->sa_flags = bswap32(k->sa_flags);
-#else
-        oact->sa_flags = tswapal(k->sa_flags);
-#endif
+        __put_user(k->_sa_handler, &oact->_sa_handler);
+        __put_user(k->sa_flags, &oact->sa_flags);
 #if !defined(TARGET_MIPS)
-        oact->sa_restorer = tswapal(k->sa_restorer);
+        __put_user(k->sa_restorer, &oact->sa_restorer);
 #endif
+        /* Not swapped.  */
         oact->sa_mask = k->sa_mask;
     }
     if (act) {
         /* FIXME: This is not threadsafe.  */
-        k->_sa_handler = tswapal(act->_sa_handler);
-#if defined(TARGET_MIPS) || defined (TARGET_ALPHA)
-        k->sa_flags = bswap32(act->sa_flags);
-#else
-        k->sa_flags = tswapal(act->sa_flags);
-#endif
+        __get_user(k->_sa_handler, &act->_sa_handler);
+        __get_user(k->sa_flags, &act->sa_flags);
 #if !defined(TARGET_MIPS)
-        k->sa_restorer = tswapal(act->sa_restorer);
+        __get_user(k->sa_restorer, &act->sa_restorer);
 #endif
+        /* To be swapped in target_to_host_sigset.  */
         k->sa_mask = act->sa_mask;
 
         /* we update the host linux signal state */
@@ -2762,7 +2777,6 @@ static void setup_frame(int sig, struct target_sigaction * ka,
 give_sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
     force_sig(TARGET_SIGSEGV/*, current*/);
-    return;
 }
 
 long do_sigreturn(CPUMIPSState *regs)
@@ -2871,7 +2885,6 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
 give_sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
     force_sig(TARGET_SIGSEGV/*, current*/);
-    return;
 }
 
 long do_rt_sigreturn(CPUMIPSState *env)
@@ -4565,7 +4578,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
     signal = current_exec_domain_sig(sig);
 
-    err |= __put_user(h2g(ka->_sa_handler), &sc->handler);
+    err |= __put_user(ka->_sa_handler, &sc->handler);
     err |= __put_user(set->sig[0], &sc->oldmask);
 #if defined(TARGET_PPC64)
     err |= __put_user(set->sig[0] >> 32, &sc->_unused[3]);
@@ -4587,7 +4600,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
     /* Create a stack frame for the caller of the handler.  */
     newsp = frame_addr - SIGNAL_FRAMESIZE;
-    err |= __put_user(env->gpr[1], (target_ulong *)(uintptr_t) newsp);
+    err |= put_user(env->gpr[1], newsp, target_ulong);
 
     if (err)
         goto sigsegv;
@@ -4595,7 +4608,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     /* Set up registers for signal handler.  */
     env->gpr[1] = newsp;
     env->gpr[3] = signal;
-    env->gpr[4] = (target_ulong) h2g(sc);
+    env->gpr[4] = frame_addr + offsetof(struct target_sigframe, sctx);
     env->nip = (target_ulong) ka->_sa_handler;
     /* Signal handlers are entered in big-endian mode.  */
     env->msr &= ~MSR_LE;
diff --git a/linux-user/sparc/syscall_nr.h b/linux-user/sparc/syscall_nr.h
index f201f9f7880..061711cc033 100644
--- a/linux-user/sparc/syscall_nr.h
+++ b/linux-user/sparc/syscall_nr.h
@@ -62,8 +62,8 @@
 #define TARGET_NR_getpagesize         64 /* Common                                      */
 #define TARGET_NR_msync               65 /* Common in newer 1.3.x revs...               */
 #define TARGET_NR_vfork               66 /* Common                                      */
-#define TARGET_NR_pread               67 /* Linux Specific                              */
-#define TARGET_NR_pwrite              68 /* Linux Specific                              */
+#define TARGET_NR_pread64             67 /* Linux Specific */
+#define TARGET_NR_pwrite64            68 /* Linux Specific */
 #define TARGET_NR_geteuid32           69 /* Linux sparc32, sbrk under SunOS             */
 #define TARGET_NR_getegid32           70 /* Linux sparc32, sstk under SunOS             */
 #define TARGET_NR_mmap                71 /* Common                                      */
diff --git a/linux-user/strace.list b/linux-user/strace.list
index af3c6a0cced..08f115d8436 100644
--- a/linux-user/strace.list
+++ b/linux-user/strace.list
@@ -972,9 +972,6 @@
 #ifdef TARGET_NR_prctl
 { TARGET_NR_prctl, "prctl" , NULL, NULL, NULL },
 #endif
-#ifdef TARGET_NR_pread
-{ TARGET_NR_pread, "pread" , NULL, NULL, NULL },
-#endif
 #ifdef TARGET_NR_pread64
 { TARGET_NR_pread64, "pread64" , NULL, NULL, NULL },
 #endif
@@ -993,9 +990,6 @@
 #ifdef TARGET_NR_putpmsg
 { TARGET_NR_putpmsg, "putpmsg" , NULL, NULL, NULL },
 #endif
-#ifdef TARGET_NR_pwrite
-{ TARGET_NR_pwrite, "pwrite" , NULL, NULL, NULL },
-#endif
 #ifdef TARGET_NR_pwrite64
 { TARGET_NR_pwrite64, "pwrite64" , NULL, NULL, NULL },
 #endif
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 6257a04d0a9..693e66fc4f3 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -28,6 +28,7 @@
 #include <fcntl.h>
 #include <time.h>
 #include <limits.h>
+#include <grp.h>
 #include <sys/types.h>
 #include <sys/ipc.h>
 #include <sys/msg.h>
@@ -35,6 +36,9 @@
 #include <sys/time.h>
 #include <sys/stat.h>
 #include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/fsuid.h>
+#include <sys/personality.h>
 #include <sys/prctl.h>
 #include <sys/resource.h>
 #include <sys/mman.h>
@@ -72,7 +76,7 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <sys/epoll.h>
 #endif
 #ifdef CONFIG_ATTR
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #endif
 
 #define termios host_termios
@@ -580,19 +584,18 @@ _syscall4(int, sys_prlimit64, pid_t, pid, int, resource,
           struct host_rlimit64 *, old_limit)
 #endif
 
-extern int personality(int);
-extern int flock(int, int);
-extern int setfsuid(int);
-extern int setfsgid(int);
-extern int setgroups(int, gid_t *);
-
 /* ARM EABI and MIPS expect 64bit types aligned even on pairs or registers */
-#ifdef TARGET_ARM 
+#ifdef TARGET_ARM
 static inline int regpairs_aligned(void *cpu_env) {
     return ((((CPUARMState *)cpu_env)->eabi) == 1) ;
 }
 #elif defined(TARGET_MIPS)
 static inline int regpairs_aligned(void *cpu_env) { return 1; }
+#elif defined(TARGET_PPC) && !defined(TARGET_PPC64)
+/* SysV AVI for PPC32 expects 64bit parameters to be passed on odd/even pairs
+ * of registers which translates to the same as ARM/MIPS, because we start with
+ * r3 as arg1 */
+static inline int regpairs_aligned(void *cpu_env) { return 1; }
 #else
 static inline int regpairs_aligned(void *cpu_env) { return 0; }
 #endif
@@ -1744,55 +1747,96 @@ static abi_long do_getsockopt(int sockfd, int level, int optname,
     return ret;
 }
 
-/* FIXME
- * lock_iovec()/unlock_iovec() have a return code of 0 for success where
- * other lock functions have a return code of 0 for failure.
- */
-static abi_long lock_iovec(int type, struct iovec *vec, abi_ulong target_addr,
-                           int count, int copy)
+static struct iovec *lock_iovec(int type, abi_ulong target_addr,
+                                int count, int copy)
 {
     struct target_iovec *target_vec;
-    abi_ulong base;
+    struct iovec *vec;
+    abi_ulong total_len, max_len;
     int i;
 
-    target_vec = lock_user(VERIFY_READ, target_addr, count * sizeof(struct target_iovec), 1);
-    if (!target_vec)
-        return -TARGET_EFAULT;
-    for(i = 0;i < count; i++) {
-        base = tswapal(target_vec[i].iov_base);
-        vec[i].iov_len = tswapal(target_vec[i].iov_len);
-        if (vec[i].iov_len != 0) {
-            vec[i].iov_base = lock_user(type, base, vec[i].iov_len, copy);
-            /* Don't check lock_user return value. We must call writev even
-               if a element has invalid base address. */
+    if (count == 0) {
+        errno = 0;
+        return NULL;
+    }
+    if (count > IOV_MAX) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    vec = calloc(count, sizeof(struct iovec));
+    if (vec == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    target_vec = lock_user(VERIFY_READ, target_addr,
+                           count * sizeof(struct target_iovec), 1);
+    if (target_vec == NULL) {
+        errno = EFAULT;
+        goto fail2;
+    }
+
+    /* ??? If host page size > target page size, this will result in a
+       value larger than what we can actually support.  */
+    max_len = 0x7fffffff & TARGET_PAGE_MASK;
+    total_len = 0;
+
+    for (i = 0; i < count; i++) {
+        abi_ulong base = tswapal(target_vec[i].iov_base);
+        abi_long len = tswapal(target_vec[i].iov_len);
+
+        if (len < 0) {
+            errno = EINVAL;
+            goto fail;
+        } else if (len == 0) {
+            /* Zero length pointer is ignored.  */
+            vec[i].iov_base = 0;
         } else {
-            /* zero length pointer is ignored */
-            vec[i].iov_base = NULL;
+            vec[i].iov_base = lock_user(type, base, len, copy);
+            if (!vec[i].iov_base) {
+                errno = EFAULT;
+                goto fail;
+            }
+            if (len > max_len - total_len) {
+                len = max_len - total_len;
+            }
         }
+        vec[i].iov_len = len;
+        total_len += len;
     }
-    unlock_user (target_vec, target_addr, 0);
-    return 0;
+
+    unlock_user(target_vec, target_addr, 0);
+    return vec;
+
+ fail:
+    free(vec);
+ fail2:
+    unlock_user(target_vec, target_addr, 0);
+    return NULL;
 }
 
-static abi_long unlock_iovec(struct iovec *vec, abi_ulong target_addr,
-                             int count, int copy)
+static void unlock_iovec(struct iovec *vec, abi_ulong target_addr,
+                         int count, int copy)
 {
     struct target_iovec *target_vec;
-    abi_ulong base;
     int i;
 
-    target_vec = lock_user(VERIFY_READ, target_addr, count * sizeof(struct target_iovec), 1);
-    if (!target_vec)
-        return -TARGET_EFAULT;
-    for(i = 0;i < count; i++) {
-        if (target_vec[i].iov_base) {
-            base = tswapal(target_vec[i].iov_base);
+    target_vec = lock_user(VERIFY_READ, target_addr,
+                           count * sizeof(struct target_iovec), 1);
+    if (target_vec) {
+        for (i = 0; i < count; i++) {
+            abi_ulong base = tswapal(target_vec[i].iov_base);
+            abi_long len = tswapal(target_vec[i].iov_base);
+            if (len < 0) {
+                break;
+            }
             unlock_user(vec[i].iov_base, base, copy ? vec[i].iov_len : 0);
         }
+        unlock_user(target_vec, target_addr, 0);
     }
-    unlock_user (target_vec, target_addr, 0);
 
-    return 0;
+    free(vec);
 }
 
 /* do_socket() Must return target values and target errnos. */
@@ -1888,8 +1932,7 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
         ret = target_to_host_sockaddr(msg.msg_name, tswapal(msgp->msg_name),
                                 msg.msg_namelen);
         if (ret) {
-            unlock_user_struct(msgp, target_msg, send ? 0 : 1);
-            return ret;
+            goto out2;
         }
     } else {
         msg.msg_name = NULL;
@@ -1900,9 +1943,13 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
     msg.msg_flags = tswap32(msgp->msg_flags);
 
     count = tswapal(msgp->msg_iovlen);
-    vec = alloca(count * sizeof(struct iovec));
     target_vec = tswapal(msgp->msg_iov);
-    lock_iovec(send ? VERIFY_READ : VERIFY_WRITE, vec, target_vec, count, send);
+    vec = lock_iovec(send ? VERIFY_READ : VERIFY_WRITE,
+                     target_vec, count, send);
+    if (vec == NULL) {
+        ret = -host_to_target_errno(errno);
+        goto out2;
+    }
     msg.msg_iovlen = count;
     msg.msg_iov = vec;
 
@@ -1932,6 +1979,7 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
 
 out:
     unlock_iovec(vec, target_vec, count, !send);
+out2:
     unlock_user_struct(msgp, target_msg, send ? 0 : 1);
     return ret;
 }
@@ -3628,9 +3676,7 @@ static abi_long do_ioctl_dm(const IOCTLEntry *ie, uint8_t *buf_temp, int fd,
         unlock_user(argptr, arg, target_size);
     }
 out:
-    if (big_buf) {
-        free(big_buf);
-    }
+    g_free(big_buf);
     return ret;
 }
 
@@ -4466,6 +4512,16 @@ static int target_to_host_fcntl_cmd(int cmd)
     return -TARGET_EINVAL;
 }
 
+#define TRANSTBL_CONVERT(a) { -1, TARGET_##a, -1, a }
+static const bitmask_transtbl flock_tbl[] = {
+    TRANSTBL_CONVERT(F_RDLCK),
+    TRANSTBL_CONVERT(F_WRLCK),
+    TRANSTBL_CONVERT(F_UNLCK),
+    TRANSTBL_CONVERT(F_EXLCK),
+    TRANSTBL_CONVERT(F_SHLCK),
+    { 0, 0, 0, 0 }
+};
+
 static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
 {
     struct flock fl;
@@ -4482,7 +4538,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_GETLK:
         if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
             return -TARGET_EFAULT;
-        fl.l_type = tswap16(target_fl->l_type);
+        fl.l_type =
+                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
         fl.l_whence = tswap16(target_fl->l_whence);
         fl.l_start = tswapal(target_fl->l_start);
         fl.l_len = tswapal(target_fl->l_len);
@@ -4492,7 +4549,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
         if (ret == 0) {
             if (!lock_user_struct(VERIFY_WRITE, target_fl, arg, 0))
                 return -TARGET_EFAULT;
-            target_fl->l_type = tswap16(fl.l_type);
+            target_fl->l_type =
+                          host_to_target_bitmask(tswap16(fl.l_type), flock_tbl);
             target_fl->l_whence = tswap16(fl.l_whence);
             target_fl->l_start = tswapal(fl.l_start);
             target_fl->l_len = tswapal(fl.l_len);
@@ -4505,7 +4563,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_SETLKW:
         if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
             return -TARGET_EFAULT;
-        fl.l_type = tswap16(target_fl->l_type);
+        fl.l_type =
+                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
         fl.l_whence = tswap16(target_fl->l_whence);
         fl.l_start = tswapal(target_fl->l_start);
         fl.l_len = tswapal(target_fl->l_len);
@@ -4517,7 +4576,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_GETLK64:
         if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
             return -TARGET_EFAULT;
-        fl64.l_type = tswap16(target_fl64->l_type) >> 1;
+        fl64.l_type =
+           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
         fl64.l_whence = tswap16(target_fl64->l_whence);
         fl64.l_start = tswap64(target_fl64->l_start);
         fl64.l_len = tswap64(target_fl64->l_len);
@@ -4527,7 +4587,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
         if (ret == 0) {
             if (!lock_user_struct(VERIFY_WRITE, target_fl64, arg, 0))
                 return -TARGET_EFAULT;
-            target_fl64->l_type = tswap16(fl64.l_type) >> 1;
+            target_fl64->l_type =
+                   host_to_target_bitmask(tswap16(fl64.l_type), flock_tbl) >> 1;
             target_fl64->l_whence = tswap16(fl64.l_whence);
             target_fl64->l_start = tswap64(fl64.l_start);
             target_fl64->l_len = tswap64(fl64.l_len);
@@ -4539,7 +4600,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_SETLKW64:
         if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
             return -TARGET_EFAULT;
-        fl64.l_type = tswap16(target_fl64->l_type) >> 1;
+        fl64.l_type =
+           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
         fl64.l_whence = tswap16(target_fl64->l_whence);
         fl64.l_start = tswap64(target_fl64->l_start);
         fl64.l_len = tswap64(target_fl64->l_len);
@@ -4875,7 +4937,7 @@ static int do_futex(target_ulong uaddr, int op, int val, target_ulong timeout,
 
 /* Map host to target signal numbers for the wait family of syscalls.
    Assume all other status bits are the same.  */
-static int host_to_target_waitstatus(int status)
+int host_to_target_waitstatus(int status)
 {
     if (WIFSIGNALED(status)) {
         return host_to_target_signal(WTERMSIG(status)) | (status & ~0x7f);
@@ -4964,8 +5026,8 @@ static int open_self_maps(void *cpu_env, int fd)
 #if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_UNICORE32)
     dprintf(fd, "%08llx-%08llx rw-p %08llx 00:00 0          [stack]\n",
                 (unsigned long long)ts->info->stack_limit,
-                (unsigned long long)(ts->stack_base + (TARGET_PAGE_SIZE - 1))
-                                     & TARGET_PAGE_MASK,
+                (unsigned long long)(ts->info->start_stack +
+                                     (TARGET_PAGE_SIZE - 1)) & TARGET_PAGE_MASK,
                 (unsigned long long)0);
 #endif
 
@@ -6165,8 +6227,11 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             ret = get_errno(settimeofday(&tv, NULL));
         }
         break;
-#if defined(TARGET_NR_select) && !defined(TARGET_S390X) && !defined(TARGET_S390)
+#if defined(TARGET_NR_select)
     case TARGET_NR_select:
+#if defined(TARGET_S390X) || defined(TARGET_ALPHA)
+        ret = do_select(arg1, arg2, arg3, arg4, arg5);
+#else
         {
             struct target_sel_arg_struct *sel;
             abi_ulong inp, outp, exp, tvp;
@@ -6182,6 +6247,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user_struct(sel, arg1, 0);
             ret = do_select(nsel, inp, outp, exp, tvp);
         }
+#endif
         break;
 #endif
 #ifdef TARGET_NR_pselect6
@@ -6531,6 +6597,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             __put_user(stfs.f_fsid.__val[0], &target_stfs->f_fsid.val[0]);
             __put_user(stfs.f_fsid.__val[1], &target_stfs->f_fsid.val[1]);
             __put_user(stfs.f_namelen, &target_stfs->f_namelen);
+            __put_user(stfs.f_frsize, &target_stfs->f_frsize);
+            memset(target_stfs->f_spare, 0, sizeof(target_stfs->f_spare));
             unlock_user_struct(target_stfs, arg2, 1);
         }
         break;
@@ -6559,6 +6627,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             __put_user(stfs.f_fsid.__val[0], &target_stfs->f_fsid.val[0]);
             __put_user(stfs.f_fsid.__val[1], &target_stfs->f_fsid.val[1]);
             __put_user(stfs.f_namelen, &target_stfs->f_namelen);
+            __put_user(stfs.f_frsize, &target_stfs->f_frsize);
+            memset(target_stfs->f_spare, 0, sizeof(target_stfs->f_spare));
             unlock_user_struct(target_stfs, arg3, 1);
         }
         break;
@@ -6890,6 +6960,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(do_fork(cpu_env, arg1, arg2, arg3, arg5, arg4));
 #elif defined(TARGET_CRIS)
         ret = get_errno(do_fork(cpu_env, arg2, arg1, arg3, arg4, arg5));
+#elif defined(TARGET_MICROBLAZE)
+        ret = get_errno(do_fork(cpu_env, arg1, arg2, arg4, arg6, arg5));
 #elif defined(TARGET_S390X)
         ret = get_errno(do_fork(cpu_env, arg2, arg1, arg3, arg5, arg4));
 #else
@@ -7099,12 +7171,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif /* TARGET_NR_getdents64 */
-#if defined(TARGET_NR__newselect) || defined(TARGET_S390X)
-#ifdef TARGET_S390X
-    case TARGET_NR_select:
-#else
+#if defined(TARGET_NR__newselect)
     case TARGET_NR__newselect:
-#endif
         ret = do_select(arg1, arg2, arg3, arg4, arg5);
         break;
 #endif
@@ -7186,26 +7254,24 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     case TARGET_NR_readv:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_WRITE, vec, arg2, count, 0) < 0)
-                goto efault;
-            ret = get_errno(readv(arg1, vec, count));
-            unlock_iovec(vec, arg2, count, 1);
+            struct iovec *vec = lock_iovec(VERIFY_WRITE, arg2, arg3, 0);
+            if (vec != NULL) {
+                ret = get_errno(readv(arg1, vec, arg3));
+                unlock_iovec(vec, arg2, arg3, 1);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
     case TARGET_NR_writev:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_READ, vec, arg2, count, 1) < 0)
-                goto efault;
-            ret = get_errno(writev(arg1, vec, count));
-            unlock_iovec(vec, arg2, count, 0);
+            struct iovec *vec = lock_iovec(VERIFY_READ, arg2, arg3, 1);
+            if (vec != NULL) {
+                ret = get_errno(writev(arg1, vec, arg3));
+                unlock_iovec(vec, arg2, arg3, 0);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
     case TARGET_NR_getsid:
@@ -7397,32 +7463,22 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         goto unimplemented;
 #endif
 #endif
-#ifdef TARGET_NR_pread
-    case TARGET_NR_pread:
-        if (regpairs_aligned(cpu_env))
-            arg4 = arg5;
-        if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
-            goto efault;
-        ret = get_errno(pread(arg1, p, arg3, arg4));
-        unlock_user(p, arg2, ret);
-        break;
-    case TARGET_NR_pwrite:
-        if (regpairs_aligned(cpu_env))
-            arg4 = arg5;
-        if (!(p = lock_user(VERIFY_READ, arg2, arg3, 1)))
-            goto efault;
-        ret = get_errno(pwrite(arg1, p, arg3, arg4));
-        unlock_user(p, arg2, 0);
-        break;
-#endif
 #ifdef TARGET_NR_pread64
     case TARGET_NR_pread64:
+        if (regpairs_aligned(cpu_env)) {
+            arg4 = arg5;
+            arg5 = arg6;
+        }
         if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
             goto efault;
         ret = get_errno(pread64(arg1, p, arg3, target_offset64(arg4, arg5)));
         unlock_user(p, arg2, ret);
         break;
     case TARGET_NR_pwrite64:
+        if (regpairs_aligned(cpu_env)) {
+            arg4 = arg5;
+            arg5 = arg6;
+        }
         if (!(p = lock_user(VERIFY_READ, arg2, arg3, 1)))
             goto efault;
         ret = get_errno(pwrite64(arg1, p, arg3, target_offset64(arg4, arg5)));
@@ -8630,14 +8686,13 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #ifdef TARGET_NR_vmsplice
 	case TARGET_NR_vmsplice:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_READ, vec, arg2, count, 1) < 0)
-                goto efault;
-            ret = get_errno(vmsplice(arg1, vec, count, arg4));
-            unlock_iovec(vec, arg2, count, 0);
+            struct iovec *vec = lock_iovec(VERIFY_READ, arg2, arg3, 1);
+            if (vec != NULL) {
+                ret = get_errno(vmsplice(arg1, vec, arg3, arg4));
+                unlock_iovec(vec, arg2, arg3, 0);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
 #endif
@@ -8824,6 +8879,19 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     }
 #endif
+#ifdef TARGET_NR_gethostname
+    case TARGET_NR_gethostname:
+    {
+        char *name = lock_user(VERIFY_WRITE, arg1, arg2, 0);
+        if (name) {
+            ret = get_errno(gethostname(name, arg2));
+            unlock_user(name, arg1, arg2);
+        } else {
+            ret = -TARGET_EFAULT;
+        }
+        break;
+    }
+#endif
     default:
     unimplemented:
         gemu_log("qemu: Unsupported syscall: %d\n", num);
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index a98cbf7b80f..92c01a96035 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -4,6 +4,10 @@
    most of them stay the same, so we handle it by putting ifdefs if
    necessary */
 
+#ifndef SYSCALL_DEFS_H
+#define SYSCALL_DEFS_H 1
+
+
 #include "syscall_nr.h"
 
 #define SOCKOP_socket           1
@@ -540,7 +544,7 @@ int do_sigaction(int sig, const struct target_sigaction *act,
 struct target_old_sigaction {
     abi_ulong _sa_handler;
     abi_ulong sa_mask;
-    abi_ulong sa_flags;
+    int32_t sa_flags;
 };
 
 struct target_rt_sigaction {
@@ -2013,6 +2017,12 @@ struct target_statfs64 {
 #define TARGET_F_SETLKW        9
 #define TARGET_F_SETOWN        5       /*  for sockets. */
 #define TARGET_F_GETOWN        6       /*  for sockets. */
+
+#define TARGET_F_RDLCK         1
+#define TARGET_F_WRLCK         2
+#define TARGET_F_UNLCK         8
+#define TARGET_F_EXLCK         16
+#define TARGET_F_SHLCK         32
 #elif defined(TARGET_MIPS)
 #define TARGET_F_GETLK         14
 #define TARGET_F_SETLK         6
@@ -2027,6 +2037,18 @@ struct target_statfs64 {
 #define TARGET_F_GETOWN        9       /*  for sockets. */
 #endif
 
+#ifndef TARGET_F_RDLCK
+#define TARGET_F_RDLCK         0
+#define TARGET_F_WRLCK         1
+#define TARGET_F_UNLCK         2
+#endif
+
+#ifndef TARGET_F_EXLCK
+#define TARGET_F_EXLCK         4
+#define TARGET_F_SHLCK         8
+#endif
+
+
 #define TARGET_F_SETSIG        10      /*  for sockets. */
 #define TARGET_F_GETSIG        11      /*  for sockets. */
 
@@ -2425,3 +2447,5 @@ struct target_ucred {
     uint32_t uid;
     uint32_t gid;
 };
+
+#endif
diff --git a/linux-user/unicore32/syscall_nr.h b/linux-user/unicore32/syscall_nr.h
index 9c72d84d804..486b8c45a03 100644
--- a/linux-user/unicore32/syscall_nr.h
+++ b/linux-user/unicore32/syscall_nr.h
@@ -187,8 +187,8 @@
 #define TARGET_NR_rt_sigtimedwait               177
 #define TARGET_NR_rt_sigqueueinfo               178
 #define TARGET_NR_rt_sigsuspend                 179
-#define TARGET_NR_pread                         180
-#define TARGET_NR_pwrite                        181
+#define TARGET_NR_pread64                       180
+#define TARGET_NR_pwrite64                      181
 #define TARGET_NR_chown                         182
 #define TARGET_NR_getcwd                        183
 #define TARGET_NR_capget                        184
diff --git a/main-loop.c b/main-loop.c
index eb3b6e62536..6f52ac39bc7 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -23,77 +23,14 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "slirp/slirp.h"
-#include "main-loop.h"
+#include "qemu/main-loop.h"
+#include "block/aio.h"
 
 #ifndef _WIN32
 
-#include "compatfd.h"
-
-static int io_thread_fd = -1;
-
-void qemu_notify_event(void)
-{
-    /* Write 8 bytes to be compatible with eventfd.  */
-    static const uint64_t val = 1;
-    ssize_t ret;
-
-    if (io_thread_fd == -1) {
-        return;
-    }
-    do {
-        ret = write(io_thread_fd, &val, sizeof(val));
-    } while (ret < 0 && errno == EINTR);
-
-    /* EAGAIN is fine, a read must be pending.  */
-    if (ret < 0 && errno != EAGAIN) {
-        fprintf(stderr, "qemu_notify_event: write() failed: %s\n",
-                strerror(errno));
-        exit(1);
-    }
-}
-
-static void qemu_event_read(void *opaque)
-{
-    int fd = (intptr_t)opaque;
-    ssize_t len;
-    char buffer[512];
-
-    /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
-    do {
-        len = read(fd, buffer, sizeof(buffer));
-    } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
-}
-
-static int qemu_event_init(void)
-{
-    int err;
-    int fds[2];
-
-    err = qemu_eventfd(fds);
-    if (err == -1) {
-        return -errno;
-    }
-    err = fcntl_setfl(fds[0], O_NONBLOCK);
-    if (err < 0) {
-        goto fail;
-    }
-    err = fcntl_setfl(fds[1], O_NONBLOCK);
-    if (err < 0) {
-        goto fail;
-    }
-    qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
-                         (void *)(intptr_t)fds[0]);
-
-    io_thread_fd = fds[1];
-    return 0;
-
-fail:
-    close(fds[0]);
-    close(fds[1]);
-    return err;
-}
+#include "qemu/compatfd.h"
 
 /* If we have signalfd, we mask out the signals we want to handle and then
  * use signalfd to listen for them.  We rely on whatever the current signal
@@ -164,57 +101,42 @@ static int qemu_signal_init(void)
 
 #else /* _WIN32 */
 
-static HANDLE qemu_event_handle = NULL;
-
-static void dummy_event_handler(void *opaque)
-{
-}
-
-static int qemu_event_init(void)
+static int qemu_signal_init(void)
 {
-    qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (!qemu_event_handle) {
-        fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
-        return -1;
-    }
-    qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
     return 0;
 }
+#endif
+
+static AioContext *qemu_aio_context;
 
 void qemu_notify_event(void)
 {
-    if (!qemu_event_handle) {
+    if (!qemu_aio_context) {
         return;
     }
-    if (!SetEvent(qemu_event_handle)) {
-        fprintf(stderr, "qemu_notify_event: SetEvent failed: %ld\n",
-                GetLastError());
-        exit(1);
-    }
+    aio_notify(qemu_aio_context);
 }
 
-static int qemu_signal_init(void)
-{
-    return 0;
-}
-#endif
-
-int main_loop_init(void)
+int qemu_init_main_loop(void)
 {
     int ret;
+    GSource *src;
 
-    qemu_mutex_lock_iothread();
-    ret = qemu_signal_init();
-    if (ret) {
-        return ret;
+    init_clocks();
+    if (init_timer_alarm() < 0) {
+        fprintf(stderr, "could not initialize alarm timer\n");
+        exit(1);
     }
 
-    /* Note eventfd must be drained before signalfd handlers run */
-    ret = qemu_event_init();
+    ret = qemu_signal_init();
     if (ret) {
         return ret;
     }
 
+    qemu_aio_context = aio_context_new();
+    src = aio_get_g_source(qemu_aio_context);
+    g_source_attach(src, NULL);
+    g_source_unref(src);
     return 0;
 }
 
@@ -400,14 +322,15 @@ void qemu_del_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque)
 
 void qemu_fd_register(int fd)
 {
-    WSAEventSelect(fd, qemu_event_handle, FD_READ | FD_ACCEPT | FD_CLOSE |
+    WSAEventSelect(fd, event_notifier_get_handle(&qemu_aio_context->notifier),
+                   FD_READ | FD_ACCEPT | FD_CLOSE |
                    FD_CONNECT | FD_WRITE | FD_OOB);
 }
 
 static int os_host_main_loop_wait(uint32_t timeout)
 {
     GMainContext *context = g_main_context_default();
-    int ret, i;
+    int select_ret, g_poll_ret, ret, i;
     PollingEntry *pe;
     WaitObjects *w = &wait_objects;
     gint poll_timeout;
@@ -422,13 +345,6 @@ static int os_host_main_loop_wait(uint32_t timeout)
         return ret;
     }
 
-    if (nfds >= 0) {
-        ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv0);
-        if (ret != 0) {
-            timeout = 0;
-        }
-    }
-
     g_main_context_prepare(context, &max_priority);
     n_poll_fds = g_main_context_query(context, max_priority, &poll_timeout,
                                       poll_fds, ARRAY_SIZE(poll_fds));
@@ -444,9 +360,9 @@ static int os_host_main_loop_wait(uint32_t timeout)
     }
 
     qemu_mutex_unlock_iothread();
-    ret = g_poll(poll_fds, n_poll_fds + w->num, poll_timeout);
+    g_poll_ret = g_poll(poll_fds, n_poll_fds + w->num, poll_timeout);
     qemu_mutex_lock_iothread();
-    if (ret > 0) {
+    if (g_poll_ret > 0) {
         for (i = 0; i < w->num; i++) {
             w->revents[i] = poll_fds[n_poll_fds + i].revents;
         }
@@ -461,12 +377,18 @@ static int os_host_main_loop_wait(uint32_t timeout)
         g_main_context_dispatch(context);
     }
 
-    /* If an edge-triggered socket event occurred, select will return a
-     * positive result on the next iteration.  We do not need to do anything
-     * here.
+    /* Call select after g_poll to avoid a useless iteration and therefore
+     * improve socket latency.
      */
 
-    return ret;
+    if (nfds >= 0) {
+        select_ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv0);
+        if (select_ret != 0) {
+            timeout = 0;
+        }
+    }
+
+    return select_ret || g_poll_ret;
 }
 #endif
 
@@ -477,8 +399,6 @@ int main_loop_wait(int nonblocking)
 
     if (nonblocking) {
         timeout = 0;
-    } else {
-        qemu_bh_update_timeout(&timeout);
     }
 
     /* poll any events */
@@ -501,9 +421,36 @@ int main_loop_wait(int nonblocking)
 
     qemu_run_all_timers();
 
-    /* Check bottom-halves last in case any of the earlier events triggered
-       them.  */
-    qemu_bh_poll();
-
     return ret;
 }
+
+/* Functions to operate on the main QEMU AioContext.  */
+
+QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque)
+{
+    return aio_bh_new(qemu_aio_context, cb, opaque);
+}
+
+bool qemu_aio_wait(void)
+{
+    return aio_poll(qemu_aio_context, true);
+}
+
+#ifdef CONFIG_POSIX
+void qemu_aio_set_fd_handler(int fd,
+                             IOHandler *io_read,
+                             IOHandler *io_write,
+                             AioFlushHandler *io_flush,
+                             void *opaque)
+{
+    aio_set_fd_handler(qemu_aio_context, fd, io_read, io_write, io_flush,
+                       opaque);
+}
+#endif
+
+void qemu_aio_set_event_notifier(EventNotifier *notifier,
+                                 EventNotifierHandler *io_read,
+                                 AioFlushEventNotifierHandler *io_flush)
+{
+    aio_set_event_notifier(qemu_aio_context, notifier, io_read, io_flush);
+}
diff --git a/memory.c b/memory.c
index d528d1f7f3b..410c5f80b42 100644
--- a/memory.c
+++ b/memory.c
@@ -13,23 +13,25 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "memory.h"
-#include "exec-memory.h"
-#include "ioport.h"
-#include "bitops.h"
-#include "kvm.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "exec/ioport.h"
+#include "qemu/bitops.h"
+#include "sysemu/kvm.h"
 #include <assert.h>
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "exec/memory-internal.h"
 
-unsigned memory_region_transaction_depth = 0;
-static bool memory_region_update_pending = false;
+static unsigned memory_region_transaction_depth;
+static bool memory_region_update_pending;
 static bool global_dirty_log = false;
 
 static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
     = QTAILQ_HEAD_INITIALIZER(memory_listeners);
 
+static QTAILQ_HEAD(, AddressSpace) address_spaces
+    = QTAILQ_HEAD_INITIALIZER(address_spaces);
+
 typedef struct AddrRange AddrRange;
 
 /*
@@ -98,13 +100,17 @@ static bool memory_listener_match(MemoryListener *listener,
         switch (_direction) {                                           \
         case Forward:                                                   \
             QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
-                _listener->_callback(_listener, ##_args);               \
+                if (_listener->_callback) {                             \
+                    _listener->_callback(_listener, ##_args);           \
+                }                                                       \
             }                                                           \
             break;                                                      \
         case Reverse:                                                   \
             QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
                                    memory_listeners, link) {            \
-                _listener->_callback(_listener, ##_args);               \
+                if (_listener->_callback) {                             \
+                    _listener->_callback(_listener, ##_args);           \
+                }                                                       \
             }                                                           \
             break;                                                      \
         default:                                                        \
@@ -119,7 +125,8 @@ static bool memory_listener_match(MemoryListener *listener,
         switch (_direction) {                                           \
         case Forward:                                                   \
             QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
-                if (memory_listener_match(_listener, _section)) {       \
+                if (_listener->_callback                                \
+                    && memory_listener_match(_listener, _section)) {    \
                     _listener->_callback(_listener, _section, ##_args); \
                 }                                                       \
             }                                                           \
@@ -127,7 +134,8 @@ static bool memory_listener_match(MemoryListener *listener,
         case Reverse:                                                   \
             QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
                                    memory_listeners, link) {            \
-                if (memory_listener_match(_listener, _section)) {       \
+                if (_listener->_callback                                \
+                    && memory_listener_match(_listener, _section)) {    \
                     _listener->_callback(_listener, _section, ##_args); \
                 }                                                       \
             }                                                           \
@@ -140,7 +148,7 @@ static bool memory_listener_match(MemoryListener *listener,
 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback)            \
     MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
         .mr = (fr)->mr,                                                 \
-        .address_space = (as)->root,                                    \
+        .address_space = (as),                                          \
         .offset_within_region = (fr)->offset_in_region,                 \
         .size = int128_get64((fr)->addr.size),                          \
         .offset_within_address_space = int128_get64((fr)->addr.start),  \
@@ -202,7 +210,7 @@ typedef struct FlatView FlatView;
 /* Range of memory in the global map.  Addresses are absolute. */
 struct FlatRange {
     MemoryRegion *mr;
-    target_phys_addr_t offset_in_region;
+    hwaddr offset_in_region;
     AddrRange addr;
     uint8_t dirty_log_mask;
     bool readable;
@@ -218,17 +226,8 @@ struct FlatView {
     unsigned nr_allocated;
 };
 
-typedef struct AddressSpace AddressSpace;
 typedef struct AddressSpaceOps AddressSpaceOps;
 
-/* A system address space - I/O, memory, etc. */
-struct AddressSpace {
-    MemoryRegion *root;
-    FlatView current_map;
-    int ioeventfd_nb;
-    MemoryRegionIoeventfd *ioeventfds;
-};
-
 #define FOR_EACH_FLAT_RANGE(var, view)          \
     for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
 
@@ -302,7 +301,7 @@ static void flatview_simplify(FlatView *view)
 }
 
 static void memory_region_read_accessor(void *opaque,
-                                        target_phys_addr_t addr,
+                                        hwaddr addr,
                                         uint64_t *value,
                                         unsigned size,
                                         unsigned shift,
@@ -311,12 +310,15 @@ static void memory_region_read_accessor(void *opaque,
     MemoryRegion *mr = opaque;
     uint64_t tmp;
 
+    if (mr->flush_coalesced_mmio) {
+        qemu_flush_coalesced_mmio_buffer();
+    }
     tmp = mr->ops->read(mr->opaque, addr, size);
     *value |= (tmp & mask) << shift;
 }
 
 static void memory_region_write_accessor(void *opaque,
-                                         target_phys_addr_t addr,
+                                         hwaddr addr,
                                          uint64_t *value,
                                          unsigned size,
                                          unsigned shift,
@@ -325,17 +327,20 @@ static void memory_region_write_accessor(void *opaque,
     MemoryRegion *mr = opaque;
     uint64_t tmp;
 
+    if (mr->flush_coalesced_mmio) {
+        qemu_flush_coalesced_mmio_buffer();
+    }
     tmp = (*value >> shift) & mask;
     mr->ops->write(mr->opaque, addr, tmp, size);
 }
 
-static void access_with_adjusted_size(target_phys_addr_t addr,
+static void access_with_adjusted_size(hwaddr addr,
                                       uint64_t *value,
                                       unsigned size,
                                       unsigned access_size_min,
                                       unsigned access_size_max,
                                       void (*access)(void *opaque,
-                                                     target_phys_addr_t addr,
+                                                     hwaddr addr,
                                                      uint64_t *value,
                                                      unsigned size,
                                                      unsigned shift,
@@ -360,8 +365,6 @@ static void access_with_adjusted_size(target_phys_addr_t addr,
     }
 }
 
-static AddressSpace address_space_memory;
-
 static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset,
                                              unsigned width, bool write)
 {
@@ -450,18 +453,17 @@ const IORangeOps memory_region_iorange_ops = {
     .destructor = memory_region_iorange_destructor,
 };
 
-static AddressSpace address_space_io;
-
 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
 {
+    AddressSpace *as;
+
     while (mr->parent) {
         mr = mr->parent;
     }
-    if (mr == address_space_memory.root) {
-        return &address_space_memory;
-    }
-    if (mr == address_space_io.root) {
-        return &address_space_io;
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (mr == as->root) {
+            return as;
+        }
     }
     abort();
 }
@@ -477,7 +479,7 @@ static void render_memory_region(FlatView *view,
 {
     MemoryRegion *subregion;
     unsigned i;
-    target_phys_addr_t offset_in_region;
+    hwaddr offset_in_region;
     Int128 remain;
     Int128 now;
     FlatRange fr;
@@ -538,12 +540,12 @@ static void render_memory_region(FlatView *view,
             offset_in_region += int128_get64(now);
             int128_subfrom(&remain, now);
         }
-        if (int128_eq(base, view->ranges[i].addr.start)) {
-            now = int128_min(remain, view->ranges[i].addr.size);
-            int128_addto(&base, now);
-            offset_in_region += int128_get64(now);
-            int128_subfrom(&remain, now);
-        }
+        now = int128_sub(int128_min(int128_add(base, remain),
+                                    addrrange_end(view->ranges[i].addr)),
+                         base);
+        int128_addto(&base, now);
+        offset_in_region += int128_get64(now);
+        int128_subfrom(&remain, now);
     }
     if (int128_nz(remain)) {
         fr.mr = mr;
@@ -563,8 +565,10 @@ static FlatView generate_memory_topology(MemoryRegion *mr)
 
     flatview_init(&view);
 
-    render_memory_region(&view, mr, int128_zero(),
-                         addrrange_make(int128_zero(), int128_2_64()), false);
+    if (mr) {
+        render_memory_region(&view, mr, int128_zero(),
+                             addrrange_make(int128_zero(), int128_2_64()), false);
+    }
     flatview_simplify(&view);
 
     return view;
@@ -592,7 +596,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
                                                   fds_new[inew]))) {
             fd = &fds_old[iold];
             section = (MemoryRegionSection) {
-                .address_space = as->root,
+                .address_space = as,
                 .offset_within_address_space = int128_get64(fd->addr.start),
                 .size = int128_get64(fd->addr.size),
             };
@@ -605,7 +609,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
                                                          fds_old[iold]))) {
             fd = &fds_new[inew];
             section = (MemoryRegionSection) {
-                .address_space = as->root,
+                .address_space = as,
                 .offset_within_address_space = int128_get64(fd->addr.start),
                 .size = int128_get64(fd->addr.size),
             };
@@ -627,7 +631,7 @@ static void address_space_update_ioeventfds(AddressSpace *as)
     AddrRange tmp;
     unsigned i;
 
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
             tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
                                   int128_sub(fr->addr.start,
@@ -715,53 +719,38 @@ static void address_space_update_topology_pass(AddressSpace *as,
 
 static void address_space_update_topology(AddressSpace *as)
 {
-    FlatView old_view = as->current_map;
+    FlatView old_view = *as->current_map;
     FlatView new_view = generate_memory_topology(as->root);
 
     address_space_update_topology_pass(as, old_view, new_view, false);
     address_space_update_topology_pass(as, old_view, new_view, true);
 
-    as->current_map = new_view;
+    *as->current_map = new_view;
     flatview_destroy(&old_view);
     address_space_update_ioeventfds(as);
 }
 
-static void memory_region_update_topology(MemoryRegion *mr)
-{
-    if (memory_region_transaction_depth) {
-        memory_region_update_pending |= !mr || mr->enabled;
-        return;
-    }
-
-    if (mr && !mr->enabled) {
-        return;
-    }
-
-    MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
-
-    if (address_space_memory.root) {
-        address_space_update_topology(&address_space_memory);
-    }
-    if (address_space_io.root) {
-        address_space_update_topology(&address_space_io);
-    }
-
-    MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
-
-    memory_region_update_pending = false;
-}
-
 void memory_region_transaction_begin(void)
 {
+    qemu_flush_coalesced_mmio_buffer();
     ++memory_region_transaction_depth;
 }
 
 void memory_region_transaction_commit(void)
 {
+    AddressSpace *as;
+
     assert(memory_region_transaction_depth);
     --memory_region_transaction_depth;
     if (!memory_region_transaction_depth && memory_region_update_pending) {
-        memory_region_update_topology(NULL);
+        memory_region_update_pending = false;
+        MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
+
+        QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+            address_space_update_topology(as);
+        }
+
+        MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
     }
 }
 
@@ -826,10 +815,11 @@ void memory_region_init(MemoryRegion *mr,
     mr->dirty_log_mask = 0;
     mr->ioeventfd_nb = 0;
     mr->ioeventfds = NULL;
+    mr->flush_coalesced_mmio = false;
 }
 
 static bool memory_region_access_valid(MemoryRegion *mr,
-                                       target_phys_addr_t addr,
+                                       hwaddr addr,
                                        unsigned size,
                                        bool is_write)
 {
@@ -855,7 +845,7 @@ static bool memory_region_access_valid(MemoryRegion *mr,
 }
 
 static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
-                                             target_phys_addr_t addr,
+                                             hwaddr addr,
                                              unsigned size)
 {
     uint64_t data = 0;
@@ -896,7 +886,7 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
 }
 
 static uint64_t memory_region_dispatch_read(MemoryRegion *mr,
-                                            target_phys_addr_t addr,
+                                            hwaddr addr,
                                             unsigned size)
 {
     uint64_t ret;
@@ -907,7 +897,7 @@ static uint64_t memory_region_dispatch_read(MemoryRegion *mr,
 }
 
 static void memory_region_dispatch_write(MemoryRegion *mr,
-                                         target_phys_addr_t addr,
+                                         hwaddr addr,
                                          uint64_t data,
                                          unsigned size)
 {
@@ -969,7 +959,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
 void memory_region_init_alias(MemoryRegion *mr,
                               const char *name,
                               MemoryRegion *orig,
-                              target_phys_addr_t offset,
+                              hwaddr offset,
                               uint64_t size)
 {
     memory_region_init(mr, name, size);
@@ -992,7 +982,7 @@ void memory_region_init_rom_device(MemoryRegion *mr,
     mr->ram_addr = qemu_ram_alloc(size, mr);
 }
 
-static uint64_t invalid_read(void *opaque, target_phys_addr_t addr,
+static uint64_t invalid_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MemoryRegion *mr = opaque;
@@ -1004,7 +994,7 @@ static uint64_t invalid_read(void *opaque, target_phys_addr_t addr,
     return -1U;
 }
 
-static void invalid_write(void *opaque, target_phys_addr_t addr, uint64_t data,
+static void invalid_write(void *opaque, hwaddr addr, uint64_t data,
                           unsigned size)
 {
     MemoryRegion *mr = opaque;
@@ -1031,6 +1021,7 @@ void memory_region_init_reservation(MemoryRegion *mr,
 void memory_region_destroy(MemoryRegion *mr)
 {
     assert(QTAILQ_EMPTY(&mr->subregions));
+    assert(memory_region_transaction_depth == 0);
     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);
@@ -1069,33 +1060,53 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;
 
+    memory_region_transaction_begin();
     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
-bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size, unsigned client)
+bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size, unsigned client)
 {
     assert(mr->terminates);
     return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size,
                                          1 << client);
 }
 
-void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size)
+void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size)
 {
     assert(mr->terminates);
     return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1);
 }
 
+bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
+                                        hwaddr size, unsigned client)
+{
+    bool ret;
+    assert(mr->terminates);
+    ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size,
+                                        1 << client);
+    if (ret) {
+        cpu_physical_memory_reset_dirty(mr->ram_addr + addr,
+                                        mr->ram_addr + addr + size,
+                                        1 << client);
+    }
+    return ret;
+}
+
+
 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 {
+    AddressSpace *as;
     FlatRange *fr;
 
-    FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
-        if (fr->mr == mr) {
-            MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory,
-                                          Forward, log_sync);
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        FOR_EACH_FLAT_RANGE(fr, as->current_map) {
+            if (fr->mr == mr) {
+                MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
+            }
         }
     }
 }
@@ -1103,21 +1114,25 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
 {
     if (mr->readonly != readonly) {
+        memory_region_transaction_begin();
         mr->readonly = readonly;
-        memory_region_update_topology(mr);
+        memory_region_update_pending |= mr->enabled;
+        memory_region_transaction_commit();
     }
 }
 
 void memory_region_rom_device_set_readable(MemoryRegion *mr, bool readable)
 {
     if (mr->readable != readable) {
+        memory_region_transaction_begin();
         mr->readable = readable;
-        memory_region_update_topology(mr);
+        memory_region_update_pending |= mr->enabled;
+        memory_region_transaction_commit();
     }
 }
 
-void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                               target_phys_addr_t size, unsigned client)
+void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
+                               hwaddr size, unsigned client)
 {
     assert(mr->terminates);
     cpu_physical_memory_reset_dirty(mr->ram_addr + addr,
@@ -1136,16 +1151,24 @@ void *memory_region_get_ram_ptr(MemoryRegion *mr)
     return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK);
 }
 
-static void memory_region_update_coalesced_range(MemoryRegion *mr)
+static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
 {
     FlatRange *fr;
     CoalescedMemoryRange *cmr;
     AddrRange tmp;
+    MemoryRegionSection section;
 
-    FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         if (fr->mr == mr) {
-            qemu_unregister_coalesced_mmio(int128_get64(fr->addr.start),
-                                           int128_get64(fr->addr.size));
+            section = (MemoryRegionSection) {
+                .address_space = as,
+                .offset_within_address_space = int128_get64(fr->addr.start),
+                .size = int128_get64(fr->addr.size),
+            };
+
+            MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, &section,
+                                 int128_get64(fr->addr.start),
+                                 int128_get64(fr->addr.size));
             QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
                 tmp = addrrange_shift(cmr->addr,
                                       int128_sub(fr->addr.start,
@@ -1154,13 +1177,23 @@ static void memory_region_update_coalesced_range(MemoryRegion *mr)
                     continue;
                 }
                 tmp = addrrange_intersection(tmp, fr->addr);
-                qemu_register_coalesced_mmio(int128_get64(tmp.start),
-                                             int128_get64(tmp.size));
+                MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, &section,
+                                     int128_get64(tmp.start),
+                                     int128_get64(tmp.size));
             }
         }
     }
 }
 
+static void memory_region_update_coalesced_range(MemoryRegion *mr)
+{
+    AddressSpace *as;
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        memory_region_update_coalesced_range_as(mr, as);
+    }
+}
+
 void memory_region_set_coalescing(MemoryRegion *mr)
 {
     memory_region_clear_coalescing(mr);
@@ -1168,7 +1201,7 @@ void memory_region_set_coalescing(MemoryRegion *mr)
 }
 
 void memory_region_add_coalescing(MemoryRegion *mr,
-                                  target_phys_addr_t offset,
+                                  hwaddr offset,
                                   uint64_t size)
 {
     CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr));
@@ -1176,12 +1209,16 @@ void memory_region_add_coalescing(MemoryRegion *mr,
     cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
     QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
     memory_region_update_coalesced_range(mr);
+    memory_region_set_flush_coalesced(mr);
 }
 
 void memory_region_clear_coalescing(MemoryRegion *mr)
 {
     CoalescedMemoryRange *cmr;
 
+    qemu_flush_coalesced_mmio_buffer();
+    mr->flush_coalesced_mmio = false;
+
     while (!QTAILQ_EMPTY(&mr->coalesced)) {
         cmr = QTAILQ_FIRST(&mr->coalesced);
         QTAILQ_REMOVE(&mr->coalesced, cmr, link);
@@ -1190,8 +1227,21 @@ void memory_region_clear_coalescing(MemoryRegion *mr)
     memory_region_update_coalesced_range(mr);
 }
 
+void memory_region_set_flush_coalesced(MemoryRegion *mr)
+{
+    mr->flush_coalesced_mmio = true;
+}
+
+void memory_region_clear_flush_coalesced(MemoryRegion *mr)
+{
+    qemu_flush_coalesced_mmio_buffer();
+    if (QTAILQ_EMPTY(&mr->coalesced)) {
+        mr->flush_coalesced_mmio = false;
+    }
+}
+
 void memory_region_add_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -1206,6 +1256,8 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
+    adjust_endianness(mr, &mrfd.data, size);
+    memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
             break;
@@ -1217,11 +1269,12 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
             sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
     mr->ioeventfds[i] = mrfd;
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 void memory_region_del_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -1236,6 +1289,8 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
+    adjust_endianness(mr, &mrfd.data, size);
+    memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
             break;
@@ -1247,15 +1302,18 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     --mr->ioeventfd_nb;
     mr->ioeventfds = g_realloc(mr->ioeventfds,
                                   sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 static void memory_region_add_subregion_common(MemoryRegion *mr,
-                                               target_phys_addr_t offset,
+                                               hwaddr offset,
                                                MemoryRegion *subregion)
 {
     MemoryRegion *other;
 
+    memory_region_transaction_begin();
+
     assert(!subregion->parent);
     subregion->parent = mr;
     subregion->addr = offset;
@@ -1288,12 +1346,13 @@ static void memory_region_add_subregion_common(MemoryRegion *mr,
     }
     QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
 done:
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled && subregion->enabled;
+    memory_region_transaction_commit();
 }
 
 
 void memory_region_add_subregion(MemoryRegion *mr,
-                                 target_phys_addr_t offset,
+                                 hwaddr offset,
                                  MemoryRegion *subregion)
 {
     subregion->may_overlap = false;
@@ -1302,7 +1361,7 @@ void memory_region_add_subregion(MemoryRegion *mr,
 }
 
 void memory_region_add_subregion_overlap(MemoryRegion *mr,
-                                         target_phys_addr_t offset,
+                                         hwaddr offset,
                                          MemoryRegion *subregion,
                                          unsigned priority)
 {
@@ -1314,10 +1373,12 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr,
 void memory_region_del_subregion(MemoryRegion *mr,
                                  MemoryRegion *subregion)
 {
+    memory_region_transaction_begin();
     assert(subregion->parent == mr);
     subregion->parent = NULL;
     QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled && subregion->enabled;
+    memory_region_transaction_commit();
 }
 
 void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
@@ -1325,11 +1386,13 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
     if (enabled == mr->enabled) {
         return;
     }
+    memory_region_transaction_begin();
     mr->enabled = enabled;
-    memory_region_update_topology(NULL);
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
 }
 
-void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr)
+void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
 {
     MemoryRegion *parent = mr->parent;
     unsigned priority = mr->priority;
@@ -1350,18 +1413,18 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr)
     memory_region_transaction_commit();
 }
 
-void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset)
+void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
 {
-    target_phys_addr_t old_offset = mr->alias_offset;
-
     assert(mr->alias);
-    mr->alias_offset = offset;
 
-    if (offset == old_offset || !mr->parent) {
+    if (offset == mr->alias_offset) {
         return;
     }
 
-    memory_region_update_topology(mr);
+    memory_region_transaction_begin();
+    mr->alias_offset = offset;
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
@@ -1384,12 +1447,12 @@ static int cmp_flatrange_addr(const void *addr_, const void *fr_)
 
 static FlatRange *address_space_lookup(AddressSpace *as, AddrRange addr)
 {
-    return bsearch(&addr, as->current_map.ranges, as->current_map.nr,
+    return bsearch(&addr, as->current_map->ranges, as->current_map->nr,
                    sizeof(FlatRange), cmp_flatrange_addr);
 }
 
 MemoryRegionSection memory_region_find(MemoryRegion *address_space,
-                                       target_phys_addr_t addr, uint64_t size)
+                                       hwaddr addr, uint64_t size)
 {
     AddressSpace *as = memory_region_to_address_space(address_space);
     AddrRange range = addrrange_make(int128_make64(addr),
@@ -1401,7 +1464,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
         return ret;
     }
 
-    while (fr > as->current_map.ranges
+    while (fr > as->current_map->ranges
            && addrrange_intersects(fr[-1].addr, range)) {
         --fr;
     }
@@ -1422,7 +1485,7 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space)
     AddressSpace *as = memory_region_to_address_space(address_space);
     FlatRange *fr;
 
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
     }
 }
@@ -1445,29 +1508,35 @@ static void listener_add_address_space(MemoryListener *listener,
     FlatRange *fr;
 
     if (listener->address_space_filter
-        && listener->address_space_filter != as->root) {
+        && listener->address_space_filter != as) {
         return;
     }
 
     if (global_dirty_log) {
-        listener->log_global_start(listener);
+        if (listener->log_global_start) {
+            listener->log_global_start(listener);
+        }
     }
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         MemoryRegionSection section = {
             .mr = fr->mr,
-            .address_space = as->root,
+            .address_space = as,
             .offset_within_region = fr->offset_in_region,
             .size = int128_get64(fr->addr.size),
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
-        listener->region_add(listener, &section);
+        if (listener->region_add) {
+            listener->region_add(listener, &section);
+        }
     }
 }
 
-void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
+void memory_listener_register(MemoryListener *listener, AddressSpace *filter)
 {
     MemoryListener *other = NULL;
+    AddressSpace *as;
 
     listener->address_space_filter = filter;
     if (QTAILQ_EMPTY(&memory_listeners)
@@ -1482,8 +1551,10 @@ void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
         }
         QTAILQ_INSERT_BEFORE(other, listener, link);
     }
-    listener_add_address_space(listener, &address_space_memory);
-    listener_add_address_space(listener, &address_space_io);
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        listener_add_address_space(listener, as);
+    }
 }
 
 void memory_listener_unregister(MemoryListener *listener)
@@ -1491,24 +1562,36 @@ void memory_listener_unregister(MemoryListener *listener)
     QTAILQ_REMOVE(&memory_listeners, listener, link);
 }
 
-void set_system_memory_map(MemoryRegion *mr)
+void address_space_init(AddressSpace *as, MemoryRegion *root)
 {
-    address_space_memory.root = mr;
-    memory_region_update_topology(NULL);
+    memory_region_transaction_begin();
+    as->root = root;
+    as->current_map = g_new(FlatView, 1);
+    flatview_init(as->current_map);
+    QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
+    as->name = NULL;
+    memory_region_transaction_commit();
+    address_space_init_dispatch(as);
 }
 
-void set_system_io_map(MemoryRegion *mr)
+void address_space_destroy(AddressSpace *as)
 {
-    address_space_io.root = mr;
-    memory_region_update_topology(NULL);
+    /* Flush out anything from MemoryListeners listening in on this */
+    memory_region_transaction_begin();
+    as->root = NULL;
+    memory_region_transaction_commit();
+    QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
+    address_space_destroy_dispatch(as);
+    flatview_destroy(as->current_map);
+    g_free(as->current_map);
 }
 
-uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)
+uint64_t io_mem_read(MemoryRegion *mr, hwaddr addr, unsigned size)
 {
     return memory_region_dispatch_read(mr, addr, size);
 }
 
-void io_mem_write(MemoryRegion *mr, target_phys_addr_t addr,
+void io_mem_write(MemoryRegion *mr, hwaddr addr,
                   uint64_t val, unsigned size)
 {
     memory_region_dispatch_write(mr, addr, val, size);
@@ -1526,7 +1609,7 @@ typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead;
 
 static void mtree_print_mr(fprintf_function mon_printf, void *f,
                            const MemoryRegion *mr, unsigned int level,
-                           target_phys_addr_t base,
+                           hwaddr base,
                            MemoryRegionListHead *alias_print_queue)
 {
     MemoryRegionList *new_ml, *ml, *next_ml;
@@ -1534,7 +1617,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
     const MemoryRegion *submr;
     unsigned int i;
 
-    if (!mr) {
+    if (!mr || !mr->enabled) {
         return;
     }
 
@@ -1564,7 +1647,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    "-" TARGET_FMT_plx "\n",
                    base + mr->addr,
                    base + mr->addr
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1,
+                   + (hwaddr)int128_get64(mr->size) - 1,
                    mr->priority,
                    mr->readable ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->readable) ? 'W'
@@ -1573,13 +1656,13 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    mr->alias->name,
                    mr->alias_offset,
                    mr->alias_offset
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1);
+                   + (hwaddr)int128_get64(mr->size) - 1);
     } else {
         mon_printf(f,
                    TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n",
                    base + mr->addr,
                    base + mr->addr
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1,
+                   + (hwaddr)int128_get64(mr->size) - 1,
                    mr->priority,
                    mr->readable ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->readable) ? 'W'
@@ -1620,16 +1703,16 @@ void mtree_info(fprintf_function mon_printf, void *f)
 {
     MemoryRegionListHead ml_head;
     MemoryRegionList *ml, *ml2;
+    AddressSpace *as;
 
     QTAILQ_INIT(&ml_head);
 
-    mon_printf(f, "memory\n");
-    mtree_print_mr(mon_printf, f, address_space_memory.root, 0, 0, &ml_head);
-
-    if (address_space_io.root &&
-        !QTAILQ_EMPTY(&address_space_io.root->subregions)) {
-        mon_printf(f, "I/O\n");
-        mtree_print_mr(mon_printf, f, address_space_io.root, 0, 0, &ml_head);
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (!as->name) {
+            continue;
+        }
+        mon_printf(f, "%s\n", as->name);
+        mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head);
     }
 
     mon_printf(f, "aliases\n");
diff --git a/memory_mapping-stub.c b/memory_mapping-stub.c
index 76be34d89fd..24d5d673719 100644
--- a/memory_mapping-stub.c
+++ b/memory_mapping-stub.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 int qemu_get_guest_memory_mapping(MemoryMappingList *list)
 {
diff --git a/memory_mapping.c b/memory_mapping.c
index 6f5a2e3f711..ff45b3a2393 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 static void memory_mapping_list_add_mapping_sorted(MemoryMappingList *list,
                                                    MemoryMapping *mapping)
@@ -30,8 +30,8 @@ static void memory_mapping_list_add_mapping_sorted(MemoryMappingList *list,
 }
 
 static void create_new_memory_mapping(MemoryMappingList *list,
-                                      target_phys_addr_t phys_addr,
-                                      target_phys_addr_t virt_addr,
+                                      hwaddr phys_addr,
+                                      hwaddr virt_addr,
                                       ram_addr_t length)
 {
     MemoryMapping *memory_mapping;
@@ -46,8 +46,8 @@ static void create_new_memory_mapping(MemoryMappingList *list,
 }
 
 static inline bool mapping_contiguous(MemoryMapping *map,
-                                      target_phys_addr_t phys_addr,
-                                      target_phys_addr_t virt_addr)
+                                      hwaddr phys_addr,
+                                      hwaddr virt_addr)
 {
     return phys_addr == map->phys_addr + map->length &&
            virt_addr == map->virt_addr + map->length;
@@ -58,7 +58,7 @@ static inline bool mapping_contiguous(MemoryMapping *map,
  * [phys_addr, phys_addr + length) have intersection?
  */
 static inline bool mapping_have_same_region(MemoryMapping *map,
-                                            target_phys_addr_t phys_addr,
+                                            hwaddr phys_addr,
                                             ram_addr_t length)
 {
     return !(phys_addr + length < map->phys_addr ||
@@ -71,8 +71,8 @@ static inline bool mapping_have_same_region(MemoryMapping *map,
  * intersection are the same?
  */
 static inline bool mapping_conflict(MemoryMapping *map,
-                                    target_phys_addr_t phys_addr,
-                                    target_phys_addr_t virt_addr)
+                                    hwaddr phys_addr,
+                                    hwaddr virt_addr)
 {
     return virt_addr - map->virt_addr != phys_addr - map->phys_addr;
 }
@@ -83,7 +83,7 @@ static inline bool mapping_conflict(MemoryMapping *map,
  * in the intersection are the same.
  */
 static inline void mapping_merge(MemoryMapping *map,
-                                 target_phys_addr_t virt_addr,
+                                 hwaddr virt_addr,
                                  ram_addr_t length)
 {
     if (virt_addr < map->virt_addr) {
@@ -98,8 +98,8 @@ static inline void mapping_merge(MemoryMapping *map,
 }
 
 void memory_mapping_list_add_merge_sorted(MemoryMappingList *list,
-                                          target_phys_addr_t phys_addr,
-                                          target_phys_addr_t virt_addr,
+                                          hwaddr phys_addr,
+                                          hwaddr virt_addr,
                                           ram_addr_t length)
 {
     MemoryMapping *memory_mapping, *last_mapping;
@@ -200,7 +200,7 @@ int qemu_get_guest_memory_mapping(MemoryMappingList *list)
      * If the guest doesn't use paging, the virtual address is equal to physical
      * address.
      */
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         offset = block->offset;
         length = block->length;
         create_new_memory_mapping(list, offset, offset, length);
@@ -213,7 +213,7 @@ void qemu_get_guest_simple_memory_mapping(MemoryMappingList *list)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         create_new_memory_mapping(list, block->offset, 0, block->length);
     }
 }
diff --git a/migration-exec.c b/migration-exec.c
index 6c97db973c9..a051a6e668e 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -16,11 +16,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 #include <sys/types.h>
 #include <sys/wait.h>
 
@@ -48,35 +47,28 @@ static int exec_close(MigrationState *s)
 {
     int ret = 0;
     DPRINTF("exec_close\n");
-    if (s->opaque) {
-        ret = qemu_fclose(s->opaque);
-        s->opaque = NULL;
-        s->fd = -1;
-        if (ret >= 0 && !(WIFEXITED(ret) && WEXITSTATUS(ret) == 0)) {
-            /* close succeeded, but non-zero exit code: */
-            ret = -EIO; /* fake errno value */
-        }
+    ret = qemu_fclose(s->opaque);
+    s->opaque = NULL;
+    s->fd = -1;
+    if (ret >= 0 && !(WIFEXITED(ret) && WEXITSTATUS(ret) == 0)) {
+        /* close succeeded, but non-zero exit code: */
+        ret = -EIO; /* fake errno value */
     }
     return ret;
 }
 
-int exec_start_outgoing_migration(MigrationState *s, const char *command)
+void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp)
 {
     FILE *f;
 
     f = popen(command, "w");
     if (f == NULL) {
-        DPRINTF("Unable to popen exec target\n");
-        goto err_after_popen;
+        error_setg_errno(errp, errno, "failed to popen the migration target");
+        return;
     }
 
     s->fd = fileno(f);
-    if (s->fd == -1) {
-        DPRINTF("Unable to retrieve file descriptor for popen'd handle\n");
-        goto err_after_open;
-    }
-
-    socket_set_nonblock(s->fd);
+    assert(s->fd != -1);
 
     s->opaque = qemu_popen(f, "w");
 
@@ -85,36 +77,27 @@ int exec_start_outgoing_migration(MigrationState *s, const char *command)
     s->write = file_write;
 
     migrate_fd_connect(s);
-    return 0;
-
-err_after_open:
-    pclose(f);
-err_after_popen:
-    return -1;
 }
 
 static void exec_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    qemu_fclose(f);
 }
 
-int exec_start_incoming_migration(const char *command)
+void exec_start_incoming_migration(const char *command, Error **errp)
 {
     QEMUFile *f;
 
     DPRINTF("Attempting to start an incoming migration\n");
     f = qemu_popen_cmd(command, "r");
     if(f == NULL) {
-        DPRINTF("Unable to apply qemu wrapper to popen file\n");
-        return -errno;
+        error_setg_errno(errp, errno, "failed to popen the migration source");
+        return;
     }
 
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL,
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL,
 			 exec_accept_incoming_migration, NULL, f);
-
-    return 0;
 }
diff --git a/migration-fd.c b/migration-fd.c
index 50138edb34a..a99e0e39713 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -14,13 +14,11 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "monitor.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_FD
 
@@ -48,42 +46,33 @@ static int fd_close(MigrationState *s)
     int ret;
 
     DPRINTF("fd_close\n");
-    if (s->fd != -1) {
-        ret = fstat(s->fd, &st);
-        if (ret == 0 && S_ISREG(st.st_mode)) {
-            /*
-             * If the file handle is a regular file make sure the
-             * data is flushed to disk before signaling success.
-             */
-            ret = fsync(s->fd);
-            if (ret != 0) {
-                ret = -errno;
-                perror("migration-fd: fsync");
-                return ret;
-            }
-        }
-        ret = close(s->fd);
-        s->fd = -1;
+    ret = fstat(s->fd, &st);
+    if (ret == 0 && S_ISREG(st.st_mode)) {
+        /*
+         * If the file handle is a regular file make sure the
+         * data is flushed to disk before signaling success.
+         */
+        ret = fsync(s->fd);
         if (ret != 0) {
             ret = -errno;
-            perror("migration-fd: close");
+            perror("migration-fd: fsync");
             return ret;
         }
     }
-    return 0;
+    ret = close(s->fd);
+    s->fd = -1;
+    if (ret != 0) {
+        ret = -errno;
+        perror("migration-fd: close");
+    }
+    return ret;
 }
 
-int fd_start_outgoing_migration(MigrationState *s, const char *fdname)
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp)
 {
-    s->fd = monitor_get_fd(cur_mon, fdname);
+    s->fd = monitor_get_fd(cur_mon, fdname, errp);
     if (s->fd == -1) {
-        DPRINTF("fd_migration: invalid file descriptor identifier\n");
-        goto err_after_get_fd;
-    }
-
-    if (fcntl(s->fd, F_SETFL, O_NONBLOCK) == -1) {
-        DPRINTF("Unable to set nonblocking mode on file descriptor\n");
-        goto err_after_open;
+        return;
     }
 
     s->get_error = fd_errno;
@@ -91,24 +80,17 @@ int fd_start_outgoing_migration(MigrationState *s, const char *fdname)
     s->close = fd_close;
 
     migrate_fd_connect(s);
-    return 0;
-
-err_after_open:
-    close(s->fd);
-err_after_get_fd:
-    return -1;
 }
 
 static void fd_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    qemu_fclose(f);
 }
 
-int fd_start_incoming_migration(const char *infd)
+void fd_start_incoming_migration(const char *infd, Error **errp)
 {
     int fd;
     QEMUFile *f;
@@ -118,11 +100,9 @@ int fd_start_incoming_migration(const char *infd)
     fd = strtol(infd, NULL, 0);
     f = qemu_fdopen(fd, "rb");
     if(f == NULL) {
-        DPRINTF("Unable to apply qemu wrapper to file descriptor\n");
-        return -errno;
+        error_setg_errno(errp, errno, "failed to open the source descriptor");
+        return;
     }
 
     qemu_set_fd_handler2(fd, NULL, fd_accept_incoming_migration, NULL, f);
-
-    return 0;
 }
diff --git a/migration-tcp.c b/migration-tcp.c
index ac891c38a3b..e78a296137d 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -14,11 +14,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_TCP
 
@@ -44,64 +43,35 @@ static int tcp_close(MigrationState *s)
 {
     int r = 0;
     DPRINTF("tcp_close\n");
-    if (s->fd != -1) {
-        if (close(s->fd) < 0) {
-            r = -errno;
-        }
-        s->fd = -1;
+    if (closesocket(s->fd) < 0) {
+        r = -socket_error();
     }
     return r;
 }
 
-static void tcp_wait_for_connect(void *opaque)
+static void tcp_wait_for_connect(int fd, void *opaque)
 {
     MigrationState *s = opaque;
-    int val, ret;
-    socklen_t valsize = sizeof(val);
-
-    DPRINTF("connect completed\n");
-    do {
-        ret = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
-    } while (ret == -1 && (socket_error()) == EINTR);
 
-    if (ret < 0) {
+    if (fd < 0) {
+        DPRINTF("migrate connect error\n");
+        s->fd = -1;
         migrate_fd_error(s);
-        return;
-    }
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
-    if (val == 0)
+    } else {
+        DPRINTF("migrate connect success\n");
+        s->fd = fd;
+        socket_set_block(s->fd);
         migrate_fd_connect(s);
-    else {
-        DPRINTF("error connecting %d\n", val);
-        migrate_fd_error(s);
     }
 }
 
-int tcp_start_outgoing_migration(MigrationState *s, const char *host_port,
-                                 Error **errp)
+void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp)
 {
-    bool in_progress;
-
     s->get_error = socket_errno;
     s->write = socket_write;
     s->close = tcp_close;
 
-    s->fd = inet_connect(host_port, false, &in_progress, errp);
-    if (error_is_set(errp)) {
-        migrate_fd_error(s);
-        return -1;
-    }
-
-    if (in_progress) {
-        DPRINTF("connect in progress\n");
-        qemu_set_fd_handler2(s->fd, NULL, NULL, tcp_wait_for_connect, s);
-    } else {
-        migrate_fd_connect(s);
-    }
-
-    return 0;
+    s->fd = inet_nonblocking_connect(host_port, tcp_wait_for_connect, s, errp);
 }
 
 static void tcp_accept_incoming_migration(void *opaque)
@@ -115,12 +85,14 @@ static void tcp_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && socket_error() == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    closesocket(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -130,26 +102,21 @@ static void tcp_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
+    return;
+
 out:
-    close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
+    closesocket(c);
 }
 
-int tcp_start_incoming_migration(const char *host_port, Error **errp)
+void tcp_start_incoming_migration(const char *host_port, Error **errp)
 {
     int s;
 
     s = inet_listen(host_port, NULL, 256, SOCK_STREAM, 0, errp);
-
     if (s < 0) {
-        return -1;
+        return;
     }
 
     qemu_set_fd_handler2(s, NULL, tcp_accept_incoming_migration, NULL,
                          (void *)(intptr_t)s);
-
-    return 0;
 }
diff --git a/migration-unix.c b/migration-unix.c
index 169de886775..218835a7a44 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -14,11 +14,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_UNIX
 
@@ -44,78 +43,35 @@ static int unix_close(MigrationState *s)
 {
     int r = 0;
     DPRINTF("unix_close\n");
-    if (s->fd != -1) {
-        if (close(s->fd) < 0) {
-            r = -errno;
-        }
-        s->fd = -1;
+    if (close(s->fd) < 0) {
+        r = -errno;
     }
     return r;
 }
 
-static void unix_wait_for_connect(void *opaque)
+static void unix_wait_for_connect(int fd, void *opaque)
 {
     MigrationState *s = opaque;
-    int val, ret;
-    socklen_t valsize = sizeof(val);
-
-    DPRINTF("connect completed\n");
-    do {
-        ret = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
-    } while (ret == -1 && errno == EINTR);
 
-    if (ret < 0) {
+    if (fd < 0) {
+        DPRINTF("migrate connect error\n");
+        s->fd = -1;
         migrate_fd_error(s);
-        return;
-    }
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
-    if (val == 0)
+    } else {
+        DPRINTF("migrate connect success\n");
+        s->fd = fd;
+        socket_set_block(s->fd);
         migrate_fd_connect(s);
-    else {
-        DPRINTF("error connecting %d\n", val);
-        migrate_fd_error(s);
     }
 }
 
-int unix_start_outgoing_migration(MigrationState *s, const char *path)
+void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
 {
-    struct sockaddr_un addr;
-    int ret;
-
-    addr.sun_family = AF_UNIX;
-    snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path);
     s->get_error = unix_errno;
     s->write = unix_write;
     s->close = unix_close;
 
-    s->fd = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (s->fd == -1) {
-        DPRINTF("Unable to open socket");
-        return -errno;
-    }
-
-    socket_set_nonblock(s->fd);
-
-    do {
-        ret = connect(s->fd, (struct sockaddr *)&addr, sizeof(addr));
-        if (ret == -1) {
-            ret = -errno;
-        }
-        if (ret == -EINPROGRESS || ret == -EWOULDBLOCK) {
-	    qemu_set_fd_handler2(s->fd, NULL, NULL, unix_wait_for_connect, s);
-            return 0;
-        }
-    } while (ret == -EINTR);
-
-    if (ret < 0) {
-        DPRINTF("connect failed\n");
-        migrate_fd_error(s);
-        return ret;
-    }
-    migrate_fd_connect(s);
-    return 0;
+    s->fd = unix_nonblocking_connect(path, unix_wait_for_connect, s, errp);
 }
 
 static void unix_accept_incoming_migration(void *opaque)
@@ -129,12 +85,14 @@ static void unix_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && errno == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    close(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -144,51 +102,21 @@ static void unix_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
+    return;
+
 out:
     close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
 }
 
-int unix_start_incoming_migration(const char *path)
+void unix_start_incoming_migration(const char *path, Error **errp)
 {
-    struct sockaddr_un addr;
     int s;
-    int ret;
-
-    DPRINTF("Attempting to start an incoming migration\n");
 
-    s = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (s == -1) {
-        fprintf(stderr, "Could not open unix socket: %s\n", strerror(errno));
-        return -errno;
-    }
-
-    memset(&addr, 0, sizeof(addr));
-    addr.sun_family = AF_UNIX;
-    snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path);
-
-    unlink(addr.sun_path);
-    if (bind(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
-        ret = -errno;
-        fprintf(stderr, "bind(unix:%s): %s\n", addr.sun_path, strerror(errno));
-        goto err;
-    }
-    if (listen(s, 1) == -1) {
-        fprintf(stderr, "listen(unix:%s): %s\n", addr.sun_path,
-                strerror(errno));
-        ret = -errno;
-        goto err;
+    s = unix_listen(path, NULL, 0, errp);
+    if (s < 0) {
+        return;
     }
 
     qemu_set_fd_handler2(s, NULL, unix_accept_incoming_migration, NULL,
                          (void *)(intptr_t)s);
-
-    return 0;
-
-err:
-    close(s);
-    return ret;
 }
diff --git a/migration.c b/migration.c
index 1edeec5ddd1..77c1971b774 100644
--- a/migration.c
+++ b/migration.c
@@ -14,13 +14,14 @@
  */
 
 #include "qemu-common.h"
-#include "migration.h"
-#include "monitor.h"
-#include "buffered_file.h"
-#include "sysemu.h"
-#include "block.h"
-#include "qemu_socket.h"
-#include "block-migration.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
+#include "migration/qemu-file.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
+#include "qemu/sockets.h"
+#include "migration/block.h"
+#include "qemu/thread.h"
 #include "qmp-commands.h"
 
 //#define DEBUG_MIGRATION
@@ -43,6 +44,11 @@ enum {
 
 #define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
 
+/* Amount of time to allocate to each "chunk" of bandwidth-throttled
+ * data. */
+#define BUFFER_DELAY     100
+#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
+
 /* Migration XBZRLE default cache size */
 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
 
@@ -53,7 +59,7 @@ static NotifierList migration_state_notifiers =
    migrations at once.  For now we don't need to add
    dynamic creation of migration */
 
-static MigrationState *migrate_get_current(void)
+MigrationState *migrate_get_current(void)
 {
     static MigrationState current_migration = {
         .state = MIG_STATE_SETUP,
@@ -64,31 +70,34 @@ static MigrationState *migrate_get_current(void)
     return &current_migration;
 }
 
-int qemu_start_incoming_migration(const char *uri, Error **errp)
+void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
     const char *p;
-    int ret;
 
     if (strstart(uri, "tcp:", &p))
-        ret = tcp_start_incoming_migration(p, errp);
+        tcp_start_incoming_migration(p, errp);
 #if !defined(WIN32)
     else if (strstart(uri, "exec:", &p))
-        ret =  exec_start_incoming_migration(p);
+        exec_start_incoming_migration(p, errp);
     else if (strstart(uri, "unix:", &p))
-        ret = unix_start_incoming_migration(p);
+        unix_start_incoming_migration(p, errp);
     else if (strstart(uri, "fd:", &p))
-        ret = fd_start_incoming_migration(p);
+        fd_start_incoming_migration(p, errp);
 #endif
     else {
-        fprintf(stderr, "unknown migration protocol: %s\n", uri);
-        ret = -EPROTONOSUPPORT;
+        error_setg(errp, "unknown migration protocol: %s\n", uri);
     }
-    return ret;
 }
 
-void process_incoming_migration(QEMUFile *f)
+static void process_incoming_migration_co(void *opaque)
 {
-    if (qemu_loadvm_state(f) < 0) {
+    QEMUFile *f = opaque;
+    int ret;
+
+    ret = qemu_loadvm_state(f);
+    qemu_set_fd_handler(qemu_get_fd(f), NULL, NULL, NULL);
+    qemu_fclose(f);
+    if (ret < 0) {
         fprintf(stderr, "load of migration failed\n");
         exit(0);
     }
@@ -102,10 +111,27 @@ void process_incoming_migration(QEMUFile *f)
     if (autostart) {
         vm_start();
     } else {
-        runstate_set(RUN_STATE_PRELAUNCH);
+        runstate_set(RUN_STATE_PAUSED);
     }
 }
 
+static void enter_migration_coroutine(void *opaque)
+{
+    Coroutine *co = opaque;
+    qemu_coroutine_enter(co, NULL);
+}
+
+void process_incoming_migration(QEMUFile *f)
+{
+    Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
+    int fd = qemu_get_fd(f);
+
+    assert(fd != -1);
+    socket_set_nonblock(fd);
+    qemu_set_fd_handler(fd, enter_migration_coroutine, NULL, co);
+    qemu_coroutine_enter(co, f);
+}
+
 /* amount of nanoseconds we are willing to wait for migration to be down.
  * the choice of nanoseconds is because it is the maximum resolution that
  * get_clock() can achieve. It is an internal measure. All user-visible
@@ -169,6 +195,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->has_total_time = true;
         info->total_time = qemu_get_clock_ms(rt_clock)
             - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
 
         info->has_ram = true;
         info->ram = g_malloc0(sizeof(*info->ram));
@@ -178,6 +206,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->ram->duplicate = dup_mig_pages_transferred();
         info->ram->normal = norm_mig_pages_transferred();
         info->ram->normal_bytes = norm_mig_bytes_transferred();
+        info->ram->dirty_pages_rate = s->dirty_pages_rate;
+
 
         if (blk_mig_active()) {
             info->has_disk = true;
@@ -195,6 +225,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->has_status = true;
         info->status = g_strdup("completed");
         info->total_time = s->total_time;
+        info->has_downtime = true;
+        info->downtime = s->downtime;
 
         info->has_ram = true;
         info->ram = g_malloc0(sizeof(*info->ram));
@@ -240,19 +272,13 @@ static int migrate_fd_cleanup(MigrationState *s)
 {
     int ret = 0;
 
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
     if (s->file) {
         DPRINTF("closing file\n");
         ret = qemu_fclose(s->file);
         s->file = NULL;
     }
 
-    if (s->fd != -1) {
-        close(s->fd);
-        s->fd = -1;
-    }
-
+    assert(s->fd == -1);
     return ret;
 }
 
@@ -276,21 +302,9 @@ static void migrate_fd_completed(MigrationState *s)
     notifier_list_notify(&migration_state_notifiers, s);
 }
 
-static void migrate_fd_put_notify(void *opaque)
-{
-    MigrationState *s = opaque;
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    qemu_file_put_notify(s->file);
-    if (s->file && qemu_file_get_error(s->file)) {
-        migrate_fd_error(s);
-    }
-}
-
-static ssize_t migrate_fd_put_buffer(void *opaque, const void *data,
+static ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
                                      size_t size)
 {
-    MigrationState *s = opaque;
     ssize_t ret;
 
     if (s->state != MIG_STATE_ACTIVE) {
@@ -304,48 +318,9 @@ static ssize_t migrate_fd_put_buffer(void *opaque, const void *data,
     if (ret == -1)
         ret = -(s->get_error(s));
 
-    if (ret == -EAGAIN) {
-        qemu_set_fd_handler2(s->fd, NULL, NULL, migrate_fd_put_notify, s);
-    }
-
     return ret;
 }
 
-static void migrate_fd_put_ready(void *opaque)
-{
-    MigrationState *s = opaque;
-    int ret;
-
-    if (s->state != MIG_STATE_ACTIVE) {
-        DPRINTF("put_ready returning because of non-active state\n");
-        return;
-    }
-
-    DPRINTF("iterate\n");
-    ret = qemu_savevm_state_iterate(s->file);
-    if (ret < 0) {
-        migrate_fd_error(s);
-    } else if (ret == 1) {
-        int old_vm_running = runstate_is_running();
-
-        DPRINTF("done iterating\n");
-        qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-        vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
-
-        if (qemu_savevm_state_complete(s->file) < 0) {
-            migrate_fd_error(s);
-        } else {
-            migrate_fd_completed(s);
-        }
-        s->total_time = qemu_get_clock_ms(rt_clock) - s->total_time;
-        if (s->state != MIG_STATE_COMPLETED) {
-            if (old_vm_running) {
-                vm_start();
-            }
-        }
-    }
-}
-
 static void migrate_fd_cancel(MigrationState *s)
 {
     if (s->state != MIG_STATE_ACTIVE)
@@ -355,40 +330,19 @@ static void migrate_fd_cancel(MigrationState *s)
 
     s->state = MIG_STATE_CANCELLED;
     notifier_list_notify(&migration_state_notifiers, s);
-    qemu_savevm_state_cancel(s->file);
+    qemu_savevm_state_cancel();
 
     migrate_fd_cleanup(s);
 }
 
-static void migrate_fd_wait_for_unfreeze(void *opaque)
+int migrate_fd_close(MigrationState *s)
 {
-    MigrationState *s = opaque;
-    int ret;
-
-    DPRINTF("wait for unfreeze\n");
-    if (s->state != MIG_STATE_ACTIVE)
-        return;
-
-    do {
-        fd_set wfds;
-
-        FD_ZERO(&wfds);
-        FD_SET(s->fd, &wfds);
-
-        ret = select(s->fd + 1, NULL, &wfds, NULL, NULL);
-    } while (ret == -1 && (s->get_error(s)) == EINTR);
-
-    if (ret == -1) {
-        qemu_file_set_error(s->file, -s->get_error(s));
+    int rc = 0;
+    if (s->fd != -1) {
+        rc = s->close(s);
+        s->fd = -1;
     }
-}
-
-static int migrate_fd_close(void *opaque)
-{
-    MigrationState *s = opaque;
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    return s->close(s);
+    return rc;
 }
 
 void add_migration_state_change_notifier(Notifier *notify)
@@ -417,28 +371,6 @@ bool migration_has_failed(MigrationState *s)
             s->state == MIG_STATE_ERROR);
 }
 
-void migrate_fd_connect(MigrationState *s)
-{
-    int ret;
-
-    s->state = MIG_STATE_ACTIVE;
-    s->file = qemu_fopen_ops_buffered(s,
-                                      s->bandwidth_limit,
-                                      migrate_fd_put_buffer,
-                                      migrate_fd_put_ready,
-                                      migrate_fd_wait_for_unfreeze,
-                                      migrate_fd_close);
-
-    DPRINTF("beginning savevm\n");
-    ret = qemu_savevm_state_begin(s->file, &s->params);
-    if (ret < 0) {
-        DPRINTF("failed, %d\n", ret);
-        migrate_fd_error(s);
-        return;
-    }
-    migrate_fd_put_ready(s);
-}
-
 static MigrationState *migrate_init(const MigrationParams *params)
 {
     MigrationState *s = migrate_get_current();
@@ -479,10 +411,10 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
                  bool has_inc, bool inc, bool has_detach, bool detach,
                  Error **errp)
 {
+    Error *local_err = NULL;
     MigrationState *s = migrate_get_current();
     MigrationParams params;
     const char *p;
-    int ret;
 
     params.blk = blk;
     params.shared = inc;
@@ -504,30 +436,25 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     s = migrate_init(&params);
 
     if (strstart(uri, "tcp:", &p)) {
-        ret = tcp_start_outgoing_migration(s, p, errp);
+        tcp_start_outgoing_migration(s, p, &local_err);
 #if !defined(WIN32)
     } else if (strstart(uri, "exec:", &p)) {
-        ret = exec_start_outgoing_migration(s, p);
+        exec_start_outgoing_migration(s, p, &local_err);
     } else if (strstart(uri, "unix:", &p)) {
-        ret = unix_start_outgoing_migration(s, p);
+        unix_start_outgoing_migration(s, p, &local_err);
     } else if (strstart(uri, "fd:", &p)) {
-        ret = fd_start_outgoing_migration(s, p);
+        fd_start_outgoing_migration(s, p, &local_err);
 #endif
     } else {
         error_set(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol");
         return;
     }
 
-    if (ret < 0) {
-        if (!error_is_set(errp)) {
-            DPRINTF("migration failed: %s\n", strerror(-ret));
-            /* FIXME: we should return meaningful errors */
-            error_set(errp, QERR_UNDEFINED_ERROR);
-        }
+    if (local_err) {
+        migrate_fd_error(s);
+        error_propagate(errp, local_err);
         return;
     }
-
-    notifier_list_notify(&migration_state_notifiers, s);
 }
 
 void qmp_migrate_cancel(Error **errp)
@@ -591,3 +518,277 @@ int64_t migrate_xbzrle_cache_size(void)
 
     return s->xbzrle_cache_size;
 }
+
+/* migration thread support */
+
+
+static ssize_t buffered_flush(MigrationState *s)
+{
+    size_t offset = 0;
+    ssize_t ret = 0;
+
+    DPRINTF("flushing %zu byte(s) of data\n", s->buffer_size);
+
+    while (s->bytes_xfer < s->xfer_limit && offset < s->buffer_size) {
+        size_t to_send = MIN(s->buffer_size - offset, s->xfer_limit - s->bytes_xfer);
+        ret = migrate_fd_put_buffer(s, s->buffer + offset, to_send);
+        if (ret <= 0) {
+            DPRINTF("error flushing data, %zd\n", ret);
+            break;
+        } else {
+            DPRINTF("flushed %zd byte(s)\n", ret);
+            offset += ret;
+            s->bytes_xfer += ret;
+        }
+    }
+
+    DPRINTF("flushed %zu of %zu byte(s)\n", offset, s->buffer_size);
+    memmove(s->buffer, s->buffer + offset, s->buffer_size - offset);
+    s->buffer_size -= offset;
+
+    if (ret < 0) {
+        return ret;
+    }
+    return offset;
+}
+
+static int buffered_put_buffer(void *opaque, const uint8_t *buf,
+                               int64_t pos, int size)
+{
+    MigrationState *s = opaque;
+    ssize_t error;
+
+    DPRINTF("putting %d bytes at %" PRId64 "\n", size, pos);
+
+    error = qemu_file_get_error(s->file);
+    if (error) {
+        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
+        return error;
+    }
+
+    if (size <= 0) {
+        return size;
+    }
+
+    if (size > (s->buffer_capacity - s->buffer_size)) {
+        DPRINTF("increasing buffer capacity from %zu by %zu\n",
+                s->buffer_capacity, size + 1024);
+
+        s->buffer_capacity += size + 1024;
+
+        s->buffer = g_realloc(s->buffer, s->buffer_capacity);
+    }
+
+    memcpy(s->buffer + s->buffer_size, buf, size);
+    s->buffer_size += size;
+
+    return size;
+}
+
+static int buffered_close(void *opaque)
+{
+    MigrationState *s = opaque;
+    ssize_t ret = 0;
+    int ret2;
+
+    DPRINTF("closing\n");
+
+    s->xfer_limit = INT_MAX;
+    while (!qemu_file_get_error(s->file) && s->buffer_size) {
+        ret = buffered_flush(s);
+        if (ret < 0) {
+            break;
+        }
+    }
+
+    ret2 = migrate_fd_close(s);
+    if (ret >= 0) {
+        ret = ret2;
+    }
+    s->complete = true;
+    return ret;
+}
+
+static int buffered_get_fd(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    return s->fd;
+}
+
+/*
+ * The meaning of the return values is:
+ *   0: We can continue sending
+ *   1: Time to stop
+ *   negative: There has been an error
+ */
+static int buffered_rate_limit(void *opaque)
+{
+    MigrationState *s = opaque;
+    int ret;
+
+    ret = qemu_file_get_error(s->file);
+    if (ret) {
+        return ret;
+    }
+
+    if (s->bytes_xfer >= s->xfer_limit) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static int64_t buffered_set_rate_limit(void *opaque, int64_t new_rate)
+{
+    MigrationState *s = opaque;
+    if (qemu_file_get_error(s->file)) {
+        goto out;
+    }
+    if (new_rate > SIZE_MAX) {
+        new_rate = SIZE_MAX;
+    }
+
+    s->xfer_limit = new_rate / XFER_LIMIT_RATIO;
+
+out:
+    return s->xfer_limit;
+}
+
+static int64_t buffered_get_rate_limit(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    return s->xfer_limit;
+}
+
+static void *buffered_file_thread(void *opaque)
+{
+    MigrationState *s = opaque;
+    int64_t initial_time = qemu_get_clock_ms(rt_clock);
+    int64_t max_size = 0;
+    bool last_round = false;
+    int ret;
+
+    qemu_mutex_lock_iothread();
+    DPRINTF("beginning savevm\n");
+    ret = qemu_savevm_state_begin(s->file, &s->params);
+    if (ret < 0) {
+        DPRINTF("failed, %d\n", ret);
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
+    qemu_mutex_unlock_iothread();
+
+    while (true) {
+        int64_t current_time = qemu_get_clock_ms(rt_clock);
+        uint64_t pending_size;
+
+        qemu_mutex_lock_iothread();
+        if (s->state != MIG_STATE_ACTIVE) {
+            DPRINTF("put_ready returning because of non-active state\n");
+            qemu_mutex_unlock_iothread();
+            break;
+        }
+        if (s->complete) {
+            qemu_mutex_unlock_iothread();
+            break;
+        }
+        if (s->bytes_xfer < s->xfer_limit) {
+            DPRINTF("iterate\n");
+            pending_size = qemu_savevm_state_pending(s->file, max_size);
+            DPRINTF("pending size %lu max %lu\n", pending_size, max_size);
+            if (pending_size && pending_size >= max_size) {
+                ret = qemu_savevm_state_iterate(s->file);
+                if (ret < 0) {
+                    qemu_mutex_unlock_iothread();
+                    break;
+                }
+            } else {
+                int old_vm_running = runstate_is_running();
+                int64_t start_time, end_time;
+
+                DPRINTF("done iterating\n");
+                start_time = qemu_get_clock_ms(rt_clock);
+                qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+                if (old_vm_running) {
+                    vm_stop(RUN_STATE_FINISH_MIGRATE);
+                } else {
+                    vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+                ret = qemu_savevm_state_complete(s->file);
+                if (ret < 0) {
+                    qemu_mutex_unlock_iothread();
+                    break;
+                } else {
+                    migrate_fd_completed(s);
+                }
+                end_time = qemu_get_clock_ms(rt_clock);
+                s->total_time = end_time - s->total_time;
+                s->downtime = end_time - start_time;
+                if (s->state != MIG_STATE_COMPLETED) {
+                    if (old_vm_running) {
+                        vm_start();
+                    }
+                }
+                last_round = true;
+            }
+        }
+        qemu_mutex_unlock_iothread();
+        if (current_time >= initial_time + BUFFER_DELAY) {
+            uint64_t transferred_bytes = s->bytes_xfer;
+            uint64_t time_spent = current_time - initial_time;
+            double bandwidth = transferred_bytes / time_spent;
+            max_size = bandwidth * migrate_max_downtime() / 1000000;
+
+            DPRINTF("transferred %" PRIu64 " time_spent %" PRIu64
+                    " bandwidth %g max_size %" PRId64 "\n",
+                    transferred_bytes, time_spent, bandwidth, max_size);
+
+            s->bytes_xfer = 0;
+            initial_time = current_time;
+        }
+        if (!last_round && (s->bytes_xfer >= s->xfer_limit)) {
+            /* usleep expects microseconds */
+            g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
+        }
+        ret = buffered_flush(s);
+        if (ret < 0) {
+            break;
+        }
+    }
+
+out:
+    if (ret < 0) {
+        migrate_fd_error(s);
+    }
+    g_free(s->buffer);
+    return NULL;
+}
+
+static const QEMUFileOps buffered_file_ops = {
+    .get_fd =         buffered_get_fd,
+    .put_buffer =     buffered_put_buffer,
+    .close =          buffered_close,
+    .rate_limit =     buffered_rate_limit,
+    .get_rate_limit = buffered_get_rate_limit,
+    .set_rate_limit = buffered_set_rate_limit,
+};
+
+void migrate_fd_connect(MigrationState *s)
+{
+    s->state = MIG_STATE_ACTIVE;
+    s->bytes_xfer = 0;
+    s->buffer = NULL;
+    s->buffer_size = 0;
+    s->buffer_capacity = 0;
+
+    s->xfer_limit = s->bandwidth_limit / XFER_LIMIT_RATIO;
+    s->complete = false;
+
+    s->file = qemu_fopen_ops(s, &buffered_file_ops);
+
+    qemu_thread_create(&s->thread, buffered_file_thread, s,
+                       QEMU_THREAD_DETACHED);
+    notifier_list_notify(&migration_state_notifiers, s);
+}
diff --git a/monitor.c b/monitor.c
index b17b1bb8497..20bd19b05fd 100644
--- a/monitor.c
+++ b/monitor.c
@@ -27,35 +27,35 @@
 #include "hw/usb.h"
 #include "hw/pcmcia.h"
 #include "hw/pc.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/watchdog.h"
 #include "hw/loader.h"
-#include "gdbstub.h"
-#include "net.h"
+#include "exec/gdbstub.h"
+#include "net/net.h"
 #include "net/slirp.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "ui/qemu-spice.h"
-#include "sysemu.h"
-#include "monitor.h"
-#include "readline.h"
-#include "console.h"
-#include "blockdev.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "monitor/readline.h"
+#include "ui/console.h"
+#include "sysemu/blockdev.h"
 #include "audio/audio.h"
-#include "disas.h"
-#include "balloon.h"
-#include "qemu-timer.h"
-#include "migration.h"
-#include "kvm.h"
-#include "acl.h"
-#include "qint.h"
-#include "qfloat.h"
-#include "qlist.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qjson.h"
-#include "json-streamer.h"
-#include "json-parser.h"
-#include "osdep.h"
+#include "disas/disas.h"
+#include "sysemu/balloon.h"
+#include "qemu/timer.h"
+#include "migration/migration.h"
+#include "sysemu/kvm.h"
+#include "qemu/acl.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "trace/control.h"
@@ -63,10 +63,10 @@
 #include "trace/simple.h"
 #endif
 #include "ui/qemu-spice.h"
-#include "memory.h"
+#include "exec/memory.h"
 #include "qmp-commands.h"
 #include "hmp.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 /* for pic/irq_info */
 #if defined(TARGET_SPARC)
@@ -123,13 +123,17 @@ typedef struct mon_cmd_t {
     const char *help;
     void (*user_print)(Monitor *mon, const QObject *data);
     union {
-        void (*info)(Monitor *mon);
         void (*cmd)(Monitor *mon, const QDict *qdict);
         int  (*cmd_new)(Monitor *mon, const QDict *params, QObject **ret_data);
         int  (*cmd_async)(Monitor *mon, const QDict *params,
                           MonitorCompletion *cb, void *opaque);
     } mhandler;
     int flags;
+    /* @sub_table is a list of 2nd level of commands. If it do not exist,
+     * mhandler should be used. If it exist, sub_table[?].mhandler should be
+     * used, and mhandler of 1st level plays the role of help function.
+     */
+    struct mon_cmd_t *sub_table;
 } mon_cmd_t;
 
 /* file descriptors passed via SCM_RIGHTS */
@@ -270,6 +274,7 @@ static void monitor_puts(Monitor *mon, const char *str)
     char c;
 
     for(;;) {
+        assert(mon->outbuf_index < sizeof(mon->outbuf) - 1);
         c = *str++;
         if (c == '\0')
             break;
@@ -450,11 +455,14 @@ static const char *monitor_event_names[] = {
     [QEVENT_SPICE_DISCONNECTED] = "SPICE_DISCONNECTED",
     [QEVENT_BLOCK_JOB_COMPLETED] = "BLOCK_JOB_COMPLETED",
     [QEVENT_BLOCK_JOB_CANCELLED] = "BLOCK_JOB_CANCELLED",
+    [QEVENT_BLOCK_JOB_ERROR] = "BLOCK_JOB_ERROR",
+    [QEVENT_BLOCK_JOB_READY] = "BLOCK_JOB_READY",
     [QEVENT_DEVICE_TRAY_MOVED] = "DEVICE_TRAY_MOVED",
     [QEVENT_SUSPEND] = "SUSPEND",
     [QEVENT_SUSPEND_DISK] = "SUSPEND_DISK",
     [QEVENT_WAKEUP] = "WAKEUP",
     [QEVENT_BALLOON_CHANGE] = "BALLOON_CHANGE",
+    [QEVENT_SPICE_MIGRATE_COMPLETED] = "SPICE_MIGRATE_COMPLETED",
 };
 QEMU_BUILD_BUG_ON(ARRAY_SIZE(monitor_event_names) != QEVENT_MAX)
 
@@ -803,28 +811,8 @@ static void user_async_cmd_handler(Monitor *mon, const mon_cmd_t *cmd,
     }
 }
 
-static void do_info(Monitor *mon, const QDict *qdict)
+static void do_info_help(Monitor *mon, const QDict *qdict)
 {
-    const mon_cmd_t *cmd;
-    const char *item = qdict_get_try_str(qdict, "item");
-
-    if (!item) {
-        goto help;
-    }
-
-    for (cmd = info_cmds; cmd->name != NULL; cmd++) {
-        if (compare_cmd(item, cmd->name))
-            break;
-    }
-
-    if (cmd->name == NULL) {
-        goto help;
-    }
-
-    cmd->mhandler.info(mon);
-    return;
-
-help:
     help_cmd(mon, "info");
 }
 
@@ -868,9 +856,11 @@ EventInfoList *qmp_query_events(Error **errp)
 int monitor_set_cpu(int cpu_index)
 {
     CPUArchState *env;
+    CPUState *cpu;
 
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index == cpu_index) {
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
+        if (cpu->cpu_index == cpu_index) {
             cur_mon->mon_cpu = env;
             return 0;
         }
@@ -889,28 +879,23 @@ static CPUArchState *mon_get_cpu(void)
 
 int monitor_get_cpu_index(void)
 {
-    return mon_get_cpu()->cpu_index;
+    CPUState *cpu = ENV_GET_CPU(mon_get_cpu());
+    return cpu->cpu_index;
 }
 
-static void do_info_registers(Monitor *mon)
+static void do_info_registers(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
     env = mon_get_cpu();
-#ifdef TARGET_I386
-    cpu_dump_state(env, (FILE *)mon, monitor_fprintf,
-                   X86_DUMP_FPU);
-#else
-    cpu_dump_state(env, (FILE *)mon, monitor_fprintf,
-                   0);
-#endif
+    cpu_dump_state(env, (FILE *)mon, monitor_fprintf, CPU_DUMP_FPU);
 }
 
-static void do_info_jit(Monitor *mon)
+static void do_info_jit(Monitor *mon, const QDict *qdict)
 {
     dump_exec_info((FILE *)mon, monitor_fprintf);
 }
 
-static void do_info_history(Monitor *mon)
+static void do_info_history(Monitor *mon, const QDict *qdict)
 {
     int i;
     const char *str;
@@ -929,7 +914,7 @@ static void do_info_history(Monitor *mon)
 
 #if defined(TARGET_PPC)
 /* XXX: not implemented in other targets */
-static void do_info_cpu_stats(Monitor *mon)
+static void do_info_cpu_stats(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -938,50 +923,11 @@ static void do_info_cpu_stats(Monitor *mon)
 }
 #endif
 
-static void do_trace_print_events(Monitor *mon)
+static void do_trace_print_events(Monitor *mon, const QDict *qdict)
 {
     trace_print_events((FILE *)mon, &monitor_fprintf);
 }
 
-static int add_graphics_client(Monitor *mon, const QDict *qdict, QObject **ret_data)
-{
-    const char *protocol  = qdict_get_str(qdict, "protocol");
-    const char *fdname = qdict_get_str(qdict, "fdname");
-    CharDriverState *s;
-
-    if (strcmp(protocol, "spice") == 0) {
-        int fd = monitor_get_fd(mon, fdname);
-        int skipauth = qdict_get_try_bool(qdict, "skipauth", 0);
-        int tls = qdict_get_try_bool(qdict, "tls", 0);
-        if (!using_spice) {
-            /* correct one? spice isn't a device ,,, */
-            qerror_report(QERR_DEVICE_NOT_ACTIVE, "spice");
-            return -1;
-        }
-        if (qemu_spice_display_add_client(fd, skipauth, tls) < 0) {
-            close(fd);
-        }
-        return 0;
-#ifdef CONFIG_VNC
-    } else if (strcmp(protocol, "vnc") == 0) {
-	int fd = monitor_get_fd(mon, fdname);
-        int skipauth = qdict_get_try_bool(qdict, "skipauth", 0);
-	vnc_display_add_client(NULL, fd, skipauth);
-	return 0;
-#endif
-    } else if ((s = qemu_chr_find(protocol)) != NULL) {
-	int fd = monitor_get_fd(mon, fdname);
-	if (qemu_chr_add_client(s, fd) < 0) {
-	    qerror_report(QERR_ADD_CLIENT_FAILED);
-	    return -1;
-	}
-	return 0;
-    }
-
-    qerror_report(QERR_INVALID_PARAMETER, "protocol");
-    return -1;
-}
-
 static int client_migrate_info(Monitor *mon, const QDict *qdict,
                                MonitorCompletion cb, void *opaque)
 {
@@ -1016,12 +962,6 @@ static int client_migrate_info(Monitor *mon, const QDict *qdict,
     return -1;
 }
 
-static int do_screen_dump(Monitor *mon, const QDict *qdict, QObject **ret_data)
-{
-    vga_hw_screen_dump(qdict_get_str(qdict, "filename"));
-    return 0;
-}
-
 static void do_logfile(Monitor *mon, const QDict *qdict)
 {
     cpu_set_log_filename(qdict_get_str(qdict, "filename"));
@@ -1108,7 +1048,7 @@ static void monitor_printc(Monitor *mon, int c)
 }
 
 static void memory_dump(Monitor *mon, int count, int format, int wsize,
-                        target_phys_addr_t addr, int is_physical)
+                        hwaddr addr, int is_physical)
 {
     CPUArchState *env;
     int l, line_size, i, max_digits, len;
@@ -1242,7 +1182,7 @@ static void do_physical_memory_dump(Monitor *mon, const QDict *qdict)
     int count = qdict_get_int(qdict, "count");
     int format = qdict_get_int(qdict, "format");
     int size = qdict_get_int(qdict, "size");
-    target_phys_addr_t addr = qdict_get_int(qdict, "addr");
+    hwaddr addr = qdict_get_int(qdict, "addr");
 
     memory_dump(mon, count, format, size, addr, 1);
 }
@@ -1250,21 +1190,21 @@ static void do_physical_memory_dump(Monitor *mon, const QDict *qdict)
 static void do_print(Monitor *mon, const QDict *qdict)
 {
     int format = qdict_get_int(qdict, "format");
-    target_phys_addr_t val = qdict_get_int(qdict, "val");
+    hwaddr val = qdict_get_int(qdict, "val");
 
     switch(format) {
     case 'o':
-        monitor_printf(mon, "%#" TARGET_PRIoPHYS, val);
+        monitor_printf(mon, "%#" HWADDR_PRIo, val);
         break;
     case 'x':
-        monitor_printf(mon, "%#" TARGET_PRIxPHYS, val);
+        monitor_printf(mon, "%#" HWADDR_PRIx, val);
         break;
     case 'u':
-        monitor_printf(mon, "%" TARGET_PRIuPHYS, val);
+        monitor_printf(mon, "%" HWADDR_PRIu, val);
         break;
     default:
     case 'd':
-        monitor_printf(mon, "%" TARGET_PRIdPHYS, val);
+        monitor_printf(mon, "%" HWADDR_PRId, val);
         break;
     case 'c':
         monitor_printc(mon, val);
@@ -1290,245 +1230,6 @@ static void do_sum(Monitor *mon, const QDict *qdict)
     monitor_printf(mon, "%05d\n", sum);
 }
 
-typedef struct {
-    int keycode;
-    const char *name;
-} KeyDef;
-
-static const KeyDef key_defs[] = {
-    { 0x2a, "shift" },
-    { 0x36, "shift_r" },
-
-    { 0x38, "alt" },
-    { 0xb8, "alt_r" },
-    { 0x64, "altgr" },
-    { 0xe4, "altgr_r" },
-    { 0x1d, "ctrl" },
-    { 0x9d, "ctrl_r" },
-
-    { 0xdd, "menu" },
-
-    { 0x01, "esc" },
-
-    { 0x02, "1" },
-    { 0x03, "2" },
-    { 0x04, "3" },
-    { 0x05, "4" },
-    { 0x06, "5" },
-    { 0x07, "6" },
-    { 0x08, "7" },
-    { 0x09, "8" },
-    { 0x0a, "9" },
-    { 0x0b, "0" },
-    { 0x0c, "minus" },
-    { 0x0d, "equal" },
-    { 0x0e, "backspace" },
-
-    { 0x0f, "tab" },
-    { 0x10, "q" },
-    { 0x11, "w" },
-    { 0x12, "e" },
-    { 0x13, "r" },
-    { 0x14, "t" },
-    { 0x15, "y" },
-    { 0x16, "u" },
-    { 0x17, "i" },
-    { 0x18, "o" },
-    { 0x19, "p" },
-    { 0x1a, "bracket_left" },
-    { 0x1b, "bracket_right" },
-    { 0x1c, "ret" },
-
-    { 0x1e, "a" },
-    { 0x1f, "s" },
-    { 0x20, "d" },
-    { 0x21, "f" },
-    { 0x22, "g" },
-    { 0x23, "h" },
-    { 0x24, "j" },
-    { 0x25, "k" },
-    { 0x26, "l" },
-    { 0x27, "semicolon" },
-    { 0x28, "apostrophe" },
-    { 0x29, "grave_accent" },
-
-    { 0x2b, "backslash" },
-    { 0x2c, "z" },
-    { 0x2d, "x" },
-    { 0x2e, "c" },
-    { 0x2f, "v" },
-    { 0x30, "b" },
-    { 0x31, "n" },
-    { 0x32, "m" },
-    { 0x33, "comma" },
-    { 0x34, "dot" },
-    { 0x35, "slash" },
-
-    { 0x37, "asterisk" },
-
-    { 0x39, "spc" },
-    { 0x3a, "caps_lock" },
-    { 0x3b, "f1" },
-    { 0x3c, "f2" },
-    { 0x3d, "f3" },
-    { 0x3e, "f4" },
-    { 0x3f, "f5" },
-    { 0x40, "f6" },
-    { 0x41, "f7" },
-    { 0x42, "f8" },
-    { 0x43, "f9" },
-    { 0x44, "f10" },
-    { 0x45, "num_lock" },
-    { 0x46, "scroll_lock" },
-
-    { 0xb5, "kp_divide" },
-    { 0x37, "kp_multiply" },
-    { 0x4a, "kp_subtract" },
-    { 0x4e, "kp_add" },
-    { 0x9c, "kp_enter" },
-    { 0x53, "kp_decimal" },
-    { 0x54, "sysrq" },
-
-    { 0x52, "kp_0" },
-    { 0x4f, "kp_1" },
-    { 0x50, "kp_2" },
-    { 0x51, "kp_3" },
-    { 0x4b, "kp_4" },
-    { 0x4c, "kp_5" },
-    { 0x4d, "kp_6" },
-    { 0x47, "kp_7" },
-    { 0x48, "kp_8" },
-    { 0x49, "kp_9" },
-
-    { 0x56, "<" },
-
-    { 0x57, "f11" },
-    { 0x58, "f12" },
-
-    { 0xb7, "print" },
-
-    { 0xc7, "home" },
-    { 0xc9, "pgup" },
-    { 0xd1, "pgdn" },
-    { 0xcf, "end" },
-
-    { 0xcb, "left" },
-    { 0xc8, "up" },
-    { 0xd0, "down" },
-    { 0xcd, "right" },
-
-    { 0xd2, "insert" },
-    { 0xd3, "delete" },
-#if defined(TARGET_SPARC) && !defined(TARGET_SPARC64)
-    { 0xf0, "stop" },
-    { 0xf1, "again" },
-    { 0xf2, "props" },
-    { 0xf3, "undo" },
-    { 0xf4, "front" },
-    { 0xf5, "copy" },
-    { 0xf6, "open" },
-    { 0xf7, "paste" },
-    { 0xf8, "find" },
-    { 0xf9, "cut" },
-    { 0xfa, "lf" },
-    { 0xfb, "help" },
-    { 0xfc, "meta_l" },
-    { 0xfd, "meta_r" },
-    { 0xfe, "compose" },
-#endif
-    { 0, NULL },
-};
-
-static int get_keycode(const char *key)
-{
-    const KeyDef *p;
-    char *endp;
-    int ret;
-
-    for(p = key_defs; p->name != NULL; p++) {
-        if (!strcmp(key, p->name))
-            return p->keycode;
-    }
-    if (strstart(key, "0x", NULL)) {
-        ret = strtoul(key, &endp, 0);
-        if (*endp == '\0' && ret >= 0x01 && ret <= 0xff)
-            return ret;
-    }
-    return -1;
-}
-
-#define MAX_KEYCODES 16
-static uint8_t keycodes[MAX_KEYCODES];
-static int nb_pending_keycodes;
-static QEMUTimer *key_timer;
-
-static void release_keys(void *opaque)
-{
-    int keycode;
-
-    while (nb_pending_keycodes > 0) {
-        nb_pending_keycodes--;
-        keycode = keycodes[nb_pending_keycodes];
-        if (keycode & 0x80)
-            kbd_put_keycode(0xe0);
-        kbd_put_keycode(keycode | 0x80);
-    }
-}
-
-static void do_sendkey(Monitor *mon, const QDict *qdict)
-{
-    char keyname_buf[16];
-    char *separator;
-    int keyname_len, keycode, i;
-    const char *string = qdict_get_str(qdict, "string");
-    int has_hold_time = qdict_haskey(qdict, "hold_time");
-    int hold_time = qdict_get_try_int(qdict, "hold_time", -1);
-
-    if (nb_pending_keycodes > 0) {
-        qemu_del_timer(key_timer);
-        release_keys(NULL);
-    }
-    if (!has_hold_time)
-        hold_time = 100;
-    i = 0;
-    while (1) {
-        separator = strchr(string, '-');
-        keyname_len = separator ? separator - string : strlen(string);
-        if (keyname_len > 0) {
-            pstrcpy(keyname_buf, sizeof(keyname_buf), string);
-            if (keyname_len > sizeof(keyname_buf) - 1) {
-                monitor_printf(mon, "invalid key: '%s...'\n", keyname_buf);
-                return;
-            }
-            if (i == MAX_KEYCODES) {
-                monitor_printf(mon, "too many keys\n");
-                return;
-            }
-            keyname_buf[keyname_len] = 0;
-            keycode = get_keycode(keyname_buf);
-            if (keycode < 0) {
-                monitor_printf(mon, "unknown key: '%s'\n", keyname_buf);
-                return;
-            }
-            keycodes[i++] = keycode;
-        }
-        if (!separator)
-            break;
-        string = separator + 1;
-    }
-    nb_pending_keycodes = i;
-    /* key down events */
-    for (i = 0; i < nb_pending_keycodes; i++) {
-        keycode = keycodes[i];
-        if (keycode & 0x80)
-            kbd_put_keycode(0xe0);
-        kbd_put_keycode(keycode & 0x7f);
-    }
-    /* delayed key up events */
-    qemu_mod_timer(key_timer, qemu_get_clock_ns(vm_clock) +
-                   muldiv64(get_ticks_per_sec(), hold_time, 1000));
-}
-
 static int mouse_button_state;
 
 static void do_mouse_move(Monitor *mon, const QDict *qdict)
@@ -1625,9 +1326,9 @@ static void do_boot_set(Monitor *mon, const QDict *qdict)
 }
 
 #if defined(TARGET_I386)
-static void print_pte(Monitor *mon, target_phys_addr_t addr,
-                      target_phys_addr_t pte,
-                      target_phys_addr_t mask)
+static void print_pte(Monitor *mon, hwaddr addr,
+                      hwaddr pte,
+                      hwaddr mask)
 {
 #ifdef TARGET_X86_64
     if (addr & (1ULL << 47)) {
@@ -1696,7 +1397,7 @@ static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
                     if (pde & PG_PSE_MASK) {
                         /* 2M pages with PAE, CR4.PSE is ignored */
                         print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
-                                  ~((target_phys_addr_t)(1 << 20) - 1));
+                                  ~((hwaddr)(1 << 20) - 1));
                     } else {
                         pt_addr = pde & 0x3fffffffff000ULL;
                         for (l3 = 0; l3 < 512; l3++) {
@@ -1706,7 +1407,7 @@ static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
                                 print_pte(mon, (l1 << 30 ) + (l2 << 21)
                                           + (l3 << 12),
                                           pte & ~PG_PSE_MASK,
-                                          ~(target_phys_addr_t)0xfff);
+                                          ~(hwaddr)0xfff);
                             }
                         }
                     }
@@ -1774,7 +1475,7 @@ static void tlb_info_64(Monitor *mon, CPUArchState *env)
 }
 #endif
 
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -1798,9 +1499,9 @@ static void tlb_info(Monitor *mon)
     }
 }
 
-static void mem_print(Monitor *mon, target_phys_addr_t *pstart,
+static void mem_print(Monitor *mon, hwaddr *pstart,
                       int *plast_prot,
-                      target_phys_addr_t end, int prot)
+                      hwaddr end, int prot)
 {
     int prot1;
     prot1 = *plast_prot;
@@ -1826,7 +1527,7 @@ static void mem_info_32(Monitor *mon, CPUArchState *env)
     unsigned int l1, l2;
     int prot, last_prot;
     uint32_t pgd, pde, pte;
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     pgd = env->cr[3] & ~0xfff;
     last_prot = 0;
@@ -1859,7 +1560,7 @@ static void mem_info_32(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 32, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
 }
 
 static void mem_info_pae32(Monitor *mon, CPUArchState *env)
@@ -1868,7 +1569,7 @@ static void mem_info_pae32(Monitor *mon, CPUArchState *env)
     int prot, last_prot;
     uint64_t pdpe, pde, pte;
     uint64_t pdp_addr, pd_addr, pt_addr;
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     pdp_addr = env->cr[3] & ~0x1f;
     last_prot = 0;
@@ -1914,7 +1615,7 @@ static void mem_info_pae32(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 32, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
 }
 
 
@@ -1993,11 +1694,11 @@ static void mem_info_64(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 48, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
 }
 #endif
 
-static void mem_info(Monitor *mon)
+static void mem_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -2036,7 +1737,7 @@ static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
                    tlb->d, tlb->wt);
 }
 
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env = mon_get_cpu();
     int i;
@@ -2052,7 +1753,7 @@ static void tlb_info(Monitor *mon)
 #endif
 
 #if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA)
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env1 = mon_get_cpu();
 
@@ -2060,22 +1761,24 @@ static void tlb_info(Monitor *mon)
 }
 #endif
 
-static void do_info_mtree(Monitor *mon)
+static void do_info_mtree(Monitor *mon, const QDict *qdict)
 {
     mtree_info((fprintf_function)monitor_printf, mon);
 }
 
-static void do_info_numa(Monitor *mon)
+static void do_info_numa(Monitor *mon, const QDict *qdict)
 {
     int i;
     CPUArchState *env;
+    CPUState *cpu;
 
     monitor_printf(mon, "%d nodes\n", nb_numa_nodes);
     for (i = 0; i < nb_numa_nodes; i++) {
         monitor_printf(mon, "node %d cpus:", i);
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            if (env->numa_node == i) {
-                monitor_printf(mon, " %d", env->cpu_index);
+            cpu = ENV_GET_CPU(env);
+            if (cpu->numa_node == i) {
+                monitor_printf(mon, " %d", cpu->cpu_index);
             }
         }
         monitor_printf(mon, "\n");
@@ -2089,7 +1792,7 @@ static void do_info_numa(Monitor *mon)
 int64_t qemu_time;
 int64_t dev_time;
 
-static void do_info_profile(Monitor *mon)
+static void do_info_profile(Monitor *mon, const QDict *qdict)
 {
     int64_t total;
     total = qemu_time;
@@ -2103,7 +1806,7 @@ static void do_info_profile(Monitor *mon)
     dev_time = 0;
 }
 #else
-static void do_info_profile(Monitor *mon)
+static void do_info_profile(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "Internal profiler not compiled\n");
 }
@@ -2112,7 +1815,7 @@ static void do_info_profile(Monitor *mon)
 /* Capture support */
 static QLIST_HEAD (capture_list_head, CaptureState) capture_head;
 
-static void do_info_capture(Monitor *mon)
+static void do_info_capture(Monitor *mon, const QDict *qdict)
 {
     int i;
     CaptureState *s;
@@ -2275,7 +1978,9 @@ static void do_acl_remove(Monitor *mon, const QDict *qdict)
 #if defined(TARGET_I386)
 static void do_inject_mce(Monitor *mon, const QDict *qdict)
 {
-    CPUArchState *cenv;
+    X86CPU *cpu;
+    CPUX86State *cenv;
+    CPUState *cs;
     int cpu_index = qdict_get_int(qdict, "cpu_index");
     int bank = qdict_get_int(qdict, "bank");
     uint64_t status = qdict_get_int(qdict, "status");
@@ -2288,8 +1993,10 @@ static void do_inject_mce(Monitor *mon, const QDict *qdict)
         flags |= MCE_INJECT_BROADCAST;
     }
     for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
-        if (cenv->cpu_index == cpu_index) {
-            cpu_x86_inject_mce(mon, cenv, bank, status, mcg_status, addr, misc,
+        cpu = x86_env_get_cpu(cenv);
+        cs = CPU(cpu);
+        if (cs->cpu_index == cpu_index) {
+            cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
                                flags);
             break;
         }
@@ -2362,7 +2069,7 @@ static void do_loadvm(Monitor *mon, const QDict *qdict)
     }
 }
 
-int monitor_get_fd(Monitor *mon, const char *fdname)
+int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp)
 {
     mon_fd_t *monfd;
 
@@ -2383,6 +2090,7 @@ int monitor_get_fd(Monitor *mon, const char *fdname)
         return fd;
     }
 
+    error_setg(errp, "File descriptor named '%s' has not been found", fdname);
     return -1;
 }
 
@@ -2392,8 +2100,9 @@ static void monitor_fdset_cleanup(MonFdset *mon_fdset)
     MonFdsetFd *mon_fdset_fd_next;
 
     QLIST_FOREACH_SAFE(mon_fdset_fd, &mon_fdset->fds, next, mon_fdset_fd_next) {
-        if (mon_fdset_fd->removed ||
-                (QLIST_EMPTY(&mon_fdset->dup_fds) && mon_refcount == 0)) {
+        if ((mon_fdset_fd->removed ||
+                (QLIST_EMPTY(&mon_fdset->dup_fds) && mon_refcount == 0)) &&
+                runstate_is_running()) {
             close(mon_fdset_fd->fd);
             g_free(mon_fdset_fd->opaque);
             QLIST_REMOVE(mon_fdset_fd, next);
@@ -2422,8 +2131,6 @@ AddfdInfo *qmp_add_fd(bool has_fdset_id, int64_t fdset_id, bool has_opaque,
 {
     int fd;
     Monitor *mon = cur_mon;
-    MonFdset *mon_fdset;
-    MonFdsetFd *mon_fdset_fd;
     AddfdInfo *fdinfo;
 
     fd = qemu_chr_fe_get_msgfd(mon->chr);
@@ -2432,57 +2139,11 @@ AddfdInfo *qmp_add_fd(bool has_fdset_id, int64_t fdset_id, bool has_opaque,
         goto error;
     }
 
-    if (has_fdset_id) {
-        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
-            if (mon_fdset->id == fdset_id) {
-                break;
-            }
-        }
-        if (mon_fdset == NULL) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
-                      "an existing fdset-id");
-            goto error;
-        }
-    } else {
-        int64_t fdset_id_prev = -1;
-        MonFdset *mon_fdset_cur = QLIST_FIRST(&mon_fdsets);
-
-        /* Use first available fdset ID */
-        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
-            mon_fdset_cur = mon_fdset;
-            if (fdset_id_prev == mon_fdset_cur->id - 1) {
-                fdset_id_prev = mon_fdset_cur->id;
-                continue;
-            }
-            break;
-        }
-
-        mon_fdset = g_malloc0(sizeof(*mon_fdset));
-        mon_fdset->id = fdset_id_prev + 1;
-
-        /* The fdset list is ordered by fdset ID */
-        if (mon_fdset->id == 0) {
-            QLIST_INSERT_HEAD(&mon_fdsets, mon_fdset, next);
-        } else if (mon_fdset->id < mon_fdset_cur->id) {
-            QLIST_INSERT_BEFORE(mon_fdset_cur, mon_fdset, next);
-        } else {
-            QLIST_INSERT_AFTER(mon_fdset_cur, mon_fdset, next);
-        }
-    }
-
-    mon_fdset_fd = g_malloc0(sizeof(*mon_fdset_fd));
-    mon_fdset_fd->fd = fd;
-    mon_fdset_fd->removed = false;
-    if (has_opaque) {
-        mon_fdset_fd->opaque = g_strdup(opaque);
+    fdinfo = monitor_fdset_add_fd(fd, has_fdset_id, fdset_id,
+                                  has_opaque, opaque, errp);
+    if (fdinfo) {
+        return fdinfo;
     }
-    QLIST_INSERT_HEAD(&mon_fdset->fds, mon_fdset_fd, next);
-
-    fdinfo = g_malloc0(sizeof(*fdinfo));
-    fdinfo->fdset_id = mon_fdset->id;
-    fdinfo->fd = mon_fdset_fd->fd;
-
-    return fdinfo;
 
 error:
     if (fd != -1) {
@@ -2568,6 +2229,87 @@ FdsetInfoList *qmp_query_fdsets(Error **errp)
     return fdset_list;
 }
 
+AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
+                                bool has_opaque, const char *opaque,
+                                Error **errp)
+{
+    MonFdset *mon_fdset = NULL;
+    MonFdsetFd *mon_fdset_fd;
+    AddfdInfo *fdinfo;
+
+    if (has_fdset_id) {
+        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+            /* Break if match found or match impossible due to ordering by ID */
+            if (fdset_id <= mon_fdset->id) {
+                if (fdset_id < mon_fdset->id) {
+                    mon_fdset = NULL;
+                }
+                break;
+            }
+        }
+    }
+
+    if (mon_fdset == NULL) {
+        int64_t fdset_id_prev = -1;
+        MonFdset *mon_fdset_cur = QLIST_FIRST(&mon_fdsets);
+
+        if (has_fdset_id) {
+            if (fdset_id < 0) {
+                error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
+                          "a non-negative value");
+                return NULL;
+            }
+            /* Use specified fdset ID */
+            QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+                mon_fdset_cur = mon_fdset;
+                if (fdset_id < mon_fdset_cur->id) {
+                    break;
+                }
+            }
+        } else {
+            /* Use first available fdset ID */
+            QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+                mon_fdset_cur = mon_fdset;
+                if (fdset_id_prev == mon_fdset_cur->id - 1) {
+                    fdset_id_prev = mon_fdset_cur->id;
+                    continue;
+                }
+                break;
+            }
+        }
+
+        mon_fdset = g_malloc0(sizeof(*mon_fdset));
+        if (has_fdset_id) {
+            mon_fdset->id = fdset_id;
+        } else {
+            mon_fdset->id = fdset_id_prev + 1;
+        }
+
+        /* The fdset list is ordered by fdset ID */
+        if (!mon_fdset_cur) {
+            QLIST_INSERT_HEAD(&mon_fdsets, mon_fdset, next);
+        } else if (mon_fdset->id < mon_fdset_cur->id) {
+            QLIST_INSERT_BEFORE(mon_fdset_cur, mon_fdset, next);
+        } else {
+            QLIST_INSERT_AFTER(mon_fdset_cur, mon_fdset, next);
+        }
+    }
+
+    mon_fdset_fd = g_malloc0(sizeof(*mon_fdset_fd));
+    mon_fdset_fd->fd = fd;
+    mon_fdset_fd->removed = false;
+    if (has_opaque) {
+        mon_fdset_fd->opaque = g_strdup(opaque);
+    }
+    QLIST_INSERT_HEAD(&mon_fdset->fds, mon_fdset_fd, next);
+
+    fdinfo = g_malloc0(sizeof(*fdinfo));
+    fdinfo->fdset_id = mon_fdset->id;
+    fdinfo->fd = mon_fdset_fd->fd;
+
+    return fdinfo;
+}
+
 int monitor_fdset_get_fd(int64_t fdset_id, int flags)
 {
 #ifndef _WIN32
@@ -2651,11 +2393,25 @@ int monitor_fdset_dup_fd_remove(int dup_fd)
     return monitor_fdset_dup_fd_find_remove(dup_fd, true);
 }
 
-/* mon_cmds and info_cmds would be sorted at runtime */
-static mon_cmd_t mon_cmds[] = {
-#include "hmp-commands.h"
-    { NULL, NULL, },
-};
+int monitor_handle_fd_param(Monitor *mon, const char *fdname)
+{
+    int fd;
+    Error *local_err = NULL;
+
+    if (!qemu_isdigit(fdname[0]) && mon) {
+
+        fd = monitor_get_fd(mon, fdname, &local_err);
+        if (fd == -1) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+            return -1;
+        }
+    } else {
+        fd = qemu_parse_fd(fdname);
+    }
+
+    return fd;
+}
 
 /* Please update hmp-commands.hx when adding or changing commands */
 static mon_cmd_t info_cmds[] = {
@@ -2664,63 +2420,63 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the version of QEMU",
-        .mhandler.info = hmp_info_version,
+        .mhandler.cmd = hmp_info_version,
     },
     {
         .name       = "network",
         .args_type  = "",
         .params     = "",
         .help       = "show the network state",
-        .mhandler.info = do_info_network,
+        .mhandler.cmd = do_info_network,
     },
     {
         .name       = "chardev",
         .args_type  = "",
         .params     = "",
         .help       = "show the character devices",
-        .mhandler.info = hmp_info_chardev,
+        .mhandler.cmd = hmp_info_chardev,
     },
     {
         .name       = "block",
         .args_type  = "",
         .params     = "",
         .help       = "show the block devices",
-        .mhandler.info = hmp_info_block,
+        .mhandler.cmd = hmp_info_block,
     },
     {
         .name       = "blockstats",
         .args_type  = "",
         .params     = "",
         .help       = "show block device statistics",
-        .mhandler.info = hmp_info_blockstats,
+        .mhandler.cmd = hmp_info_blockstats,
     },
     {
         .name       = "block-jobs",
         .args_type  = "",
         .params     = "",
         .help       = "show progress of ongoing block device operations",
-        .mhandler.info = hmp_info_block_jobs,
+        .mhandler.cmd = hmp_info_block_jobs,
     },
     {
         .name       = "registers",
         .args_type  = "",
         .params     = "",
         .help       = "show the cpu registers",
-        .mhandler.info = do_info_registers,
+        .mhandler.cmd = do_info_registers,
     },
     {
         .name       = "cpus",
         .args_type  = "",
         .params     = "",
         .help       = "show infos for each CPU",
-        .mhandler.info = hmp_info_cpus,
+        .mhandler.cmd = hmp_info_cpus,
     },
     {
         .name       = "history",
         .args_type  = "",
         .params     = "",
         .help       = "show the command line history",
-        .mhandler.info = do_info_history,
+        .mhandler.cmd = do_info_history,
     },
 #if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
     defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
@@ -2730,11 +2486,11 @@ static mon_cmd_t info_cmds[] = {
         .params     = "",
         .help       = "show the interrupts statistics (if available)",
 #ifdef TARGET_SPARC
-        .mhandler.info = sun4m_irq_info,
+        .mhandler.cmd = sun4m_irq_info,
 #elif defined(TARGET_LM32)
-        .mhandler.info = lm32_irq_info,
+        .mhandler.cmd = lm32_irq_info,
 #else
-        .mhandler.info = irq_info,
+        .mhandler.cmd = irq_info,
 #endif
     },
     {
@@ -2743,11 +2499,11 @@ static mon_cmd_t info_cmds[] = {
         .params     = "",
         .help       = "show i8259 (PIC) state",
 #ifdef TARGET_SPARC
-        .mhandler.info = sun4m_pic_info,
+        .mhandler.cmd = sun4m_pic_info,
 #elif defined(TARGET_LM32)
-        .mhandler.info = lm32_do_pic_info,
+        .mhandler.cmd = lm32_do_pic_info,
 #else
-        .mhandler.info = pic_info,
+        .mhandler.cmd = pic_info,
 #endif
     },
 #endif
@@ -2756,7 +2512,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show PCI info",
-        .mhandler.info = hmp_info_pci,
+        .mhandler.cmd = hmp_info_pci,
     },
 #if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
     defined(TARGET_PPC) || defined(TARGET_XTENSA)
@@ -2765,7 +2521,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show virtual to physical memory mappings",
-        .mhandler.info = tlb_info,
+        .mhandler.cmd = tlb_info,
     },
 #endif
 #if defined(TARGET_I386)
@@ -2774,7 +2530,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the active virtual memory mappings",
-        .mhandler.info = mem_info,
+        .mhandler.cmd = mem_info,
     },
 #endif
     {
@@ -2782,91 +2538,91 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show memory tree",
-        .mhandler.info = do_info_mtree,
+        .mhandler.cmd = do_info_mtree,
     },
     {
         .name       = "jit",
         .args_type  = "",
         .params     = "",
         .help       = "show dynamic compiler info",
-        .mhandler.info = do_info_jit,
+        .mhandler.cmd = do_info_jit,
     },
     {
         .name       = "kvm",
         .args_type  = "",
         .params     = "",
         .help       = "show KVM information",
-        .mhandler.info = hmp_info_kvm,
+        .mhandler.cmd = hmp_info_kvm,
     },
     {
         .name       = "numa",
         .args_type  = "",
         .params     = "",
         .help       = "show NUMA information",
-        .mhandler.info = do_info_numa,
+        .mhandler.cmd = do_info_numa,
     },
     {
         .name       = "usb",
         .args_type  = "",
         .params     = "",
         .help       = "show guest USB devices",
-        .mhandler.info = usb_info,
+        .mhandler.cmd = usb_info,
     },
     {
         .name       = "usbhost",
         .args_type  = "",
         .params     = "",
         .help       = "show host USB devices",
-        .mhandler.info = usb_host_info,
+        .mhandler.cmd = usb_host_info,
     },
     {
         .name       = "profile",
         .args_type  = "",
         .params     = "",
         .help       = "show profiling information",
-        .mhandler.info = do_info_profile,
+        .mhandler.cmd = do_info_profile,
     },
     {
         .name       = "capture",
         .args_type  = "",
         .params     = "",
         .help       = "show capture information",
-        .mhandler.info = do_info_capture,
+        .mhandler.cmd = do_info_capture,
     },
     {
         .name       = "snapshots",
         .args_type  = "",
         .params     = "",
         .help       = "show the currently saved VM snapshots",
-        .mhandler.info = do_info_snapshots,
+        .mhandler.cmd = do_info_snapshots,
     },
     {
         .name       = "status",
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM status (running|paused)",
-        .mhandler.info = hmp_info_status,
+        .mhandler.cmd = hmp_info_status,
     },
     {
         .name       = "pcmcia",
         .args_type  = "",
         .params     = "",
         .help       = "show guest PCMCIA status",
-        .mhandler.info = pcmcia_info,
+        .mhandler.cmd = pcmcia_info,
     },
     {
         .name       = "mice",
         .args_type  = "",
         .params     = "",
         .help       = "show which guest mouse is receiving events",
-        .mhandler.info = hmp_info_mice,
+        .mhandler.cmd = hmp_info_mice,
     },
     {
         .name       = "vnc",
         .args_type  = "",
         .params     = "",
         .help       = "show the vnc server status",
-        .mhandler.info = hmp_info_vnc,
+        .mhandler.cmd = hmp_info_vnc,
     },
 #if defined(CONFIG_SPICE)
     {
@@ -2874,7 +2630,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the spice server status",
-        .mhandler.info = hmp_info_spice,
+        .mhandler.cmd = hmp_info_spice,
     },
 #endif
     {
@@ -2882,14 +2638,14 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM name",
-        .mhandler.info = hmp_info_name,
+        .mhandler.cmd = hmp_info_name,
     },
     {
         .name       = "uuid",
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM UUID",
-        .mhandler.info = hmp_info_uuid,
+        .mhandler.cmd = hmp_info_uuid,
     },
 #if defined(TARGET_PPC)
     {
@@ -2897,7 +2653,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show CPU statistics",
-        .mhandler.info = do_info_cpu_stats,
+        .mhandler.cmd = do_info_cpu_stats,
     },
 #endif
 #if defined(CONFIG_SLIRP)
@@ -2906,7 +2662,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show user network stack connection states",
-        .mhandler.info = do_info_usernet,
+        .mhandler.cmd = do_info_usernet,
     },
 #endif
     {
@@ -2914,62 +2670,68 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show migration status",
-        .mhandler.info = hmp_info_migrate,
+        .mhandler.cmd = hmp_info_migrate,
     },
     {
         .name       = "migrate_capabilities",
         .args_type  = "",
         .params     = "",
         .help       = "show current migration capabilities",
-        .mhandler.info = hmp_info_migrate_capabilities,
+        .mhandler.cmd = hmp_info_migrate_capabilities,
     },
     {
         .name       = "migrate_cache_size",
         .args_type  = "",
         .params     = "",
         .help       = "show current migration xbzrle cache size",
-        .mhandler.info = hmp_info_migrate_cache_size,
+        .mhandler.cmd = hmp_info_migrate_cache_size,
     },
     {
         .name       = "balloon",
         .args_type  = "",
         .params     = "",
         .help       = "show balloon information",
-        .mhandler.info = hmp_info_balloon,
+        .mhandler.cmd = hmp_info_balloon,
     },
     {
         .name       = "qtree",
         .args_type  = "",
         .params     = "",
         .help       = "show device tree",
-        .mhandler.info = do_info_qtree,
+        .mhandler.cmd = do_info_qtree,
     },
     {
         .name       = "qdm",
         .args_type  = "",
         .params     = "",
         .help       = "show qdev device model list",
-        .mhandler.info = do_info_qdm,
+        .mhandler.cmd = do_info_qdm,
     },
     {
         .name       = "roms",
         .args_type  = "",
         .params     = "",
         .help       = "show roms",
-        .mhandler.info = do_info_roms,
+        .mhandler.cmd = do_info_roms,
     },
     {
         .name       = "trace-events",
         .args_type  = "",
         .params     = "",
         .help       = "show available trace-events & their state",
-        .mhandler.info = do_trace_print_events,
+        .mhandler.cmd = do_trace_print_events,
     },
     {
         .name       = NULL,
     },
 };
 
+/* mon_cmds and info_cmds would be sorted at runtime */
+static mon_cmd_t mon_cmds[] = {
+#include "hmp-commands.h"
+    { NULL, NULL, },
+};
+
 static const mon_cmd_t qmp_cmds[] = {
 #include "qmp-commands-old.h"
     { /* NULL */ },
@@ -3485,11 +3247,7 @@ static int64_t expr_unary(Monitor *mon)
         break;
     default:
         errno = 0;
-#if TARGET_PHYS_ADDR_BITS > 32
         n = strtoull(pch, &p, 0);
-#else
-        n = strtoul(pch, &p, 0);
-#endif
         if (errno == ERANGE) {
             expr_error(mon, "number too large");
         }
@@ -3768,18 +3526,27 @@ static const mon_cmd_t *search_dispatch_table(const mon_cmd_t *disp_table,
     return NULL;
 }
 
-static const mon_cmd_t *monitor_find_command(const char *cmdname)
-{
-    return search_dispatch_table(mon_cmds, cmdname);
-}
-
 static const mon_cmd_t *qmp_find_cmd(const char *cmdname)
 {
     return search_dispatch_table(qmp_cmds, cmdname);
 }
 
+/*
+ * Parse @cmdline according to command table @table.
+ * If @cmdline is blank, return NULL.
+ * If it can't be parsed, report to @mon, and return NULL.
+ * Else, insert command arguments into @qdict, and return the command.
+ * If sub-command table exist, and if @cmdline contains addtional string for
+ * sub-command, this function will try search sub-command table. if no
+ * addtional string for sub-command exist, this function will return the found
+ * one in @table.
+ * Do not assume the returned command points into @table!  It doesn't
+ * when the command is a sub-command.
+ */
 static const mon_cmd_t *monitor_parse_command(Monitor *mon,
                                               const char *cmdline,
+                                              int start,
+                                              mon_cmd_t *table,
                                               QDict *qdict)
 {
     const char *p, *typestr;
@@ -3790,20 +3557,35 @@ static const mon_cmd_t *monitor_parse_command(Monitor *mon,
     char *key;
 
 #ifdef DEBUG
-    monitor_printf(mon, "command='%s'\n", cmdline);
+    monitor_printf(mon, "command='%s', start='%d'\n", cmdline, start);
 #endif
 
     /* extract the command name */
-    p = get_command_name(cmdline, cmdname, sizeof(cmdname));
+    p = get_command_name(cmdline + start, cmdname, sizeof(cmdname));
     if (!p)
         return NULL;
 
-    cmd = monitor_find_command(cmdname);
+    cmd = search_dispatch_table(table, cmdname);
     if (!cmd) {
-        monitor_printf(mon, "unknown command: '%s'\n", cmdname);
+        monitor_printf(mon, "unknown command: '%.*s'\n",
+                       (int)(p - cmdline), cmdline);
         return NULL;
     }
 
+    /* filter out following useless space */
+    while (qemu_isspace(*p)) {
+        p++;
+    }
+    /* search sub command */
+    if (cmd->sub_table != NULL) {
+        /* check if user set additional command */
+        if (*p == '\0') {
+            return cmd;
+        }
+        return monitor_parse_command(mon, cmdline, p - cmdline,
+                                     cmd->sub_table, qdict);
+    }
+
     /* parse the parameters */
     typestr = cmd->args_type;
     for(;;) {
@@ -4159,7 +3941,7 @@ static void handle_user_command(Monitor *mon, const char *cmdline)
 
     qdict = qdict_new();
 
-    cmd = monitor_parse_command(mon, cmdline, qdict);
+    cmd = monitor_parse_command(mon, cmdline, 0, mon_cmds, qdict);
     if (!cmd)
         goto out;
 
@@ -4315,7 +4097,6 @@ static void monitor_find_completion(const char *cmdline)
     int nb_args, i, len;
     const char *ptype, *str;
     const mon_cmd_t *cmd;
-    const KeyDef *key;
 
     parse_cmdline(cmdline, &nb_args, args);
 #ifdef DEBUG_COMPLETION
@@ -4389,8 +4170,8 @@ static void monitor_find_completion(const char *cmdline)
                 if (sep)
                     str = sep + 1;
                 readline_set_completion_index(cur_mon->rs, strlen(str));
-                for(key = key_defs; key->name != NULL; key++) {
-                    cmd_completion(str, key->name);
+                for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+                    cmd_completion(str, QKeyCode_lookup[i]);
                 }
             } else if (!strcmp(cmd->name, "help|?")) {
                 readline_set_completion_index(cur_mon->rs, strlen(str));
@@ -4926,7 +4707,6 @@ void monitor_init(CharDriverState *chr, int flags)
     Monitor *mon;
 
     if (is_first_init) {
-        key_timer = qemu_new_timer_ns(vm_clock, release_keys, NULL);
         monitor_protocol_event_init();
         is_first_init = 0;
     }
@@ -5026,3 +4806,25 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
 
     return monitor_read_bdrv_key_start(mon, bs, completion_cb, opaque);
 }
+
+QemuOptsList qemu_mon_opts = {
+    .name = "mon",
+    .implied_opt_name = "chardev",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_mon_opts.head),
+    .desc = {
+        {
+            .name = "mode",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "chardev",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "default",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "pretty",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
diff --git a/nbd.c b/nbd.c
index 0dd60c5f4c7..0698a023add 100644
--- a/nbd.c
+++ b/nbd.c
@@ -16,10 +16,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "nbd.h"
-#include "block.h"
+#include "block/nbd.h"
+#include "block/block.h"
 
-#include "qemu-coroutine.h"
+#include "block/coroutine.h"
 
 #include <errno.h>
 #include <string.h>
@@ -36,8 +36,8 @@
 #include <linux/fs.h>
 #endif
 
-#include "qemu_socket.h"
-#include "qemu-queue.h"
+#include "qemu/sockets.h"
+#include "qemu/queue.h"
 
 //#define DEBUG_NBD
 
@@ -57,9 +57,12 @@
 
 /* This is all part of the "official" NBD API */
 
+#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
 #define NBD_REPLY_SIZE          (4 + 4 + 8)
 #define NBD_REQUEST_MAGIC       0x25609513
 #define NBD_REPLY_MAGIC         0x67446698
+#define NBD_OPTS_MAGIC          0x49484156454F5054LL
+#define NBD_CLIENT_MAGIC        0x0000420281861253LL
 
 #define NBD_SET_SOCK            _IO(0xab, 0)
 #define NBD_SET_BLKSIZE         _IO(0xab, 1)
@@ -75,6 +78,49 @@
 
 #define NBD_OPT_EXPORT_NAME     (1 << 0)
 
+/* Definitions for opaque data types */
+
+typedef struct NBDRequest NBDRequest;
+
+struct NBDRequest {
+    QSIMPLEQ_ENTRY(NBDRequest) entry;
+    NBDClient *client;
+    uint8_t *data;
+};
+
+struct NBDExport {
+    int refcount;
+    void (*close)(NBDExport *exp);
+
+    BlockDriverState *bs;
+    char *name;
+    off_t dev_offset;
+    off_t size;
+    uint32_t nbdflags;
+    QTAILQ_HEAD(, NBDClient) clients;
+    QSIMPLEQ_HEAD(, NBDRequest) requests;
+    QTAILQ_ENTRY(NBDExport) next;
+};
+
+static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
+
+struct NBDClient {
+    int refcount;
+    void (*close)(NBDClient *client);
+
+    NBDExport *exp;
+    int sock;
+
+    Coroutine *recv_coroutine;
+
+    CoMutex send_lock;
+    Coroutine *send_coroutine;
+
+    QTAILQ_ENTRY(NBDClient) next;
+    int nb_requests;
+    bool closing;
+};
+
 /* That's all folks */
 
 ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
@@ -162,7 +208,14 @@ int tcp_socket_outgoing(const char *address, uint16_t port)
 
 int tcp_socket_outgoing_spec(const char *address_and_port)
 {
-    return inet_connect(address_and_port, true, NULL, NULL);
+    Error *local_err = NULL;
+    int fd = inet_connect(address_and_port, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int tcp_socket_incoming(const char *address, uint16_t port)
@@ -174,29 +227,57 @@ int tcp_socket_incoming(const char *address, uint16_t port)
 
 int tcp_socket_incoming_spec(const char *address_and_port)
 {
-    char *ostr  = NULL;
-    int olen = 0;
-    return inet_listen(address_and_port, ostr, olen, SOCK_STREAM, 0, NULL);
+    Error *local_err = NULL;
+    int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int unix_socket_incoming(const char *path)
 {
-    char *ostr = NULL;
-    int olen = 0;
+    Error *local_err = NULL;
+    int fd = unix_listen(path, NULL, 0, &local_err);
 
-    return unix_listen(path, ostr, olen);
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int unix_socket_outgoing(const char *path)
 {
-    return unix_connect(path);
+    Error *local_err = NULL;
+    int fd = unix_connect(path, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
-/* Basic flow
+/* Basic flow for negotiation
 
    Server         Client
-
    Negotiate
+
+   or
+
+   Server         Client
+   Negotiate #1
+                  Option
+   Negotiate #2
+
+   ----
+
+   followed by
+
+   Server         Client
                   Request
    Response
                   Request
@@ -204,36 +285,152 @@ int unix_socket_outgoing(const char *path)
                   ...
    ...
                   Request (type == 2)
+
 */
 
-static int nbd_send_negotiate(int csock, off_t size, uint32_t flags)
+static int nbd_receive_options(NBDClient *client)
 {
+    int csock = client->sock;
+    char name[256];
+    uint32_t tmp, length;
+    uint64_t magic;
+    int rc;
+
+    /* Client sends:
+        [ 0 ..   3]   reserved (0)
+        [ 4 ..  11]   NBD_OPTS_MAGIC
+        [12 ..  15]   NBD_OPT_EXPORT_NAME
+        [16 ..  19]   length
+        [20 ..  xx]   export name (length bytes)
+     */
+
+    rc = -EINVAL;
+    if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking reserved");
+    if (tmp != 0) {
+        LOG("Bad reserved received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking reserved");
+    if (magic != be64_to_cpu(NBD_OPTS_MAGIC)) {
+        LOG("Bad magic received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking option");
+    if (tmp != be32_to_cpu(NBD_OPT_EXPORT_NAME)) {
+        LOG("Bad option received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &length, sizeof(length)) != sizeof(length)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking length");
+    length = be32_to_cpu(length);
+    if (length > 255) {
+        LOG("Bad length received");
+        goto fail;
+    }
+    if (read_sync(csock, name, length) != length) {
+        LOG("read failed");
+        goto fail;
+    }
+    name[length] = '\0';
+
+    client->exp = nbd_export_find(name);
+    if (!client->exp) {
+        LOG("export not found");
+        goto fail;
+    }
+
+    QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
+    nbd_export_get(client->exp);
+
+    TRACE("Option negotiation succeeded.");
+    rc = 0;
+fail:
+    return rc;
+}
+
+static int nbd_send_negotiate(NBDClient *client)
+{
+    int csock = client->sock;
     char buf[8 + 8 + 8 + 128];
     int rc;
+    const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
+                         NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
 
-    /* Negotiate
-        [ 0 ..   7]   passwd   ("NBDMAGIC")
-        [ 8 ..  15]   magic    (0x00420281861253)
+    /* Negotiation header without options:
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
-        [24 ..  27]   flags
-        [28 .. 151]   reserved (0)
+        [24 ..  25]   server flags (0)
+        [24 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
+
+       Negotiation header with options, part 1:
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
+        [16 ..  17]   server flags (0)
+
+       part 2 (after options are sent):
+        [18 ..  25]   size
+        [26 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
      */
 
     socket_set_block(csock);
     rc = -EINVAL;
 
     TRACE("Beginning negotiation.");
+    memset(buf, 0, sizeof(buf));
     memcpy(buf, "NBDMAGIC", 8);
-    cpu_to_be64w((uint64_t*)(buf + 8), 0x00420281861253LL);
-    cpu_to_be64w((uint64_t*)(buf + 16), size);
-    cpu_to_be32w((uint32_t*)(buf + 24),
-                 flags | NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
-                 NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
-    memset(buf + 28, 0, 124);
-
-    if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
-        LOG("write failed");
-        goto fail;
+    if (client->exp) {
+        assert ((client->exp->nbdflags & ~65535) == 0);
+        cpu_to_be64w((uint64_t*)(buf + 8), NBD_CLIENT_MAGIC);
+        cpu_to_be64w((uint64_t*)(buf + 16), client->exp->size);
+        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
+    } else {
+        cpu_to_be64w((uint64_t*)(buf + 8), NBD_OPTS_MAGIC);
+    }
+
+    if (client->exp) {
+        if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
+            LOG("write failed");
+            goto fail;
+        }
+    } else {
+        if (write_sync(csock, buf, 18) != 18) {
+            LOG("write failed");
+            goto fail;
+        }
+        rc = nbd_receive_options(client);
+        if (rc < 0) {
+            LOG("option negotiation failed");
+            goto fail;
+        }
+
+        assert ((client->exp->nbdflags & ~65535) == 0);
+        cpu_to_be64w((uint64_t*)(buf + 18), client->exp->size);
+        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
+        if (write_sync(csock, buf + 18, sizeof(buf) - 18) != sizeof(buf) - 18) {
+            LOG("write failed");
+            goto fail;
+        }
     }
 
     TRACE("Negotiation succeeded.");
@@ -295,7 +492,7 @@ int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
         uint32_t namesize;
 
         TRACE("Checking magic (opts_magic)");
-        if (magic != 0x49484156454F5054LL) {
+        if (magic != NBD_OPTS_MAGIC) {
             LOG("Bad magic received");
             goto fail;
         }
@@ -334,7 +531,7 @@ int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
     } else {
         TRACE("Checking magic (cli_magic)");
 
-        if (magic != 0x00420281861253LL) {
+        if (magic != NBD_CLIENT_MAGIC) {
             LOG("Bad magic received");
             goto fail;
         }
@@ -399,24 +596,23 @@ int nbd_init(int fd, int csock, uint32_t flags, off_t size, size_t blocksize)
         return -serrno;
     }
 
-    if (flags & NBD_FLAG_READ_ONLY) {
-        int read_only = 1;
-        TRACE("Setting readonly attribute");
+    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0) {
+        if (errno == ENOTTY) {
+            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
+            TRACE("Setting readonly attribute");
 
-        if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+                int serrno = errno;
+                LOG("Failed setting read-only attribute");
+                return -serrno;
+            }
+        } else {
             int serrno = errno;
-            LOG("Failed setting read-only attribute");
+            LOG("Failed setting flags");
             return -serrno;
         }
     }
 
-    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0
-        && errno != ENOTTY) {
-        int serrno = errno;
-        LOG("Failed setting flags");
-        return -serrno;
-    }
-
     TRACE("Negotiation ended");
 
     return 0;
@@ -477,7 +673,7 @@ int nbd_client(int fd)
 
 ssize_t nbd_send_request(int csock, struct nbd_request *request)
 {
-    uint8_t buf[4 + 4 + 8 + 8 + 4];
+    uint8_t buf[NBD_REQUEST_SIZE];
     ssize_t ret;
 
     cpu_to_be32w((uint32_t*)buf, NBD_REQUEST_MAGIC);
@@ -504,7 +700,7 @@ ssize_t nbd_send_request(int csock, struct nbd_request *request)
 
 static ssize_t nbd_receive_request(int csock, struct nbd_request *request)
 {
-    uint8_t buf[4 + 4 + 8 + 8 + 4];
+    uint8_t buf[NBD_REQUEST_SIZE];
     uint32_t magic;
     ssize_t ret;
 
@@ -582,7 +778,7 @@ ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply)
 
 static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
 {
-    uint8_t buf[4 + 4 + 8];
+    uint8_t buf[NBD_REPLY_SIZE];
     ssize_t ret;
 
     /* Reply
@@ -610,58 +806,47 @@ static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
 
 #define MAX_NBD_REQUESTS 16
 
-typedef struct NBDRequest NBDRequest;
-
-struct NBDRequest {
-    QSIMPLEQ_ENTRY(NBDRequest) entry;
-    NBDClient *client;
-    uint8_t *data;
-};
-
-struct NBDExport {
-    BlockDriverState *bs;
-    off_t dev_offset;
-    off_t size;
-    uint32_t nbdflags;
-    QSIMPLEQ_HEAD(, NBDRequest) requests;
-};
-
-struct NBDClient {
-    int refcount;
-    void (*close)(NBDClient *client);
-
-    NBDExport *exp;
-    int sock;
-
-    Coroutine *recv_coroutine;
-
-    CoMutex send_lock;
-    Coroutine *send_coroutine;
-
-    int nb_requests;
-};
-
-static void nbd_client_get(NBDClient *client)
+void nbd_client_get(NBDClient *client)
 {
     client->refcount++;
 }
 
-static void nbd_client_put(NBDClient *client)
+void nbd_client_put(NBDClient *client)
 {
     if (--client->refcount == 0) {
+        /* The last reference should be dropped by client->close,
+         * which is called by nbd_client_close.
+         */
+        assert(client->closing);
+
+        qemu_set_fd_handler2(client->sock, NULL, NULL, NULL, NULL);
+        close(client->sock);
+        client->sock = -1;
+        if (client->exp) {
+            QTAILQ_REMOVE(&client->exp->clients, client, next);
+            nbd_export_put(client->exp);
+        }
         g_free(client);
     }
 }
 
-static void nbd_client_close(NBDClient *client)
+void nbd_client_close(NBDClient *client)
 {
-    qemu_set_fd_handler2(client->sock, NULL, NULL, NULL, NULL);
-    close(client->sock);
-    client->sock = -1;
+    if (client->closing) {
+        return;
+    }
+
+    client->closing = true;
+
+    /* Force requests to finish.  They will drop their own references,
+     * then we'll close the socket and free the NBDClient.
+     */
+    shutdown(client->sock, 2);
+
+    /* Also tell the client, so that they release their reference.  */
     if (client->close) {
         client->close(client);
     }
-    nbd_client_put(client);
 }
 
 static NBDRequest *nbd_request_get(NBDClient *client)
@@ -695,28 +880,109 @@ static void nbd_request_put(NBDRequest *req)
 }
 
 NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags)
+                          off_t size, uint32_t nbdflags,
+                          void (*close)(NBDExport *))
 {
     NBDExport *exp = g_malloc0(sizeof(NBDExport));
     QSIMPLEQ_INIT(&exp->requests);
+    exp->refcount = 1;
+    QTAILQ_INIT(&exp->clients);
     exp->bs = bs;
     exp->dev_offset = dev_offset;
     exp->nbdflags = nbdflags;
     exp->size = size == -1 ? bdrv_getlength(bs) : size;
+    exp->close = close;
     return exp;
 }
 
+NBDExport *nbd_export_find(const char *name)
+{
+    NBDExport *exp;
+    QTAILQ_FOREACH(exp, &exports, next) {
+        if (strcmp(name, exp->name) == 0) {
+            return exp;
+        }
+    }
+
+    return NULL;
+}
+
+void nbd_export_set_name(NBDExport *exp, const char *name)
+{
+    if (exp->name == name) {
+        return;
+    }
+
+    nbd_export_get(exp);
+    if (exp->name != NULL) {
+        g_free(exp->name);
+        exp->name = NULL;
+        QTAILQ_REMOVE(&exports, exp, next);
+        nbd_export_put(exp);
+    }
+    if (name != NULL) {
+        nbd_export_get(exp);
+        exp->name = g_strdup(name);
+        QTAILQ_INSERT_TAIL(&exports, exp, next);
+    }
+    nbd_export_put(exp);
+}
+
 void nbd_export_close(NBDExport *exp)
 {
-    while (!QSIMPLEQ_EMPTY(&exp->requests)) {
-        NBDRequest *first = QSIMPLEQ_FIRST(&exp->requests);
-        QSIMPLEQ_REMOVE_HEAD(&exp->requests, entry);
-        qemu_vfree(first->data);
-        g_free(first);
+    NBDClient *client, *next;
+
+    nbd_export_get(exp);
+    QTAILQ_FOREACH_SAFE(client, &exp->clients, next, next) {
+        nbd_client_close(client);
+    }
+    nbd_export_set_name(exp, NULL);
+    nbd_export_put(exp);
+}
+
+void nbd_export_get(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    exp->refcount++;
+}
+
+void nbd_export_put(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    if (exp->refcount == 1) {
+        nbd_export_close(exp);
     }
 
-    bdrv_close(exp->bs);
-    g_free(exp);
+    if (--exp->refcount == 0) {
+        assert(exp->name == NULL);
+
+        if (exp->close) {
+            exp->close(exp);
+        }
+
+        while (!QSIMPLEQ_EMPTY(&exp->requests)) {
+            NBDRequest *first = QSIMPLEQ_FIRST(&exp->requests);
+            QSIMPLEQ_REMOVE_HEAD(&exp->requests, entry);
+            qemu_vfree(first->data);
+            g_free(first);
+        }
+
+        g_free(exp);
+    }
+}
+
+BlockDriverState *nbd_export_get_blockdev(NBDExport *exp)
+{
+    return exp->bs;
+}
+
+void nbd_export_close_all(void)
+{
+    NBDExport *exp, *next;
+
+    QTAILQ_FOREACH_SAFE(exp, &exports, next, next) {
+        nbd_export_close(exp);
+    }
 }
 
 static int nbd_can_read(void *opaque);
@@ -805,14 +1071,18 @@ out:
 static void nbd_trip(void *opaque)
 {
     NBDClient *client = opaque;
-    NBDRequest *req = nbd_request_get(client);
     NBDExport *exp = client->exp;
+    NBDRequest *req;
     struct nbd_request request;
     struct nbd_reply reply;
     ssize_t ret;
 
     TRACE("Reading request.");
+    if (client->closing) {
+        return;
+    }
 
+    req = nbd_request_get(client);
     ret = nbd_co_receive_request(req, &request);
     if (ret == -EAGAIN) {
         goto done;
@@ -974,15 +1244,21 @@ NBDClient *nbd_client_new(NBDExport *exp, int csock,
                           void (*close)(NBDClient *))
 {
     NBDClient *client;
-    if (nbd_send_negotiate(csock, exp->size, exp->nbdflags) < 0) {
-        return NULL;
-    }
     client = g_malloc0(sizeof(NBDClient));
     client->refcount = 1;
     client->exp = exp;
     client->sock = csock;
+    if (nbd_send_negotiate(client) < 0) {
+        g_free(client);
+        return NULL;
+    }
     client->close = close;
     qemu_co_mutex_init(&client->send_lock);
     qemu_set_fd_handler2(csock, nbd_can_read, nbd_read, NULL, client);
+
+    if (exp) {
+        QTAILQ_INSERT_TAIL(&exp->clients, client, next);
+        nbd_export_get(exp);
+    }
     return client;
 }
diff --git a/net/Makefile.objs b/net/Makefile.objs
index cf041877179..a08cd14e2e4 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -1,4 +1,4 @@
-common-obj-y = queue.o checksum.o util.o hub.o
+common-obj-y = net.o queue.o checksum.o util.o hub.o
 common-obj-y += socket.o
 common-obj-y += dump.o
 common-obj-$(CONFIG_POSIX) += tap.o
diff --git a/net/vde.h b/net/clients.h
index 6ce6698937f..77932942bdd 100644
--- a/net/vde.h
+++ b/net/clients.h
@@ -21,17 +21,35 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef QEMU_NET_VDE_H
-#define QEMU_NET_VDE_H
+#ifndef QEMU_NET_CLIENTS_H
+#define QEMU_NET_CLIENTS_H
 
-#include "qemu-common.h"
+#include "net/net.h"
 #include "qapi-types.h"
 
-#ifdef CONFIG_VDE
+int net_init_dump(const NetClientOptions *opts, const char *name,
+                  NetClientState *peer);
 
-int net_init_vde(const NetClientOptions *opts, const char *name,
+#ifdef CONFIG_SLIRP
+int net_init_slirp(const NetClientOptions *opts, const char *name,
+                   NetClientState *peer);
+#endif
+
+int net_init_hubport(const NetClientOptions *opts, const char *name,
+                     NetClientState *peer);
+
+int net_init_socket(const NetClientOptions *opts, const char *name,
+                    NetClientState *peer);
+
+int net_init_tap(const NetClientOptions *opts, const char *name,
                  NetClientState *peer);
 
-#endif /* CONFIG_VDE */
+int net_init_bridge(const NetClientOptions *opts, const char *name,
+                    NetClientState *peer);
+
+#ifdef CONFIG_VDE
+int net_init_vde(const NetClientOptions *opts, const char *name,
+                 NetClientState *peer);
+#endif
 
-#endif /* QEMU_NET_VDE_H */
+#endif /* QEMU_NET_CLIENTS_H */
diff --git a/net/dump.c b/net/dump.c
index 004231d4818..41197217205 100644
--- a/net/dump.c
+++ b/net/dump.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 
-#include "dump.h"
+#include "clients.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-log.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+#include "qemu/timer.h"
 #include "hub.h"
 
 typedef struct DumpState {
diff --git a/net/dump.h b/net/dump.h
deleted file mode 100644
index 33f152b460c..00000000000
--- a/net/dump.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef QEMU_NET_DUMP_H
-#define QEMU_NET_DUMP_H
-
-#include "net.h"
-#include "qapi-types.h"
-
-int net_init_dump(const NetClientOptions *opts, const char *name,
-                  NetClientState *peer);
-
-#endif /* QEMU_NET_DUMP_H */
diff --git a/net/hub.c b/net/hub.c
index ac157e32eed..a24c9d17f77 100644
--- a/net/hub.c
+++ b/net/hub.c
@@ -12,10 +12,11 @@
  *
  */
 
-#include "monitor.h"
-#include "net.h"
+#include "monitor/monitor.h"
+#include "net/net.h"
+#include "clients.h"
 #include "hub.h"
-#include "iov.h"
+#include "qemu/iov.h"
 
 /*
  * A hub broadcasts incoming packets to all its ports except the source port.
@@ -97,12 +98,12 @@ static int net_hub_port_can_receive(NetClientState *nc)
             continue;
         }
 
-        if (!qemu_can_send_packet(&port->nc)) {
-            return 0;
+        if (qemu_can_send_packet(&port->nc)) {
+            return 1;
         }
     }
 
-    return 1;
+    return 0;
 }
 
 static ssize_t net_hub_port_receive(NetClientState *nc,
@@ -255,7 +256,7 @@ void net_hub_info(Monitor *mon)
 /**
  * Get the hub id that a client is connected to
  *
- * @id              Pointer for hub id output, may be NULL
+ * @id: Pointer for hub id output, may be NULL
  */
 int net_hub_id_for_client(NetClientState *nc, int *id)
 {
diff --git a/net/hub.h b/net/hub.h
index 26a1ade1f9e..583ada89d8a 100644
--- a/net/hub.h
+++ b/net/hub.h
@@ -17,13 +17,9 @@
 
 #include "qemu-common.h"
 
-int net_init_hubport(const NetClientOptions *opts, const char *name,
-                     NetClientState *peer);
 NetClientState *net_hub_add_port(int hub_id, const char *name);
 NetClientState *net_hub_find_client_by_name(int hub_id, const char *name);
 void net_hub_info(Monitor *mon);
-int net_hub_id_for_client(NetClientState *nc, int *id);
 void net_hub_check_clients(void);
-NetClientState *net_hub_port_find(int hub_id);
 
 #endif /* NET_HUB_H */
diff --git a/net.c b/net/net.c
index 60043ddec69..cdd9b04989c 100644
--- a/net.c
+++ b/net/net.c
@@ -21,26 +21,24 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net.h"
-
 #include "config-host.h"
 
-#include "net/tap.h"
-#include "net/socket.h"
-#include "net/dump.h"
+#include "net/net.h"
+#include "clients.h"
+#include "hub.h"
 #include "net/slirp.h"
-#include "net/vde.h"
-#include "net/hub.h"
-#include "net/util.h"
-#include "monitor.h"
+#include "util.h"
+
+#include "monitor/monitor.h"
 #include "qemu-common.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
+#include "qemu/config-file.h"
 #include "qmp-commands.h"
 #include "hw/qdev.h"
-#include "iov.h"
+#include "qemu/iov.h"
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
@@ -357,7 +355,12 @@ void qemu_flush_queued_packets(NetClientState *nc)
 {
     nc->receive_disabled = 0;
 
-    qemu_net_queue_flush(nc->send_queue);
+    if (qemu_net_queue_flush(nc->send_queue)) {
+        /* We emptied the queue successfully, signal to the IO thread to repoll
+         * the file descriptor (for tap, for example).
+         */
+        qemu_notify_event();
+    }
 }
 
 static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
@@ -418,16 +421,27 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                                 void *opaque)
 {
     NetClientState *nc = opaque;
+    int ret;
 
     if (nc->link_down) {
         return iov_size(iov, iovcnt);
     }
 
+    if (nc->receive_disabled) {
+        return 0;
+    }
+
     if (nc->info->receive_iov) {
-        return nc->info->receive_iov(nc, iov, iovcnt);
+        ret = nc->info->receive_iov(nc, iov, iovcnt);
     } else {
-        return nc_sendv_compat(nc, iov, iovcnt);
+        ret = nc_sendv_compat(nc, iov, iovcnt);
     }
+
+    if (ret == 0) {
+        nc->receive_disabled = 1;
+    }
+
+    return ret;
 }
 
 ssize_t qemu_sendv_packet_async(NetClientState *sender,
@@ -522,24 +536,6 @@ int qemu_find_nic_model(NICInfo *nd, const char * const *models,
     return -1;
 }
 
-int net_handle_fd_param(Monitor *mon, const char *param)
-{
-    int fd;
-
-    if (!qemu_isdigit(param[0]) && mon) {
-
-        fd = monitor_get_fd(mon, param);
-        if (fd == -1) {
-            error_report("No file descriptor named %s found", param);
-            return -1;
-        }
-    } else {
-        fd = qemu_parse_fd(param);
-    }
-
-    return fd;
-}
-
 static int net_init_nic(const NetClientOptions *opts, const char *name,
                         NetClientState *peer)
 {
@@ -832,6 +828,7 @@ exit_err:
 void qmp_netdev_del(const char *id, Error **errp)
 {
     NetClientState *nc;
+    QemuOpts *opts;
 
     nc = qemu_find_netdev(id);
     if (!nc) {
@@ -839,8 +836,14 @@ void qmp_netdev_del(const char *id, Error **errp)
         return;
     }
 
+    opts = qemu_opts_find(qemu_find_opts_err("netdev", NULL), id);
+    if (!opts) {
+        error_setg(errp, "Device '%s' is not a netdev", id);
+        return;
+    }
+
     qemu_del_net_client(nc);
-    qemu_opts_del(qemu_opts_find(qemu_find_opts_err("netdev", errp), id));
+    qemu_opts_del(opts);
 }
 
 void print_net_client(Monitor *mon, NetClientState *nc)
@@ -849,7 +852,7 @@ void print_net_client(Monitor *mon, NetClientState *nc)
                    NetClientOptionsKind_lookup[nc->info->type], nc->info_str);
 }
 
-void do_info_network(Monitor *mon)
+void do_info_network(Monitor *mon, const QDict *qdict)
 {
     NetClientState *nc, *peer;
     NetClientOptionsKind type;
@@ -1051,3 +1054,29 @@ unsigned compute_mcast_idx(const uint8_t *ep)
     }
     return crc >> 26;
 }
+
+QemuOptsList qemu_netdev_opts = {
+    .name = "netdev",
+    .implied_opt_name = "type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_netdev_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any params
+         * validation will happen later
+         */
+        { /* end of list */ }
+    },
+};
+
+QemuOptsList qemu_net_opts = {
+    .name = "net",
+    .implied_opt_name = "type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_net_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any params
+         * validation will happen later
+         */
+        { /* end of list */ }
+    },
+};
diff --git a/net/queue.c b/net/queue.c
index e8030aafe43..6eaf5b63c01 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -22,8 +22,8 @@
  */
 
 #include "net/queue.h"
-#include "qemu-queue.h"
-#include "net.h"
+#include "qemu/queue.h"
+#include "net/net.h"
 
 /* The delivery handler may only return zero if it will call
  * qemu_net_queue_flush() when it determines that it is once again able
@@ -83,12 +83,12 @@ void qemu_del_net_queue(NetQueue *queue)
     g_free(queue);
 }
 
-static ssize_t qemu_net_queue_append(NetQueue *queue,
-                                     NetClientState *sender,
-                                     unsigned flags,
-                                     const uint8_t *buf,
-                                     size_t size,
-                                     NetPacketSent *sent_cb)
+static void qemu_net_queue_append(NetQueue *queue,
+                                  NetClientState *sender,
+                                  unsigned flags,
+                                  const uint8_t *buf,
+                                  size_t size,
+                                  NetPacketSent *sent_cb)
 {
     NetPacket *packet;
 
@@ -100,16 +100,14 @@ static ssize_t qemu_net_queue_append(NetQueue *queue,
     memcpy(packet->data, buf, size);
 
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
-
-    return size;
 }
 
-static ssize_t qemu_net_queue_append_iov(NetQueue *queue,
-                                         NetClientState *sender,
-                                         unsigned flags,
-                                         const struct iovec *iov,
-                                         int iovcnt,
-                                         NetPacketSent *sent_cb)
+static void qemu_net_queue_append_iov(NetQueue *queue,
+                                      NetClientState *sender,
+                                      unsigned flags,
+                                      const struct iovec *iov,
+                                      int iovcnt,
+                                      NetPacketSent *sent_cb)
 {
     NetPacket *packet;
     size_t max_len = 0;
@@ -133,8 +131,6 @@ static ssize_t qemu_net_queue_append_iov(NetQueue *queue,
     }
 
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
-
-    return packet->size;
 }
 
 static ssize_t qemu_net_queue_deliver(NetQueue *queue,
@@ -177,7 +173,8 @@ ssize_t qemu_net_queue_send(NetQueue *queue,
     ssize_t ret;
 
     if (queue->delivering || !qemu_can_send_packet(sender)) {
-        return qemu_net_queue_append(queue, sender, flags, data, size, sent_cb);
+        qemu_net_queue_append(queue, sender, flags, data, size, sent_cb);
+        return 0;
     }
 
     ret = qemu_net_queue_deliver(queue, sender, flags, data, size);
@@ -201,8 +198,8 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue,
     ssize_t ret;
 
     if (queue->delivering || !qemu_can_send_packet(sender)) {
-        return qemu_net_queue_append_iov(queue, sender, flags,
-                                         iov, iovcnt, sent_cb);
+        qemu_net_queue_append_iov(queue, sender, flags, iov, iovcnt, sent_cb);
+        return 0;
     }
 
     ret = qemu_net_queue_deliver_iov(queue, sender, flags, iov, iovcnt);
@@ -228,7 +225,7 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from)
     }
 }
 
-void qemu_net_queue_flush(NetQueue *queue)
+bool qemu_net_queue_flush(NetQueue *queue)
 {
     while (!QTAILQ_EMPTY(&queue->packets)) {
         NetPacket *packet;
@@ -244,7 +241,7 @@ void qemu_net_queue_flush(NetQueue *queue)
                                      packet->size);
         if (ret == 0) {
             QTAILQ_INSERT_HEAD(&queue->packets, packet, entry);
-            break;
+            return false;
         }
 
         if (packet->sent_cb) {
@@ -253,4 +250,5 @@ void qemu_net_queue_flush(NetQueue *queue)
 
         g_free(packet);
     }
+    return true;
 }
diff --git a/net/slirp.c b/net/slirp.c
index 8db66ea5390..4df550faf6c 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -29,11 +29,13 @@
 #include <pwd.h>
 #include <sys/wait.h>
 #endif
-#include "net.h"
-#include "net/hub.h"
-#include "monitor.h"
-#include "qemu_socket.h"
+#include "net/net.h"
+#include "clients.h"
+#include "hub.h"
+#include "monitor/monitor.h"
+#include "qemu/sockets.h"
 #include "slirp/libslirp.h"
+#include "char/char.h"
 
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
@@ -135,7 +137,7 @@ static int net_slirp_init(NetClientState *peer, const char *model,
                           const char *vhostname, const char *tftp_export,
                           const char *bootfile, const char *vdhcp_start,
                           const char *vnameserver, const char *smb_export,
-                          const char *vsmbserver)
+                          const char *vsmbserver, const char **dnssearch)
 {
     /* default settings according to historic slirp */
     struct in_addr net  = { .s_addr = htonl(0x0a000200) }; /* 10.0.2.0 */
@@ -241,7 +243,7 @@ static int net_slirp_init(NetClientState *peer, const char *model,
     s = DO_UPCAST(SlirpState, nc, nc);
 
     s->slirp = slirp_init(restricted, net, mask, host, vhostname,
-                          tftp_export, bootfile, dhcp, dns, s);
+                          tftp_export, bootfile, dhcp, dns, dnssearch, s);
     QTAILQ_INSERT_TAIL(&slirp_stacks, s, entry);
 
     for (config = slirp_configs; config; config = config->next) {
@@ -668,7 +670,7 @@ static int slirp_guestfwd(SlirpState *s, const char *config_str,
     return -1;
 }
 
-void do_info_usernet(Monitor *mon)
+void do_info_usernet(Monitor *mon, const QDict *qdict)
 {
     SlirpState *s;
 
@@ -698,6 +700,31 @@ net_init_slirp_configs(const StringList *fwd, int flags)
     }
 }
 
+static const char **slirp_dnssearch(const StringList *dnsname)
+{
+    const StringList *c = dnsname;
+    size_t i = 0, num_opts = 0;
+    const char **ret;
+
+    while (c) {
+        num_opts++;
+        c = c->next;
+    }
+
+    if (num_opts == 0) {
+        return NULL;
+    }
+
+    ret = g_malloc((num_opts + 1) * sizeof(*ret));
+    c = dnsname;
+    while (c) {
+        ret[i++] = c->value->str;
+        c = c->next;
+    }
+    ret[i] = NULL;
+    return ret;
+}
+
 int net_init_slirp(const NetClientOptions *opts, const char *name,
                    NetClientState *peer)
 {
@@ -705,6 +732,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     char *vnet;
     int ret;
     const NetdevUserOptions *user;
+    const char **dnssearch;
 
     assert(opts->kind == NET_CLIENT_OPTIONS_KIND_USER);
     user = opts->user;
@@ -713,6 +741,8 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
            user->has_ip  ? g_strdup_printf("%s/24", user->ip) :
            NULL;
 
+    dnssearch = slirp_dnssearch(user->dnssearch);
+
     /* all optional fields are initialized to "all bits zero" */
 
     net_init_slirp_configs(user->hostfwd, SLIRP_CFG_HOSTFWD);
@@ -721,7 +751,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     ret = net_slirp_init(peer, "user", name, user->q_restrict, vnet,
                          user->host, user->hostname, user->tftp,
                          user->bootfile, user->dhcpstart, user->dns, user->smb,
-                         user->smbserver);
+                         user->smbserver, dnssearch);
 
     while (slirp_configs) {
         config = slirp_configs;
@@ -730,6 +760,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     }
 
     g_free(vnet);
+    g_free(dnssearch);
 
     return ret;
 }
diff --git a/net/socket.c b/net/socket.c
index c172c249be6..396dc8c0b10 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -21,17 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net/socket.h"
-
 #include "config-host.h"
 
-#include "net.h"
-#include "monitor.h"
-#include "qemu-char.h"
+#include "net/net.h"
+#include "clients.h"
+#include "monitor/monitor.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-option.h"
-#include "qemu_socket.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
 
 typedef struct NetSocketState {
     NetClientState nc;
@@ -40,29 +39,106 @@ typedef struct NetSocketState {
     int state; /* 0 = getting length, 1 = getting data */
     unsigned int index;
     unsigned int packet_len;
+    unsigned int send_index;      /* number of bytes sent (only SOCK_STREAM) */
     uint8_t buf[4096];
     struct sockaddr_in dgram_dst; /* contains inet host and port destination iff connectionless (SOCK_DGRAM) */
+    IOHandler *send_fn;           /* differs between SOCK_STREAM/SOCK_DGRAM */
+    bool read_poll;               /* waiting to receive data? */
+    bool write_poll;              /* waiting to transmit data? */
 } NetSocketState;
 
 static void net_socket_accept(void *opaque);
+static void net_socket_writable(void *opaque);
+
+/* Only read packets from socket when peer can receive them */
+static int net_socket_can_send(void *opaque)
+{
+    NetSocketState *s = opaque;
+
+    return qemu_can_send_packet(&s->nc);
+}
+
+static void net_socket_update_fd_handler(NetSocketState *s)
+{
+    qemu_set_fd_handler2(s->fd,
+                         s->read_poll  ? net_socket_can_send : NULL,
+                         s->read_poll  ? s->send_fn : NULL,
+                         s->write_poll ? net_socket_writable : NULL,
+                         s);
+}
+
+static void net_socket_read_poll(NetSocketState *s, bool enable)
+{
+    s->read_poll = enable;
+    net_socket_update_fd_handler(s);
+}
+
+static void net_socket_write_poll(NetSocketState *s, bool enable)
+{
+    s->write_poll = enable;
+    net_socket_update_fd_handler(s);
+}
+
+static void net_socket_writable(void *opaque)
+{
+    NetSocketState *s = opaque;
+
+    net_socket_write_poll(s, false);
+
+    qemu_flush_queued_packets(&s->nc);
+}
 
-/* XXX: we consider we can send the whole packet without blocking */
 static ssize_t net_socket_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
-    uint32_t len;
-    len = htonl(size);
-
-    send_all(s->fd, (const uint8_t *)&len, sizeof(len));
-    return send_all(s->fd, buf, size);
+    uint32_t len = htonl(size);
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len  = sizeof(len),
+        }, {
+            .iov_base = (void *)buf,
+            .iov_len  = size,
+        },
+    };
+    size_t remaining;
+    ssize_t ret;
+
+    remaining = iov_size(iov, 2) - s->send_index;
+    ret = iov_send(s->fd, iov, 2, s->send_index, remaining);
+
+    if (ret == -1 && errno == EAGAIN) {
+        ret = 0; /* handled further down */
+    }
+    if (ret == -1) {
+        s->send_index = 0;
+        return -errno;
+    }
+    if (ret < (ssize_t)remaining) {
+        s->send_index += ret;
+        net_socket_write_poll(s, true);
+        return 0;
+    }
+    s->send_index = 0;
+    return size;
 }
 
 static ssize_t net_socket_receive_dgram(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
+    ssize_t ret;
+
+    do {
+        ret = qemu_sendto(s->fd, buf, size, 0,
+                          (struct sockaddr *)&s->dgram_dst,
+                          sizeof(s->dgram_dst));
+    } while (ret == -1 && errno == EINTR);
 
-    return sendto(s->fd, (const void *)buf, size, 0,
-                  (struct sockaddr *)&s->dgram_dst, sizeof(s->dgram_dst));
+    if (ret == -1 && errno == EAGAIN) {
+        net_socket_write_poll(s, true);
+        return 0;
+    }
+    return ret;
 }
 
 static void net_socket_send(void *opaque)
@@ -81,7 +157,8 @@ static void net_socket_send(void *opaque)
     } else if (size == 0) {
         /* end of connection */
     eoc:
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         if (s->listen_fd != -1) {
             qemu_set_fd_handler(s->listen_fd, net_socket_accept, NULL, s);
         }
@@ -152,7 +229,8 @@ static void net_socket_send_dgram(void *opaque)
         return;
     if (size == 0) {
         /* end of connection */
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         return;
     }
     qemu_send_packet(&s->nc, s->buf, size);
@@ -243,7 +321,8 @@ static void net_socket_cleanup(NetClientState *nc)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
     if (s->fd != -1) {
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         close(s->fd);
         s->fd = -1;
     }
@@ -314,8 +393,8 @@ static NetSocketState *net_socket_fd_init_dgram(NetClientState *peer,
 
     s->fd = fd;
     s->listen_fd = -1;
-
-    qemu_set_fd_handler(s->fd, net_socket_send_dgram, NULL, s);
+    s->send_fn = net_socket_send_dgram;
+    net_socket_read_poll(s, true);
 
     /* mcast: save bound address as dst */
     if (is_connected) {
@@ -332,7 +411,8 @@ err:
 static void net_socket_connect(void *opaque)
 {
     NetSocketState *s = opaque;
-    qemu_set_fd_handler(s->fd, net_socket_send, NULL, s);
+    s->send_fn = net_socket_send;
+    net_socket_read_poll(s, true);
 }
 
 static NetClientInfo net_socket_info = {
@@ -629,7 +709,7 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
     if (sock->has_fd) {
         int fd;
 
-        fd = net_handle_fd_param(cur_mon, sock->fd);
+        fd = monitor_handle_fd_param(cur_mon, sock->fd);
         if (fd == -1 || !net_socket_fd_init(peer, "socket", name, fd, 1)) {
             return -1;
         }
@@ -666,7 +746,7 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
         error_report("localaddr= is mandatory with udp=");
         return -1;
     }
-    if (net_socket_udp_init(peer, "udp", name, sock->udp, sock->localaddr) ==
+    if (net_socket_udp_init(peer, "socket", name, sock->udp, sock->localaddr) ==
         -1) {
         return -1;
     }
diff --git a/net/socket.h b/net/socket.h
deleted file mode 100644
index 3f8a0924592..00000000000
--- a/net/socket.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef QEMU_NET_SOCKET_H
-#define QEMU_NET_SOCKET_H
-
-#include "net.h"
-#include "qapi-types.h"
-
-int net_init_socket(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
-
-#endif /* QEMU_NET_SOCKET_H */
diff --git a/net/tap-aix.c b/net/tap-aix.c
index f27c17729e0..aff6c527e9f 100644
--- a/net/tap-aix.c
+++ b/net/tap-aix.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index a3b717dd1c4..01c705b4c03 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -22,10 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "qemu-error.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 
 #ifdef __NetBSD__
 #include <sys/ioctl.h>
diff --git a/net/tap-haiku.c b/net/tap-haiku.c
index 34739d15624..08cc034cee3 100644
--- a/net/tap-haiku.c
+++ b/net/tap-haiku.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
diff --git a/net/tap-linux.c b/net/tap-linux.c
index c6521bec344..059f5f34ab2 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -23,15 +23,16 @@
  * THE SOFTWARE.
  */
 
+#include "tap_int.h"
+#include "tap-linux.h"
 #include "net/tap.h"
-#include "net/tap-linux.h"
 
 #include <net/if.h>
 #include <sys/ioctl.h>
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #define PATH_NET_TUN "/dev/net/tun"
 
@@ -39,6 +40,7 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required
 {
     struct ifreq ifr;
     int fd, ret;
+    int len = sizeof(struct virtio_net_hdr);
 
     TFR(fd = open(PATH_NET_TUN, O_RDWR));
     if (fd < 0) {
@@ -65,6 +67,13 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required
             close(fd);
             return -1;
         }
+        /*
+         * Make sure vnet header size has the default value: for a persistent
+         * tap it might have been modified e.g. by another instance of qemu.
+         * Ignore errors since old kernels do not support this ioctl: in this
+         * case the header size implicitly has the correct value.
+         */
+        ioctl(fd, TUNSETVNETHDRSZ, &len);
     }
 
     if (ifname[0] != '\0')
diff --git a/net/tap-linux.h b/net/tap-linux.h
index 659e98122b5..cb2a6d480ae 100644
--- a/net/tap-linux.h
+++ b/net/tap-linux.h
@@ -13,8 +13,8 @@
  *  GNU General Public License for more details.
  */
 
-#ifndef QEMU_TAP_H
-#define QEMU_TAP_H
+#ifndef QEMU_TAP_LINUX_H
+#define QEMU_TAP_LINUX_H
 
 #include <stdint.h>
 #ifdef __linux__
@@ -44,20 +44,4 @@
 #define TUN_F_TSO_ECN	0x08	/* I can handle TSO with ECN bits. */
 #define TUN_F_UFO	0x10	/* I can handle UFO packets */
 
-struct virtio_net_hdr
-{
-    uint8_t flags;
-    uint8_t gso_type;
-    uint16_t hdr_len;
-    uint16_t gso_size;
-    uint16_t csum_start;
-    uint16_t csum_offset;
-};
-
-struct virtio_net_hdr_mrg_rxbuf
-{
-    struct virtio_net_hdr hdr;
-    uint16_t num_buffers;   /* Number of merged rx buffers */
-};
-
 #endif /* QEMU_TAP_H */
diff --git a/net/tap-solaris.c b/net/tap-solaris.c
index 5d6ac42f249..486a7ea838c 100644
--- a/net/tap-solaris.c
+++ b/net/tap-solaris.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
-#include "sysemu.h"
+#include "tap_int.h"
+#include "sysemu/sysemu.h"
 
 #include <sys/stat.h>
 #include <sys/ethernet.h>
@@ -38,7 +38,7 @@
 #include <net/if.h>
 #include <syslog.h>
 #include <stropts.h>
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen)
 {
diff --git a/net/tap-win32.c b/net/tap-win32.c
index c0ea954ca14..265369c3c54 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -26,12 +26,14 @@
  *  distribution); if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 
 #include "qemu-common.h"
-#include "net.h"
-#include "sysemu.h"
-#include "qemu-error.h"
+#include "clients.h"            /* net_init_tap */
+#include "net/net.h"
+#include "net/tap.h"            /* tap_has_ufo, ... */
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 #include <stdio.h>
 #include <windows.h>
 #include <winioctl.h>
@@ -564,7 +566,7 @@ static void tap_win32_free_buffer(tap_win32_overlapped_t *overlapped,
 }
 
 static int tap_win32_open(tap_win32_overlapped_t **phandle,
-                          const char *prefered_name)
+                          const char *preferred_name)
 {
     char device_path[256];
     char device_guid[0x100];
@@ -580,8 +582,9 @@ static int tap_win32_open(tap_win32_overlapped_t **phandle,
     DWORD version_len;
     DWORD idThread;
 
-    if (prefered_name != NULL)
-        snprintf(name_buffer, sizeof(name_buffer), "%s", prefered_name);
+    if (preferred_name != NULL) {
+        snprintf(name_buffer, sizeof(name_buffer), "%s", preferred_name);
+    }
 
     rc = get_device_guid(device_guid, sizeof(device_guid), name_buffer, sizeof(name_buffer));
     if (rc)
@@ -751,3 +754,13 @@ struct vhost_net *tap_get_vhost_net(NetClientState *nc)
 {
     return NULL;
 }
+
+int tap_has_vnet_hdr_len(NetClientState *nc, int len)
+{
+    return 0;
+}
+
+void tap_set_vnet_hdr_len(NetClientState *nc, int len)
+{
+    assert(0);
+}
diff --git a/net/tap.c b/net/tap.c
index 19715257941..eb40c42d7d9 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 
 #include "config-host.h"
 
@@ -33,14 +33,14 @@
 #include <sys/socket.h>
 #include <net/if.h>
 
-#include "net.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "net/net.h"
+#include "clients.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
-#include "net/tap-linux.h"
+#include "net/tap.h"
 
 #include "hw/vhost_net.h"
 
@@ -340,6 +340,13 @@ static TAPState *net_tap_fd_init(NetClientState *peer,
     s->using_vnet_hdr = 0;
     s->has_ufo = tap_probe_has_ufo(s->fd);
     tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
+    /*
+     * Make sure host header length is set correctly in tap:
+     * it might have been modified by another instance of qemu.
+     */
+    if (tap_probe_vnet_hdr_len(s->fd, s->host_vnet_hdr_len)) {
+        tap_fd_set_vnet_hdr_len(s->fd, s->host_vnet_hdr_len);
+    }
     tap_read_poll(s, 1);
     s->vhost_net = NULL;
     return s;
@@ -610,7 +617,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
             return -1;
         }
 
-        fd = net_handle_fd_param(cur_mon, tap->fd);
+        fd = monitor_handle_fd_param(cur_mon, tap->fd);
         if (fd == -1) {
             return -1;
         }
@@ -686,7 +693,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
         int vhostfd;
 
         if (tap->has_vhostfd) {
-            vhostfd = net_handle_fd_param(cur_mon, tap->vhostfd);
+            vhostfd = monitor_handle_fd_param(cur_mon, tap->vhostfd);
             if (vhostfd == -1) {
                 return -1;
             }
diff --git a/net/tap.h b/net/tap_int.h
index 0fb018c4b7c..1dffe12a452 100644
--- a/net/tap.h
+++ b/net/tap_int.h
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef QEMU_NET_TAP_H
-#define QEMU_NET_TAP_H
+#ifndef QEMU_TAP_H
+#define QEMU_TAP_H
 
 #include "qemu-common.h"
 #include "qapi-types.h"
@@ -32,20 +32,10 @@
 #define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
 #define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
 
-int net_init_tap(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer);
-
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required);
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen);
 
-int tap_has_ufo(NetClientState *nc);
-int tap_has_vnet_hdr(NetClientState *nc);
-int tap_has_vnet_hdr_len(NetClientState *nc, int len);
-void tap_using_vnet_hdr(NetClientState *nc, int using_vnet_hdr);
-void tap_set_offload(NetClientState *nc, int csum, int tso4, int tso6, int ecn, int ufo);
-void tap_set_vnet_hdr_len(NetClientState *nc, int len);
-
 int tap_set_sndbuf(int fd, const NetdevTapOptions *tap);
 int tap_probe_vnet_hdr(int fd);
 int tap_probe_vnet_hdr_len(int fd, int len);
@@ -53,12 +43,4 @@ int tap_probe_has_ufo(int fd);
 void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo);
 void tap_fd_set_vnet_hdr_len(int fd, int len);
 
-int tap_get_fd(NetClientState *nc);
-
-struct vhost_net;
-struct vhost_net *tap_get_vhost_net(NetClientState *nc);
-
-int net_init_bridge(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
-
-#endif /* QEMU_NET_TAP_H */
+#endif /* QEMU_TAP_H */
diff --git a/net/util.c b/net/util.c
index 1e9afbc1ae1..7e9507679d7 100644
--- a/net/util.c
+++ b/net/util.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/util.h"
+#include "util.h"
 #include <errno.h>
 #include <stdlib.h>
 
diff --git a/net/vde.c b/net/vde.c
index b91a6c799bd..4dea32d07a5 100644
--- a/net/vde.c
+++ b/net/vde.c
@@ -21,16 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net/vde.h"
-
 #include "config-host.h"
 
 #include <libvdeplug.h>
 
-#include "net.h"
-#include "qemu-char.h"
+#include "net/net.h"
+#include "clients.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
+#include "qemu/option.h"
+#include "qemu/main-loop.h"
 
 typedef struct VDEState {
     NetClientState nc;
diff --git a/os-posix.c b/os-posix.c
index 79fa2288e48..5c645189025 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -36,7 +36,7 @@
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "net/slirp.h"
 #include "qemu-options.h"
 
@@ -148,8 +148,7 @@ void os_set_proc_name(const char *s)
     char name[16];
     if (!s)
         return;
-    name[sizeof(name) - 1] = 0;
-    strncpy(name, s, sizeof(name));
+    pstrcpy(name, sizeof(name), s);
     /* Could rewrite argv[0] too, but that's a bit more complicated.
        This simple way is enough for `top'. */
     if (prctl(PR_SET_NAME, name)) {
@@ -194,7 +193,6 @@ void os_parse_cmd_args(int index, const char *optarg)
         break;
 #endif
     }
-    return;
 }
 
 static void change_process_uid(void)
@@ -360,3 +358,8 @@ int qemu_create_pidfile(const char *filename)
     /* keep pidfile open & locked forever */
     return 0;
 }
+
+bool is_daemonized(void)
+{
+    return daemonize;
+}
diff --git a/os-win32.c b/os-win32.c
index 13892ba3206..9673a81c7db 100644
--- a/os-win32.c
+++ b/os-win32.c
@@ -30,7 +30,7 @@
 #include <errno.h>
 #include <sys/time.h>
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qemu-options.h"
 
 /***********************************************************/
diff --git a/page_cache.c b/page_cache.c
index 0294f7e9f61..ba5640bd731 100644
--- a/page_cache.c
+++ b/page_cache.c
@@ -24,7 +24,7 @@
 #include <strings.h>
 
 #include "qemu-common.h"
-#include "qemu/page_cache.h"
+#include "migration/page_cache.h"
 
 #ifdef DEBUG_CACHE
 #define DPRINTF(fmt, ...) \
diff --git a/pc-bios/README b/pc-bios/README
index 303713099e6..eff3de7615f 100644
--- a/pc-bios/README
+++ b/pc-bios/README
@@ -17,7 +17,7 @@
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
   https://github.com/dgibson/SLOF, and the image currently in qemu is
-  built from git tag qemu-slof-20120731.
+  built from git tag qemu-slof-20121018.
 
 - sgabios (the Serial Graphics Adapter option ROM) provides a means for
   legacy x86 software to communicate with an attached serial console as
diff --git a/pc-bios/acpi-dsdt.aml b/pc-bios/acpi-dsdt.aml
new file mode 100644
index 00000000000..75dfd1e310a
--- /dev/null
+++ b/pc-bios/acpi-dsdt.aml
diff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index eac67cb05a8..924bee30d54 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
diff --git a/pc-bios/multiboot.bin b/pc-bios/multiboot.bin
index f74a6e142fd..7b3c1745a43 100644
--- a/pc-bios/multiboot.bin
+++ b/pc-bios/multiboot.bin
diff --git a/pc-bios/optionrom/multiboot.S b/pc-bios/optionrom/multiboot.S
index f08222a3c61..003bcfb49fa 100644
--- a/pc-bios/optionrom/multiboot.S
+++ b/pc-bios/optionrom/multiboot.S
@@ -75,6 +75,13 @@ run_multiboot:
 	shr		$4, %eax
 	mov		%ax, %fs
 
+	/* Account for the EBDA in the multiboot structure's e801
+	 * map.
+	 */
+	int		$0x12
+	cwtl
+	movl		%eax, %fs:4
+
 	/* ES = mmap_addr */
 	mov 		%fs:48, %eax
 	shr		$4, %eax
diff --git a/pc-bios/q35-acpi-dsdt.aml b/pc-bios/q35-acpi-dsdt.aml
new file mode 100644
index 00000000000..cf7b0857628
--- /dev/null
+++ b/pc-bios/q35-acpi-dsdt.aml
diff --git a/pc-bios/slof.bin b/pc-bios/slof.bin
index 84ba6b83f3c..3410f4fff40 100644
--- a/pc-bios/slof.bin
+++ b/pc-bios/slof.bin
diff --git a/pci-ids.txt b/pci-ids.txt
deleted file mode 100644
index 73125a8bd7c..00000000000
--- a/pci-ids.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-
-PCI IDs for qemu
-================
-
-Red Hat, Inc. donates a part of its device ID range to qemu, to be used for
-virtual devices.  The vendor ID is 1af4 (formerly Qumranet ID).
-
-The 1000 -> 10ff device ID range is used for VirtIO devices.
-
-The 1100 device ID is used as PCI Subsystem ID for existing hardware
-devices emulated by qemu.
-
-All other device IDs are reserved.
-
-
-VirtIO Device IDs
------------------
-
-1af4:1000  network device
-1af4:1001  block device
-1af4:1002  balloon device
-1af4:1003  console device
-
-1af4:1004  Reserved.
-   to      Contact Gerd Hoffmann <kraxel@redhat.com> to get a
-1af4:10ef  device ID assigned for your new virtio device.
-
-1af4:10f0  Available for experimental usage without registration.  Must get
-   to      official ID when the code leaves the test lab (i.e. when seeking
-1af4:10ff  upstream merge or shipping a distro/product) to avoid conflicts.
-
diff --git a/pflib.c b/pflib.c
deleted file mode 100644
index 987e11001a2..00000000000
--- a/pflib.c
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * PixelFormat conversion library.
- *
- * Author: Gerd Hoffmann <kraxel@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-#include "qemu-common.h"
-#include "console.h"
-#include "pflib.h"
-
-typedef struct QemuPixel QemuPixel;
-
-typedef void (*pf_convert)(QemuPfConv *conv,
-                           void *dst, void *src, uint32_t cnt);
-typedef void (*pf_convert_from)(PixelFormat *pf,
-                                QemuPixel *dst, void *src, uint32_t cnt);
-typedef void (*pf_convert_to)(PixelFormat *pf,
-                              void *dst, QemuPixel *src, uint32_t cnt);
-
-struct QemuPfConv {
-    pf_convert        convert;
-    PixelFormat       src;
-    PixelFormat       dst;
-
-    /* for copy_generic() */
-    pf_convert_from   conv_from;
-    pf_convert_to     conv_to;
-    QemuPixel         *conv_buf;
-    uint32_t          conv_cnt;
-};
-
-struct QemuPixel {
-    uint8_t red;
-    uint8_t green;
-    uint8_t blue;
-    uint8_t alpha;
-};
-
-/* ----------------------------------------------------------------------- */
-/* PixelFormat -> QemuPixel conversions                                    */
-
-static void conv_16_to_pixel(PixelFormat *pf,
-                             QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint16_t *src16 = src;
-
-    while (cnt > 0) {
-        dst->red   = ((*src16 & pf->rmask) >> pf->rshift) << (8 - pf->rbits);
-        dst->green = ((*src16 & pf->gmask) >> pf->gshift) << (8 - pf->gbits);
-        dst->blue  = ((*src16 & pf->bmask) >> pf->bshift) << (8 - pf->bbits);
-        dst->alpha = ((*src16 & pf->amask) >> pf->ashift) << (8 - pf->abits);
-        dst++, src16++, cnt--;
-    }
-}
-
-/* assumes pf->{r,g,b,a}bits == 8 */
-static void conv_32_to_pixel_fast(PixelFormat *pf,
-                                  QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint32_t *src32 = src;
-
-    while (cnt > 0) {
-        dst->red   = (*src32 & pf->rmask) >> pf->rshift;
-        dst->green = (*src32 & pf->gmask) >> pf->gshift;
-        dst->blue  = (*src32 & pf->bmask) >> pf->bshift;
-        dst->alpha = (*src32 & pf->amask) >> pf->ashift;
-        dst++, src32++, cnt--;
-    }
-}
-
-static void conv_32_to_pixel_generic(PixelFormat *pf,
-                                     QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint32_t *src32 = src;
-
-    while (cnt > 0) {
-        if (pf->rbits < 8) {
-            dst->red   = ((*src32 & pf->rmask) >> pf->rshift) << (8 - pf->rbits);
-        } else {
-            dst->red   = ((*src32 & pf->rmask) >> pf->rshift) >> (pf->rbits - 8);
-        }
-        if (pf->gbits < 8) {
-            dst->green = ((*src32 & pf->gmask) >> pf->gshift) << (8 - pf->gbits);
-        } else {
-            dst->green = ((*src32 & pf->gmask) >> pf->gshift) >> (pf->gbits - 8);
-        }
-        if (pf->bbits < 8) {
-            dst->blue  = ((*src32 & pf->bmask) >> pf->bshift) << (8 - pf->bbits);
-        } else {
-            dst->blue  = ((*src32 & pf->bmask) >> pf->bshift) >> (pf->bbits - 8);
-        }
-        if (pf->abits < 8) {
-            dst->alpha = ((*src32 & pf->amask) >> pf->ashift) << (8 - pf->abits);
-        } else {
-            dst->alpha = ((*src32 & pf->amask) >> pf->ashift) >> (pf->abits - 8);
-        }
-        dst++, src32++, cnt--;
-    }
-}
-
-/* ----------------------------------------------------------------------- */
-/* QemuPixel -> PixelFormat conversions                                    */
-
-static void conv_pixel_to_16(PixelFormat *pf,
-                             void *dst, QemuPixel *src, uint32_t cnt)
-{
-    uint16_t *dst16 = dst;
-
-    while (cnt > 0) {
-        *dst16  = ((uint16_t)src->red   >> (8 - pf->rbits)) << pf->rshift;
-        *dst16 |= ((uint16_t)src->green >> (8 - pf->gbits)) << pf->gshift;
-        *dst16 |= ((uint16_t)src->blue  >> (8 - pf->bbits)) << pf->bshift;
-        *dst16 |= ((uint16_t)src->alpha >> (8 - pf->abits)) << pf->ashift;
-        dst16++, src++, cnt--;
-    }
-}
-
-static void conv_pixel_to_32(PixelFormat *pf,
-                             void *dst, QemuPixel *src, uint32_t cnt)
-{
-    uint32_t *dst32 = dst;
-
-    while (cnt > 0) {
-        *dst32  = ((uint32_t)src->red   >> (8 - pf->rbits)) << pf->rshift;
-        *dst32 |= ((uint32_t)src->green >> (8 - pf->gbits)) << pf->gshift;
-        *dst32 |= ((uint32_t)src->blue  >> (8 - pf->bbits)) << pf->bshift;
-        *dst32 |= ((uint32_t)src->alpha >> (8 - pf->abits)) << pf->ashift;
-        dst32++, src++, cnt--;
-    }
-}
-
-/* ----------------------------------------------------------------------- */
-/* PixelFormat -> PixelFormat conversions                                  */
-
-static void convert_copy(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    uint32_t bytes = cnt * conv->src.bytes_per_pixel;
-    memcpy(dst, src, bytes);
-}
-
-static void convert_generic(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    if (conv->conv_cnt < cnt) {
-        conv->conv_cnt = cnt;
-        conv->conv_buf = g_realloc(conv->conv_buf, sizeof(QemuPixel) * conv->conv_cnt);
-    }
-    conv->conv_from(&conv->src, conv->conv_buf, src, cnt);
-    conv->conv_to(&conv->dst, dst, conv->conv_buf, cnt);
-}
-
-/* ----------------------------------------------------------------------- */
-/* public interface                                                        */
-
-QemuPfConv *qemu_pf_conv_get(PixelFormat *dst, PixelFormat *src)
-{
-    QemuPfConv *conv = g_malloc0(sizeof(QemuPfConv));
-
-    conv->src = *src;
-    conv->dst = *dst;
-
-    if (memcmp(&conv->src, &conv->dst, sizeof(PixelFormat)) == 0) {
-        /* formats identical, can simply copy */
-        conv->convert = convert_copy;
-    } else {
-        /* generic two-step conversion: src -> QemuPixel -> dst  */
-        switch (conv->src.bytes_per_pixel) {
-        case 2:
-            conv->conv_from = conv_16_to_pixel;
-            break;
-        case 4:
-            if (conv->src.rbits == 8 && conv->src.gbits == 8 && conv->src.bbits == 8) {
-                conv->conv_from = conv_32_to_pixel_fast;
-            } else {
-                conv->conv_from = conv_32_to_pixel_generic;
-            }
-            break;
-        default:
-            goto err;
-        }
-        switch (conv->dst.bytes_per_pixel) {
-        case 2:
-            conv->conv_to = conv_pixel_to_16;
-            break;
-        case 4:
-            conv->conv_to = conv_pixel_to_32;
-            break;
-        default:
-            goto err;
-        }
-        conv->convert = convert_generic;
-    }
-    return conv;
-
-err:
-    g_free(conv);
-    return NULL;
-}
-
-void qemu_pf_conv_run(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    conv->convert(conv, dst, src, cnt);
-}
-
-void qemu_pf_conv_put(QemuPfConv *conv)
-{
-    if (conv) {
-        g_free(conv->conv_buf);
-        g_free(conv);
-    }
-}
diff --git a/pflib.h b/pflib.h
deleted file mode 100644
index b70c313acfd..00000000000
--- a/pflib.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#ifndef __QEMU_PFLIB_H
-#define __QEMU_PFLIB_H
-
-/*
- * PixelFormat conversion library.
- *
- * Author: Gerd Hoffmann <kraxel@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-typedef struct QemuPfConv QemuPfConv;
-
-QemuPfConv *qemu_pf_conv_get(PixelFormat *dst, PixelFormat *src);
-void qemu_pf_conv_run(QemuPfConv *conv, void *dst, void *src, uint32_t cnt);
-void qemu_pf_conv_put(QemuPfConv *conv);
-
-#endif
diff --git a/pixman b/pixman
new file mode 160000
+Subproject 97336fad32acf802003855cd8bd6477fa49a12e
diff --git a/posix-aio-compat.c b/posix-aio-compat.c
deleted file mode 100644
index 96e4daf5059..00000000000
--- a/posix-aio-compat.c
+++ /dev/null
@@ -1,679 +0,0 @@
-/*
- * QEMU posix-aio emulation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <errno.h>
-#include <time.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "qemu-queue.h"
-#include "osdep.h"
-#include "sysemu.h"
-#include "qemu-common.h"
-#include "trace.h"
-#include "block_int.h"
-#include "iov.h"
-
-#include "block/raw-posix-aio.h"
-
-static void do_spawn_thread(void);
-
-struct qemu_paiocb {
-    BlockDriverAIOCB common;
-    int aio_fildes;
-    union {
-        struct iovec *aio_iov;
-        void *aio_ioctl_buf;
-    };
-    int aio_niov;
-    size_t aio_nbytes;
-#define aio_ioctl_cmd   aio_nbytes /* for QEMU_AIO_IOCTL */
-    off_t aio_offset;
-
-    QTAILQ_ENTRY(qemu_paiocb) node;
-    int aio_type;
-    ssize_t ret;
-    int active;
-    struct qemu_paiocb *next;
-};
-
-typedef struct PosixAioState {
-    int rfd, wfd;
-    struct qemu_paiocb *first_aio;
-} PosixAioState;
-
-
-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
-static pthread_t thread_id;
-static pthread_attr_t attr;
-static int max_threads = 64;
-static int cur_threads = 0;
-static int idle_threads = 0;
-static int new_threads = 0;     /* backlog of threads we need to create */
-static int pending_threads = 0; /* threads created but not running yet */
-static QEMUBH *new_thread_bh;
-static QTAILQ_HEAD(, qemu_paiocb) request_list;
-
-#ifdef CONFIG_PREADV
-static int preadv_present = 1;
-#else
-static int preadv_present = 0;
-#endif
-
-static void die2(int err, const char *what)
-{
-    fprintf(stderr, "%s failed: %s\n", what, strerror(err));
-    abort();
-}
-
-static void die(const char *what)
-{
-    die2(errno, what);
-}
-
-static void mutex_lock(pthread_mutex_t *mutex)
-{
-    int ret = pthread_mutex_lock(mutex);
-    if (ret) die2(ret, "pthread_mutex_lock");
-}
-
-static void mutex_unlock(pthread_mutex_t *mutex)
-{
-    int ret = pthread_mutex_unlock(mutex);
-    if (ret) die2(ret, "pthread_mutex_unlock");
-}
-
-static int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex,
-                           struct timespec *ts)
-{
-    int ret = pthread_cond_timedwait(cond, mutex, ts);
-    if (ret && ret != ETIMEDOUT) die2(ret, "pthread_cond_timedwait");
-    return ret;
-}
-
-static void cond_signal(pthread_cond_t *cond)
-{
-    int ret = pthread_cond_signal(cond);
-    if (ret) die2(ret, "pthread_cond_signal");
-}
-
-static void thread_create(pthread_t *thread, pthread_attr_t *attr,
-                          void *(*start_routine)(void*), void *arg)
-{
-    int ret = pthread_create(thread, attr, start_routine, arg);
-    if (ret) die2(ret, "pthread_create");
-}
-
-static ssize_t handle_aiocb_ioctl(struct qemu_paiocb *aiocb)
-{
-    int ret;
-
-    ret = ioctl(aiocb->aio_fildes, aiocb->aio_ioctl_cmd, aiocb->aio_ioctl_buf);
-    if (ret == -1)
-        return -errno;
-
-    /*
-     * This looks weird, but the aio code only considers a request
-     * successful if it has written the full number of bytes.
-     *
-     * Now we overload aio_nbytes as aio_ioctl_cmd for the ioctl command,
-     * so in fact we return the ioctl command here to make posix_aio_read()
-     * happy..
-     */
-    return aiocb->aio_nbytes;
-}
-
-static ssize_t handle_aiocb_flush(struct qemu_paiocb *aiocb)
-{
-    int ret;
-
-    ret = qemu_fdatasync(aiocb->aio_fildes);
-    if (ret == -1)
-        return -errno;
-    return 0;
-}
-
-#ifdef CONFIG_PREADV
-
-static ssize_t
-qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return preadv(fd, iov, nr_iov, offset);
-}
-
-static ssize_t
-qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return pwritev(fd, iov, nr_iov, offset);
-}
-
-#else
-
-static ssize_t
-qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return -ENOSYS;
-}
-
-static ssize_t
-qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return -ENOSYS;
-}
-
-#endif
-
-static ssize_t handle_aiocb_rw_vector(struct qemu_paiocb *aiocb)
-{
-    ssize_t len;
-
-    do {
-        if (aiocb->aio_type & QEMU_AIO_WRITE)
-            len = qemu_pwritev(aiocb->aio_fildes,
-                               aiocb->aio_iov,
-                               aiocb->aio_niov,
-                               aiocb->aio_offset);
-         else
-            len = qemu_preadv(aiocb->aio_fildes,
-                              aiocb->aio_iov,
-                              aiocb->aio_niov,
-                              aiocb->aio_offset);
-    } while (len == -1 && errno == EINTR);
-
-    if (len == -1)
-        return -errno;
-    return len;
-}
-
-/*
- * Read/writes the data to/from a given linear buffer.
- *
- * Returns the number of bytes handles or -errno in case of an error. Short
- * reads are only returned if the end of the file is reached.
- */
-static ssize_t handle_aiocb_rw_linear(struct qemu_paiocb *aiocb, char *buf)
-{
-    ssize_t offset = 0;
-    ssize_t len;
-
-    while (offset < aiocb->aio_nbytes) {
-         if (aiocb->aio_type & QEMU_AIO_WRITE)
-             len = pwrite(aiocb->aio_fildes,
-                          (const char *)buf + offset,
-                          aiocb->aio_nbytes - offset,
-                          aiocb->aio_offset + offset);
-         else
-             len = pread(aiocb->aio_fildes,
-                         buf + offset,
-                         aiocb->aio_nbytes - offset,
-                         aiocb->aio_offset + offset);
-
-         if (len == -1 && errno == EINTR)
-             continue;
-         else if (len == -1) {
-             offset = -errno;
-             break;
-         } else if (len == 0)
-             break;
-
-         offset += len;
-    }
-
-    return offset;
-}
-
-static ssize_t handle_aiocb_rw(struct qemu_paiocb *aiocb)
-{
-    ssize_t nbytes;
-    char *buf;
-
-    if (!(aiocb->aio_type & QEMU_AIO_MISALIGNED)) {
-        /*
-         * If there is just a single buffer, and it is properly aligned
-         * we can just use plain pread/pwrite without any problems.
-         */
-        if (aiocb->aio_niov == 1)
-             return handle_aiocb_rw_linear(aiocb, aiocb->aio_iov->iov_base);
-
-        /*
-         * We have more than one iovec, and all are properly aligned.
-         *
-         * Try preadv/pwritev first and fall back to linearizing the
-         * buffer if it's not supported.
-         */
-        if (preadv_present) {
-            nbytes = handle_aiocb_rw_vector(aiocb);
-            if (nbytes == aiocb->aio_nbytes)
-                return nbytes;
-            if (nbytes < 0 && nbytes != -ENOSYS)
-                return nbytes;
-            preadv_present = 0;
-        }
-
-        /*
-         * XXX(hch): short read/write.  no easy way to handle the reminder
-         * using these interfaces.  For now retry using plain
-         * pread/pwrite?
-         */
-    }
-
-    /*
-     * Ok, we have to do it the hard way, copy all segments into
-     * a single aligned buffer.
-     */
-    buf = qemu_blockalign(aiocb->common.bs, aiocb->aio_nbytes);
-    if (aiocb->aio_type & QEMU_AIO_WRITE) {
-        char *p = buf;
-        int i;
-
-        for (i = 0; i < aiocb->aio_niov; ++i) {
-            memcpy(p, aiocb->aio_iov[i].iov_base, aiocb->aio_iov[i].iov_len);
-            p += aiocb->aio_iov[i].iov_len;
-        }
-    }
-
-    nbytes = handle_aiocb_rw_linear(aiocb, buf);
-    if (!(aiocb->aio_type & QEMU_AIO_WRITE)) {
-        char *p = buf;
-        size_t count = aiocb->aio_nbytes, copy;
-        int i;
-
-        for (i = 0; i < aiocb->aio_niov && count; ++i) {
-            copy = count;
-            if (copy > aiocb->aio_iov[i].iov_len)
-                copy = aiocb->aio_iov[i].iov_len;
-            memcpy(aiocb->aio_iov[i].iov_base, p, copy);
-            p     += copy;
-            count -= copy;
-        }
-    }
-    qemu_vfree(buf);
-
-    return nbytes;
-}
-
-static void posix_aio_notify_event(void);
-
-static void *aio_thread(void *unused)
-{
-    mutex_lock(&lock);
-    pending_threads--;
-    mutex_unlock(&lock);
-    do_spawn_thread();
-
-    while (1) {
-        struct qemu_paiocb *aiocb;
-        ssize_t ret = 0;
-        qemu_timeval tv;
-        struct timespec ts;
-
-        qemu_gettimeofday(&tv);
-        ts.tv_sec = tv.tv_sec + 10;
-        ts.tv_nsec = 0;
-
-        mutex_lock(&lock);
-
-        while (QTAILQ_EMPTY(&request_list) &&
-               !(ret == ETIMEDOUT)) {
-            idle_threads++;
-            ret = cond_timedwait(&cond, &lock, &ts);
-            idle_threads--;
-        }
-
-        if (QTAILQ_EMPTY(&request_list))
-            break;
-
-        aiocb = QTAILQ_FIRST(&request_list);
-        QTAILQ_REMOVE(&request_list, aiocb, node);
-        aiocb->active = 1;
-        mutex_unlock(&lock);
-
-        switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
-        case QEMU_AIO_READ:
-            ret = handle_aiocb_rw(aiocb);
-            if (ret >= 0 && ret < aiocb->aio_nbytes && aiocb->common.bs->growable) {
-                /* A short read means that we have reached EOF. Pad the buffer
-                 * with zeros for bytes after EOF. */
-                iov_memset(aiocb->aio_iov, aiocb->aio_niov, ret,
-                           0, aiocb->aio_nbytes - ret);
-
-                ret = aiocb->aio_nbytes;
-            }
-            break;
-        case QEMU_AIO_WRITE:
-            ret = handle_aiocb_rw(aiocb);
-            break;
-        case QEMU_AIO_FLUSH:
-            ret = handle_aiocb_flush(aiocb);
-            break;
-        case QEMU_AIO_IOCTL:
-            ret = handle_aiocb_ioctl(aiocb);
-            break;
-        default:
-            fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
-            ret = -EINVAL;
-            break;
-        }
-
-        mutex_lock(&lock);
-        aiocb->ret = ret;
-        mutex_unlock(&lock);
-
-        posix_aio_notify_event();
-    }
-
-    cur_threads--;
-    mutex_unlock(&lock);
-
-    return NULL;
-}
-
-static void do_spawn_thread(void)
-{
-    sigset_t set, oldset;
-
-    mutex_lock(&lock);
-    if (!new_threads) {
-        mutex_unlock(&lock);
-        return;
-    }
-
-    new_threads--;
-    pending_threads++;
-
-    mutex_unlock(&lock);
-
-    /* block all signals */
-    if (sigfillset(&set)) die("sigfillset");
-    if (sigprocmask(SIG_SETMASK, &set, &oldset)) die("sigprocmask");
-
-    thread_create(&thread_id, &attr, aio_thread, NULL);
-
-    if (sigprocmask(SIG_SETMASK, &oldset, NULL)) die("sigprocmask restore");
-}
-
-static void spawn_thread_bh_fn(void *opaque)
-{
-    do_spawn_thread();
-}
-
-static void spawn_thread(void)
-{
-    cur_threads++;
-    new_threads++;
-    /* If there are threads being created, they will spawn new workers, so
-     * we don't spend time creating many threads in a loop holding a mutex or
-     * starving the current vcpu.
-     *
-     * If there are no idle threads, ask the main thread to create one, so we
-     * inherit the correct affinity instead of the vcpu affinity.
-     */
-    if (!pending_threads) {
-        qemu_bh_schedule(new_thread_bh);
-    }
-}
-
-static void qemu_paio_submit(struct qemu_paiocb *aiocb)
-{
-    aiocb->ret = -EINPROGRESS;
-    aiocb->active = 0;
-    mutex_lock(&lock);
-    if (idle_threads == 0 && cur_threads < max_threads)
-        spawn_thread();
-    QTAILQ_INSERT_TAIL(&request_list, aiocb, node);
-    mutex_unlock(&lock);
-    cond_signal(&cond);
-}
-
-static ssize_t qemu_paio_return(struct qemu_paiocb *aiocb)
-{
-    ssize_t ret;
-
-    mutex_lock(&lock);
-    ret = aiocb->ret;
-    mutex_unlock(&lock);
-
-    return ret;
-}
-
-static int qemu_paio_error(struct qemu_paiocb *aiocb)
-{
-    ssize_t ret = qemu_paio_return(aiocb);
-
-    if (ret < 0)
-        ret = -ret;
-    else
-        ret = 0;
-
-    return ret;
-}
-
-static void posix_aio_read(void *opaque)
-{
-    PosixAioState *s = opaque;
-    struct qemu_paiocb *acb, **pacb;
-    int ret;
-    ssize_t len;
-
-    /* read all bytes from signal pipe */
-    for (;;) {
-        char bytes[16];
-
-        len = read(s->rfd, bytes, sizeof(bytes));
-        if (len == -1 && errno == EINTR)
-            continue; /* try again */
-        if (len == sizeof(bytes))
-            continue; /* more to read */
-        break;
-    }
-
-    for(;;) {
-        pacb = &s->first_aio;
-        for(;;) {
-            acb = *pacb;
-            if (!acb)
-                return;
-
-            ret = qemu_paio_error(acb);
-            if (ret == ECANCELED) {
-                /* remove the request */
-                *pacb = acb->next;
-                qemu_aio_release(acb);
-            } else if (ret != EINPROGRESS) {
-                /* end of aio */
-                if (ret == 0) {
-                    ret = qemu_paio_return(acb);
-                    if (ret == acb->aio_nbytes)
-                        ret = 0;
-                    else
-                        ret = -EINVAL;
-                } else {
-                    ret = -ret;
-                }
-
-                trace_paio_complete(acb, acb->common.opaque, ret);
-
-                /* remove the request */
-                *pacb = acb->next;
-                /* call the callback */
-                acb->common.cb(acb->common.opaque, ret);
-                qemu_aio_release(acb);
-                break;
-            } else {
-                pacb = &acb->next;
-            }
-        }
-    }
-}
-
-static int posix_aio_flush(void *opaque)
-{
-    PosixAioState *s = opaque;
-    return !!s->first_aio;
-}
-
-static PosixAioState *posix_aio_state;
-
-static void posix_aio_notify_event(void)
-{
-    char byte = 0;
-    ssize_t ret;
-
-    ret = write(posix_aio_state->wfd, &byte, sizeof(byte));
-    if (ret < 0 && errno != EAGAIN)
-        die("write()");
-}
-
-static void paio_remove(struct qemu_paiocb *acb)
-{
-    struct qemu_paiocb **pacb;
-
-    /* remove the callback from the queue */
-    pacb = &posix_aio_state->first_aio;
-    for(;;) {
-        if (*pacb == NULL) {
-            fprintf(stderr, "paio_remove: aio request not found!\n");
-            break;
-        } else if (*pacb == acb) {
-            *pacb = acb->next;
-            qemu_aio_release(acb);
-            break;
-        }
-        pacb = &(*pacb)->next;
-    }
-}
-
-static void paio_cancel(BlockDriverAIOCB *blockacb)
-{
-    struct qemu_paiocb *acb = (struct qemu_paiocb *)blockacb;
-    int active = 0;
-
-    trace_paio_cancel(acb, acb->common.opaque);
-
-    mutex_lock(&lock);
-    if (!acb->active) {
-        QTAILQ_REMOVE(&request_list, acb, node);
-        acb->ret = -ECANCELED;
-    } else if (acb->ret == -EINPROGRESS) {
-        active = 1;
-    }
-    mutex_unlock(&lock);
-
-    if (active) {
-        /* fail safe: if the aio could not be canceled, we wait for
-           it */
-        while (qemu_paio_error(acb) == EINPROGRESS)
-            ;
-    }
-
-    paio_remove(acb);
-}
-
-static AIOPool raw_aio_pool = {
-    .aiocb_size         = sizeof(struct qemu_paiocb),
-    .cancel             = paio_cancel,
-};
-
-BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
-{
-    struct qemu_paiocb *acb;
-
-    acb = qemu_aio_get(&raw_aio_pool, bs, cb, opaque);
-    acb->aio_type = type;
-    acb->aio_fildes = fd;
-
-    if (qiov) {
-        acb->aio_iov = qiov->iov;
-        acb->aio_niov = qiov->niov;
-    }
-    acb->aio_nbytes = nb_sectors * 512;
-    acb->aio_offset = sector_num * 512;
-
-    acb->next = posix_aio_state->first_aio;
-    posix_aio_state->first_aio = acb;
-
-    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
-    qemu_paio_submit(acb);
-    return &acb->common;
-}
-
-BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    struct qemu_paiocb *acb;
-
-    acb = qemu_aio_get(&raw_aio_pool, bs, cb, opaque);
-    acb->aio_type = QEMU_AIO_IOCTL;
-    acb->aio_fildes = fd;
-    acb->aio_offset = 0;
-    acb->aio_ioctl_buf = buf;
-    acb->aio_ioctl_cmd = req;
-
-    acb->next = posix_aio_state->first_aio;
-    posix_aio_state->first_aio = acb;
-
-    qemu_paio_submit(acb);
-    return &acb->common;
-}
-
-int paio_init(void)
-{
-    PosixAioState *s;
-    int fds[2];
-    int ret;
-
-    if (posix_aio_state)
-        return 0;
-
-    s = g_malloc(sizeof(PosixAioState));
-
-    s->first_aio = NULL;
-    if (qemu_pipe(fds) == -1) {
-        fprintf(stderr, "failed to create pipe\n");
-        g_free(s);
-        return -1;
-    }
-
-    s->rfd = fds[0];
-    s->wfd = fds[1];
-
-    fcntl(s->rfd, F_SETFL, O_NONBLOCK);
-    fcntl(s->wfd, F_SETFL, O_NONBLOCK);
-
-    qemu_aio_set_fd_handler(s->rfd, posix_aio_read, NULL, posix_aio_flush, s);
-
-    ret = pthread_attr_init(&attr);
-    if (ret)
-        die2(ret, "pthread_attr_init");
-
-    ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-    if (ret)
-        die2(ret, "pthread_attr_setdetachstate");
-
-    QTAILQ_INIT(&request_list);
-    new_thread_bh = qemu_bh_new(spawn_thread_bh_fn, NULL);
-
-    posix_aio_state = s;
-    return 0;
-}
diff --git a/qapi-schema.json b/qapi-schema.json
index bd8ad74495b..3a4817b3910 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -22,15 +22,36 @@
 # @KVMMissingCap: the requested operation can't be fulfilled because a
 #                 required KVM capability is missing
 #
-# @MigrationExpected: the requested operation can't be fulfilled because a
-#                     migration process is expected
-#
 # Since: 1.2
 ##
 { 'enum': 'ErrorClass',
   'data': [ 'GenericError', 'CommandNotFound', 'DeviceEncrypted',
-            'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap',
-            'MigrationExpected' ] }
+            'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap' ] }
+
+##
+# @add_client
+#
+# Allow client connections for VNC, Spice and socket based
+# character devices to be passed in to QEMU via SCM_RIGHTS.
+#
+# @protocol: protocol name. Valid names are "vnc", "spice" or the
+#            name of a character device (eg. from -chardev id=XXXX)
+#
+# @fdname: file descriptor name previously passed via 'getfd' command
+#
+# @skipauth: #optional whether to skip authentication. Only applies
+#            to "vnc" and "spice" protocols
+#
+# @tls: #optional whether to perform TLS. Only applies to the "spice"
+#       protocol
+#
+# Returns: nothing on success.
+#
+# Since: 0.14.0
+##
+{ 'command': 'add_client',
+  'data': { 'protocol': 'str', 'fdname': 'str', '*skipauth': 'bool',
+            '*tls': 'bool' } }
 
 ##
 # @NameInfo:
@@ -118,13 +139,17 @@
 ##
 # @RunState
 #
-# An enumation of VM run states.
+# An enumeration of VM run states.
 #
 # @debug: QEMU is running on a debugger
 #
 # @finish-migrate: guest is paused to finish the migration process
 #
-# @inmigrate: guest is paused waiting for an incoming migration
+# @inmigrate: guest is paused waiting for an incoming migration.  Note
+# that this state does not tell whether the machine will start at the
+# end of the migration.  This depends on the command-line -S option and
+# any invocation of 'stop' or 'cont' that has happened since QEMU was
+# started.
 #
 # @internal-error: An internal error that prevents further guest execution
 # has occurred
@@ -156,6 +181,70 @@
             'running', 'save-vm', 'shutdown', 'suspended', 'watchdog' ] }
 
 ##
+# @SnapshotInfo
+#
+# @id: unique snapshot id
+#
+# @name: user chosen name
+#
+# @vm-state-size: size of the VM state
+#
+# @date-sec: UTC date of the snapshot in seconds
+#
+# @date-nsec: fractional part in nano seconds to be used with date-sec
+#
+# @vm-clock-sec: VM clock relative to boot in seconds
+#
+# @vm-clock-nsec: fractional part in nano seconds to be used with vm-clock-sec
+#
+# Since: 1.3
+#
+##
+
+{ 'type': 'SnapshotInfo',
+  'data': { 'id': 'str', 'name': 'str', 'vm-state-size': 'int',
+            'date-sec': 'int', 'date-nsec': 'int',
+            'vm-clock-sec': 'int', 'vm-clock-nsec': 'int' } }
+
+##
+# @ImageInfo:
+#
+# Information about a QEMU image file
+#
+# @filename: name of the image file
+#
+# @format: format of the image file
+#
+# @virtual-size: maximum capacity in bytes of the image
+#
+# @actual-size: #optional actual size on disk in bytes of the image
+#
+# @dirty-flag: #optional true if image is not cleanly closed
+#
+# @cluster-size: #optional size of a cluster in bytes
+#
+# @encrypted: #optional true if the image is encrypted
+#
+# @backing-filename: #optional name of the backing file
+#
+# @full-backing-filename: #optional full path of the backing file
+#
+# @backing-filename-format: #optional the format of the backing file
+#
+# @snapshots: #optional list of VM snapshots
+#
+# Since: 1.3
+#
+##
+
+{ 'type': 'ImageInfo',
+  'data': {'filename': 'str', 'format': 'str', '*dirty-flag': 'bool',
+           '*actual-size': 'int', 'virtual-size': 'int',
+           '*cluster-size': 'int', '*encrypted': 'bool',
+           '*backing-filename': 'str', '*full-backing-filename': 'str',
+           '*backing-filename-format': 'str', '*snapshots': ['SnapshotInfo'] } }
+
+##
 # @StatusInfo:
 #
 # Information about VCPU run state
@@ -236,6 +325,80 @@
 { 'command': 'query-chardev', 'returns': ['ChardevInfo'] }
 
 ##
+# @DataFormat:
+#
+# An enumeration of data format.
+#
+# @utf8: The data format is 'utf8'.
+#
+# @base64: The data format is 'base64'.
+#
+# Since: 1.4
+##
+{ 'enum': 'DataFormat'
+  'data': [ 'utf8', 'base64' ] }
+
+##
+# @memchar-write:
+#
+# Provide writing interface for memchardev. Write data to char
+# device 'memory'.
+#
+# @device: the name of the memory char device.
+#
+# @size: the size to write in bytes.
+#
+# @data: the source data write to memchar.
+#
+# @format: #optional the format of the data write to chardev 'memory',
+#          by default is 'utf8'.
+#
+# Returns: Nothing on success
+#          If @device is not a valid char device, DeviceNotFound
+#
+# Since: 1.4
+##
+{ 'command': 'memchar-write',
+  'data': {'device': 'str', 'size': 'int', 'data': 'str',
+           '*format': 'DataFormat'} }
+
+##
+# @MemCharRead
+#
+# Result of QMP command memchar-read.
+#
+# @data: The data read from memchar as string.
+#
+# @count: The numbers of bytes read from.
+#
+# Since: 1.4
+##
+{ 'type': 'MemCharRead',
+  'data': { 'data': 'str', 'count': 'int' } }
+
+##
+# @memchar-read:
+#
+# Provide read interface for memchardev. Read from the char
+# device 'memory' and return the data.
+#
+# @device: the name of the memory char device.
+#
+# @size: the size to read in bytes.
+#
+# @format: #optional the format of the data want to read from
+#          memchardev, by default is 'utf8'.
+#
+# Returns: @MemCharRead
+#          If @device is not a valid memchr device, DeviceNotFound
+#
+# Since: 1.4
+##
+{ 'command': 'memchar-read',
+  'data': {'device': 'str', 'size': 'int', '*format': 'DataFormat'},
+  'returns': 'MemCharRead' }
+
+##
 # @CommandInfo:
 #
 # Information about a QMP command
@@ -294,13 +457,17 @@
 #
 # @normal : number of normal pages (since 1.2)
 #
-# @normal-bytes : number of normal bytes sent (since 1.2)
+# @normal-bytes: number of normal bytes sent (since 1.2)
+#
+# @dirty-pages-rate: number of pages dirtied by second by the
+#        guest (since 1.3)
 #
 # Since: 0.14.0
 ##
 { 'type': 'MigrationStats',
   'data': {'transferred': 'int', 'remaining': 'int', 'total': 'int' ,
-           'duplicate': 'int', 'normal': 'int', 'normal-bytes': 'int' } }
+           'duplicate': 'int', 'normal': 'int', 'normal-bytes': 'int',
+           'dirty-pages-rate' : 'int' } }
 
 ##
 # @XBZRLECacheStats
@@ -349,13 +516,23 @@
 #        If migration has ended, it returns the total migration
 #        time. (since 1.2)
 #
+# @downtime: #optional only present when migration finishes correctly
+#        total downtime in milliseconds for the guest.
+#        (since 1.3)
+#
+# @expected-downtime: #optional only present while migration is active
+#        expected downtime in milliseconds for the guest in last walk
+#        of the dirty bitmap. (since 1.3)
+#
 # Since: 0.14.0
 ##
 { 'type': 'MigrationInfo',
   'data': {'*status': 'str', '*ram': 'MigrationStats',
            '*disk': 'MigrationStats',
            '*xbzrle-cache': 'XBZRLECacheStats',
-           '*total-time': 'int'} }
+           '*total-time': 'int',
+           '*expected-downtime': 'int',
+           '*downtime': 'int'} }
 
 ##
 # @query-migrate
@@ -558,6 +735,20 @@
 { 'enum': 'BlockDeviceIoStatus', 'data': [ 'ok', 'failed', 'nospace' ] }
 
 ##
+# @BlockDirtyInfo:
+#
+# Block dirty bitmap information.
+#
+# @count: number of dirty bytes according to the dirty bitmap
+#
+# @granularity: granularity of the dirty bitmap in bytes (since 1.4)
+#
+# Since: 1.3
+##
+{ 'type': 'BlockDirtyInfo',
+  'data': {'count': 'int', 'granularity': 'int'} }
+
+##
 # @BlockInfo:
 #
 # Block device information.  This structure describes a virtual device and
@@ -576,6 +767,9 @@
 # @tray_open: #optional True if the device has a tray and it is open
 #             (only present if removable is true)
 #
+# @dirty: #optional dirty bitmap information (only present if the dirty
+#         bitmap is enabled)
+#
 # @io-status: #optional @BlockDeviceIoStatus. Only present if the device
 #             supports it and the VM is configured to stop on errors
 #
@@ -587,7 +781,8 @@
 { 'type': 'BlockInfo',
   'data': {'device': 'str', 'type': 'str', 'removable': 'bool',
            'locked': 'bool', '*inserted': 'BlockDeviceInfo',
-           '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus'} }
+           '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus',
+           '*dirty': 'BlockDirtyInfo' } }
 
 ##
 # @query-block:
@@ -785,7 +980,7 @@
 ##
 # @SpiceQueryMouseMode
 #
-# An enumation of Spice mouse states.
+# An enumeration of Spice mouse states.
 #
 # @client: Mouse cursor position is determined by the client.
 #
@@ -808,6 +1003,9 @@
 #
 # @enabled: true if the SPICE server is enabled, false otherwise
 #
+# @migrated: true if the last guest migration completed and spice
+#            migration had completed as well. false otherwise.
+#
 # @host: #optional The hostname the SPICE server is bound to.  This depends on
 #        the name resolution on the host and may be an IP address.
 #
@@ -833,7 +1031,7 @@
 # Since: 0.14.0
 ##
 { 'type': 'SpiceInfo',
-  'data': {'enabled': 'bool', '*host': 'str', '*port': 'int',
+  'data': {'enabled': 'bool', 'migrated': 'bool', '*host': 'str', '*port': 'int',
            '*tls-port': 'int', '*auth': 'str', '*compiled-version': 'str',
            'mouse-mode': 'SpiceQueryMouseMode', '*channels': ['SpiceChannel']} }
 
@@ -855,28 +1053,10 @@
 #
 # @actual: the number of bytes the balloon currently contains
 #
-# @mem_swapped_in: #optional number of pages swapped in within the guest
-#
-# @mem_swapped_out: #optional number of pages swapped out within the guest
-#
-# @major_page_faults: #optional number of major page faults within the guest
-#
-# @minor_page_faults: #optional number of minor page faults within the guest
-#
-# @free_mem: #optional amount of memory (in bytes) free in the guest
-#
-# @total_mem: #optional amount of memory (in bytes) visible to the guest
-#
 # Since: 0.14.0
 #
-# Notes: all current versions of QEMU do not fill out optional information in
-#        this structure.
 ##
-{ 'type': 'BalloonInfo',
-  'data': {'actual': 'int', '*mem_swapped_in': 'int',
-           '*mem_swapped_out': 'int', '*major_page_faults': 'int',
-           '*minor_page_faults': 'int', '*free_mem': 'int',
-           '*total_mem': 'int'} }
+{ 'type': 'BalloonInfo', 'data': {'actual': 'int' } }
 
 ##
 # @query-balloon:
@@ -1021,6 +1201,46 @@
 { 'command': 'query-pci', 'returns': ['PciInfo'] }
 
 ##
+# @BlockdevOnError:
+#
+# An enumeration of possible behaviors for errors on I/O operations.
+# The exact meaning depends on whether the I/O was initiated by a guest
+# or by a block job
+#
+# @report: for guest operations, report the error to the guest;
+#          for jobs, cancel the job
+#
+# @ignore: ignore the error, only report a QMP event (BLOCK_IO_ERROR
+#          or BLOCK_JOB_ERROR)
+#
+# @enospc: same as @stop on ENOSPC, same as @report otherwise.
+#
+# @stop: for guest operations, stop the virtual machine;
+#        for jobs, pause the job
+#
+# Since: 1.3
+##
+{ 'enum': 'BlockdevOnError',
+  'data': ['report', 'ignore', 'enospc', 'stop'] }
+
+##
+# @MirrorSyncMode:
+#
+# An enumeration of possible behaviors for the initial synchronization
+# phase of storage mirroring.
+#
+# @top: copies data in the topmost image to the destination
+#
+# @full: copies data from all images to the destination
+#
+# @none: only copy data written from now on
+#
+# Since: 1.3
+##
+{ 'enum': 'MirrorSyncMode',
+  'data': ['top', 'full', 'none'] }
+
+##
 # @BlockJobInfo:
 #
 # Information about a long-running block device operation.
@@ -1031,15 +1251,24 @@
 #
 # @len: the maximum progress value
 #
+# @busy: false if the job is known to be in a quiescent state, with
+#        no pending I/O.  Since 1.3.
+#
+# @paused: whether the job is paused or, if @busy is true, will
+#          pause itself as soon as possible.  Since 1.3.
+#
 # @offset: the current progress value
 #
 # @speed: the rate limit, bytes per second
 #
+# @io-status: the status of the job (since 1.3)
+#
 # Since: 1.1
 ##
 { 'type': 'BlockJobInfo',
   'data': {'type': 'str', 'device': 'str', 'len': 'int',
-           'offset': 'int', 'speed': 'int'} }
+           'offset': 'int', 'busy': 'bool', 'paused': 'bool', 'speed': 'int',
+           'io-status': 'BlockDeviceIoStatus'} }
 
 ##
 # @query-block-jobs:
@@ -1072,7 +1301,9 @@
 # Since:  0.14.0
 #
 # Notes:  This function will succeed even if the guest is already in the stopped
-#         state
+#         state.  In "inmigrate" state, it will ensure that the guest
+#         remains paused once migration finishes, as if the -S option was
+#         passed on the command line.
 ##
 { 'command': 'stop' }
 
@@ -1161,11 +1392,14 @@
 # Since:  0.14.0
 #
 # Returns:  If successful, nothing
-#           If the QEMU is waiting for an incoming migration, MigrationExpected
 #           If QEMU was started with an encrypted block device and a key has
 #              not yet been set, DeviceEncrypted.
 #
-# Notes:  This command will succeed if the guest is currently running.
+# Notes:  This command will succeed if the guest is currently running.  It
+#         will also succeed if the guest is in the "inmigrate" state; in
+#         this case, the effect of the command is to make sure the guest
+#         starts once migration finishes, removing the effect of the -S
+#         command line option if it was passed.
 ##
 { 'command': 'cont' }
 
@@ -1307,7 +1541,7 @@
 # @format: #optional the format of the snapshot image, default is 'qcow2'.
 #
 # @mode: #optional whether and how QEMU should create a new image, default is
-# 'absolute-paths'.
+#        'absolute-paths'.
 ##
 { 'type': 'BlockdevSnapshot',
   'data': { 'device': 'str', 'snapshot-file': 'str', '*format': 'str',
@@ -1361,7 +1595,7 @@
 # @format: #optional the format of the snapshot image, default is 'qcow2'.
 #
 # @mode: #optional whether and how QEMU should create a new image, default is
-# 'absolute-paths'.
+#        'absolute-paths'.
 #
 # Returns: nothing on success
 #          If @device is not a valid block device, DeviceNotFound
@@ -1401,6 +1635,92 @@
   'returns': 'str' }
 
 ##
+# @block-commit
+#
+# Live commit of data from overlay image nodes into backing nodes - i.e.,
+# writes data between 'top' and 'base' into 'base'.
+#
+# @device:  the name of the device
+#
+# @base:   #optional The file name of the backing image to write data into.
+#                    If not specified, this is the deepest backing image
+#
+# @top:              The file name of the backing image within the image chain,
+#                    which contains the topmost data to be committed down.
+#                    Note, the active layer as 'top' is currently unsupported.
+#
+#                    If top == base, that is an error.
+#
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# Returns: Nothing on success
+#          If commit or stream is already active on this device, DeviceInUse
+#          If @device does not exist, DeviceNotFound
+#          If image commit is not supported by this device, NotSupported
+#          If @base or @top is invalid, a generic error is returned
+#          If @top is the active layer, or omitted, a generic error is returned
+#          If @speed is invalid, InvalidParameter
+#
+# Since: 1.3
+#
+##
+{ 'command': 'block-commit',
+  'data': { 'device': 'str', '*base': 'str', 'top': 'str',
+            '*speed': 'int' } }
+
+##
+# @drive-mirror
+#
+# Start mirroring a block device's writes to a new destination.
+#
+# @device:  the name of the device whose writes should be mirrored.
+#
+# @target: the target of the new image. If the file exists, or if it
+#          is a device, the existing file/device will be used as the new
+#          destination.  If it does not exist, a new file will be created.
+#
+# @format: #optional the format of the new destination, default is to
+#          probe if @mode is 'existing', else the format of the source
+#
+# @mode: #optional whether and how QEMU should create a new image, default is
+#        'absolute-paths'.
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# @sync: what parts of the disk image should be copied to the destination
+#        (all the disk, only the sectors allocated in the topmost image, or
+#        only new I/O).
+#
+# @granularity: #optional granularity of the dirty bitmap, default is 64K
+#               if the image format doesn't have clusters, 4K if the clusters
+#               are smaller than that, else the cluster size.  Must be a
+#               power of 2 between 512 and 64M (since 1.4).
+#
+# @buf-size: #optional maximum amount of data in flight from source to
+#            target (since 1.4).
+#
+# @on-source-error: #optional the action to take on an error on the source,
+#                   default 'report'.  'stop' and 'enospc' can only be used
+#                   if the block device supports io-status (see BlockInfo).
+#
+# @on-target-error: #optional the action to take on an error on the target,
+#                   default 'report' (no limitations, since this applies to
+#                   a different block device than @device).
+#
+# Returns: nothing on success
+#          If @device is not a valid block device, DeviceNotFound
+#
+# Since 1.3
+##
+{ 'command': 'drive-mirror',
+  'data': { 'device': 'str', 'target': 'str', '*format': 'str',
+            'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
+            '*speed': 'int', '*granularity': 'uint32',
+            '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
+            '*on-target-error': 'BlockdevOnError' } }
+
+##
 # @migrate_cancel
 #
 # Cancel the current executing migration process.
@@ -1736,13 +2056,18 @@
 #
 # @speed:  #optional the maximum speed, in bytes per second
 #
+# @on-error: #optional the action to take on an error (default report).
+#            'stop' and 'enospc' can only be used if the block device
+#            supports io-status (see BlockInfo).  Since 1.3.
+#
 # Returns: Nothing on success
 #          If @device does not exist, DeviceNotFound
 #
 # Since: 1.1
 ##
-{ 'command': 'block-stream', 'data': { 'device': 'str', '*base': 'str',
-                                       '*speed': 'int' } }
+{ 'command': 'block-stream',
+  'data': { 'device': 'str', '*base': 'str', '*speed': 'int',
+            '*on-error': 'BlockdevOnError' } }
 
 ##
 # @block-job-set-speed:
@@ -1786,12 +2111,84 @@
 #
 # @device: the device name
 #
+# @force: #optional whether to allow cancellation of a paused job (default
+#         false).  Since 1.3.
+#
 # Returns: Nothing on success
 #          If no background operation is active on this device, DeviceNotActive
 #
 # Since: 1.1
 ##
-{ 'command': 'block-job-cancel', 'data': { 'device': 'str' } }
+{ 'command': 'block-job-cancel', 'data': { 'device': 'str', '*force': 'bool' } }
+
+##
+# @block-job-pause:
+#
+# Pause an active background block operation.
+#
+# This command returns immediately after marking the active background block
+# operation for pausing.  It is an error to call this command if no
+# operation is in progress.  Pausing an already paused job has no cumulative
+# effect; a single block-job-resume command will resume the job.
+#
+# The operation will pause as soon as possible.  No event is emitted when
+# the operation is actually paused.  Cancelling a paused job automatically
+# resumes it.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-pause', 'data': { 'device': 'str' } }
+
+##
+# @block-job-resume:
+#
+# Resume an active background block operation.
+#
+# This command returns immediately after resuming a paused background block
+# operation.  It is an error to call this command if no operation is in
+# progress.  Resuming an already running job is not an error.
+#
+# This command also clears the error status of the job.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-resume', 'data': { 'device': 'str' } }
+
+##
+# @block-job-complete:
+#
+# Manually trigger completion of an active background block operation.  This
+# is supported for drive mirroring, where it also switches the device to
+# write to the target path only.  The ability to complete is signaled with
+# a BLOCK_JOB_READY event.
+#
+# This command completes an active background block operation synchronously.
+# The ordering of this command's return with the BLOCK_JOB_COMPLETED event
+# is not defined.  Note that if an I/O error occurs during the processing of
+# this command: 1) the command itself will fail; 2) the error will be processed
+# according to the rerror/werror arguments that were specified when starting
+# the operation.
+#
+# A cancelled or paused job cannot be completed.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-complete', 'data': { 'device': 'str' } }
 
 ##
 # @ObjectTypeInfo:
@@ -1889,6 +2286,19 @@
 { 'command': 'xen-save-devices-state', 'data': {'filename': 'str'} }
 
 ##
+# @xen-set-global-dirty-log
+#
+# Enable or disable the global dirty log mode.
+#
+# @enable: true to enable, false to disable.
+#
+# Returns: nothing
+#
+# Since: 1.3
+##
+{ 'command': 'xen-set-global-dirty-log', 'data': { 'enable': 'bool' } }
+
+##
 # @device_del:
 #
 # Remove a device from a guest
@@ -1915,26 +2325,33 @@
 # supported on i386 and x86_64.
 #
 # @paging: if true, do paging to get guest's memory mapping. This allows
-# using gdb to process the core file. However, setting @paging to false
-# may be desirable because of two reasons:
+#          using gdb to process the core file.
+#
+#          IMPORTANT: this option can make QEMU allocate several gigabytes
+#                     of RAM. This can happen for a large guest, or a
+#                     malicious guest pretending to be large.
+#
+#          Also, paging=true has the following limitations:
 #
-#   1. The guest may be in a catastrophic state or can have corrupted
-#      memory, which cannot be trusted
-#   2. The guest can be in real-mode even if paging is enabled. For example,
-#      the guest uses ACPI to sleep, and ACPI sleep state goes in real-mode
+#             1. The guest may be in a catastrophic state or can have corrupted
+#                memory, which cannot be trusted
+#             2. The guest can be in real-mode even if paging is enabled. For
+#                example, the guest uses ACPI to sleep, and ACPI sleep state
+#                goes in real-mode
 #
 # @protocol: the filename or file descriptor of the vmcore. The supported
-# protocols are:
+#            protocols are:
 #
-#   1. file: the protocol starts with "file:", and the following string is
-#      the file's path.
-#   2. fd: the protocol starts with "fd:", and the following string is the
-#      fd's name.
+#            1. file: the protocol starts with "file:", and the following
+#               string is the file's path.
+#            2. fd: the protocol starts with "fd:", and the following string
+#               is the fd's name.
 #
 # @begin: #optional if specified, the starting physical address.
 #
 # @length: #optional if specified, the memory size, in bytes. If you don't
-# want to dump all guest's memory, please specify the start @begin and @length
+#          want to dump all guest's memory, please specify the start @begin
+#          and @length
 #
 # Returns: nothing on success
 #
@@ -1943,6 +2360,7 @@
 { 'command': 'dump-guest-memory',
   'data': { 'paging': 'bool', 'protocol': 'str', '*begin': 'int',
             '*length': 'int' } }
+
 ##
 # @netdev_add:
 #
@@ -2053,6 +2471,9 @@
 #
 # @dns: #optional guest-visible address of the virtual nameserver
 #
+# @dnssearch: #optional list of DNS suffixes to search, passed as DHCP option
+#             to the guest
+#
 # @smb: #optional root directory of the built-in SMB server
 #
 # @smbserver: #optional IP address of the built-in SMB server
@@ -2075,6 +2496,7 @@
     '*bootfile':  'str',
     '*dhcpstart': 'str',
     '*dns':       'str',
+    '*dnssearch': ['String'],
     '*smb':       'str',
     '*smbserver': 'str',
     '*hostfwd':   ['String'],
@@ -2275,6 +2697,59 @@
     'opts': 'NetClientOptions' } }
 
 ##
+# @InetSocketAddress
+#
+# Captures a socket address or address range in the Internet namespace.
+#
+# @host: host part of the address
+#
+# @port: port part of the address, or lowest port if @to is present
+#
+# @to: highest port to try
+#
+# @ipv4: whether to accept IPv4 addresses, default try both IPv4 and IPv6
+#        #optional
+#
+# @ipv6: whether to accept IPv6 addresses, default try both IPv4 and IPv6
+#        #optional
+#
+# Since 1.3
+##
+{ 'type': 'InetSocketAddress',
+  'data': {
+    'host': 'str',
+    'port': 'str',
+    '*to': 'uint16',
+    '*ipv4': 'bool',
+    '*ipv6': 'bool' } }
+
+##
+# @UnixSocketAddress
+#
+# Captures a socket address in the local ("Unix socket") namespace.
+#
+# @path: filesystem path to use
+#
+# Since 1.3
+##
+{ 'type': 'UnixSocketAddress',
+  'data': {
+    'path': 'str' } }
+
+##
+# @SocketAddress
+#
+# Captures the address of a socket, which could also be a named file descriptor
+#
+# Since 1.3
+##
+{ 'union': 'SocketAddress',
+  'data': {
+    'inet': 'InetSocketAddress',
+    'unix': 'UnixSocketAddress',
+    'fd': 'String' } }
+
+##
 # @getfd:
 #
 # Receive a file descriptor via SCM rights and assign it a name
@@ -2381,7 +2856,7 @@
 #
 # Returns: @AddfdInfo on success
 #          If file descriptor was not received, FdNotSupplied
-#          If @fdset-id does not exist, InvalidParameterValue
+#          If @fdset-id is a negative value, InvalidParameterValue
 #
 # Notes: The list of fd sets is shared by all monitor connections.
 #
@@ -2493,3 +2968,223 @@
 # Since: 1.2.0
 ##
 { 'command': 'query-target', 'returns': 'TargetInfo' }
+
+##
+# @QKeyCode:
+#
+# An enumeration of key name.
+#
+# This is used by the send-key command.
+#
+# Since: 1.3.0
+##
+{ 'enum': 'QKeyCode',
+  'data': [ 'shift', 'shift_r', 'alt', 'alt_r', 'altgr', 'altgr_r', 'ctrl',
+            'ctrl_r', 'menu', 'esc', '1', '2', '3', '4', '5', '6', '7', '8',
+            '9', '0', 'minus', 'equal', 'backspace', 'tab', 'q', 'w', 'e',
+            'r', 't', 'y', 'u', 'i', 'o', 'p', 'bracket_left', 'bracket_right',
+            'ret', 'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'semicolon',
+            'apostrophe', 'grave_accent', 'backslash', 'z', 'x', 'c', 'v', 'b',
+            'n', 'm', 'comma', 'dot', 'slash', 'asterisk', 'spc', 'caps_lock',
+            'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10',
+            'num_lock', 'scroll_lock', 'kp_divide', 'kp_multiply',
+            'kp_subtract', 'kp_add', 'kp_enter', 'kp_decimal', 'sysrq', 'kp_0',
+            'kp_1', 'kp_2', 'kp_3', 'kp_4', 'kp_5', 'kp_6', 'kp_7', 'kp_8',
+            'kp_9', 'less', 'f11', 'f12', 'print', 'home', 'pgup', 'pgdn', 'end',
+            'left', 'up', 'down', 'right', 'insert', 'delete', 'stop', 'again',
+            'props', 'undo', 'front', 'copy', 'open', 'paste', 'find', 'cut',
+             'lf', 'help', 'meta_l', 'meta_r', 'compose' ] }
+
+##
+# @KeyValue
+#
+# Represents a keyboard key.
+#
+# Since: 1.3.0
+##
+{ 'union': 'KeyValue',
+  'data': {
+    'number': 'int',
+    'qcode': 'QKeyCode' } }
+
+##
+# @send-key:
+#
+# Send keys to guest.
+#
+# @keys: An array of @KeyValue elements. All @KeyValues in this array are
+#        simultaneously sent to the guest. A @KeyValue.number value is sent
+#        directly to the guest, while @KeyValue.qcode must be a valid
+#        @QKeyCode value
+#
+# @hold-time: #optional time to delay key up events, milliseconds. Defaults
+#             to 100
+#
+# Returns: Nothing on success
+#          If key is unknown or redundant, InvalidParameter
+#
+# Since: 1.3.0
+#
+##
+{ 'command': 'send-key',
+  'data': { 'keys': ['KeyValue'], '*hold-time': 'int' } }
+
+##
+# @screendump:
+#
+# Write a PPM of the VGA screen to a file.
+#
+# @filename: the path of a new PPM file to store the image
+#
+# Returns: Nothing on success
+#
+# Since: 0.14.0
+##
+{ 'command': 'screendump', 'data': {'filename': 'str'} }
+
+##
+# @nbd-server-start:
+#
+# Start an NBD server listening on the given host and port.  Block
+# devices can then be exported using @nbd-server-add.  The NBD
+# server will present them as named exports; for example, another
+# QEMU instance could refer to them as "nbd:HOST:PORT:exportname=NAME".
+#
+# @addr: Address on which to listen.
+#
+# Returns: error if the server is already running.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-start',
+  'data': { 'addr': 'SocketAddress' } }
+
+##
+# @nbd-server-add:
+#
+# Export a device to QEMU's embedded NBD server.
+#
+# @device: Block device to be exported
+#
+# @writable: Whether clients should be able to write to the device via the
+#     NBD connection (default false). #optional
+#
+# Returns: error if the device is already marked for export.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-add', 'data': {'device': 'str', '*writable': 'bool'} }
+
+##
+# @nbd-server-stop:
+#
+# Stop QEMU's embedded NBD server, and unregister all devices previously
+# added via @nbd-server-add.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-stop' }
+
+##
+# @ChardevFile:
+#
+# Configuration info for file chardevs.
+#
+# @in:  #optional The name of the input file
+# @out: The name of the output file
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevFile', 'data': { '*in' : 'str',
+                                   'out' : 'str' } }
+
+##
+# @ChardevPort:
+#
+# Configuration info for device chardevs.
+#
+# @device: The name of the special file for the device,
+#          i.e. /dev/ttyS0 on Unix or COM1: on Windows
+# @type: What kind of device this is.
+#
+# Since: 1.4
+##
+{ 'enum': 'ChardevPortKind', 'data': [ 'serial',
+                                       'parallel' ] }
+
+{ 'type': 'ChardevPort', 'data': { 'device' : 'str',
+                                   'type'   : 'ChardevPortKind'} }
+
+##
+# @ChardevSocket:
+#
+# Configuration info for socket chardevs.
+#
+# @addr: socket address to listen on (server=true)
+#        or connect to (server=false)
+# @server: #optional create server socket (default: true)
+# @wait: #optional wait for connect (not used for server
+#        sockets, default: false)
+# @nodelay: #optional set TCP_NODELAY socket option (default: false)
+# @telnet: #optional enable telnet protocol (default: false)
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevSocket', 'data': { 'addr'     : 'SocketAddress',
+                                     '*server'  : 'bool',
+                                     '*wait'    : 'bool',
+                                     '*nodelay' : 'bool',
+                                     '*telnet'  : 'bool' } }
+
+##
+# @ChardevBackend:
+#
+# Configuration info for the new chardev backend.
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevDummy', 'data': { } }
+
+{ 'union': 'ChardevBackend', 'data': { 'file'   : 'ChardevFile',
+                                       'port'   : 'ChardevPort',
+                                       'socket' : 'ChardevSocket',
+                                       'pty'    : 'ChardevDummy',
+                                       'null'   : 'ChardevDummy' } }
+
+##
+# @ChardevReturn:
+#
+# Return info about the chardev backend just created.
+#
+# Since: 1.4
+##
+{ 'type' : 'ChardevReturn', 'data': { '*pty' : 'str' } }
+
+##
+# @chardev-add:
+#
+# Add a file chardev
+#
+# @id: the chardev's ID, must be unique
+# @backend: backend type and parameters
+#
+# Returns: chardev info.
+#
+# Since: 1.4
+##
+{ 'command': 'chardev-add', 'data': {'id'      : 'str',
+                                     'backend' : 'ChardevBackend' },
+  'returns': 'ChardevReturn' }
+
+##
+# @chardev-remove:
+#
+# Remove a chardev
+#
+# @id: the chardev's ID, must exist and not be in use
+#
+# Returns: Nothing on success
+#
+# Since: 1.4
+##
+{ 'command': 'chardev-remove', 'data': {'id': 'str'} }
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index 5f5846e767e..1f9c97342c1 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,3 +1,5 @@
-qapi-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
-qapi-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
-qapi-obj-y += string-input-visitor.o string-output-visitor.o opts-visitor.o
+util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
+util-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
+util-obj-y += string-input-visitor.o string-output-visitor.o
+
+util-obj-y += opts-visitor.o
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
index e048b6c8634..174bd8bdb08 100644
--- a/qapi/opts-visitor.c
+++ b/qapi/opts-visitor.c
@@ -10,10 +10,12 @@
  *
  */
 
-#include "opts-visitor.h"
-#include "qemu-queue.h"
-#include "qemu-option-internal.h"
-#include "qapi-visit-impl.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/opts-visitor.h"
+#include "qemu/queue.h"
+#include "qemu/option_int.h"
+#include "qapi/visitor-impl.h"
 
 
 struct OptsVisitor
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index a154523731b..1334de33cc6 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -11,10 +11,11 @@
  *
  */
 
-#include "qapi-dealloc-visitor.h"
-#include "qemu-queue.h"
+#include "qapi/dealloc-visitor.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
+#include "qapi/visitor-impl.h"
 
 typedef struct StackEntry
 {
@@ -132,6 +133,11 @@ static void qapi_dealloc_type_number(Visitor *v, double *obj, const char *name,
 {
 }
 
+static void qapi_dealloc_type_size(Visitor *v, uint64_t *obj, const char *name,
+                                   Error **errp)
+{
+}
+
 static void qapi_dealloc_type_enum(Visitor *v, int *obj, const char *strings[],
                                    const char *kind, const char *name,
                                    Error **errp)
@@ -164,6 +170,7 @@ QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
     v->visitor.type_bool = qapi_dealloc_type_bool;
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
+    v->visitor.type_size = qapi_dealloc_type_size;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qapi/qapi-types-core.h b/qapi/qapi-types-core.h
deleted file mode 100644
index f781fc3ab79..00000000000
--- a/qapi/qapi-types-core.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Core Definitions for QAPI-generated Types
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QAPI_TYPES_CORE_H
-#define QAPI_TYPES_CORE_H
-
-#include "qemu-common.h"
-#include "error.h"
-#include "qerror.h"
-
-#endif
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index 7a82b637664..401ee6e597b 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -11,8 +11,10 @@
  *
  */
 
-#include "qapi/qapi-visit-core.h"
-#include "qapi/qapi-visit-impl.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/visitor.h"
+#include "qapi/visitor-impl.h"
 
 void visit_start_handle(Visitor *v, void **obj, const char *kind,
                         const char *name, Error **errp)
diff --git a/qapi/qapi-visit-impl.h b/qapi/qapi-visit-impl.h
deleted file mode 100644
index 0f3a1898fe1..00000000000
--- a/qapi/qapi-visit-impl.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Core Definitions for QAPI Visitor implementations
- *
- * Copyright (C) 2012 Red Hat, Inc.
- *
- * Author: Paolo Bonizni <pbonzini@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-#ifndef QAPI_VISITOR_IMPL_H
-#define QAPI_VISITOR_IMPL_H
-
-#include "qapi/qapi-types-core.h"
-#include "qapi/qapi-visit-core.h"
-
-void input_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name, Error **errp);
-void output_type_enum(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
-
-#endif
diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c
index 40859946862..921de33bce6 100644
--- a/qapi/qmp-dispatch.c
+++ b/qapi/qmp-dispatch.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qemu-objects.h"
-#include "qapi/qmp-core.h"
-#include "json-parser.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/dispatch.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi-types.h"
-#include "error.h"
-#include "qerror.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 
 static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp)
 {
diff --git a/qapi/qmp-input-visitor.c b/qapi/qmp-input-visitor.c
index 107d8d361b8..67fb127050a 100644
--- a/qapi/qmp-input-visitor.c
+++ b/qapi/qmp-input-visitor.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qmp-input-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qemu-queue.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
-#include "qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/qerror.h"
 
 #define QIV_STACK_SIZE 1024
 
diff --git a/qapi/qmp-output-visitor.c b/qapi/qmp-output-visitor.c
index 2bce9d5db1d..74a5684ed3d 100644
--- a/qapi/qmp-output-visitor.c
+++ b/qapi/qmp-output-visitor.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qmp-output-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qemu-queue.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
-#include "qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/qerror.h"
 
 typedef struct QStackEntry
 {
diff --git a/qapi/qmp-registry.c b/qapi/qmp-registry.c
index 54146133772..70cdbca470e 100644
--- a/qapi/qmp-registry.c
+++ b/qapi/qmp-registry.c
@@ -12,7 +12,9 @@
  *
  */
 
-#include "qapi/qmp-core.h"
+#include <glib.h>
+#include <string.h>
+#include "qapi/qmp/dispatch.h"
 
 static QTAILQ_HEAD(QmpCommandList, QmpCommand) qmp_commands =
     QTAILQ_HEAD_INITIALIZER(qmp_commands);
diff --git a/qapi/string-input-visitor.c b/qapi/string-input-visitor.c
index 497eb9a60a5..8f1bc413330 100644
--- a/qapi/string-input-visitor.c
+++ b/qapi/string-input-visitor.c
@@ -11,9 +11,9 @@
  */
 
 #include "qemu-common.h"
-#include "string-input-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qerror.h"
+#include "qapi/string-input-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qapi/qmp/qerror.h"
 
 struct StringInputVisitor
 {
diff --git a/qapi/string-output-visitor.c b/qapi/string-output-visitor.c
index 34e525eadd0..921653d4256 100644
--- a/qapi/string-output-visitor.c
+++ b/qapi/string-output-visitor.c
@@ -11,9 +11,9 @@
  */
 
 #include "qemu-common.h"
-#include "string-output-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qerror.h"
+#include "qapi/string-output-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qapi/qmp/qerror.h"
 
 struct StringOutputVisitor
 {
diff --git a/qemu-aio.h b/qemu-aio.h
deleted file mode 100644
index bfdd35f02cf..00000000000
--- a/qemu-aio.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * QEMU aio implementation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_AIO_H
-#define QEMU_AIO_H
-
-#include "qemu-common.h"
-#include "qemu-char.h"
-
-typedef struct BlockDriverAIOCB BlockDriverAIOCB;
-typedef void BlockDriverCompletionFunc(void *opaque, int ret);
-
-typedef struct AIOPool {
-    void (*cancel)(BlockDriverAIOCB *acb);
-    int aiocb_size;
-    BlockDriverAIOCB *free_aiocb;
-} AIOPool;
-
-struct BlockDriverAIOCB {
-    AIOPool *pool;
-    BlockDriverState *bs;
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-    BlockDriverAIOCB *next;
-};
-
-void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
-                   BlockDriverCompletionFunc *cb, void *opaque);
-void qemu_aio_release(void *p);
-
-/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
-typedef int (AioFlushHandler)(void *opaque);
-
-/* Flush any pending AIO operation. This function will block until all
- * outstanding AIO operations have been completed or cancelled. */
-void qemu_aio_flush(void);
-
-/* Wait for a single AIO completion to occur.  This function will wait
- * until a single AIO event has completed and it will ensure something
- * has moved before returning. This can issue new pending aio as
- * result of executing I/O completion or bh callbacks.
- *
- * Return whether there is still any pending AIO operation.  */
-bool qemu_aio_wait(void);
-
-/* Register a file descriptor and associated callbacks.  Behaves very similarly
- * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
- * be invoked when using either qemu_aio_wait() or qemu_aio_flush().
- *
- * Code that invokes AIO completion functions should rely on this function
- * instead of qemu_set_fd_handler[2].
- */
-int qemu_aio_set_fd_handler(int fd,
-                            IOHandler *io_read,
-                            IOHandler *io_write,
-                            AioFlushHandler *io_flush,
-                            void *opaque);
-
-#endif
diff --git a/qemu-bridge-helper.c b/qemu-bridge-helper.c
index 652eec99fd5..287bfd5e38c 100644
--- a/qemu-bridge-helper.c
+++ b/qemu-bridge-helper.c
@@ -39,7 +39,7 @@
 #include <linux/if_bridge.h>
 #endif
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 #include "net/tap-linux.h"
 
diff --git a/qemu-char.c b/qemu-char.c
index 398baf1e04f..ac5d62dd9a3 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -22,12 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "hw/usb.h"
 #include "hw/baum.h"
 #include "hw/msmouse.h"
@@ -95,10 +94,11 @@
 #endif
 #endif
 
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "ui/qemu-spice.h"
 
 #define READ_BUF_LEN 4096
+#define CBUFF_SIZE 65536
 
 /***********************************************************/
 /* character device */
@@ -123,19 +123,20 @@ void qemu_chr_be_event(CharDriverState *s, int event)
     s->chr_event(s->handler_opaque, event);
 }
 
-static void qemu_chr_generic_open_bh(void *opaque)
+static void qemu_chr_fire_open_event(void *opaque)
 {
     CharDriverState *s = opaque;
     qemu_chr_be_event(s, CHR_EVENT_OPENED);
-    qemu_bh_delete(s->bh);
-    s->bh = NULL;
+    qemu_free_timer(s->open_timer);
+    s->open_timer = NULL;
 }
 
 void qemu_chr_generic_open(CharDriverState *s)
 {
-    if (s->bh == NULL) {
-	s->bh = qemu_bh_new(qemu_chr_generic_open_bh, s);
-	qemu_bh_schedule(s->bh);
+    if (s->open_timer == NULL) {
+        s->open_timer = qemu_new_timer_ms(rt_clock,
+                                          qemu_chr_fire_open_event, s);
+        qemu_mod_timer(s->open_timer, qemu_get_clock_ms(rt_clock) - 1);
     }
 }
 
@@ -772,6 +773,10 @@ static CharDriverState *qemu_chr_open_stdio(QemuOpts *opts)
     if (stdio_nb_clients >= STDIO_MAX_CLIENTS) {
         return NULL;
     }
+    if (is_daemonized()) {
+        error_report("cannot use stdio with -daemonize");
+        return NULL;
+    }
     if (stdio_nb_clients == 0) {
         old_fd0_flags = fcntl(0, F_GETFL);
         tcgetattr (0, &oldtty);
@@ -852,6 +857,8 @@ static void cfmakeraw (struct termios *termios_p)
     || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
     || defined(__GLIBC__)
 
+#define HAVE_CHARDEV_TTY 1
+
 typedef struct {
     int fd;
     int connected;
@@ -980,6 +987,7 @@ static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
     CharDriverState *chr;
     PtyCharDriver *s;
     struct termios tty;
+    const char *label;
     int master_fd, slave_fd, len;
 #if defined(__OpenBSD__) || defined(__DragonFly__)
     char pty_name[PATH_MAX];
@@ -1005,7 +1013,13 @@ static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
     chr->filename = g_malloc(len);
     snprintf(chr->filename, len, "pty:%s", q_ptsname(master_fd));
     qemu_opt_set(opts, "path", q_ptsname(master_fd));
-    fprintf(stderr, "char device redirected to %s\n", q_ptsname(master_fd));
+
+    label = qemu_opts_id(opts);
+    fprintf(stderr, "char device redirected to %s%s%s%s\n",
+            q_ptsname(master_fd),
+            label ? " (label " : "",
+            label ? label      : "",
+            label ? ")"        : "");
 
     s = g_malloc0(sizeof(PtyCharDriver));
     chr->opaque = s;
@@ -1217,30 +1231,34 @@ static void qemu_chr_close_tty(CharDriverState *chr)
     }
 }
 
-static CharDriverState *qemu_chr_open_tty(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_tty_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
-    int fd;
 
-    TFR(fd = qemu_open(filename, O_RDWR | O_NONBLOCK));
-    if (fd < 0) {
-        return NULL;
-    }
     tty_serial_init(fd, 115200, 'N', 8, 1);
     chr = qemu_chr_open_fd(fd, fd);
     chr->chr_ioctl = tty_serial_ioctl;
     chr->chr_close = qemu_chr_close_tty;
     return chr;
 }
-#else  /* ! __linux__ && ! __sun__ */
-static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
+
+static CharDriverState *qemu_chr_open_tty(QemuOpts *opts)
 {
-    return NULL;
+    const char *filename = qemu_opt_get(opts, "path");
+    int fd;
+
+    TFR(fd = qemu_open(filename, O_RDWR | O_NONBLOCK));
+    if (fd < 0) {
+        return NULL;
+    }
+    return qemu_chr_open_tty_fd(fd);
 }
 #endif /* __linux__ || __sun__ */
 
 #if defined(__linux__)
+
+#define HAVE_CHARDEV_PARPORT 1
+
 typedef struct {
     int fd;
     int mode;
@@ -1350,17 +1368,10 @@ static void pp_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_pp_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
     ParallelCharDriver *drv;
-    int fd;
-
-    TFR(fd = qemu_open(filename, O_RDWR));
-    if (fd < 0) {
-        return NULL;
-    }
 
     if (ioctl(fd, PPCLAIM) < 0) {
         close(fd);
@@ -1384,6 +1395,9 @@ static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
 #endif /* __linux__ */
 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
+
+#define HAVE_CHARDEV_PARPORT 1
+
 static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
 {
     int fd = (int)(intptr_t)chr->opaque;
@@ -1421,16 +1435,9 @@ static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_pp_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
-    int fd;
-
-    fd = qemu_open(filename, O_RDWR);
-    if (fd < 0) {
-        return NULL;
-    }
 
     chr = g_malloc0(sizeof(CharDriverState));
     chr->opaque = (void *)(intptr_t)fd;
@@ -1652,9 +1659,8 @@ static int win_chr_poll(void *opaque)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_win_path(const char *filename)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
     WinCharState *s;
 
@@ -1673,6 +1679,11 @@ static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
     return chr;
 }
 
+static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
+{
+    return qemu_chr_open_win_path(qemu_opt_get(opts, "path"));
+}
+
 static int win_chr_pipe_poll(void *opaque)
 {
     CharDriverState *chr = opaque;
@@ -2097,14 +2108,14 @@ static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
 {
     CharDriverState *chr = NULL;
     NetCharDriver *s = NULL;
+    Error *local_err = NULL;
     int fd = -1;
 
     chr = g_malloc0(sizeof(CharDriverState));
     s = g_malloc0(sizeof(NetCharDriver));
 
-    fd = inet_dgram_opts(opts);
+    fd = inet_dgram_opts(opts, &local_err);
     if (fd < 0) {
-        fprintf(stderr, "inet_dgram_opts failed\n");
         goto return_err;
     }
 
@@ -2118,6 +2129,10 @@ static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
     return chr;
 
 return_err:
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
     g_free(chr);
     g_free(s);
     if (fd >= 0) {
@@ -2141,17 +2156,14 @@ typedef struct {
 
 static void tcp_chr_accept(void *opaque);
 
-static void tcp_chr_connect(void *opaque);
-
 static int tcp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     TCPCharDriver *s = chr->opaque;
     if (s->connected) {
         return send_all(s->fd, buf, len);
     } else {
-        /* (Re-)connect for unconnected writing */
-        tcp_chr_connect(chr);
-        return 0;
+        /* XXX: indicate an error ? */
+        return len;
     }
 }
 
@@ -2332,8 +2344,10 @@ static void tcp_chr_connect(void *opaque)
     TCPCharDriver *s = chr->opaque;
 
     s->connected = 1;
-    qemu_set_fd_handler2(s->fd, tcp_chr_read_poll,
-                         tcp_chr_read, NULL, chr);
+    if (s->fd >= 0) {
+        qemu_set_fd_handler2(s->fd, tcp_chr_read_poll,
+                             tcp_chr_read, NULL, chr);
+    }
     qemu_chr_generic_open(chr);
 }
 
@@ -2425,54 +2439,56 @@ static void tcp_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_socket_fd(int fd, bool do_nodelay,
+                                                bool is_listen, bool is_telnet,
+                                                bool is_waitconnect,
+                                                Error **errp)
 {
     CharDriverState *chr = NULL;
     TCPCharDriver *s = NULL;
-    int fd = -1;
-    int is_listen;
-    int is_waitconnect;
-    int do_nodelay;
-    int is_unix;
-    int is_telnet;
-
-    is_listen      = qemu_opt_get_bool(opts, "server", 0);
-    is_waitconnect = qemu_opt_get_bool(opts, "wait", 1);
-    is_telnet      = qemu_opt_get_bool(opts, "telnet", 0);
-    do_nodelay     = !qemu_opt_get_bool(opts, "delay", 1);
-    is_unix        = qemu_opt_get(opts, "path") != NULL;
-    if (!is_listen)
-        is_waitconnect = 0;
+    char host[NI_MAXHOST], serv[NI_MAXSERV];
+    const char *left = "", *right = "";
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    memset(&ss, 0, ss_len);
+    if (getsockname(fd, (struct sockaddr *) &ss, &ss_len) != 0) {
+        error_setg(errp, "getsockname: %s", strerror(errno));
+        return NULL;
+    }
 
     chr = g_malloc0(sizeof(CharDriverState));
     s = g_malloc0(sizeof(TCPCharDriver));
 
-    if (is_unix) {
-        if (is_listen) {
-            fd = unix_listen_opts(opts);
-        } else {
-            fd = unix_connect_opts(opts);
-        }
-    } else {
-        if (is_listen) {
-            fd = inet_listen_opts(opts, 0, NULL);
-        } else {
-            fd = inet_connect_opts(opts, NULL, NULL);
-        }
-    }
-    if (fd < 0) {
-        goto fail;
-    }
-
-    if (!is_waitconnect)
-        socket_set_nonblock(fd);
-
     s->connected = 0;
     s->fd = -1;
     s->listen_fd = -1;
     s->msgfd = -1;
-    s->is_unix = is_unix;
-    s->do_nodelay = do_nodelay && !is_unix;
+
+    chr->filename = g_malloc(256);
+    switch (ss.ss_family) {
+#ifndef _WIN32
+    case AF_UNIX:
+        s->is_unix = 1;
+        snprintf(chr->filename, 256, "unix:%s%s",
+                 ((struct sockaddr_un *)(&ss))->sun_path,
+                 is_listen ? ",server" : "");
+        break;
+#endif
+    case AF_INET6:
+        left  = "[";
+        right = "]";
+        /* fall through */
+    case AF_INET:
+        s->do_nodelay = do_nodelay;
+        getnameinfo((struct sockaddr *) &ss, ss_len, host, sizeof(host),
+                    serv, sizeof(serv), NI_NUMERICHOST | NI_NUMERICSERV);
+        snprintf(chr->filename, 256, "%s:%s:%s%s%s%s",
+                 is_telnet ? "telnet" : "tcp",
+                 left, host, right, serv,
+                 is_listen ? ",server" : "");
+        break;
+    }
 
     chr->opaque = s;
     chr->chr_write = tcp_chr_write;
@@ -2483,9 +2499,9 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
     if (is_listen) {
         s->listen_fd = fd;
         qemu_set_fd_handler2(s->listen_fd, NULL, tcp_chr_accept, NULL, chr);
-        if (is_telnet)
+        if (is_telnet) {
             s->do_telnetopt = 1;
-
+        }
     } else {
         s->connected = 1;
         s->fd = fd;
@@ -2493,22 +2509,6 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
         tcp_chr_connect(chr);
     }
 
-    /* for "info chardev" monitor command */
-    chr->filename = g_malloc(256);
-    if (is_unix) {
-        snprintf(chr->filename, 256, "unix:%s%s",
-                 qemu_opt_get(opts, "path"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    } else if (is_telnet) {
-        snprintf(chr->filename, 256, "telnet:%s:%s%s",
-                 qemu_opt_get(opts, "host"), qemu_opt_get(opts, "port"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    } else {
-        snprintf(chr->filename, 256, "tcp:%s:%s%s",
-                 qemu_opt_get(opts, "host"), qemu_opt_get(opts, "port"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    }
-
     if (is_listen && is_waitconnect) {
         printf("QEMU waiting for connection on: %s\n",
                chr->filename);
@@ -2516,12 +2516,67 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
         socket_set_nonblock(s->listen_fd);
     }
     return chr;
+}
+
+static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
+{
+    CharDriverState *chr = NULL;
+    Error *local_err = NULL;
+    int fd = -1;
+    int is_listen;
+    int is_waitconnect;
+    int do_nodelay;
+    int is_unix;
+    int is_telnet;
+
+    is_listen      = qemu_opt_get_bool(opts, "server", 0);
+    is_waitconnect = qemu_opt_get_bool(opts, "wait", 1);
+    is_telnet      = qemu_opt_get_bool(opts, "telnet", 0);
+    do_nodelay     = !qemu_opt_get_bool(opts, "delay", 1);
+    is_unix        = qemu_opt_get(opts, "path") != NULL;
+    if (!is_listen)
+        is_waitconnect = 0;
+
+    if (is_unix) {
+        if (is_listen) {
+            fd = unix_listen_opts(opts, &local_err);
+        } else {
+            fd = unix_connect_opts(opts, &local_err, NULL, NULL);
+        }
+    } else {
+        if (is_listen) {
+            fd = inet_listen_opts(opts, 0, &local_err);
+        } else {
+            fd = inet_connect_opts(opts, &local_err, NULL, NULL);
+        }
+    }
+    if (fd < 0) {
+        goto fail;
+    }
+
+    if (!is_waitconnect)
+        socket_set_nonblock(fd);
+
+    chr = qemu_chr_open_socket_fd(fd, do_nodelay, is_listen, is_telnet,
+                                  is_waitconnect, &local_err);
+    if (error_is_set(&local_err)) {
+        goto fail;
+    }
+    return chr;
+
 
  fail:
-    if (fd >= 0)
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    if (fd >= 0) {
         closesocket(fd);
-    g_free(s);
-    g_free(chr);
+    }
+    if (chr) {
+        g_free(chr->opaque);
+        g_free(chr);
+    }
     return NULL;
 }
 
@@ -2589,6 +2644,199 @@ size_t qemu_chr_mem_osize(const CharDriverState *chr)
     return d->outbuf_size;
 }
 
+/*********************************************************/
+/*CircularMemory chardev*/
+
+typedef struct {
+    size_t size;
+    size_t prod;
+    size_t cons;
+    uint8_t *cbuf;
+} CirMemCharDriver;
+
+static bool cirmem_chr_is_empty(const CharDriverState *chr)
+{
+    const CirMemCharDriver *d = chr->opaque;
+
+    return d->cons == d->prod;
+}
+
+static size_t qemu_chr_cirmem_count(const CharDriverState *chr)
+{
+    const CirMemCharDriver *d = chr->opaque;
+
+    return (d->prod - d->cons);
+}
+
+static int cirmem_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
+{
+    CirMemCharDriver *d = chr->opaque;
+    int i;
+
+    if (!buf || (len < 0)) {
+        return -1;
+    }
+
+    for (i = 0; i < len; i++ ) {
+        /* Avoid writing the IAC information to the queue. */
+        if ((unsigned char)buf[i] == IAC) {
+            continue;
+        }
+
+        d->cbuf[d->prod++ % d->size] = buf[i];
+        if ((d->prod - d->cons) > d->size) {
+            d->cons = d->prod - d->size;
+        }
+    }
+
+    return 0;
+}
+
+static int cirmem_chr_read(CharDriverState *chr, uint8_t *buf, int len)
+{
+    CirMemCharDriver *d = chr->opaque;
+    int i;
+
+    for (i = 0; i < len && !cirmem_chr_is_empty(chr); i++) {
+        buf[i] = d->cbuf[d->cons++ % d->size];
+    }
+
+    return i;
+}
+
+static void cirmem_chr_close(struct CharDriverState *chr)
+{
+    CirMemCharDriver *d = chr->opaque;
+
+    g_free(d->cbuf);
+    g_free(d);
+    chr->opaque = NULL;
+}
+
+static CharDriverState *qemu_chr_open_cirmemchr(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    CirMemCharDriver *d;
+
+    chr = g_malloc0(sizeof(CharDriverState));
+    d = g_malloc(sizeof(*d));
+
+    d->size = qemu_opt_get_number(opts, "maxcapacity", 0);
+    if (d->size == 0) {
+        d->size = CBUFF_SIZE;
+    }
+
+    /* The size must be power of 2 */
+    if (d->size & (d->size - 1)) {
+        fprintf(stderr, "chardev: size of memory device must be power of 2\n");
+        goto fail;
+    }
+
+    d->prod = 0;
+    d->cons = 0;
+    d->cbuf = g_malloc0(d->size);
+
+    chr->opaque = d;
+    chr->chr_write = cirmem_chr_write;
+    chr->chr_close = cirmem_chr_close;
+
+    return chr;
+
+fail:
+    g_free(d);
+    g_free(chr);
+    return NULL;
+}
+
+static bool qemu_is_chr(const CharDriverState *chr, const char *filename)
+{
+    return strcmp(chr->filename, filename);
+}
+
+void qmp_memchar_write(const char *device, int64_t size,
+                       const char *data, bool has_format,
+                       enum DataFormat format,
+                       Error **errp)
+{
+    CharDriverState *chr;
+    guchar *write_data;
+    int ret;
+    gsize write_count;
+
+    chr = qemu_chr_find(device);
+    if (!chr) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (qemu_is_chr(chr, "memory")) {
+        error_setg(errp,"%s is not memory char device", device);
+        return;
+    }
+
+    write_count = (gsize)size;
+
+    if (has_format && (format == DATA_FORMAT_BASE64)) {
+        write_data = g_base64_decode(data, &write_count);
+    } else {
+        write_data = (uint8_t *)data;
+    }
+
+    ret = cirmem_chr_write(chr, write_data, write_count);
+
+    if (ret < 0) {
+        error_setg(errp, "Failed to write to device %s", device);
+        return;
+    }
+}
+
+MemCharRead *qmp_memchar_read(const char *device, int64_t size,
+                              bool has_format, enum DataFormat format,
+                              Error **errp)
+{
+    CharDriverState *chr;
+    guchar *read_data;
+    MemCharRead *meminfo;
+    size_t count;
+
+    chr = qemu_chr_find(device);
+    if (!chr) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return NULL;
+    }
+
+    if (qemu_is_chr(chr, "memory")) {
+        error_setg(errp,"%s is not memory char device", device);
+        return NULL;
+    }
+
+    if (size <= 0) {
+        error_setg(errp, "size must be greater than zero");
+        return NULL;
+    }
+
+    meminfo = g_malloc0(sizeof(MemCharRead));
+
+    count = qemu_chr_cirmem_count(chr);
+    if (count == 0) {
+        meminfo->data = g_strdup("");
+        return meminfo;
+    }
+
+    size = size > count ? count : size;
+    read_data = g_malloc0(size + 1);
+
+    meminfo->count = cirmem_chr_read(chr, read_data, size);
+
+    if (has_format && (format == DATA_FORMAT_BASE64)) {
+        meminfo->data = g_base64_encode(read_data, (size_t)meminfo->count);
+    } else {
+        meminfo->data = (char *)read_data;
+    }
+
+    return meminfo;
+}
+
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename)
 {
     char host[65], port[33], width[8], height[8];
@@ -2643,6 +2891,11 @@ QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename)
         qemu_opt_set(opts, "path", filename);
         return opts;
     }
+    if (strstart(filename, "memory", &p)) {
+        qemu_opt_set(opts, "backend", "memory");
+        qemu_opt_set(opts, "maxcapacity", p);
+        return opts;
+    }
     if (strstart(filename, "file:", &p)) {
         qemu_opt_set(opts, "backend", "file");
         qemu_opt_set(opts, "path", p);
@@ -2717,6 +2970,22 @@ fail:
     return NULL;
 }
 
+#ifdef HAVE_CHARDEV_PARPORT
+
+static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+{
+    const char *filename = qemu_opt_get(opts, "path");
+    int fd;
+
+    fd = qemu_open(filename, O_RDWR);
+    if (fd < 0) {
+        return NULL;
+    }
+    return qemu_chr_open_pp_fd(fd);
+}
+
+#endif
+
 static const struct {
     const char *name;
     CharDriverState *(*open)(QemuOpts *opts);
@@ -2726,6 +2995,7 @@ static const struct {
     { .name = "udp",       .open = qemu_chr_open_udp },
     { .name = "msmouse",   .open = qemu_chr_open_msmouse },
     { .name = "vc",        .open = text_console_init },
+    { .name = "memory",    .open = qemu_chr_open_cirmemchr },
 #ifdef _WIN32
     { .name = "file",      .open = qemu_chr_open_win_file_out },
     { .name = "pipe",      .open = qemu_chr_open_win_pipe },
@@ -2735,57 +3005,60 @@ static const struct {
 #else
     { .name = "file",      .open = qemu_chr_open_file_out },
     { .name = "pipe",      .open = qemu_chr_open_pipe },
-    { .name = "pty",       .open = qemu_chr_open_pty },
     { .name = "stdio",     .open = qemu_chr_open_stdio },
 #endif
 #ifdef CONFIG_BRLAPI
     { .name = "braille",   .open = chr_baum_init },
 #endif
-#if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
-    || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
-    || defined(__FreeBSD_kernel__)
+#ifdef HAVE_CHARDEV_TTY
     { .name = "tty",       .open = qemu_chr_open_tty },
+    { .name = "serial",    .open = qemu_chr_open_tty },
+    { .name = "pty",       .open = qemu_chr_open_pty },
 #endif
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__) \
-    || defined(__FreeBSD_kernel__)
+#ifdef HAVE_CHARDEV_PARPORT
+    { .name = "parallel",  .open = qemu_chr_open_pp },
     { .name = "parport",   .open = qemu_chr_open_pp },
 #endif
 #ifdef CONFIG_SPICE
     { .name = "spicevmc",     .open = qemu_chr_open_spice },
+#if SPICE_SERVER_VERSION >= 0x000c02
+    { .name = "spiceport",    .open = qemu_chr_open_spice_port },
+#endif
 #endif
 };
 
 CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
-                                    void (*init)(struct CharDriverState *s))
+                                    void (*init)(struct CharDriverState *s),
+                                    Error **errp)
 {
     CharDriverState *chr;
     int i;
 
     if (qemu_opts_id(opts) == NULL) {
-        fprintf(stderr, "chardev: no id specified\n");
-        return NULL;
+        error_setg(errp, "chardev: no id specified\n");
+        goto err;
     }
 
     if (qemu_opt_get(opts, "backend") == NULL) {
-        fprintf(stderr, "chardev: \"%s\" missing backend\n",
-                qemu_opts_id(opts));
-        return NULL;
+        error_setg(errp, "chardev: \"%s\" missing backend\n",
+                   qemu_opts_id(opts));
+        goto err;
     }
     for (i = 0; i < ARRAY_SIZE(backend_table); i++) {
         if (strcmp(backend_table[i].name, qemu_opt_get(opts, "backend")) == 0)
             break;
     }
     if (i == ARRAY_SIZE(backend_table)) {
-        fprintf(stderr, "chardev: backend \"%s\" not found\n",
-                qemu_opt_get(opts, "backend"));
-        return NULL;
+        error_setg(errp, "chardev: backend \"%s\" not found\n",
+                   qemu_opt_get(opts, "backend"));
+        goto err;
     }
 
     chr = backend_table[i].open(opts);
     if (!chr) {
-        fprintf(stderr, "chardev: opening backend \"%s\" failed\n",
-                qemu_opt_get(opts, "backend"));
-        return NULL;
+        error_setg(errp, "chardev: opening backend \"%s\" failed\n",
+                   qemu_opt_get(opts, "backend"));
+        goto err;
     }
 
     if (!chr->filename)
@@ -2806,7 +3079,12 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
         chr->avail_connections = 1;
     }
     chr->label = g_strdup(qemu_opts_id(opts));
+    chr->opts = opts;
     return chr;
+
+err:
+    qemu_opts_del(opts);
+    return NULL;
 }
 
 CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
@@ -2814,6 +3092,7 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
     const char *p;
     CharDriverState *chr;
     QemuOpts *opts;
+    Error *err = NULL;
 
     if (strstart(filename, "chardev:", &p)) {
         return qemu_chr_find(p);
@@ -2823,11 +3102,14 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
     if (!opts)
         return NULL;
 
-    chr = qemu_chr_new_from_opts(opts, init);
+    chr = qemu_chr_new_from_opts(opts, init, &err);
+    if (error_is_set(&err)) {
+        fprintf(stderr, "%s\n", error_get_pretty(err));
+        error_free(err);
+    }
     if (chr && qemu_opt_get_bool(opts, "mux", 0)) {
         monitor_init(chr, MONITOR_USE_READLINE);
     }
-    qemu_opts_del(opts);
     return chr;
 }
 
@@ -2855,10 +3137,14 @@ void qemu_chr_fe_close(struct CharDriverState *chr)
 void qemu_chr_delete(CharDriverState *chr)
 {
     QTAILQ_REMOVE(&chardevs, chr, next);
-    if (chr->chr_close)
+    if (chr->chr_close) {
         chr->chr_close(chr);
+    }
     g_free(chr->filename);
     g_free(chr->label);
+    if (chr->opts) {
+        qemu_opts_del(chr->opts);
+    }
     g_free(chr);
 }
 
@@ -2901,3 +3187,278 @@ CharDriverState *qemu_char_get_next_serial(void)
     return serial_hds[next_serial++];
 }
 
+QemuOptsList qemu_chardev_opts = {
+    .name = "chardev",
+    .implied_opt_name = "backend",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_chardev_opts.head),
+    .desc = {
+        {
+            .name = "backend",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "host",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "port",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "localaddr",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "localport",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "to",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "wait",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "server",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "delay",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "telnet",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "width",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "height",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "cols",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "rows",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "mux",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "signal",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "name",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "debug",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "maxcapacity",
+            .type = QEMU_OPT_NUMBER,
+        },
+        { /* end of list */ }
+    },
+};
+
+#ifdef _WIN32
+
+static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+{
+    HANDLE out;
+
+    if (file->in) {
+        error_setg(errp, "input file not supported");
+        return NULL;
+    }
+
+    out = CreateFile(file->out, GENERIC_WRITE, FILE_SHARE_READ, NULL,
+                     OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (out == INVALID_HANDLE_VALUE) {
+        error_setg(errp, "open %s failed", file->out);
+        return NULL;
+    }
+    return qemu_chr_open_win_file(out);
+}
+
+static CharDriverState *qmp_chardev_open_port(ChardevPort *port, Error **errp)
+{
+    switch (port->type) {
+    case CHARDEV_PORT_KIND_SERIAL:
+        return qemu_chr_open_win_path(port->device);
+    default:
+        error_setg(errp, "unknown chardev port (%d)", port->type);
+        return NULL;
+    }
+}
+
+#else /* WIN32 */
+
+static int qmp_chardev_open_file_source(char *src, int flags,
+                                        Error **errp)
+{
+    int fd = -1;
+
+    TFR(fd = qemu_open(src, flags, 0666));
+    if (fd == -1) {
+        error_setg(errp, "open %s: %s", src, strerror(errno));
+    }
+    return fd;
+}
+
+static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+{
+    int flags, in = -1, out = -1;
+
+    flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
+    out = qmp_chardev_open_file_source(file->out, flags, errp);
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+
+    if (file->in) {
+        flags = O_RDONLY;
+        in = qmp_chardev_open_file_source(file->in, flags, errp);
+        if (error_is_set(errp)) {
+            qemu_close(out);
+            return NULL;
+        }
+    }
+
+    return qemu_chr_open_fd(in, out);
+}
+
+static CharDriverState *qmp_chardev_open_port(ChardevPort *port, Error **errp)
+{
+    switch (port->type) {
+#ifdef HAVE_CHARDEV_TTY
+    case CHARDEV_PORT_KIND_SERIAL:
+    {
+        int flags, fd;
+        flags = O_RDWR;
+        fd = qmp_chardev_open_file_source(port->device, flags, errp);
+        if (error_is_set(errp)) {
+            return NULL;
+        }
+        socket_set_nonblock(fd);
+        return qemu_chr_open_tty_fd(fd);
+    }
+#endif
+#ifdef HAVE_CHARDEV_PARPORT
+    case CHARDEV_PORT_KIND_PARALLEL:
+    {
+        int flags, fd;
+        flags = O_RDWR;
+        fd = qmp_chardev_open_file_source(port->device, flags, errp);
+        if (error_is_set(errp)) {
+            return NULL;
+        }
+        return qemu_chr_open_pp_fd(fd);
+    }
+#endif
+    default:
+        error_setg(errp, "unknown chardev port (%d)", port->type);
+        return NULL;
+    }
+}
+
+#endif /* WIN32 */
+
+static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
+                                                Error **errp)
+{
+    SocketAddress *addr = sock->addr;
+    bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
+    bool is_listen      = sock->has_server  ? sock->server  : true;
+    bool is_telnet      = sock->has_telnet  ? sock->telnet  : false;
+    bool is_waitconnect = sock->has_wait    ? sock->wait    : false;
+    int fd;
+
+    if (is_listen) {
+        fd = socket_listen(addr, errp);
+    } else {
+        fd = socket_connect(addr, errp, NULL, NULL);
+    }
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+    return qemu_chr_open_socket_fd(fd, do_nodelay, is_listen,
+                                   is_telnet, is_waitconnect, errp);
+}
+
+ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
+                               Error **errp)
+{
+    ChardevReturn *ret = g_new0(ChardevReturn, 1);
+    CharDriverState *chr = NULL;
+
+    chr = qemu_chr_find(id);
+    if (chr) {
+        error_setg(errp, "Chardev '%s' already exists", id);
+        g_free(ret);
+        return NULL;
+    }
+
+    switch (backend->kind) {
+    case CHARDEV_BACKEND_KIND_FILE:
+        chr = qmp_chardev_open_file(backend->file, errp);
+        break;
+    case CHARDEV_BACKEND_KIND_PORT:
+        chr = qmp_chardev_open_port(backend->port, errp);
+        break;
+    case CHARDEV_BACKEND_KIND_SOCKET:
+        chr = qmp_chardev_open_socket(backend->socket, errp);
+        break;
+#ifdef HAVE_CHARDEV_TTY
+    case CHARDEV_BACKEND_KIND_PTY:
+    {
+        /* qemu_chr_open_pty sets "path" in opts */
+        QemuOpts *opts;
+        opts = qemu_opts_create_nofail(qemu_find_opts("chardev"));
+        chr = qemu_chr_open_pty(opts);
+        ret->pty = g_strdup(qemu_opt_get(opts, "path"));
+        ret->has_pty = true;
+        qemu_opts_del(opts);
+        break;
+    }
+#endif
+    case CHARDEV_BACKEND_KIND_NULL:
+        chr = qemu_chr_open_null(NULL);
+        break;
+    default:
+        error_setg(errp, "unknown chardev backend (%d)", backend->kind);
+        break;
+    }
+
+    if (chr == NULL && !error_is_set(errp)) {
+        error_setg(errp, "Failed to create chardev");
+    }
+    if (chr) {
+        chr->label = g_strdup(id);
+        chr->avail_connections = 1;
+        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
+        return ret;
+    } else {
+        g_free(ret);
+        return NULL;
+    }
+}
+
+void qmp_chardev_remove(const char *id, Error **errp)
+{
+    CharDriverState *chr;
+
+    chr = qemu_chr_find(id);
+    if (NULL == chr) {
+        error_setg(errp, "Chardev '%s' not found", id);
+        return;
+    }
+    if (chr->chr_can_read || chr->chr_read ||
+        chr->chr_event || chr->handler_opaque) {
+        error_setg(errp, "Chardev '%s' is busy", id);
+        return;
+    }
+    qemu_chr_delete(chr);
+}
diff --git a/qemu-config.c b/qemu-config.c
deleted file mode 100644
index c05ffbc444e..00000000000
--- a/qemu-config.c
+++ /dev/null
@@ -1,889 +0,0 @@
-#include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "hw/qdev.h"
-#include "error.h"
-
-static QemuOptsList qemu_drive_opts = {
-    .name = "drive",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_drive_opts.head),
-    .desc = {
-        {
-            .name = "bus",
-            .type = QEMU_OPT_NUMBER,
-            .help = "bus number",
-        },{
-            .name = "unit",
-            .type = QEMU_OPT_NUMBER,
-            .help = "unit number (i.e. lun for scsi)",
-        },{
-            .name = "if",
-            .type = QEMU_OPT_STRING,
-            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
-        },{
-            .name = "index",
-            .type = QEMU_OPT_NUMBER,
-            .help = "index number",
-        },{
-            .name = "cyls",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of cylinders (ide disk geometry)",
-        },{
-            .name = "heads",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of heads (ide disk geometry)",
-        },{
-            .name = "secs",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of sectors (ide disk geometry)",
-        },{
-            .name = "trans",
-            .type = QEMU_OPT_STRING,
-            .help = "chs translation (auto, lba. none)",
-        },{
-            .name = "media",
-            .type = QEMU_OPT_STRING,
-            .help = "media type (disk, cdrom)",
-        },{
-            .name = "snapshot",
-            .type = QEMU_OPT_BOOL,
-            .help = "enable/disable snapshot mode",
-        },{
-            .name = "file",
-            .type = QEMU_OPT_STRING,
-            .help = "disk image",
-        },{
-            .name = "cache",
-            .type = QEMU_OPT_STRING,
-            .help = "host cache usage (none, writeback, writethrough, "
-                    "directsync, unsafe)",
-        },{
-            .name = "aio",
-            .type = QEMU_OPT_STRING,
-            .help = "host AIO implementation (threads, native)",
-        },{
-            .name = "format",
-            .type = QEMU_OPT_STRING,
-            .help = "disk format (raw, qcow2, ...)",
-        },{
-            .name = "serial",
-            .type = QEMU_OPT_STRING,
-            .help = "disk serial number",
-        },{
-            .name = "rerror",
-            .type = QEMU_OPT_STRING,
-            .help = "read error action",
-        },{
-            .name = "werror",
-            .type = QEMU_OPT_STRING,
-            .help = "write error action",
-        },{
-            .name = "addr",
-            .type = QEMU_OPT_STRING,
-            .help = "pci address (virtio only)",
-        },{
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-            .help = "open drive file as read-only",
-        },{
-            .name = "iops",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit total I/O operations per second",
-        },{
-            .name = "iops_rd",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit read operations per second",
-        },{
-            .name = "iops_wr",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit write operations per second",
-        },{
-            .name = "bps",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit total bytes per second",
-        },{
-            .name = "bps_rd",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit read bytes per second",
-        },{
-            .name = "bps_wr",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit write bytes per second",
-        },{
-            .name = "copy-on-read",
-            .type = QEMU_OPT_BOOL,
-            .help = "copy read data from backing file into image file",
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_iscsi_opts = {
-    .name = "iscsi",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_iscsi_opts.head),
-    .desc = {
-        {
-            .name = "user",
-            .type = QEMU_OPT_STRING,
-            .help = "username for CHAP authentication to target",
-        },{
-            .name = "password",
-            .type = QEMU_OPT_STRING,
-            .help = "password for CHAP authentication to target",
-        },{
-            .name = "header-digest",
-            .type = QEMU_OPT_STRING,
-            .help = "HeaderDigest setting. "
-                    "{CRC32C|CRC32C-NONE|NONE-CRC32C|NONE}",
-        },{
-            .name = "initiator-name",
-            .type = QEMU_OPT_STRING,
-            .help = "Initiator iqn name to use when connecting",
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_chardev_opts = {
-    .name = "chardev",
-    .implied_opt_name = "backend",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_chardev_opts.head),
-    .desc = {
-        {
-            .name = "backend",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "host",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "port",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localaddr",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localport",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "to",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "wait",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "server",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "delay",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "telnet",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "width",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "height",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "cols",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "rows",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "mux",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "signal",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "name",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "debug",
-            .type = QEMU_OPT_NUMBER,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_fsdev_opts = {
-    .name = "fsdev",
-    .implied_opt_name = "fsdriver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_fsdev_opts.head),
-    .desc = {
-        {
-            .name = "fsdriver",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "security_model",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "writeout",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-
-        }, {
-            .name = "socket",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "sock_fd",
-            .type = QEMU_OPT_NUMBER,
-        },
-
-        { /*End of list */ }
-    },
-};
-
-QemuOptsList qemu_virtfs_opts = {
-    .name = "virtfs",
-    .implied_opt_name = "fsdriver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_virtfs_opts.head),
-    .desc = {
-        {
-            .name = "fsdriver",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "mount_tag",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "security_model",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "writeout",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-        }, {
-            .name = "socket",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "sock_fd",
-            .type = QEMU_OPT_NUMBER,
-        },
-
-        { /*End of list */ }
-    },
-};
-
-static QemuOptsList qemu_device_opts = {
-    .name = "device",
-    .implied_opt_name = "driver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_device_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any
-         * sanity checking will happen later
-         * when setting device properties
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_netdev_opts = {
-    .name = "netdev",
-    .implied_opt_name = "type",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_netdev_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any params
-         * validation will happen later
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_net_opts = {
-    .name = "net",
-    .implied_opt_name = "type",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_net_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any params
-         * validation will happen later
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_rtc_opts = {
-    .name = "rtc",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_rtc_opts.head),
-    .desc = {
-        {
-            .name = "base",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "clock",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "driftfix",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_global_opts = {
-    .name = "global",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_global_opts.head),
-    .desc = {
-        {
-            .name = "driver",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "property",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "value",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_sandbox_opts = {
-    .name = "sandbox",
-    .implied_opt_name = "enable",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_sandbox_opts.head),
-    .desc = {
-        {
-            .name = "enable",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_mon_opts = {
-    .name = "mon",
-    .implied_opt_name = "chardev",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_mon_opts.head),
-    .desc = {
-        {
-            .name = "mode",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "chardev",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "default",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "pretty",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_trace_opts = {
-    .name = "trace",
-    .implied_opt_name = "trace",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
-    .desc = {
-        {
-            .name = "events",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "file",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_cpudef_opts = {
-    .name = "cpudef",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_cpudef_opts.head),
-    .desc = {
-        {
-            .name = "name",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "level",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "vendor",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "family",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "model",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "stepping",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "feature_edx",      /* cpuid 0000_0001.edx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "feature_ecx",      /* cpuid 0000_0001.ecx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "extfeature_edx",   /* cpuid 8000_0001.edx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "extfeature_ecx",   /* cpuid 8000_0001.ecx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "xlevel",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "model_id",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "vendor_override",
-            .type = QEMU_OPT_NUMBER,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_spice_opts = {
-    .name = "spice",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_spice_opts.head),
-    .desc = {
-        {
-            .name = "port",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "tls-port",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "addr",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "password",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "disable-ticketing",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "disable-copy-paste",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "sasl",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "x509-dir",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-key-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-key-password",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-cert-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-cacert-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-dh-key-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "tls-ciphers",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "tls-channel",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "plaintext-channel",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "image-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "jpeg-wan-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "zlib-glz-wan-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "streaming-video",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "agent-mouse",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "playback-compression",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_option_rom_opts = {
-    .name = "option-rom",
-    .implied_opt_name = "romfile",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_option_rom_opts.head),
-    .desc = {
-        {
-            .name = "bootindex",
-            .type = QEMU_OPT_NUMBER,
-        }, {
-            .name = "romfile",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_machine_opts = {
-    .name = "machine",
-    .implied_opt_name = "type",
-    .merge_lists = true,
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_machine_opts.head),
-    .desc = {
-        {
-            .name = "type",
-            .type = QEMU_OPT_STRING,
-            .help = "emulated machine"
-        }, {
-            .name = "accel",
-            .type = QEMU_OPT_STRING,
-            .help = "accelerator list",
-        }, {
-            .name = "kernel_irqchip",
-            .type = QEMU_OPT_BOOL,
-            .help = "use KVM in-kernel irqchip",
-        }, {
-            .name = "kvm_shadow_mem",
-            .type = QEMU_OPT_SIZE,
-            .help = "KVM shadow MMU size",
-        }, {
-            .name = "kernel",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel image file",
-        }, {
-            .name = "initrd",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux initial ramdisk file",
-        }, {
-            .name = "append",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel command line",
-        }, {
-            .name = "dtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel device tree file",
-        }, {
-            .name = "dumpdtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Dump current dtb to a file and quit",
-        }, {
-            .name = "phandle_start",
-            .type = QEMU_OPT_STRING,
-            .help = "The first phandle ID we may generate dynamically",
-        }, {
-            .name = "dt_compatible",
-            .type = QEMU_OPT_STRING,
-            .help = "Overrides the \"compatible\" property of the dt root node",
-        }, {
-            .name = "dump-guest-core",
-            .type = QEMU_OPT_BOOL,
-            .help = "Include guest memory in  a core dump",
-        },
-        { /* End of list */ }
-    },
-};
-
-QemuOptsList qemu_boot_opts = {
-    .name = "boot-opts",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_boot_opts.head),
-    .desc = {
-        /* the three names below are not used now */
-        {
-            .name = "order",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "once",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "menu",
-            .type = QEMU_OPT_STRING,
-        /* following are really used */
-        }, {
-            .name = "splash",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "splash-time",
-            .type = QEMU_OPT_STRING,
-        },
-        { /*End of list */ }
-    },
-};
-
-static QemuOptsList *vm_config_groups[32] = {
-    &qemu_drive_opts,
-    &qemu_chardev_opts,
-    &qemu_device_opts,
-    &qemu_netdev_opts,
-    &qemu_net_opts,
-    &qemu_rtc_opts,
-    &qemu_global_opts,
-    &qemu_mon_opts,
-    &qemu_cpudef_opts,
-    &qemu_trace_opts,
-    &qemu_option_rom_opts,
-    &qemu_machine_opts,
-    &qemu_boot_opts,
-    &qemu_iscsi_opts,
-    &qemu_sandbox_opts,
-    NULL,
-};
-
-static QemuOptsList *find_list(QemuOptsList **lists, const char *group,
-                               Error **errp)
-{
-    int i;
-
-    for (i = 0; lists[i] != NULL; i++) {
-        if (strcmp(lists[i]->name, group) == 0)
-            break;
-    }
-    if (lists[i] == NULL) {
-        error_set(errp, QERR_INVALID_OPTION_GROUP, group);
-    }
-    return lists[i];
-}
-
-QemuOptsList *qemu_find_opts(const char *group)
-{
-    QemuOptsList *ret;
-    Error *local_err = NULL;
-
-    ret = find_list(vm_config_groups, group, &local_err);
-    if (error_is_set(&local_err)) {
-        error_report("%s\n", error_get_pretty(local_err));
-        error_free(local_err);
-    }
-
-    return ret;
-}
-
-QemuOptsList *qemu_find_opts_err(const char *group, Error **errp)
-{
-    return find_list(vm_config_groups, group, errp);
-}
-
-void qemu_add_opts(QemuOptsList *list)
-{
-    int entries, i;
-
-    entries = ARRAY_SIZE(vm_config_groups);
-    entries--; /* keep list NULL terminated */
-    for (i = 0; i < entries; i++) {
-        if (vm_config_groups[i] == NULL) {
-            vm_config_groups[i] = list;
-            return;
-        }
-    }
-    fprintf(stderr, "ran out of space in vm_config_groups");
-    abort();
-}
-
-int qemu_set_option(const char *str)
-{
-    char group[64], id[64], arg[64];
-    QemuOptsList *list;
-    QemuOpts *opts;
-    int rc, offset;
-
-    rc = sscanf(str, "%63[^.].%63[^.].%63[^=]%n", group, id, arg, &offset);
-    if (rc < 3 || str[offset] != '=') {
-        error_report("can't parse: \"%s\"", str);
-        return -1;
-    }
-
-    list = qemu_find_opts(group);
-    if (list == NULL) {
-        return -1;
-    }
-
-    opts = qemu_opts_find(list, id);
-    if (!opts) {
-        error_report("there is no %s \"%s\" defined",
-                     list->name, id);
-        return -1;
-    }
-
-    if (qemu_opt_set(opts, arg, str+offset+1) == -1) {
-        return -1;
-    }
-    return 0;
-}
-
-int qemu_global_option(const char *str)
-{
-    char driver[64], property[64];
-    QemuOpts *opts;
-    int rc, offset;
-
-    rc = sscanf(str, "%63[^.].%63[^=]%n", driver, property, &offset);
-    if (rc < 2 || str[offset] != '=') {
-        error_report("can't parse: \"%s\"", str);
-        return -1;
-    }
-
-    opts = qemu_opts_create(&qemu_global_opts, NULL, 0, NULL);
-    qemu_opt_set(opts, "driver", driver);
-    qemu_opt_set(opts, "property", property);
-    qemu_opt_set(opts, "value", str+offset+1);
-    return 0;
-}
-
-struct ConfigWriteData {
-    QemuOptsList *list;
-    FILE *fp;
-};
-
-static int config_write_opt(const char *name, const char *value, void *opaque)
-{
-    struct ConfigWriteData *data = opaque;
-
-    fprintf(data->fp, "  %s = \"%s\"\n", name, value);
-    return 0;
-}
-
-static int config_write_opts(QemuOpts *opts, void *opaque)
-{
-    struct ConfigWriteData *data = opaque;
-    const char *id = qemu_opts_id(opts);
-
-    if (id) {
-        fprintf(data->fp, "[%s \"%s\"]\n", data->list->name, id);
-    } else {
-        fprintf(data->fp, "[%s]\n", data->list->name);
-    }
-    qemu_opt_foreach(opts, config_write_opt, data, 0);
-    fprintf(data->fp, "\n");
-    return 0;
-}
-
-void qemu_config_write(FILE *fp)
-{
-    struct ConfigWriteData data = { .fp = fp };
-    QemuOptsList **lists = vm_config_groups;
-    int i;
-
-    fprintf(fp, "# qemu config file\n\n");
-    for (i = 0; lists[i] != NULL; i++) {
-        data.list = lists[i];
-        qemu_opts_foreach(data.list, config_write_opts, &data, 0);
-    }
-}
-
-int qemu_config_parse(FILE *fp, QemuOptsList **lists, const char *fname)
-{
-    char line[1024], group[64], id[64], arg[64], value[1024];
-    Location loc;
-    QemuOptsList *list = NULL;
-    Error *local_err = NULL;
-    QemuOpts *opts = NULL;
-    int res = -1, lno = 0;
-
-    loc_push_none(&loc);
-    while (fgets(line, sizeof(line), fp) != NULL) {
-        loc_set_file(fname, ++lno);
-        if (line[0] == '\n') {
-            /* skip empty lines */
-            continue;
-        }
-        if (line[0] == '#') {
-            /* comment */
-            continue;
-        }
-        if (sscanf(line, "[%63s \"%63[^\"]\"]", group, id) == 2) {
-            /* group with id */
-            list = find_list(lists, group, &local_err);
-            if (error_is_set(&local_err)) {
-                error_report("%s\n", error_get_pretty(local_err));
-                error_free(local_err);
-                goto out;
-            }
-            opts = qemu_opts_create(list, id, 1, NULL);
-            continue;
-        }
-        if (sscanf(line, "[%63[^]]]", group) == 1) {
-            /* group without id */
-            list = find_list(lists, group, &local_err);
-            if (error_is_set(&local_err)) {
-                error_report("%s\n", error_get_pretty(local_err));
-                error_free(local_err);
-                goto out;
-            }
-            opts = qemu_opts_create(list, NULL, 0, NULL);
-            continue;
-        }
-        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2) {
-            /* arg = value */
-            if (opts == NULL) {
-                error_report("no group defined");
-                goto out;
-            }
-            if (qemu_opt_set(opts, arg, value) != 0) {
-                goto out;
-            }
-            continue;
-        }
-        error_report("parse error");
-        goto out;
-    }
-    if (ferror(fp)) {
-        error_report("error reading file");
-        goto out;
-    }
-    res = 0;
-out:
-    loc_pop(&loc);
-    return res;
-}
-
-int qemu_read_config_file(const char *filename)
-{
-    FILE *f = fopen(filename, "r");
-    int ret;
-
-    if (f == NULL) {
-        return -errno;
-    }
-
-    ret = qemu_config_parse(f, vm_config_groups, filename);
-    fclose(f);
-
-    if (ret == 0) {
-        return 0;
-    } else {
-        return -EINVAL;
-    }
-}
diff --git a/qemu-coroutine-io.c b/qemu-coroutine-io.c
index 57349650031..e8ad1a40110 100644
--- a/qemu-coroutine-io.c
+++ b/qemu-coroutine-io.c
@@ -23,9 +23,9 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "qemu-coroutine.h"
-#include "iov.h"
+#include "qemu/sockets.h"
+#include "block/coroutine.h"
+#include "qemu/iov.h"
 
 ssize_t coroutine_fn
 qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
diff --git a/qemu-coroutine-lock.c b/qemu-coroutine-lock.c
index 26ad76bf50c..97ef01c7963 100644
--- a/qemu-coroutine-lock.c
+++ b/qemu-coroutine-lock.c
@@ -23,10 +23,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-coroutine.h"
-#include "qemu-coroutine-int.h"
-#include "qemu-queue.h"
-#include "main-loop.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
+#include "qemu/queue.h"
+#include "block/aio.h"
 #include "trace.h"
 
 static QTAILQ_HEAD(, Coroutine) unlock_bh_queue =
diff --git a/qemu-coroutine-sleep.c b/qemu-coroutine-sleep.c
index d7083ee41a4..169ce5ccc92 100644
--- a/qemu-coroutine-sleep.c
+++ b/qemu-coroutine-sleep.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qemu-coroutine.h"
-#include "qemu-timer.h"
+#include "block/coroutine.h"
+#include "qemu/timer.h"
 
 typedef struct CoSleepCB {
     QEMUTimer *ts;
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 600be2643c7..0f6e2685749 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -14,8 +14,8 @@
 
 #include "trace.h"
 #include "qemu-common.h"
-#include "qemu-coroutine.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
 
 Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
 {
diff --git a/qemu-doc.texi b/qemu-doc.texi
index 35cabbcb9e0..6d7f50d8323 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -416,11 +416,13 @@ snapshots.
 * vm_snapshots::              VM snapshots
 * qemu_img_invocation::       qemu-img Invocation
 * qemu_nbd_invocation::       qemu-nbd Invocation
+* disk_images_formats::       Disk image file formats
 * host_drives::               Using host drives
 * disk_images_fat_images::    Virtual FAT disk images
 * disk_images_nbd::           NBD access
 * disk_images_sheepdog::      Sheepdog disk images
 * disk_images_iscsi::         iSCSI LUNs
+* disk_images_gluster::       GlusterFS disk images
 @end menu
 
 @node disk_images_quickstart
@@ -506,6 +508,172 @@ state is not saved or restored properly (in particular USB).
 
 @include qemu-nbd.texi
 
+@node disk_images_formats
+@subsection Disk image file formats
+
+QEMU supports many image file formats that can be used with VMs as well as with
+any of the tools (like @code{qemu-img}). This includes the preferred formats
+raw and qcow2 as well as formats that are supported for compatibility with
+older QEMU versions or other hypervisors.
+
+Depending on the image format, different options can be passed to
+@code{qemu-img create} and @code{qemu-img convert} using the @code{-o} option.
+This section describes each format and the options that are supported for it.
+
+@table @option
+@item raw
+
+Raw disk image format. This format has the advantage of
+being simple and easily exportable to all other emulators. If your
+file system supports @emph{holes} (for example in ext2 or ext3 on
+Linux or NTFS on Windows), then only the written sectors will reserve
+space. Use @code{qemu-img info} to know the real size used by the
+image or @code{ls -ls} on Unix/Linux.
+
+@item qcow2
+QEMU image format, the most versatile format. Use it to have smaller
+images (useful if your filesystem does not supports holes, for example
+on Windows), optional AES encryption, zlib based compression and
+support of multiple VM snapshots.
+
+Supported options:
+@table @code
+@item compat
+Determines the qcow2 version to use. @code{compat=0.10} uses the traditional
+image format that can be read by any QEMU since 0.10 (this is the default).
+@code{compat=1.1} enables image format extensions that only QEMU 1.1 and
+newer understand. Amongst others, this includes zero clusters, which allow
+efficient copy-on-read for sparse images.
+
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@item backing_fmt
+Image format of the base image
+@item encryption
+If this option is set to @code{on}, the image is encrypted.
+
+Encryption uses the AES format which is very secure (128 bit keys). Use
+a long password (16 characters) to get maximum protection.
+
+@item cluster_size
+Changes the qcow2 cluster size (must be between 512 and 2M). Smaller cluster
+sizes can improve the image file size whereas larger cluster sizes generally
+provide better performance.
+
+@item preallocation
+Preallocation mode (allowed values: off, metadata). An image with preallocated
+metadata is initially larger but can improve performance when the image needs
+to grow.
+
+@item lazy_refcounts
+If this option is set to @code{on}, reference count updates are postponed with
+the goal of avoiding metadata I/O and improving performance. This is
+particularly interesting with @option{cache=writethrough} which doesn't batch
+metadata updates. The tradeoff is that after a host crash, the reference count
+tables must be rebuilt, i.e. on the next open an (automatic) @code{qemu-img
+check -r all} is required, which may take some time.
+
+This option can only be enabled if @code{compat=1.1} is specified.
+
+@end table
+
+@item qed
+Old QEMU image format with support for backing files and compact image files
+(when your filesystem or transport medium does not support holes).
+
+When converting QED images to qcow2, you might want to consider using the
+@code{lazy_refcounts=on} option to get a more QED-like behaviour.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand).
+@item backing_fmt
+Image file format of backing file (optional).  Useful if the format cannot be
+autodetected because it has no header, like some vhd/vpc files.
+@item cluster_size
+Changes the cluster size (must be power-of-2 between 4K and 64K). Smaller
+cluster sizes can improve the image file size whereas larger cluster sizes
+generally provide better performance.
+@item table_size
+Changes the number of clusters per L1/L2 table (must be power-of-2 between 1
+and 16).  There is normally no need to change this value but this option can be
+used for performance benchmarking.
+@end table
+
+@item qcow
+Old QEMU image format with support for backing files, compact image files,
+encryption and compression.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@item encryption
+If this option is set to @code{on}, the image is encrypted.
+@end table
+
+@item cow
+User Mode Linux Copy On Write image format. It is supported only for
+compatibility with previous versions.
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@end table
+
+@item vdi
+VirtualBox 1.1 compatible image format.
+Supported options:
+@table @code
+@item static
+If this option is set to @code{on}, the image is created with metadata
+preallocation.
+@end table
+
+@item vmdk
+VMware 3 and 4 compatible image format.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand).
+@item compat6
+Create a VMDK version 6 image (instead of version 4)
+@item subformat
+Specifies which VMDK subformat to use. Valid options are
+@code{monolithicSparse} (default),
+@code{monolithicFlat},
+@code{twoGbMaxExtentSparse},
+@code{twoGbMaxExtentFlat} and
+@code{streamOptimized}.
+@end table
+
+@item vpc
+VirtualPC compatible image format (VHD).
+Supported options:
+@table @code
+@item subformat
+Specifies which VHD subformat to use. Valid options are
+@code{dynamic} (default) and @code{fixed}.
+@end table
+@end table
+
+@subsubsection Read-only formats
+More disk image file formats are supported in a read-only mode.
+@table @option
+@item bochs
+Bochs images of @code{growing} type.
+@item cloop
+Linux Compressed Loop image, useful only to reuse directly compressed
+CD-ROM images present for example in the Knoppix CD-ROMs.
+@item dmg
+Apple disk image.
+@item parallels
+Parallels disk image format.
+@end table
+
+
 @node host_drives
 @subsection Using host drives
 
@@ -610,14 +778,14 @@ QEMU can access directly to block device exported using the Network Block Device
 protocol.
 
 @example
-qemu-system-i386 linux.img -hdb nbd:my_nbd_server.mydomain.org:1024
+qemu-system-i386 linux.img -hdb nbd://my_nbd_server.mydomain.org:1024/
 @end example
 
 If the NBD server is located on the same host, you can use an unix socket instead
 of an inet socket:
 
 @example
-qemu-system-i386 linux.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 linux.img -hdb nbd+unix://?socket=/tmp/my_socket
 @end example
 
 In this case, the block device must be exported using qemu-nbd:
@@ -631,17 +799,26 @@ The use of qemu-nbd allows to share a disk between several guests:
 qemu-nbd --socket=/tmp/my_socket --share=2 my_disk.qcow2
 @end example
 
+@noindent
 and then you can use it with two guests:
 @example
-qemu-system-i386 linux1.img -hdb nbd:unix:/tmp/my_socket
-qemu-system-i386 linux2.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 linux1.img -hdb nbd+unix://?socket=/tmp/my_socket
+qemu-system-i386 linux2.img -hdb nbd+unix://?socket=/tmp/my_socket
 @end example
 
-If the nbd-server uses named exports (since NBD 2.9.18), you must use the
-"exportname" option:
+If the nbd-server uses named exports (supported since NBD 2.9.18, or with QEMU's
+own embedded NBD server), you must specify an export name in the URI:
 @example
-qemu-system-i386 -cdrom nbd:localhost:exportname=debian-500-ppc-netinst
-qemu-system-i386 -cdrom nbd:localhost:exportname=openSUSE-11.1-ppc-netinst
+qemu-system-i386 -cdrom nbd://localhost/debian-500-ppc-netinst
+qemu-system-i386 -cdrom nbd://localhost/openSUSE-11.1-ppc-netinst
+@end example
+
+The URI syntax for NBD is supported since QEMU 1.3.  An alternative syntax is
+also available.  Here are some example of the older syntax:
+@example
+qemu-system-i386 linux.img -hdb nbd:my_nbd_server.mydomain.org:1024
+qemu-system-i386 linux2.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 -cdrom nbd:localhost:10809:exportname=debian-500-ppc-netinst
 @end example
 
 @node disk_images_sheepdog
@@ -805,7 +982,55 @@ qemu-system-i386 -iscsi initiator-name=iqn.qemu.test:my-initiator \
     -cdrom iscsi://127.0.0.1/iqn.qemu.test/2
 @end example
 
+@node disk_images_gluster
+@subsection GlusterFS disk images
+
+GlusterFS is an user space distributed file system.
+
+You can boot from the GlusterFS disk image with the command:
+@example
+qemu-system-x86_64 -drive file=gluster[+@var{transport}]://[@var{server}[:@var{port}]]/@var{volname}/@var{image}[?socket=...]
+@end example
+
+@var{gluster} is the protocol.
+
+@var{transport} specifies the transport type used to connect to gluster
+management daemon (glusterd). Valid transport types are
+tcp, unix and rdma. If a transport type isn't specified, then tcp
+type is assumed.
+
+@var{server} specifies the server where the volume file specification for
+the given volume resides. This can be either hostname, ipv4 address
+or ipv6 address. ipv6 address needs to be within square brackets [ ].
+If transport type is unix, then @var{server} field should not be specifed.
+Instead @var{socket} field needs to be populated with the path to unix domain
+socket.
+
+@var{port} is the port number on which glusterd is listening. This is optional
+and if not specified, QEMU will send 0 which will make gluster to use the
+default port. If the transport type is unix, then @var{port} should not be
+specified.
 
+@var{volname} is the name of the gluster volume which contains the disk image.
+
+@var{image} is the path to the actual disk image that resides on gluster volume.
+
+You can create a GlusterFS disk image with the command:
+@example
+qemu-img create gluster://@var{server}/@var{volname}/@var{image} @var{size}
+@end example
+
+Examples
+@example
+qemu-system-x86_64 -drive file=gluster://1.2.3.4/testvol/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://1.2.3.4/testvol/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
+qemu-system-x86_64 -drive file=gluster+rdma://1.2.3.4:24007/testvol/a.img
+@end example
 
 @node pcsys_network
 @section Network emulation
diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index 39419a0314b..a18136302d2 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -34,9 +34,9 @@ STEXI
 ETEXI
 
 DEF("info", img_info,
-    "info [-f fmt] filename")
+    "info [-f fmt] [--output=ofmt] [--backing-chain] filename")
 STEXI
-@item info [-f @var{fmt}] @var{filename}
+@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 ETEXI
 
 DEF("snapshot", img_snapshot,
diff --git a/qemu-img.c b/qemu-img.c
index b41e670a618..85d3740b9ca 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -21,12 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp/qjson.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-error.h"
-#include "osdep.h"
-#include "sysemu.h"
-#include "block_int.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
+#include "block/block_int.h"
+#include <getopt.h>
 #include <stdio.h>
 
 #ifdef _WIN32
@@ -84,12 +88,13 @@ static void help(void)
            "  '-p' show progress of command (only certain commands)\n"
            "  '-S' indicates the consecutive number of bytes that must contain only zeros\n"
            "       for qemu-img to create a sparse image during conversion\n"
+           "  '--output' takes the format in which the output must be done (human or json)\n"
            "\n"
            "Parameters to check subcommand:\n"
            "  '-r' tries to repair any inconsistencies that are found during the check.\n"
            "       '-r leaks' repairs only cluster leaks, whereas '-r all' fixes all\n"
            "       kinds of errors, with a higher risk of choosing the wrong fix or\n"
-           "       hiding corruption that has already occured.\n"
+           "       hiding corruption that has already occurred.\n"
            "\n"
            "Parameters to snapshot subcommand:\n"
            "  'snapshot' is the name of the snapshot to create, apply or delete\n"
@@ -221,7 +226,8 @@ static int print_block_option_help(const char *filename, const char *fmt)
 
 static BlockDriverState *bdrv_new_open(const char *filename,
                                        const char *fmt,
-                                       int flags)
+                                       int flags,
+                                       bool require_io)
 {
     BlockDriverState *bs;
     BlockDriver *drv;
@@ -246,7 +252,7 @@ static BlockDriverState *bdrv_new_open(const char *filename,
         goto fail;
     }
 
-    if (bdrv_is_encrypted(bs)) {
+    if (bdrv_is_encrypted(bs) && require_io) {
         printf("Disk image '%s' is encrypted.\n", filename);
         if (read_password(password, sizeof(password)) < 0) {
             error_report("No password given");
@@ -288,13 +294,14 @@ static int add_old_style_options(const char *fmt, QEMUOptionParameter *list,
 
 static int img_create(int argc, char **argv)
 {
-    int c, ret = 0;
+    int c;
     uint64_t img_size = -1;
     const char *fmt = "raw";
     const char *base_fmt = NULL;
     const char *filename;
     const char *base_filename = NULL;
     char *options = NULL;
+    Error *local_err = NULL;
 
     for(;;) {
         c = getopt(argc, argv, "F:b:f:he6o:");
@@ -341,26 +348,30 @@ static int img_create(int argc, char **argv)
         char *end;
         sval = strtosz_suffix(argv[optind++], &end, STRTOSZ_DEFSUFFIX_B);
         if (sval < 0 || *end) {
-            error_report("Invalid image size specified! You may use k, M, G or "
-                  "T suffixes for ");
-            error_report("kilobytes, megabytes, gigabytes and terabytes.");
-            ret = -1;
-            goto out;
+            if (sval == -ERANGE) {
+                error_report("Image size must be less than 8 EiB!");
+            } else {
+                error_report("Invalid image size specified! You may use k, M, "
+                      "G or T suffixes for ");
+                error_report("kilobytes, megabytes, gigabytes and terabytes.");
+            }
+            return 1;
         }
         img_size = (uint64_t)sval;
     }
 
     if (options && is_help_option(options)) {
-        ret = print_block_option_help(filename, fmt);
-        goto out;
+        return print_block_option_help(filename, fmt);
     }
 
-    ret = bdrv_img_create(filename, fmt, base_filename, base_fmt,
-                          options, img_size, BDRV_O_FLAGS);
-out:
-    if (ret) {
+    bdrv_img_create(filename, fmt, base_filename, base_fmt,
+                    options, img_size, BDRV_O_FLAGS, &local_err);
+    if (error_is_set(&local_err)) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         return 1;
     }
+
     return 0;
 }
 
@@ -413,7 +424,7 @@ static int img_check(int argc, char **argv)
     }
     filename = argv[optind++];
 
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -520,7 +531,7 @@ static int img_commit(int argc, char **argv)
         return -1;
     }
 
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -668,7 +679,7 @@ static int img_convert(int argc, char **argv)
     QEMUOptionParameter *out_baseimg_param;
     char *options = NULL;
     const char *snapshot_name = NULL;
-    float local_progress;
+    float local_progress = 0;
     int min_sparse = 8; /* Need at least 4k of zeros for sparse detection */
 
     fmt = NULL;
@@ -762,7 +773,7 @@ static int img_convert(int argc, char **argv)
 
     total_sectors = 0;
     for (bs_i = 0; bs_i < bs_n; bs_i++) {
-        bs[bs_i] = bdrv_new_open(argv[optind + bs_i], fmt, BDRV_O_FLAGS);
+        bs[bs_i] = bdrv_new_open(argv[optind + bs_i], fmt, BDRV_O_FLAGS, true);
         if (!bs[bs_i]) {
             error_report("Could not open '%s'", argv[optind + bs_i]);
             ret = -1;
@@ -881,7 +892,7 @@ static int img_convert(int argc, char **argv)
         return -1;
     }
 
-    out_bs = bdrv_new_open(out_filename, out_fmt, flags);
+    out_bs = bdrv_new_open(out_filename, out_fmt, flags, true);
     if (!out_bs) {
         ret = -1;
         goto out;
@@ -908,8 +919,10 @@ static int img_convert(int argc, char **argv)
         sector_num = 0;
 
         nb_sectors = total_sectors;
-        local_progress = (float)100 /
-            (nb_sectors / MIN(nb_sectors, cluster_sectors));
+        if (nb_sectors != 0) {
+            local_progress = (float)100 /
+                (nb_sectors / MIN(nb_sectors, cluster_sectors));
+        }
 
         for(;;) {
             int64_t bs_num;
@@ -980,8 +993,10 @@ static int img_convert(int argc, char **argv)
 
         sector_num = 0; // total number of sectors converted so far
         nb_sectors = total_sectors - sector_num;
-        local_progress = (float)100 /
-            (nb_sectors / MIN(nb_sectors, IO_BUF_SIZE / 512));
+        if (nb_sectors != 0) {
+            local_progress = (float)100 /
+                (nb_sectors / MIN(nb_sectors, IO_BUF_SIZE / 512));
+        }
 
         for(;;) {
             nb_sectors = total_sectors - sector_num;
@@ -1102,21 +1117,312 @@ static void dump_snapshots(BlockDriverState *bs)
     g_free(sn_tab);
 }
 
-static int img_info(int argc, char **argv)
+static void dump_json_image_info_list(ImageInfoList *list)
+{
+    Error *errp = NULL;
+    QString *str;
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj;
+    visit_type_ImageInfoList(qmp_output_get_visitor(ov),
+                             &list, NULL, &errp);
+    obj = qmp_output_get_qobject(ov);
+    str = qobject_to_json_pretty(obj);
+    assert(str != NULL);
+    printf("%s\n", qstring_get_str(str));
+    qobject_decref(obj);
+    qmp_output_visitor_cleanup(ov);
+    QDECREF(str);
+}
+
+static void collect_snapshots(BlockDriverState *bs , ImageInfo *info)
+{
+    int i, sn_count;
+    QEMUSnapshotInfo *sn_tab = NULL;
+    SnapshotInfoList *info_list, *cur_item = NULL;
+    sn_count = bdrv_snapshot_list(bs, &sn_tab);
+
+    for (i = 0; i < sn_count; i++) {
+        info->has_snapshots = true;
+        info_list = g_new0(SnapshotInfoList, 1);
+
+        info_list->value                = g_new0(SnapshotInfo, 1);
+        info_list->value->id            = g_strdup(sn_tab[i].id_str);
+        info_list->value->name          = g_strdup(sn_tab[i].name);
+        info_list->value->vm_state_size = sn_tab[i].vm_state_size;
+        info_list->value->date_sec      = sn_tab[i].date_sec;
+        info_list->value->date_nsec     = sn_tab[i].date_nsec;
+        info_list->value->vm_clock_sec  = sn_tab[i].vm_clock_nsec / 1000000000;
+        info_list->value->vm_clock_nsec = sn_tab[i].vm_clock_nsec % 1000000000;
+
+        /* XXX: waiting for the qapi to support qemu-queue.h types */
+        if (!cur_item) {
+            info->snapshots = cur_item = info_list;
+        } else {
+            cur_item->next = info_list;
+            cur_item = info_list;
+        }
+
+    }
+
+    g_free(sn_tab);
+}
+
+static void dump_json_image_info(ImageInfo *info)
+{
+    Error *errp = NULL;
+    QString *str;
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj;
+    visit_type_ImageInfo(qmp_output_get_visitor(ov),
+                         &info, NULL, &errp);
+    obj = qmp_output_get_qobject(ov);
+    str = qobject_to_json_pretty(obj);
+    assert(str != NULL);
+    printf("%s\n", qstring_get_str(str));
+    qobject_decref(obj);
+    qmp_output_visitor_cleanup(ov);
+    QDECREF(str);
+}
+
+static void collect_image_info(BlockDriverState *bs,
+                   ImageInfo *info,
+                   const char *filename,
+                   const char *fmt)
 {
-    int c;
-    const char *filename, *fmt;
-    BlockDriverState *bs;
-    char size_buf[128], dsize_buf[128];
     uint64_t total_sectors;
-    int64_t allocated_size;
     char backing_filename[1024];
     char backing_filename2[1024];
     BlockDriverInfo bdi;
 
+    bdrv_get_geometry(bs, &total_sectors);
+
+    info->filename        = g_strdup(filename);
+    info->format          = g_strdup(bdrv_get_format_name(bs));
+    info->virtual_size    = total_sectors * 512;
+    info->actual_size     = bdrv_get_allocated_file_size(bs);
+    info->has_actual_size = info->actual_size >= 0;
+    if (bdrv_is_encrypted(bs)) {
+        info->encrypted = true;
+        info->has_encrypted = true;
+    }
+    if (bdrv_get_info(bs, &bdi) >= 0) {
+        if (bdi.cluster_size != 0) {
+            info->cluster_size = bdi.cluster_size;
+            info->has_cluster_size = true;
+        }
+        info->dirty_flag = bdi.is_dirty;
+        info->has_dirty_flag = true;
+    }
+    bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
+    if (backing_filename[0] != '\0') {
+        info->backing_filename = g_strdup(backing_filename);
+        info->has_backing_filename = true;
+        bdrv_get_full_backing_filename(bs, backing_filename2,
+                                       sizeof(backing_filename2));
+
+        if (strcmp(backing_filename, backing_filename2) != 0) {
+            info->full_backing_filename =
+                        g_strdup(backing_filename2);
+            info->has_full_backing_filename = true;
+        }
+
+        if (bs->backing_format[0]) {
+            info->backing_filename_format = g_strdup(bs->backing_format);
+            info->has_backing_filename_format = true;
+        }
+    }
+}
+
+static void dump_human_image_info(ImageInfo *info)
+{
+    char size_buf[128], dsize_buf[128];
+    if (!info->has_actual_size) {
+        snprintf(dsize_buf, sizeof(dsize_buf), "unavailable");
+    } else {
+        get_human_readable_size(dsize_buf, sizeof(dsize_buf),
+                                info->actual_size);
+    }
+    get_human_readable_size(size_buf, sizeof(size_buf), info->virtual_size);
+    printf("image: %s\n"
+           "file format: %s\n"
+           "virtual size: %s (%" PRId64 " bytes)\n"
+           "disk size: %s\n",
+           info->filename, info->format, size_buf,
+           info->virtual_size,
+           dsize_buf);
+
+    if (info->has_encrypted && info->encrypted) {
+        printf("encrypted: yes\n");
+    }
+
+    if (info->has_cluster_size) {
+        printf("cluster_size: %" PRId64 "\n", info->cluster_size);
+    }
+
+    if (info->has_dirty_flag && info->dirty_flag) {
+        printf("cleanly shut down: no\n");
+    }
+
+    if (info->has_backing_filename) {
+        printf("backing file: %s", info->backing_filename);
+        if (info->has_full_backing_filename) {
+            printf(" (actual path: %s)", info->full_backing_filename);
+        }
+        putchar('\n');
+        if (info->has_backing_filename_format) {
+            printf("backing file format: %s\n", info->backing_filename_format);
+        }
+    }
+
+    if (info->has_snapshots) {
+        SnapshotInfoList *elem;
+        char buf[256];
+
+        printf("Snapshot list:\n");
+        printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), NULL));
+
+        /* Ideally bdrv_snapshot_dump() would operate on SnapshotInfoList but
+         * we convert to the block layer's native QEMUSnapshotInfo for now.
+         */
+        for (elem = info->snapshots; elem; elem = elem->next) {
+            QEMUSnapshotInfo sn = {
+                .vm_state_size = elem->value->vm_state_size,
+                .date_sec = elem->value->date_sec,
+                .date_nsec = elem->value->date_nsec,
+                .vm_clock_nsec = elem->value->vm_clock_sec * 1000000000ULL +
+                                 elem->value->vm_clock_nsec,
+            };
+
+            pstrcpy(sn.id_str, sizeof(sn.id_str), elem->value->id);
+            pstrcpy(sn.name, sizeof(sn.name), elem->value->name);
+            printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), &sn));
+        }
+    }
+}
+
+static void dump_human_image_info_list(ImageInfoList *list)
+{
+    ImageInfoList *elem;
+    bool delim = false;
+
+    for (elem = list; elem; elem = elem->next) {
+        if (delim) {
+            printf("\n");
+        }
+        delim = true;
+
+        dump_human_image_info(elem->value);
+    }
+}
+
+static gboolean str_equal_func(gconstpointer a, gconstpointer b)
+{
+    return strcmp(a, b) == 0;
+}
+
+/**
+ * Open an image file chain and return an ImageInfoList
+ *
+ * @filename: topmost image filename
+ * @fmt: topmost image format (may be NULL to autodetect)
+ * @chain: true  - enumerate entire backing file chain
+ *         false - only topmost image file
+ *
+ * Returns a list of ImageInfo objects or NULL if there was an error opening an
+ * image file.  If there was an error a message will have been printed to
+ * stderr.
+ */
+static ImageInfoList *collect_image_info_list(const char *filename,
+                                              const char *fmt,
+                                              bool chain)
+{
+    ImageInfoList *head = NULL;
+    ImageInfoList **last = &head;
+    GHashTable *filenames;
+
+    filenames = g_hash_table_new_full(g_str_hash, str_equal_func, NULL, NULL);
+
+    while (filename) {
+        BlockDriverState *bs;
+        ImageInfo *info;
+        ImageInfoList *elem;
+
+        if (g_hash_table_lookup_extended(filenames, filename, NULL, NULL)) {
+            error_report("Backing file '%s' creates an infinite loop.",
+                         filename);
+            goto err;
+        }
+        g_hash_table_insert(filenames, (gpointer)filename, NULL);
+
+        bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_NO_BACKING,
+                           false);
+        if (!bs) {
+            goto err;
+        }
+
+        info = g_new0(ImageInfo, 1);
+        collect_image_info(bs, info, filename, fmt);
+        collect_snapshots(bs, info);
+
+        elem = g_new0(ImageInfoList, 1);
+        elem->value = info;
+        *last = elem;
+        last = &elem->next;
+
+        bdrv_delete(bs);
+
+        filename = fmt = NULL;
+        if (chain) {
+            if (info->has_full_backing_filename) {
+                filename = info->full_backing_filename;
+            } else if (info->has_backing_filename) {
+                filename = info->backing_filename;
+            }
+            if (info->has_backing_filename_format) {
+                fmt = info->backing_filename_format;
+            }
+        }
+    }
+    g_hash_table_destroy(filenames);
+    return head;
+
+err:
+    qapi_free_ImageInfoList(head);
+    g_hash_table_destroy(filenames);
+    return NULL;
+}
+
+enum {
+    OPTION_OUTPUT = 256,
+    OPTION_BACKING_CHAIN = 257,
+};
+
+typedef enum OutputFormat {
+    OFORMAT_JSON,
+    OFORMAT_HUMAN,
+} OutputFormat;
+
+static int img_info(int argc, char **argv)
+{
+    int c;
+    OutputFormat output_format = OFORMAT_HUMAN;
+    bool chain = false;
+    const char *filename, *fmt, *output;
+    ImageInfoList *list;
+
     fmt = NULL;
+    output = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:h");
+        int option_index = 0;
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"format", required_argument, 0, 'f'},
+            {"output", required_argument, 0, OPTION_OUTPUT},
+            {"backing-chain", no_argument, 0, OPTION_BACKING_CHAIN},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:h",
+                        long_options, &option_index);
         if (c == -1) {
             break;
         }
@@ -1128,6 +1434,12 @@ static int img_info(int argc, char **argv)
         case 'f':
             fmt = optarg;
             break;
+        case OPTION_OUTPUT:
+            output = optarg;
+            break;
+        case OPTION_BACKING_CHAIN:
+            chain = true;
+            break;
         }
     }
     if (optind >= argc) {
@@ -1135,49 +1447,34 @@ static int img_info(int argc, char **argv)
     }
     filename = argv[optind++];
 
-    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_NO_BACKING);
-    if (!bs) {
+    if (output && !strcmp(output, "json")) {
+        output_format = OFORMAT_JSON;
+    } else if (output && !strcmp(output, "human")) {
+        output_format = OFORMAT_HUMAN;
+    } else if (output) {
+        error_report("--output must be used with human or json as argument.");
         return 1;
     }
-    bdrv_get_geometry(bs, &total_sectors);
-    get_human_readable_size(size_buf, sizeof(size_buf), total_sectors * 512);
-    allocated_size = bdrv_get_allocated_file_size(bs);
-    if (allocated_size < 0) {
-        snprintf(dsize_buf, sizeof(dsize_buf), "unavailable");
-    } else {
-        get_human_readable_size(dsize_buf, sizeof(dsize_buf),
-                                allocated_size);
-    }
-    printf("image: %s\n"
-           "file format: %s\n"
-           "virtual size: %s (%" PRId64 " bytes)\n"
-           "disk size: %s\n",
-           filename, bdrv_get_format_name(bs), size_buf,
-           (total_sectors * 512),
-           dsize_buf);
-    if (bdrv_is_encrypted(bs)) {
-        printf("encrypted: yes\n");
-    }
-    if (bdrv_get_info(bs, &bdi) >= 0) {
-        if (bdi.cluster_size != 0) {
-            printf("cluster_size: %d\n", bdi.cluster_size);
-        }
-        if (bdi.is_dirty) {
-            printf("cleanly shut down: no\n");
-        }
+
+    list = collect_image_info_list(filename, fmt, chain);
+    if (!list) {
+        return 1;
     }
-    bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
-    if (backing_filename[0] != '\0') {
-        bdrv_get_full_backing_filename(bs, backing_filename2,
-                                       sizeof(backing_filename2));
-        printf("backing file: %s", backing_filename);
-        if (strcmp(backing_filename, backing_filename2) != 0) {
-            printf(" (actual path: %s)", backing_filename2);
+
+    switch (output_format) {
+    case OFORMAT_HUMAN:
+        dump_human_image_info_list(list);
+        break;
+    case OFORMAT_JSON:
+        if (chain) {
+            dump_json_image_info_list(list);
+        } else {
+            dump_json_image_info(list->value);
         }
-        putchar('\n');
+        break;
     }
-    dump_snapshots(bs);
-    bdrv_delete(bs);
+
+    qapi_free_ImageInfoList(list);
     return 0;
 }
 
@@ -1248,7 +1545,7 @@ static int img_snapshot(int argc, char **argv)
     filename = argv[optind++];
 
     /* Open the image */
-    bs = bdrv_new_open(filename, NULL, bdrv_oflags);
+    bs = bdrv_new_open(filename, NULL, bdrv_oflags, true);
     if (!bs) {
         return 1;
     }
@@ -1366,7 +1663,7 @@ static int img_rebase(int argc, char **argv)
      * Ignore the old backing file for unsafe rebase in case we want to correct
      * the reference to a renamed or moved backing file.
      */
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -1409,13 +1706,15 @@ static int img_rebase(int argc, char **argv)
             error_report("Could not open old backing file '%s'", backing_name);
             goto out;
         }
-
-        bs_new_backing = bdrv_new("new_backing");
-        ret = bdrv_open(bs_new_backing, out_baseimg, BDRV_O_FLAGS,
+        if (out_baseimg[0]) {
+            bs_new_backing = bdrv_new("new_backing");
+            ret = bdrv_open(bs_new_backing, out_baseimg, BDRV_O_FLAGS,
                         new_backing_drv);
-        if (ret) {
-            error_report("Could not open new backing file '%s'", out_baseimg);
-            goto out;
+            if (ret) {
+                error_report("Could not open new backing file '%s'",
+                             out_baseimg);
+                goto out;
+            }
         }
     }
 
@@ -1431,22 +1730,27 @@ static int img_rebase(int argc, char **argv)
     if (!unsafe) {
         uint64_t num_sectors;
         uint64_t old_backing_num_sectors;
-        uint64_t new_backing_num_sectors;
+        uint64_t new_backing_num_sectors = 0;
         uint64_t sector;
         int n;
         uint8_t * buf_old;
         uint8_t * buf_new;
-        float local_progress;
+        float local_progress = 0;
 
         buf_old = qemu_blockalign(bs, IO_BUF_SIZE);
         buf_new = qemu_blockalign(bs, IO_BUF_SIZE);
 
         bdrv_get_geometry(bs, &num_sectors);
         bdrv_get_geometry(bs_old_backing, &old_backing_num_sectors);
-        bdrv_get_geometry(bs_new_backing, &new_backing_num_sectors);
+        if (bs_new_backing) {
+            bdrv_get_geometry(bs_new_backing, &new_backing_num_sectors);
+        }
+
+        if (num_sectors != 0) {
+            local_progress = (float)100 /
+                (num_sectors / MIN(num_sectors, IO_BUF_SIZE / 512));
+        }
 
-        local_progress = (float)100 /
-            (num_sectors / MIN(num_sectors, IO_BUF_SIZE / 512));
         for (sector = 0; sector < num_sectors; sector += n) {
 
             /* How many sectors can we handle with the next read? */
@@ -1480,7 +1784,7 @@ static int img_rebase(int argc, char **argv)
                 }
             }
 
-            if (sector >= new_backing_num_sectors) {
+            if (sector >= new_backing_num_sectors || !bs_new_backing) {
                 memset(buf_new, 0, n * BDRV_SECTOR_SIZE);
             } else {
                 if (sector + n > new_backing_num_sectors) {
@@ -1526,7 +1830,12 @@ static int img_rebase(int argc, char **argv)
      * backing file are overwritten in the COW file now, so the visible content
      * doesn't change when we switch the backing file.
      */
-    ret = bdrv_change_backing_file(bs, out_baseimg, out_basefmt);
+    if (out_baseimg && *out_baseimg) {
+        ret = bdrv_change_backing_file(bs, out_baseimg, out_basefmt);
+    } else {
+        ret = bdrv_change_backing_file(bs, NULL, NULL);
+    }
+
     if (ret == -ENOSPC) {
         error_report("Could not change the backing file to '%s': No "
                      "space left in the file header", out_baseimg);
@@ -1629,7 +1938,7 @@ static int img_resize(int argc, char **argv)
     }
 
     /* Parse size */
-    param = qemu_opts_create(&resize_options, NULL, 0, NULL);
+    param = qemu_opts_create_nofail(&resize_options);
     if (qemu_opt_set(param, BLOCK_OPT_SIZE, size)) {
         /* Error message already printed when size parsing fails */
         ret = -1;
@@ -1639,7 +1948,7 @@ static int img_resize(int argc, char **argv)
     n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0);
     qemu_opts_del(param);
 
-    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR);
+    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR, true);
     if (!bs) {
         ret = -1;
         goto out;
@@ -1697,14 +2006,13 @@ int main(int argc, char **argv)
 
     error_set_progname(argv[0]);
 
+    qemu_init_main_loop();
     bdrv_init();
     if (argc < 2)
         help();
     cmdname = argv[1];
     argc--; argv++;
 
-    qemu_init_main_loop();
-
     /* find the command */
     for(cmd = img_cmds; cmd->name != NULL; cmd++) {
         if (!strcmp(cmdname, cmd->name)) {
diff --git a/qemu-img.texi b/qemu-img.texi
index 6b42e35fe7c..00fca8da865 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -28,6 +28,10 @@ Command parameters:
 is the disk image format. It is guessed automatically in most cases. See below
 for a description of the supported disk formats.
 
+@item --backing-chain
+will enumerate information about backing files in a disk image chain. Refer
+below for further description.
+
 @item size
 is the disk image size in bytes. Optional suffixes @code{k} or @code{K}
 (kilobyte, 1024) @code{M} (megabyte, 1024k) and @code{G} (gigabyte, 1024M)
@@ -87,7 +91,7 @@ Perform a consistency check on the disk image @var{filename}.
 If @code{-r} is specified, qemu-img tries to repair any inconsistencies found
 during the check. @code{-r leaks} repairs only cluster leaks, whereas
 @code{-r all} fixes all kinds of errors, with a higher risk of choosing the
-wrong fix or hiding corruption that has already occured.
+wrong fix or hiding corruption that has already occurred.
 
 Only the formats @code{qcow2}, @code{qed} and @code{vdi} support
 consistency checks.
@@ -129,12 +133,28 @@ created as a copy on write image of the specified base image; the
 @var{backing_file} should have the same content as the input's base image,
 however the path, image format, etc may differ.
 
-@item info [-f @var{fmt}] @var{filename}
+@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 
 Give information about the disk image @var{filename}. Use it in
 particular to know the size reserved on disk which can be different
 from the displayed size. If VM snapshots are stored in the disk image,
-they are displayed too.
+they are displayed too. The command can output in the format @var{ofmt}
+which is either @code{human} or @code{json}.
+
+If a disk image has a backing file chain, information about each disk image in
+the chain can be recursively enumerated by using the option @code{--backing-chain}.
+
+For instance, if you have an image chain like:
+
+@example
+base.qcow2 <- snap1.qcow2 <- snap2.qcow2
+@end example
+
+To enumerate information about each disk image in the above chain, starting from top to base, do:
+
+@example
+qemu-img info --backing-chain snap2.qcow2
+@end example
 
 @item snapshot [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot} ] @var{filename}
 
@@ -147,7 +167,9 @@ Changes the backing file of an image. Only the formats @code{qcow2} and
 
 The backing file is changed to @var{backing_file} and (if the image format of
 @var{filename} supports this) the backing file format is changed to
-@var{backing_fmt}.
+@var{backing_fmt}. If @var{backing_file} is specified as ``'' (the empty
+string), then the image is rebased onto no backing file (i.e. it will exist
+independently of any backing file).
 
 There are two different modes in which @code{rebase} can operate:
 @table @option
@@ -204,7 +226,10 @@ After using this command to grow a disk image, you must use file system and
 partitioning tools inside the VM to actually begin using the new space on the
 device.
 @end table
+@c man end
 
+@ignore
+@c man begin NOTES
 Supported image file formats:
 
 @table @option
@@ -225,6 +250,13 @@ support of multiple VM snapshots.
 
 Supported options:
 @table @code
+@item compat
+Determines the qcow2 version to use. @code{compat=0.10} uses the traditional
+image format that can be read by any QEMU since 0.10 (this is the default).
+@code{compat=1.1} enables image format extensions that only QEMU 1.1 and
+newer understand. Amongst others, this includes zero clusters, which allow
+efficient copy-on-read for sparse images.
+
 @item backing_file
 File name of a base image (see @option{create} subcommand)
 @item backing_fmt
@@ -245,73 +277,33 @@ Preallocation mode (allowed values: off, metadata). An image with preallocated
 metadata is initially larger but can improve performance when the image needs
 to grow.
 
-@end table
+@item lazy_refcounts
+If this option is set to @code{on}, reference count updates are postponed with
+the goal of avoiding metadata I/O and improving performance. This is
+particularly interesting with @option{cache=writethrough} which doesn't batch
+metadata updates. The tradeoff is that after a host crash, the reference count
+tables must be rebuilt, i.e. on the next open an (automatic) @code{qemu-img
+check -r all} is required, which may take some time.
 
-@item qed
-Image format with support for backing files and compact image files (when your
-filesystem or transport medium does not support holes).  Good performance due
-to less metadata than the more featureful qcow2 format, especially with
-cache=writethrough or cache=directsync.  Consider using qcow2 which will soon
-have a similar optimization and is most actively developed.
+This option can only be enabled if @code{compat=1.1} is specified.
 
-Supported options:
-@table @code
-@item backing_file
-File name of a base image (see @option{create} subcommand).
-@item backing_fmt
-Image file format of backing file (optional).  Useful if the format cannot be
-autodetected because it has no header, like some vhd/vpc files.
-@item cluster_size
-Changes the cluster size (must be power-of-2 between 4K and 64K). Smaller
-cluster sizes can improve the image file size whereas larger cluster sizes
-generally provide better performance.
-@item table_size
-Changes the number of clusters per L1/L2 table (must be power-of-2 between 1
-and 16).  There is normally no need to change this value but this option can be
-used for performance benchmarking.
 @end table
 
-@item qcow
-Old QEMU image format. Left for compatibility.
+@item Other
+QEMU also supports various other image file formats for compatibility with
+older QEMU versions or other hypervisors, including VMDK, VDI, VHD (vpc), qcow1
+and QED. For a full list of supported formats see @code{qemu-img --help}.
+For a more detailed description of these formats, see the QEMU Emulation User
+Documentation.
 
-Supported options:
-@table @code
-@item backing_file
-File name of a base image (see @option{create} subcommand)
-@item encryption
-If this option is set to @code{on}, the image is encrypted.
-@end table
-
-@item cow
-User Mode Linux Copy On Write image format. Used to be the only growable
-image format in QEMU. It is supported only for compatibility with
-previous versions. It does not work on win32.
-@item vdi
-VirtualBox 1.1 compatible image format.
-@item vmdk
-VMware 3 and 4 compatible image format.
-
-Supported options:
-@table @code
-@item backing_fmt
-Image format of the base image
-@item compat6
-Create a VMDK version 6 image (instead of version 4)
-@end table
-
-@item vpc
-VirtualPC compatible image format (VHD).
-
-@item cloop
-Linux Compressed Loop image, useful only to reuse directly compressed
-CD-ROM images present for example in the Knoppix CD-ROMs.
+The main purpose of the block drivers for these formats is image conversion.
+For running VMs, it is recommended to convert the disk images to either raw or
+qcow2 in order to achieve good performance.
 @end table
 
 
 @c man end
 
-@ignore
-
 @setfilename qemu-img
 @settitle QEMU disk image utility
 
diff --git a/qemu-io.c b/qemu-io.c
index d0f4fb70c79..61880932b3e 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -15,8 +15,8 @@
 #include <libgen.h>
 
 #include "qemu-common.h"
-#include "main-loop.h"
-#include "block_int.h"
+#include "qemu/main-loop.h"
+#include "block/block_int.h"
 #include "cmd.h"
 #include "trace/control.h"
 
@@ -265,6 +265,18 @@ static int do_co_write_zeroes(int64_t offset, int count, int *total)
     }
 }
 
+static int do_write_compressed(char *buf, int64_t offset, int count, int *total)
+{
+    int ret;
+
+    ret = bdrv_write_compressed(bs, offset >> 9, (uint8_t *)buf, count >> 9);
+    if (ret < 0) {
+        return ret;
+    }
+    *total = count;
+    return 1;
+}
+
 static int do_load_vmstate(char *buf, int64_t offset, int count, int *total)
 {
     *total = bdrv_load_vmstate(bs, (uint8_t *)buf, offset, count);
@@ -687,6 +699,7 @@ static void write_help(void)
 " Writes into a segment of the currently open file, using a buffer\n"
 " filled with a set pattern (0xcdcdcdcd).\n"
 " -b, -- write to the VM state rather than the virtual disk\n"
+" -c, -- write compressed data with bdrv_write_compressed\n"
 " -p, -- use bdrv_pwrite to write the file\n"
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
@@ -703,7 +716,7 @@ static const cmdinfo_t write_cmd = {
     .cfunc      = write_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-bCpqz] [-P pattern ] off len",
+    .args       = "[-bcCpqz] [-P pattern ] off len",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = write_help,
 };
@@ -712,6 +725,7 @@ static int write_f(int argc, char **argv)
 {
     struct timeval t1, t2;
     int Cflag = 0, pflag = 0, qflag = 0, bflag = 0, Pflag = 0, zflag = 0;
+    int cflag = 0;
     int c, cnt;
     char *buf = NULL;
     int64_t offset;
@@ -720,11 +734,14 @@ static int write_f(int argc, char **argv)
     int total = 0;
     int pattern = 0xcd;
 
-    while ((c = getopt(argc, argv, "bCpP:qz")) != EOF) {
+    while ((c = getopt(argc, argv, "bcCpP:qz")) != EOF) {
         switch (c) {
         case 'b':
             bflag = 1;
             break;
+        case 'c':
+            cflag = 1;
+            break;
         case 'C':
             Cflag = 1;
             break;
@@ -801,6 +818,8 @@ static int write_f(int argc, char **argv)
         cnt = do_save_vmstate(buf, offset, count, &total);
     } else if (zflag) {
         cnt = do_co_write_zeroes(offset, count, &total);
+    } else if (cflag) {
+        cnt = do_write_compressed(buf, offset, count, &total);
     } else {
         cnt = do_write(buf, offset, count, &total);
     }
@@ -1362,7 +1381,7 @@ static int aio_write_f(int argc, char **argv)
 
 static int aio_flush_f(int argc, char **argv)
 {
-    qemu_aio_flush();
+    bdrv_drain_all();
     return 0;
 }
 
@@ -1652,6 +1671,67 @@ static const cmdinfo_t map_cmd = {
        .oneline        = "prints the allocated areas of a file",
 };
 
+static int break_f(int argc, char **argv)
+{
+    int ret;
+
+    ret = bdrv_debug_breakpoint(bs, argv[1], argv[2]);
+    if (ret < 0) {
+        printf("Could not set breakpoint: %s\n", strerror(-ret));
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t break_cmd = {
+       .name           = "break",
+       .argmin         = 2,
+       .argmax         = 2,
+       .cfunc          = break_f,
+       .args           = "event tag",
+       .oneline        = "sets a breakpoint on event and tags the stopped "
+                         "request as tag",
+};
+
+static int resume_f(int argc, char **argv)
+{
+    int ret;
+
+    ret = bdrv_debug_resume(bs, argv[1]);
+    if (ret < 0) {
+        printf("Could not resume request: %s\n", strerror(-ret));
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t resume_cmd = {
+       .name           = "resume",
+       .argmin         = 1,
+       .argmax         = 1,
+       .cfunc          = resume_f,
+       .args           = "tag",
+       .oneline        = "resumes the request tagged as tag",
+};
+
+static int wait_break_f(int argc, char **argv)
+{
+    while (!bdrv_debug_is_suspended(bs, argv[1])) {
+        qemu_aio_wait();
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t wait_break_cmd = {
+       .name           = "wait_break",
+       .argmin         = 1,
+       .argmax         = 1,
+       .cfunc          = wait_break_f,
+       .args           = "tag",
+       .oneline        = "waits for the suspension of a request",
+};
+
 static int abort_f(int argc, char **argv)
 {
     abort();
@@ -1892,9 +1972,8 @@ int main(int argc, char **argv)
         exit(1);
     }
 
-    bdrv_init();
-
     qemu_init_main_loop();
+    bdrv_init();
 
     /* initialize commands */
     quit_init();
@@ -1916,6 +1995,9 @@ int main(int argc, char **argv)
     add_command(&discard_cmd);
     add_command(&alloc_cmd);
     add_command(&map_cmd);
+    add_command(&break_cmd);
+    add_command(&resume_cmd);
+    add_command(&wait_break_cmd);
     add_command(&abort_cmd);
 
     add_args_command(init_args_command);
diff --git a/qemu-log.c b/qemu-log.c
index 396aafdf627..b655b305eab 100644
--- a/qemu-log.c
+++ b/qemu-log.c
@@ -18,7 +18,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #ifdef WIN32
 static const char *logfilename = "qemu.log";
@@ -116,6 +116,9 @@ const CPULogItem cpu_log_items[] = {
       "show all i/o ports accesses" },
     { LOG_UNIMP, "unimp",
       "log unimplemented functionality" },
+    { LOG_GUEST_ERROR, "guest_errors",
+      "log when the guest OS does something invalid (eg accessing a\n"
+      "non-existent register)" },
     { 0, NULL, NULL },
 };
 
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 1c1cf6a4630..0a6091b6a8b 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -17,8 +17,8 @@
  */
 
 #include "qemu-common.h"
-#include "block.h"
-#include "nbd.h"
+#include "block/block.h"
+#include "block/nbd.h"
 
 #include <stdarg.h>
 #include <stdio.h>
@@ -41,8 +41,8 @@ static NBDExport *exp;
 static int verbose;
 static char *srcpath;
 static char *sockpath;
-static bool sigterm_reported;
-static bool nbd_started;
+static int persistent = 0;
+static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
 static int shared = 1;
 static int nb_fds;
 
@@ -186,7 +186,7 @@ static int find_partition(BlockDriverState *bs, int partition,
 
 static void termsig_handler(int signum)
 {
-    sigterm_reported = true;
+    state = TERMINATE;
     qemu_notify_event();
 }
 
@@ -269,10 +269,20 @@ static int nbd_can_accept(void *opaque)
     return nb_fds < shared;
 }
 
+static void nbd_export_closed(NBDExport *exp)
+{
+    assert(state == TERMINATING);
+    state = TERMINATED;
+}
+
 static void nbd_client_closed(NBDClient *client)
 {
     nb_fds--;
+    if (nb_fds == 0 && !persistent && state == RUNNING) {
+        state = TERMINATE;
+    }
     qemu_notify_event();
+    nbd_client_put(client);
 }
 
 static void nbd_accept(void *opaque)
@@ -282,7 +292,11 @@ static void nbd_accept(void *opaque)
     socklen_t addr_len = sizeof(addr);
 
     int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    nbd_started = true;
+    if (state >= TERMINATE) {
+        close(fd);
+        return;
+    }
+
     if (fd >= 0 && nbd_client_new(exp, fd, nbd_client_closed)) {
         nb_fds++;
     }
@@ -329,7 +343,6 @@ int main(int argc, char **argv)
     int partition = -1;
     int ret;
     int fd;
-    int persistent = 0;
     bool seen_cache = false;
 #ifdef CONFIG_LINUX_AIO
     bool seen_aio = false;
@@ -526,6 +539,7 @@ int main(int argc, char **argv)
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
+    qemu_init_main_loop();
     bdrv_init();
     atexit(bdrv_close_all);
 
@@ -546,7 +560,7 @@ int main(int argc, char **argv)
         }
     }
 
-    exp = nbd_export_new(bs, dev_offset, fd_size, nbdflags);
+    exp = nbd_export_new(bs, dev_offset, fd_size, nbdflags, nbd_export_closed);
 
     if (sockpath) {
         fd = unix_socket_incoming(sockpath);
@@ -571,7 +585,6 @@ int main(int argc, char **argv)
         memset(&client_thread, 0, sizeof(client_thread));
     }
 
-    qemu_init_main_loop();
     qemu_set_fd_handler2(fd, nbd_can_accept, nbd_accept, NULL,
                          (void *)(uintptr_t)fd);
 
@@ -581,11 +594,18 @@ int main(int argc, char **argv)
         err(EXIT_FAILURE, "Could not chdir to root directory");
     }
 
+    state = RUNNING;
     do {
         main_loop_wait(false);
-    } while (!sigterm_reported && (persistent || !nbd_started || nb_fds > 0));
+        if (state == TERMINATE) {
+            state = TERMINATING;
+            nbd_export_close(exp);
+            nbd_export_put(exp);
+            exp = NULL;
+        }
+    } while (state != TERMINATED);
 
-    nbd_export_close(exp);
+    bdrv_close(bs);
     if (sockpath) {
         unlink(sockpath);
     }
diff --git a/qemu-options.hx b/qemu-options.hx
index 3c411c427e8..2d44137bf9a 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -6,10 +6,6 @@ HXCOMM construct option structures, enums and help message for specified
 HXCOMM architectures.
 HXCOMM HXCOMM can be used for comments, discarded from both texi and C
 
-HXCOMM TODO : when we are able to change -help output without breaking
-HXCOMM libvirt we should update the help options which refer to -cpu ?,
-HXCOMM -driver ?, etc to use the preferred -cpu help etc instead.
-
 DEFHEADING(Standard options:)
 STEXI
 @table @option
@@ -33,17 +29,18 @@ ETEXI
 
 DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
     "-machine [type=]name[,prop[=value][,...]]\n"
-    "                selects emulated machine (-machine ? for list)\n"
+    "                selects emulated machine ('-machine help' for list)\n"
     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
     "                supported accelerators are kvm, xen, tcg (default: tcg)\n"
     "                kernel_irqchip=on|off controls accelerated irqchip support\n"
     "                kvm_shadow_mem=size of KVM shadow MMU\n"
-    "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n",
+    "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
+    "                mem-merge=on|off controls memory merge support (default: on)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
 @findex -machine
-Select the emulated machine by @var{name}. Use @code{-machine ?} to list
+Select the emulated machine by @var{name}. Use @code{-machine help} to list
 available machines. Supported machine properties are:
 @table @option
 @item accel=@var{accels1}[:@var{accels2}[:...]]
@@ -57,6 +54,10 @@ Enables in-kernel irqchip support for the chosen accelerator when available.
 Defines the size of the KVM shadow MMU.
 @item dump-guest-core=on|off
 Include guest memory in a core dump. The default is on.
+@item mem-merge=on|off
+Enables or disables memory merge support. This feature, when supported by
+the host, de-duplicates identical memory pages among VMs instances
+(enabled by default).
 @end table
 ETEXI
 
@@ -64,11 +65,11 @@ HXCOMM Deprecated by -machine
 DEF("M", HAS_ARG, QEMU_OPTION_M, "", QEMU_ARCH_ALL)
 
 DEF("cpu", HAS_ARG, QEMU_OPTION_cpu,
-    "-cpu cpu        select CPU (-cpu ? for list)\n", QEMU_ARCH_ALL)
+    "-cpu cpu        select CPU ('-cpu help' for list)\n", QEMU_ARCH_ALL)
 STEXI
 @item -cpu @var{model}
 @findex -cpu
-Select CPU model (-cpu ? for list and additional feature selection)
+Select CPU model (@code{-cpu help} for list and additional feature selection)
 ETEXI
 
 DEF("smp", HAS_ARG, QEMU_OPTION_smp,
@@ -205,33 +206,33 @@ Open drive @option{file} as read-only. Guest write attempts will fail.
 file sectors into the image file.
 @end table
 
-By default, writethrough caching is used for all block device.  This means that
-the host page cache will be used to read and write data but write notification
-will be sent to the guest only when the data has been reported as written by
-the storage subsystem.
+By default, the @option{cache=writeback} mode is used. It will report data
+writes as completed as soon as the data is present in the host page cache.
+This is safe as long as your guest OS makes sure to correctly flush disk caches
+where needed. If your guest OS does not handle volatile disk write caches
+correctly and your host crashes or loses power, then the guest may experience
+data corruption.
 
-Writeback caching will report data writes as completed as soon as the data is
-present in the host page cache.  This is safe as long as you trust your host.
-If your host crashes or loses power, then the guest may experience data
-corruption.
+For such guests, you should consider using @option{cache=writethrough}. This
+means that the host page cache will be used to read and write data, but write
+notification will be sent to the guest only after QEMU has made sure to flush
+each write to the disk. Be aware that this has a major impact on performance.
 
 The host page cache can be avoided entirely with @option{cache=none}.  This will
-attempt to do disk IO directly to the guests memory.  QEMU may still perform
-an internal copy of the data.
+attempt to do disk IO directly to the guest's memory.  QEMU may still perform
+an internal copy of the data. Note that this is considered a writeback mode and
+the guest OS must handle the disk write cache correctly in order to avoid data
+corruption on host crashes.
 
 The host page cache can be avoided while only sending write notifications to
-the guest when the data has been reported as written by the storage subsystem
-using @option{cache=directsync}.
-
-Some block drivers perform badly with @option{cache=writethrough}, most notably,
-qcow2.  If performance is more important than correctness,
-@option{cache=writeback} should be used with qcow2.
+the guest when the data has been flushed to the disk using
+@option{cache=directsync}.
 
 In case you don't care about data integrity over host failures, use
-cache=unsafe. This option tells QEMU that it never needs to write any data
-to the disk but can instead keeps things in cache. If anything goes wrong,
+@option{cache=unsafe}. This option tells QEMU that it never needs to write any
+data to the disk but can instead keep things in cache. If anything goes wrong,
 like your host losing power, the disk storage getting disconnected accidentally,
-etc. you're image will most probably be rendered unusable.   When using
+etc. your image will most probably be rendered unusable.   When using
 the @option{-snapshot} option, unsafe caching is always used.
 
 Copy-on-read avoids accessing the same backing file sectors repeatedly and is
@@ -252,6 +253,14 @@ qemu-system-i386 -drive file=file,index=2,media=disk
 qemu-system-i386 -drive file=file,index=3,media=disk
 @end example
 
+You can open an image using pre-opened file descriptors from an fd set:
+@example
+qemu-system-i386
+-add-fd fd=3,set=2,opaque="rdwr:/path/to/file"
+-add-fd fd=4,set=2,opaque="rdonly:/path/to/file"
+-drive file=/dev/fdset/2,index=0,media=disk
+@end example
+
 You can connect a CDROM to the slave of ide0:
 @example
 qemu-system-i386 -drive file=file,if=ide,index=1,media=cdrom
@@ -284,6 +293,34 @@ qemu-system-i386 -hda a -hdb b
 @end example
 ETEXI
 
+DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd,
+    "-add-fd fd=fd,set=set[,opaque=opaque]\n"
+    "                Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL)
+STEXI
+@item -add-fd fd=@var{fd},set=@var{set}[,opaque=@var{opaque}]
+@findex -add-fd
+
+Add a file descriptor to an fd set.  Valid options are:
+
+@table @option
+@item fd=@var{fd}
+This option defines the file descriptor of which a duplicate is added to fd set.
+The file descriptor cannot be stdin, stdout, or stderr.
+@item set=@var{set}
+This option defines the ID of the fd set to add the file descriptor to.
+@item opaque=@var{opaque}
+This option defines a free-form string that can be used to describe @var{fd}.
+@end table
+
+You can open an image using pre-opened file descriptors from an fd set:
+@example
+qemu-system-i386
+-add-fd fd=3,set=2,opaque="rdwr:/path/to/file"
+-add-fd fd=4,set=2,opaque="rdonly:/path/to/file"
+-drive file=/dev/fdset/2,index=0,media=disk
+@end example
+ETEXI
+
 DEF("set", HAS_ARG, QEMU_OPTION_set,
     "-set group.id.arg=value\n"
     "                set <arg> parameter for item <id> of type <group>\n"
@@ -339,13 +376,14 @@ ETEXI
 
 DEF("boot", HAS_ARG, QEMU_OPTION_boot,
     "-boot [order=drives][,once=drives][,menu=on|off]\n"
-    "      [,splash=sp_name][,splash-time=sp_time]\n"
+    "      [,splash=sp_name][,splash-time=sp_time][,reboot-timeout=rb_time]\n"
     "                'drives': floppy (a), hard disk (c), CD-ROM (d), network (n)\n"
     "                'sp_name': the file's name that would be passed to bios as logo picture, if menu=on\n"
-    "                'sp_time': the period that splash picture last if menu=on, unit is ms\n",
+    "                'sp_time': the period that splash picture last if menu=on, unit is ms\n"
+    "                'rb_timeout': the timeout before guest reboot when boot failed, unit is ms\n",
     QEMU_ARCH_ALL)
 STEXI
-@item -boot [order=@var{drives}][,once=@var{drives}][,menu=on|off][,splash=@var{sp_name}][,splash-time=@var{sp_time}]
+@item -boot [order=@var{drives}][,once=@var{drives}][,menu=on|off][,splash=@var{sp_name}][,splash-time=@var{sp_time}][,reboot-timeout=@var{rb_timeout}]
 @findex -boot
 Specify boot order @var{drives} as a string of drive letters. Valid
 drive letters depend on the target achitecture. The x86 PC uses: a, b
@@ -364,6 +402,11 @@ limitation: The splash file could be a jpeg file or a BMP file in 24 BPP
 format(true color). The resolution should be supported by the SVGA mode, so
 the recommended is 320x240, 640x480, 800x640.
 
+A timeout could be passed to bios, guest will pause for @var{rb_timeout} ms
+when boot failed, then reboot. If @var{rb_timeout} is '-1', guest will not
+reboot, qemu passes '-1' to bios by default. Currently Seabios for X86
+system support it.
+
 @example
 # try to boot from network first, then from hard disk
 qemu-system-i386 -boot order=nc
@@ -452,12 +495,12 @@ ETEXI
 DEF("soundhw", HAS_ARG, QEMU_OPTION_soundhw,
     "-soundhw c1,... enable audio support\n"
     "                and only specified sound cards (comma separated list)\n"
-    "                use -soundhw ? to get the list of supported cards\n"
-    "                use -soundhw all to enable all of them\n", QEMU_ARCH_ALL)
+    "                use '-soundhw help' to get the list of supported cards\n"
+    "                use '-soundhw all' to enable all of them\n", QEMU_ARCH_ALL)
 STEXI
 @item -soundhw @var{card1}[,@var{card2},...] or -soundhw all
 @findex -soundhw
-Enable audio and selected sound hardware. Use ? to print all
+Enable audio and selected sound hardware. Use 'help' to print all
 available sound hardware.
 
 @example
@@ -466,7 +509,7 @@ qemu-system-i386 -soundhw es1370 disk.img
 qemu-system-i386 -soundhw ac97 disk.img
 qemu-system-i386 -soundhw hda disk.img
 qemu-system-i386 -soundhw all disk.img
-qemu-system-i386 -soundhw ?
+qemu-system-i386 -soundhw help
 @end example
 
 Note that Linux's i810_audio OSS kernel (for AC97) module might
@@ -555,16 +598,16 @@ DEF("device", HAS_ARG, QEMU_OPTION_device,
     "-device driver[,prop[=value][,...]]\n"
     "                add device (based on driver)\n"
     "                prop=value,... sets driver properties\n"
-    "                use -device ? to print all possible drivers\n"
-    "                use -device driver,? to print all possible properties\n",
+    "                use '-device help' to print all possible drivers\n"
+    "                use '-device driver,help' to print all possible properties\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -device @var{driver}[,@var{prop}[=@var{value}][,...]]
 @findex -device
 Add device @var{driver}.  @var{prop}=@var{value} sets driver
 properties.  Valid properties depend on the driver.  To get help on
-possible drivers and properties, use @code{-device ?} and
-@code{-device @var{driver},?}.
+possible drivers and properties, use @code{-device help} and
+@code{-device @var{driver},help}.
 ETEXI
 
 DEFHEADING()
@@ -838,7 +881,23 @@ Enable SDL.
 ETEXI
 
 DEF("spice", HAS_ARG, QEMU_OPTION_spice,
-    "-spice <args>   enable spice\n", QEMU_ARCH_ALL)
+    "-spice [port=port][,tls-port=secured-port][,x509-dir=<dir>]\n"
+    "       [,x509-key-file=<file>][,x509-key-password=<file>]\n"
+    "       [,x509-cert-file=<file>][,x509-cacert-file=<file>]\n"
+    "       [,x509-dh-key-file=<file>][,addr=addr][,ipv4|ipv6]\n"
+    "       [,tls-ciphers=<list>]\n"
+    "       [,tls-channel=[main|display|cursor|inputs|record|playback]]\n"
+    "       [,plaintext-channel=[main|display|cursor|inputs|record|playback]]\n"
+    "       [,sasl][,password=<secret>][,disable-ticketing]\n"
+    "       [,image-compression=[auto_glz|auto_lz|quic|glz|lz|off]]\n"
+    "       [,jpeg-wan-compression=[auto|never|always]]\n"
+    "       [,zlib-glz-wan-compression=[auto|never|always]]\n"
+    "       [,streaming-video=[off|all|filter]][,disable-copy-paste]\n"
+    "       [,agent-mouse=[on|off]][,playback-compression=[on|off]]\n"
+    "       [,seamless-migration=[on|off]]\n"
+    "   enable spice\n"
+    "   at least one of {port, tls-port} is mandatory\n",
+    QEMU_ARCH_ALL)
 STEXI
 @item -spice @var{option}[,@var{option}[,...]]
 @findex -spice
@@ -920,6 +979,9 @@ Enable/disable passing mouse events via vdagent.  Default is on.
 @item playback-compression=[on|off]
 Enable/disable audio stream compression (using celt 0.5.1).  Default is on.
 
+@item seamless-migration=[on|off]
+Enable/disable spice seamless migration. Default is off.
+
 @end table
 ETEXI
 
@@ -1034,6 +1096,14 @@ client is specified by the @var{display}. For reverse network
 connections (@var{host}:@var{d},@code{reverse}), the @var{d} argument
 is a TCP port number, not a display number.
 
+@item websocket
+
+Opens an additional TCP listening port dedicated to VNC Websocket connections.
+By defintion the Websocket port is 5700+@var{display}. If @var{host} is
+specified connections will only be allowed from this host.
+As an alternative the Websocket port could be specified by using
+@code{websocket}=@var{port}.
+
 @item password
 
 Require that password based authentication is used for client connections.
@@ -1256,8 +1326,8 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                create a new Network Interface Card and connect it to VLAN 'n'\n"
 #ifdef CONFIG_SLIRP
     "-net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off]\n"
-    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f]\n"
-    "         [,hostfwd=rule][,guestfwd=rule]"
+    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,dnssearch=domain][,tftp=dir]\n"
+    "         [,bootfile=f][,hostfwd=rule][,guestfwd=rule]"
 #ifndef _WIN32
                                              "[,smb=dir[,smbserver=addr]]\n"
 #endif
@@ -1269,7 +1339,7 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                connect the host TAP network interface to VLAN 'n'\n"
 #else
     "-net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostforce=on|off]\n"
-    "                connect the host TAP network interface to VLAN 'n' \n"
+    "                connect the host TAP network interface to VLAN 'n'\n"
     "                use network scripts 'file' (default=" DEFAULT_NETWORK_SCRIPT ")\n"
     "                to configure it and 'dfile' (default=" DEFAULT_NETWORK_DOWN_SCRIPT ")\n"
     "                to deconfigure it\n"
@@ -1335,9 +1405,10 @@ Valid values for @var{type} are
 @code{virtio}, @code{i82551}, @code{i82557b}, @code{i82559er},
 @code{ne2k_pci}, @code{ne2k_isa}, @code{pcnet}, @code{rtl8139},
 @code{e1000}, @code{smc91c111}, @code{lance} and @code{mcf_fec}.
-Not all devices are supported on all targets.  Use -net nic,model=?
+Not all devices are supported on all targets.  Use @code{-net nic,model=help}
 for a list of available devices for your target.
 
+@item -netdev user,id=@var{id}[,@var{option}][,@var{option}][,...]
 @item -net user[,@var{option}][,@var{option}][,...]
 Use the user mode network stack which requires no administrator
 privilege to run. Valid options are:
@@ -1346,6 +1417,7 @@ privilege to run. Valid options are:
 @item vlan=@var{n}
 Connect user mode stack to VLAN @var{n} (@var{n} = 0 is the default).
 
+@item id=@var{id}
 @item name=@var{name}
 Assign symbolic name for use in monitor commands.
 
@@ -1364,7 +1436,7 @@ able to contact the host and no guest IP packets will be routed over the host
 to the outside. This option does not affect any explicitly set forwarding rules.
 
 @item hostname=@var{name}
-Specifies the client hostname reported by the builtin DHCP server.
+Specifies the client hostname reported by the built-in DHCP server.
 
 @item dhcpstart=@var{addr}
 Specify the first of the 16 IPs the built-in DHCP server can assign. Default
@@ -1375,6 +1447,18 @@ Specify the guest-visible address of the virtual nameserver. The address must
 be different from the host address. Default is the 3rd IP in the guest network,
 i.e. x.x.x.3.
 
+@item dnssearch=@var{domain}
+Provides an entry for the domain-search list sent by the built-in
+DHCP server. More than one domain suffix can be transmitted by specifying
+this option multiple times. If supported, this will cause the guest to
+automatically try to append the given domain suffix(es) in case a domain name
+can not be resolved.
+
+Example:
+@example
+qemu -net user,dnssearch=mgmt.example.org,dnssearch=example.org [...]
+@end example
+
 @item tftp=@var{dir}
 When using the user mode network stack, activate a built-in TFTP
 server. The files in @var{dir} will be exposed as the root of a TFTP server.
@@ -1471,6 +1555,7 @@ processed and applied to -net user. Mixing them with the new configuration
 syntax gives undefined results. Their use for new applications is discouraged
 as they will be removed from future versions.
 
+@item -netdev tap,id=@var{id}[,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
 @item -net tap[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
 Connect the host TAP network interface @var{name} to VLAN @var{n}.
 
@@ -1510,6 +1595,7 @@ qemu-system-i386 linux.img \
                  -net nic -net tap,"helper=/usr/local/libexec/qemu-bridge-helper"
 @end example
 
+@item -netdev bridge,id=@var{id}[,br=@var{bridge}][,helper=@var{helper}]
 @item -net bridge[,vlan=@var{n}][,name=@var{name}][,br=@var{bridge}][,helper=@var{helper}]
 Connect a host TAP network interface to a host bridge device.
 
@@ -1532,6 +1618,7 @@ qemu-system-i386 linux.img -net bridge -net nic,model=virtio
 qemu-system-i386 linux.img -net bridge,br=qemubr0 -net nic,model=virtio
 @end example
 
+@item -netdev socket,id=@var{id}[,fd=@var{h}][,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
 @item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}] [,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
 
 Connect the VLAN @var{n} to a remote VLAN in another QEMU virtual
@@ -1554,6 +1641,7 @@ qemu-system-i386 linux.img \
                  -net socket,connect=127.0.0.1:1234
 @end example
 
+@item -netdev socket,id=@var{id}[,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
 @item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
 
 Create a VLAN @var{n} shared with another QEMU virtual
@@ -1605,6 +1693,7 @@ qemu-system-i386 linux.img \
                  -net socket,mcast=239.192.168.1:1102,localaddr=1.2.3.4
 @end example
 
+@item -netdev vde,id=@var{id}[,sock=@var{socketpath}][,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
 @item -net vde[,vlan=@var{n}][,name=@var{name}][,sock=@var{socketpath}] [,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
 Connect VLAN @var{n} to PORT @var{n} of a vde switch running on host and
 listening for incoming connections on @var{socketpath}. Use GROUP @var{groupname}
@@ -1647,6 +1736,7 @@ DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
     "-chardev msmouse,id=id[,mux=on|off]\n"
     "-chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]]\n"
     "         [,mux=on|off]\n"
+    "-chardev memory,id=id,maxcapacity=maxcapacity\n"
     "-chardev file,id=id,path=path[,mux=on|off]\n"
     "-chardev pipe,id=id,path=path[,mux=on|off]\n"
 #ifdef _WIN32
@@ -1661,13 +1751,16 @@ DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
 #endif
 #if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
         || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+    "-chardev serial,id=id,path=path[,mux=on|off]\n"
     "-chardev tty,id=id,path=path[,mux=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
+    "-chardev parallel,id=id,path=path[,mux=on|off]\n"
     "-chardev parport,id=id,path=path[,mux=on|off]\n"
 #endif
 #if defined(CONFIG_SPICE)
     "-chardev spicevmc,id=id,name=name[,debug=debug]\n"
+    "-chardev spiceport,id=id,name=name[,debug=debug]\n"
 #endif
     , QEMU_ARCH_ALL
 )
@@ -1685,6 +1778,7 @@ Backend is one of:
 @option{udp},
 @option{msmouse},
 @option{vc},
+@option{memory},
 @option{file},
 @option{pipe},
 @option{console},
@@ -1693,8 +1787,10 @@ Backend is one of:
 @option{stdio},
 @option{braille},
 @option{tty},
+@option{parallel},
 @option{parport},
 @option{spicevmc}.
+@option{spiceport}.
 The specific backend will determine the applicable options.
 
 All devices must have an id, which can be any string up to 127 characters long.
@@ -1791,6 +1887,14 @@ the console, in pixels.
 @option{cols} and @option{rows} specify that the console be sized to fit a text
 console with the given dimensions.
 
+@item -chardev memory ,id=@var{id} ,maxcapacity=@var{maxcapacity}
+
+Create a circular buffer with fixed size indicated by optionally @option{maxcapacity}
+which will be default 64K if it is not given.
+
+@option{maxcapacity} specifies the max capacity of the size of circular buffer
+to create. Should be power of 2.
+
 @item -chardev file ,id=@var{id} ,path=@var{path}
 
 Log all traffic received from the guest to a file.
@@ -1827,8 +1931,8 @@ take any options.
 
 Send traffic from the guest to a serial device on the host.
 
-@option{serial} is
-only available on Windows hosts.
+On Unix hosts serial will actually accept any tty device,
+not only serial lines.
 
 @option{path} specifies the name of the serial device to open.
 
@@ -1854,16 +1958,15 @@ Connect to a local BrlAPI server. @option{braille} does not take any options.
 
 @item -chardev tty ,id=@var{id} ,path=@var{path}
 
-Connect to a local tty device.
-
 @option{tty} is only available on Linux, Sun, FreeBSD, NetBSD, OpenBSD and
-DragonFlyBSD hosts.
+DragonFlyBSD hosts.  It is an alias for -serial.
 
 @option{path} specifies the path to the tty. @option{path} is required.
 
+@item -chardev parallel ,id=@var{id} ,path=@var{path}
 @item -chardev parport ,id=@var{id} ,path=@var{path}
 
-@option{parport} is only available on Linux, FreeBSD and DragonFlyBSD hosts.
+@option{parallel} is only available on Linux, FreeBSD and DragonFlyBSD hosts.
 
 Connect to a local parallel port.
 
@@ -1880,6 +1983,17 @@ required.
 
 Connect to a spice virtual machine channel, such as vdiport.
 
+@item -chardev spiceport ,id=@var{id} ,debug=@var{debug}, name=@var{name}
+
+@option{spiceport} is only available when spice support is built in.
+
+@option{debug} debug level for spicevmc
+
+@option{name} name of spice port to connect to
+
+Connect to a spice port, allowing a Spice client to handle the traffic
+identified by a name (preferably a fqdn).
+
 @end table
 ETEXI
 
@@ -1985,6 +2099,23 @@ qemu-system-i386 --drive file=sheepdog:192.0.2.1:30000:MyVirtualMachine
 
 See also @url{http://http://www.osrg.net/sheepdog/}.
 
+@item GlusterFS
+GlusterFS is an user space distributed file system.
+QEMU supports the use of GlusterFS volumes for hosting VM disk images using
+TCP, Unix Domain Sockets and RDMA transport protocols.
+
+Syntax for specifying a VM disk image on GlusterFS volume is
+@example
+gluster[+transport]://[server[:port]]/volname/image[?socket=...]
+@end example
+
+
+Example
+@example
+qemu-system-x86_84 --drive file=gluster://192.0.2.1/testvol/a.img
+@end example
+
+See also @url{http://www.gluster.org}.
 @end table
 ETEXI
 
@@ -2361,7 +2492,7 @@ Shorthand for -gdb tcp::1234, i.e. open a gdbserver on TCP port 1234
 ETEXI
 
 DEF("d", HAS_ARG, QEMU_OPTION_d, \
-    "-d item1,...    output log to /tmp/qemu.log (use -d ? for a list of log items)\n",
+    "-d item1,...    output log to /tmp/qemu.log (use '-d help' for a list of log items)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -d
@@ -2496,13 +2627,13 @@ ETEXI
 
 DEF("clock", HAS_ARG, QEMU_OPTION_clock, \
     "-clock          force the use of the given methods for timer alarm.\n" \
-    "                To see what timers are available use -clock ?\n",
+    "                To see what timers are available use '-clock help'\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -clock @var{method}
 @findex -clock
 Force the use of the given methods for timer alarm. To see what timers
-are available use -clock ?.
+are available use @code{-clock help}.
 ETEXI
 
 HXCOMM Options deprecated by -rtc
@@ -2571,7 +2702,7 @@ watchdog with a single timer, or @code{i6300esb} (Intel 6300ESB I/O
 controller hub) which is a much more featureful PCI-based dual-timer
 watchdog.  Choose a model for which your guest has drivers.
 
-Use @code{-watchdog ?} to list available hardware models.  Only one
+Use @code{-watchdog help} to list available hardware models.  Only one
 watchdog can be enabled for a guest.
 ETEXI
 
@@ -2819,6 +2950,30 @@ STEXI
 Enable FIPS 140-2 compliance mode.
 ETEXI
 
+HXCOMM Deprecated by -machine accel=tcg property
+DEF("no-kvm", 0, QEMU_OPTION_no_kvm, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated by kvm-pit driver properties
+DEF("no-kvm-pit-reinjection", 0, QEMU_OPTION_no_kvm_pit_reinjection,
+    "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated (ignored)
+DEF("no-kvm-pit", 0, QEMU_OPTION_no_kvm_pit, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated by -machine kernel_irqchip=on|off property
+DEF("no-kvm-irqchip", 0, QEMU_OPTION_no_kvm_irqchip, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated (ignored)
+DEF("tdf", 0, QEMU_OPTION_tdf,"", QEMU_ARCH_ALL)
+
+DEF("object", HAS_ARG, QEMU_OPTION_object,
+    "-object TYPENAME[,PROP1=VALUE1,...]\n"
+    "                create an new object of type TYPENAME setting properties\n"
+    "                in the order they are specified.  Note that the 'id'\n"
+    "                property must be set.  These objects are placed in the\n"
+    "                '/objects' path.\n",
+    QEMU_ARCH_ALL)
+
 HXCOMM This is the last statement. Insert new options before this line!
 STEXI
 @end table
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index 64329a3c09d..031da1dfee1 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -14,7 +14,7 @@
  */
 #include <stdio.h>
 #include <seccomp.h>
-#include "qemu-seccomp.h"
+#include "sysemu/seccomp.h"
 
 struct QemuSeccompSyscall {
     int32_t num;
@@ -26,8 +26,12 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(timer_gettime), 254 },
     { SCMP_SYS(futex), 253 },
     { SCMP_SYS(select), 252 },
+#if defined(__x86_64__)
     { SCMP_SYS(recvfrom), 251 },
     { SCMP_SYS(sendto), 250 },
+#elif defined(__i386__)
+    { SCMP_SYS(socketcall), 250 },
+#endif
     { SCMP_SYS(read), 249 },
     { SCMP_SYS(brk), 248 },
     { SCMP_SYS(clone), 247 },
@@ -36,15 +40,30 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(execve), 245 },
     { SCMP_SYS(open), 245 },
     { SCMP_SYS(ioctl), 245 },
+#if defined(__x86_64__)
+    { SCMP_SYS(socket), 245 },
+    { SCMP_SYS(setsockopt), 245 },
     { SCMP_SYS(recvmsg), 245 },
     { SCMP_SYS(sendmsg), 245 },
     { SCMP_SYS(accept), 245 },
     { SCMP_SYS(connect), 245 },
+    { SCMP_SYS(socketpair), 245 },
+    { SCMP_SYS(bind), 245 },
+    { SCMP_SYS(listen), 245 },
+    { SCMP_SYS(semget), 245 },
+#elif defined(__i386__)
+    { SCMP_SYS(ipc), 245 },
+#endif
     { SCMP_SYS(gettimeofday), 245 },
     { SCMP_SYS(readlink), 245 },
     { SCMP_SYS(access), 245 },
     { SCMP_SYS(prctl), 245 },
     { SCMP_SYS(signalfd), 245 },
+    { SCMP_SYS(getrlimit), 245 },
+    { SCMP_SYS(set_tid_address), 245 },
+    { SCMP_SYS(statfs), 245 },
+    { SCMP_SYS(unlink), 245 },
+    { SCMP_SYS(wait4), 245 },
 #if defined(__i386__)
     { SCMP_SYS(fcntl64), 245 },
     { SCMP_SYS(fstat64), 245 },
@@ -56,30 +75,33 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(sigreturn), 245 },
     { SCMP_SYS(_newselect), 245 },
     { SCMP_SYS(_llseek), 245 },
-    { SCMP_SYS(mmap2), 245},
+    { SCMP_SYS(mmap2), 245 },
     { SCMP_SYS(sigprocmask), 245 },
-#elif defined(__x86_64__)
-    { SCMP_SYS(sched_getparam), 245},
-    { SCMP_SYS(sched_getscheduler), 245},
-    { SCMP_SYS(fstat), 245},
-    { SCMP_SYS(clock_getres), 245},
-    { SCMP_SYS(sched_get_priority_min), 245},
-    { SCMP_SYS(sched_get_priority_max), 245},
-    { SCMP_SYS(stat), 245},
-    { SCMP_SYS(socket), 245},
-    { SCMP_SYS(setsockopt), 245},
-    { SCMP_SYS(uname), 245},
-    { SCMP_SYS(semget), 245},
 #endif
+    { SCMP_SYS(sched_getparam), 245 },
+    { SCMP_SYS(sched_getscheduler), 245 },
+    { SCMP_SYS(fstat), 245 },
+    { SCMP_SYS(clock_getres), 245 },
+    { SCMP_SYS(sched_get_priority_min), 245 },
+    { SCMP_SYS(sched_get_priority_max), 245 },
+    { SCMP_SYS(stat), 245 },
+    { SCMP_SYS(uname), 245 },
     { SCMP_SYS(eventfd2), 245 },
     { SCMP_SYS(dup), 245 },
+    { SCMP_SYS(dup2), 245 },
+    { SCMP_SYS(dup3), 245 },
     { SCMP_SYS(gettid), 245 },
+    { SCMP_SYS(getgid), 245 },
+    { SCMP_SYS(getegid), 245 },
+    { SCMP_SYS(getuid), 245 },
+    { SCMP_SYS(geteuid), 245 },
     { SCMP_SYS(timer_create), 245 },
     { SCMP_SYS(exit), 245 },
     { SCMP_SYS(clock_gettime), 245 },
     { SCMP_SYS(time), 245 },
     { SCMP_SYS(restart_syscall), 245 },
     { SCMP_SYS(pwrite64), 245 },
+    { SCMP_SYS(nanosleep), 245 },
     { SCMP_SYS(chown), 245 },
     { SCMP_SYS(openat), 245 },
     { SCMP_SYS(getdents), 245 },
@@ -93,8 +115,6 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(lseek), 245 },
     { SCMP_SYS(pselect6), 245 },
     { SCMP_SYS(fork), 245 },
-    { SCMP_SYS(bind), 245 },
-    { SCMP_SYS(listen), 245 },
     { SCMP_SYS(eventfd), 245 },
     { SCMP_SYS(rt_sigprocmask), 245 },
     { SCMP_SYS(write), 244 },
@@ -104,10 +124,112 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(pipe2), 242 },
     { SCMP_SYS(munmap), 242 },
     { SCMP_SYS(mremap), 242 },
+    { SCMP_SYS(fdatasync), 242 },
+    { SCMP_SYS(close), 242 },
+    { SCMP_SYS(rt_sigpending), 242 },
+    { SCMP_SYS(rt_sigtimedwait), 242 },
+    { SCMP_SYS(readv), 242 },
+    { SCMP_SYS(writev), 242 },
+    { SCMP_SYS(preadv), 242 },
+    { SCMP_SYS(pwritev), 242 },
+    { SCMP_SYS(setrlimit), 242 },
+    { SCMP_SYS(ftruncate), 242 },
+    { SCMP_SYS(lstat), 242 },
+    { SCMP_SYS(pipe), 242 },
+    { SCMP_SYS(umask), 242 },
+    { SCMP_SYS(chdir), 242 },
+    { SCMP_SYS(setitimer), 242 },
+    { SCMP_SYS(setsid), 242 },
+    { SCMP_SYS(poll), 242 },
+    { SCMP_SYS(epoll_create), 242 },
+    { SCMP_SYS(epoll_ctl), 242 },
+    { SCMP_SYS(epoll_wait), 242 },
+#if defined(__i386__)
+    { SCMP_SYS(waitpid), 242 },
+#elif defined(__x86_64__)
     { SCMP_SYS(getsockname), 242 },
     { SCMP_SYS(getpeername), 242 },
-    { SCMP_SYS(fdatasync), 242 },
-    { SCMP_SYS(close), 242 }
+    { SCMP_SYS(accept4), 242 },
+    { SCMP_SYS(newfstatat), 241 },
+    { SCMP_SYS(shutdown), 241 },
+    { SCMP_SYS(getsockopt), 241 },
+    { SCMP_SYS(semctl), 241 },
+    { SCMP_SYS(semop), 241 },
+    { SCMP_SYS(semtimedop), 241 },
+    { SCMP_SYS(epoll_ctl_old), 241 },
+    { SCMP_SYS(epoll_wait_old), 241 },
+#endif
+    { SCMP_SYS(epoll_pwait), 241 },
+    { SCMP_SYS(epoll_create1), 241 },
+    { SCMP_SYS(ppoll), 241 },
+    { SCMP_SYS(creat), 241 },
+    { SCMP_SYS(link), 241 },
+    { SCMP_SYS(getpid), 241 },
+    { SCMP_SYS(getppid), 241 },
+    { SCMP_SYS(getpgrp), 241 },
+    { SCMP_SYS(getpgid), 241 },
+    { SCMP_SYS(getsid), 241 },
+    { SCMP_SYS(getdents64), 241 },
+    { SCMP_SYS(getresuid), 241 },
+    { SCMP_SYS(getresgid), 241 },
+    { SCMP_SYS(getgroups), 241 },
+#if defined(__i386__)
+    { SCMP_SYS(getresuid32), 241 },
+    { SCMP_SYS(getresgid32), 241 },
+    { SCMP_SYS(getgroups32), 241 },
+    { SCMP_SYS(signal), 241 },
+    { SCMP_SYS(sigaction), 241 },
+    { SCMP_SYS(sigsuspend), 241 },
+    { SCMP_SYS(sigpending), 241 },
+    { SCMP_SYS(truncate64), 241 },
+    { SCMP_SYS(ftruncate64), 241 },
+    { SCMP_SYS(fchown32), 241 },
+    { SCMP_SYS(chown32), 241 },
+    { SCMP_SYS(lchown32), 241 },
+    { SCMP_SYS(statfs64), 241 },
+    { SCMP_SYS(fstatfs64), 241 },
+    { SCMP_SYS(fstatat64), 241 },
+    { SCMP_SYS(lstat64), 241 },
+    { SCMP_SYS(sendfile64), 241 },
+    { SCMP_SYS(ugetrlimit), 241 },
+#endif
+    { SCMP_SYS(alarm), 241 },
+    { SCMP_SYS(rt_sigsuspend), 241 },
+    { SCMP_SYS(rt_sigqueueinfo), 241 },
+    { SCMP_SYS(rt_tgsigqueueinfo), 241 },
+    { SCMP_SYS(sigaltstack), 241 },
+    { SCMP_SYS(signalfd4), 241 },
+    { SCMP_SYS(truncate), 241 },
+    { SCMP_SYS(fchown), 241 },
+    { SCMP_SYS(lchown), 241 },
+    { SCMP_SYS(fchownat), 241 },
+    { SCMP_SYS(fstatfs), 241 },
+    { SCMP_SYS(sendfile), 241 },
+    { SCMP_SYS(getitimer), 241 },
+    { SCMP_SYS(syncfs), 241 },
+    { SCMP_SYS(fsync), 241 },
+    { SCMP_SYS(fchdir), 241 },
+    { SCMP_SYS(flock), 241 },
+    { SCMP_SYS(msync), 241 },
+    { SCMP_SYS(sched_setparam), 241 },
+    { SCMP_SYS(sched_setscheduler), 241 },
+    { SCMP_SYS(sched_yield), 241 },
+    { SCMP_SYS(sched_rr_get_interval), 241 },
+    { SCMP_SYS(sched_setaffinity), 241 },
+    { SCMP_SYS(sched_getaffinity), 241 },
+    { SCMP_SYS(readahead), 241 },
+    { SCMP_SYS(timer_getoverrun), 241 },
+    { SCMP_SYS(unlinkat), 241 },
+    { SCMP_SYS(readlinkat), 241 },
+    { SCMP_SYS(faccessat), 241 },
+    { SCMP_SYS(get_robust_list), 241 },
+    { SCMP_SYS(splice), 241 },
+    { SCMP_SYS(vmsplice), 241 },
+    { SCMP_SYS(getcpu), 241 },
+    { SCMP_SYS(sendmmsg), 241 },
+    { SCMP_SYS(recvmmsg), 241 },
+    { SCMP_SYS(prlimit64), 241 },
+    { SCMP_SYS(waitid), 241 }
 };
 
 int seccomp_start(void)
diff --git a/qemu-sockets.c b/qemu-sockets.c
deleted file mode 100644
index 361d890da3e..00000000000
--- a/qemu-sockets.c
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- *  inet and unix socket functions for qemu
- *
- *  (c) 2008 Gerd Hoffmann <kraxel@redhat.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; under version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
-
-#include "qemu_socket.h"
-#include "qemu-common.h" /* for qemu_isdigit */
-
-#ifndef AI_ADDRCONFIG
-# define AI_ADDRCONFIG 0
-#endif
-
-static const int on=1, off=0;
-
-/* used temporarely until all users are converted to QemuOpts */
-static QemuOptsList dummy_opts = {
-    .name = "dummy",
-    .head = QTAILQ_HEAD_INITIALIZER(dummy_opts.head),
-    .desc = {
-        {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "host",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "port",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "to",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "block",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end if list */ }
-    },
-};
-
-static int inet_getport(struct addrinfo *e)
-{
-    struct sockaddr_in *i4;
-    struct sockaddr_in6 *i6;
-
-    switch (e->ai_family) {
-    case PF_INET6:
-        i6 = (void*)e->ai_addr;
-        return ntohs(i6->sin6_port);
-    case PF_INET:
-        i4 = (void*)e->ai_addr;
-        return ntohs(i4->sin_port);
-    default:
-        return 0;
-    }
-}
-
-static void inet_setport(struct addrinfo *e, int port)
-{
-    struct sockaddr_in *i4;
-    struct sockaddr_in6 *i6;
-
-    switch (e->ai_family) {
-    case PF_INET6:
-        i6 = (void*)e->ai_addr;
-        i6->sin6_port = htons(port);
-        break;
-    case PF_INET:
-        i4 = (void*)e->ai_addr;
-        i4->sin_port = htons(port);
-        break;
-    }
-}
-
-const char *inet_strfamily(int family)
-{
-    switch (family) {
-    case PF_INET6: return "ipv6";
-    case PF_INET:  return "ipv4";
-    case PF_UNIX:  return "unix";
-    }
-    return "unknown";
-}
-
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
-{
-    struct addrinfo ai,*res,*e;
-    const char *addr;
-    char port[33];
-    char uaddr[INET6_ADDRSTRLEN+1];
-    char uport[33];
-    int slisten, rc, to, port_min, port_max, p;
-
-    memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
-    ai.ai_socktype = SOCK_STREAM;
-
-    if ((qemu_opt_get(opts, "host") == NULL) ||
-        (qemu_opt_get(opts, "port") == NULL)) {
-        fprintf(stderr, "%s: host and/or port not specified\n", __FUNCTION__);
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-        return -1;
-    }
-    pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
-    addr = qemu_opt_get(opts, "host");
-
-    to = qemu_opt_get_number(opts, "to", 0);
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
-
-    /* lookup */
-    if (port_offset)
-        snprintf(port, sizeof(port), "%d", atoi(port) + port_offset);
-    rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
-    if (rc != 0) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-        return -1;
-    }
-
-    /* create socket + bind */
-    for (e = res; e != NULL; e = e->ai_next) {
-        getnameinfo((struct sockaddr*)e->ai_addr,e->ai_addrlen,
-		        uaddr,INET6_ADDRSTRLEN,uport,32,
-		        NI_NUMERICHOST | NI_NUMERICSERV);
-        slisten = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
-        if (slisten < 0) {
-            fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-                    inet_strfamily(e->ai_family), strerror(errno));
-            if (!e->ai_next) {
-                error_set(errp, QERR_SOCKET_CREATE_FAILED);
-            }
-            continue;
-        }
-
-        setsockopt(slisten,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
-#ifdef IPV6_V6ONLY
-        if (e->ai_family == PF_INET6) {
-            /* listen on both ipv4 and ipv6 */
-            setsockopt(slisten,IPPROTO_IPV6,IPV6_V6ONLY,(void*)&off,
-                sizeof(off));
-        }
-#endif
-
-        port_min = inet_getport(e);
-        port_max = to ? to + port_offset : port_min;
-        for (p = port_min; p <= port_max; p++) {
-            inet_setport(e, p);
-            if (bind(slisten, e->ai_addr, e->ai_addrlen) == 0) {
-                goto listen;
-            }
-            if (p == port_max) {
-                fprintf(stderr,"%s: bind(%s,%s,%d): %s\n", __FUNCTION__,
-                        inet_strfamily(e->ai_family), uaddr, inet_getport(e),
-                        strerror(errno));
-                if (!e->ai_next) {
-                    error_set(errp, QERR_SOCKET_BIND_FAILED);
-                }
-            }
-        }
-        closesocket(slisten);
-    }
-    fprintf(stderr, "%s: FAILED\n", __FUNCTION__);
-    freeaddrinfo(res);
-    return -1;
-
-listen:
-    if (listen(slisten,1) != 0) {
-        error_set(errp, QERR_SOCKET_LISTEN_FAILED);
-        perror("listen");
-        closesocket(slisten);
-        freeaddrinfo(res);
-        return -1;
-    }
-    snprintf(uport, sizeof(uport), "%d", inet_getport(e) - port_offset);
-    qemu_opt_set(opts, "host", uaddr);
-    qemu_opt_set(opts, "port", uport);
-    qemu_opt_set(opts, "ipv6", (e->ai_family == PF_INET6) ? "on" : "off");
-    qemu_opt_set(opts, "ipv4", (e->ai_family != PF_INET6) ? "on" : "off");
-    freeaddrinfo(res);
-    return slisten;
-}
-
-int inet_connect_opts(QemuOpts *opts, bool *in_progress, Error **errp)
-{
-    struct addrinfo ai,*res,*e;
-    const char *addr;
-    const char *port;
-    char uaddr[INET6_ADDRSTRLEN+1];
-    char uport[33];
-    int sock,rc;
-    bool block;
-
-    memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
-    ai.ai_socktype = SOCK_STREAM;
-
-    if (in_progress) {
-        *in_progress = false;
-    }
-
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
-    block = qemu_opt_get_bool(opts, "block", 0);
-    if (addr == NULL || port == NULL) {
-        fprintf(stderr, "inet_connect: host and/or port not specified\n");
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-        return -1;
-    }
-
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
-
-    /* lookup */
-    if (0 != (rc = getaddrinfo(addr, port, &ai, &res))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-	return -1;
-    }
-
-    for (e = res; e != NULL; e = e->ai_next) {
-        if (getnameinfo((struct sockaddr*)e->ai_addr,e->ai_addrlen,
-                            uaddr,INET6_ADDRSTRLEN,uport,32,
-                            NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-            fprintf(stderr,"%s: getnameinfo: oops\n", __FUNCTION__);
-            continue;
-        }
-        sock = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
-        if (sock < 0) {
-            fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-            inet_strfamily(e->ai_family), strerror(errno));
-            continue;
-        }
-        setsockopt(sock,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
-        if (!block) {
-            socket_set_nonblock(sock);
-        }
-        /* connect to peer */
-        do {
-            rc = 0;
-            if (connect(sock, e->ai_addr, e->ai_addrlen) < 0) {
-                rc = -socket_error();
-            }
-        } while (rc == -EINTR);
-
-  #ifdef _WIN32
-        if (!block && (rc == -EINPROGRESS || rc == -EWOULDBLOCK
-                       || rc == -WSAEALREADY)) {
-  #else
-        if (!block && (rc == -EINPROGRESS)) {
-  #endif
-            if (in_progress) {
-                *in_progress = true;
-            }
-        } else if (rc < 0) {
-            if (NULL == e->ai_next)
-                fprintf(stderr, "%s: connect(%s,%s,%s,%s): %s\n", __FUNCTION__,
-                        inet_strfamily(e->ai_family),
-                        e->ai_canonname, uaddr, uport, strerror(errno));
-            closesocket(sock);
-            continue;
-        }
-        freeaddrinfo(res);
-        return sock;
-    }
-    error_set(errp, QERR_SOCKET_CONNECT_FAILED);
-    freeaddrinfo(res);
-    return -1;
-}
-
-int inet_dgram_opts(QemuOpts *opts)
-{
-    struct addrinfo ai, *peer = NULL, *local = NULL;
-    const char *addr;
-    const char *port;
-    char uaddr[INET6_ADDRSTRLEN+1];
-    char uport[33];
-    int sock = -1, rc;
-
-    /* lookup peer addr */
-    memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
-    ai.ai_socktype = SOCK_DGRAM;
-
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
-    if (addr == NULL || strlen(addr) == 0) {
-        addr = "localhost";
-    }
-    if (port == NULL || strlen(port) == 0) {
-        fprintf(stderr, "inet_dgram: port not specified\n");
-        return -1;
-    }
-
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
-
-    if (0 != (rc = getaddrinfo(addr, port, &ai, &peer))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-	return -1;
-    }
-
-    /* lookup local addr */
-    memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_PASSIVE;
-    ai.ai_family = peer->ai_family;
-    ai.ai_socktype = SOCK_DGRAM;
-
-    addr = qemu_opt_get(opts, "localaddr");
-    port = qemu_opt_get(opts, "localport");
-    if (addr == NULL || strlen(addr) == 0) {
-        addr = NULL;
-    }
-    if (!port || strlen(port) == 0)
-        port = "0";
-
-    if (0 != (rc = getaddrinfo(addr, port, &ai, &local))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        return -1;
-    }
-
-    /* create socket */
-    sock = qemu_socket(peer->ai_family, peer->ai_socktype, peer->ai_protocol);
-    if (sock < 0) {
-        fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-                inet_strfamily(peer->ai_family), strerror(errno));
-        goto err;
-    }
-    setsockopt(sock,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
-
-    /* bind socket */
-    if (getnameinfo((struct sockaddr*)local->ai_addr,local->ai_addrlen,
-                    uaddr,INET6_ADDRSTRLEN,uport,32,
-                    NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-        fprintf(stderr, "%s: getnameinfo: oops\n", __FUNCTION__);
-        goto err;
-    }
-    if (bind(sock, local->ai_addr, local->ai_addrlen) < 0) {
-        fprintf(stderr,"%s: bind(%s,%s,%d): OK\n", __FUNCTION__,
-                inet_strfamily(local->ai_family), uaddr, inet_getport(local));
-        goto err;
-    }
-
-    /* connect to peer */
-    if (getnameinfo((struct sockaddr*)peer->ai_addr, peer->ai_addrlen,
-                    uaddr, INET6_ADDRSTRLEN, uport, 32,
-                    NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-        fprintf(stderr, "%s: getnameinfo: oops\n", __FUNCTION__);
-        goto err;
-    }
-    if (connect(sock,peer->ai_addr,peer->ai_addrlen) < 0) {
-        fprintf(stderr, "%s: connect(%s,%s,%s,%s): %s\n", __FUNCTION__,
-                inet_strfamily(peer->ai_family),
-                peer->ai_canonname, uaddr, uport, strerror(errno));
-        goto err;
-    }
-
-    freeaddrinfo(local);
-    freeaddrinfo(peer);
-    return sock;
-
-err:
-    if (-1 != sock)
-        closesocket(sock);
-    if (local)
-        freeaddrinfo(local);
-    if (peer)
-        freeaddrinfo(peer);
-    return -1;
-}
-
-/* compatibility wrapper */
-static int inet_parse(QemuOpts *opts, const char *str)
-{
-    const char *optstr, *h;
-    char addr[64];
-    char port[33];
-    int pos;
-
-    /* parse address */
-    if (str[0] == ':') {
-        /* no host given */
-        addr[0] = '\0';
-        if (1 != sscanf(str,":%32[^,]%n",port,&pos)) {
-            fprintf(stderr, "%s: portonly parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
-        }
-    } else if (str[0] == '[') {
-        /* IPv6 addr */
-        if (2 != sscanf(str,"[%64[^]]]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: ipv6 parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
-        }
-        qemu_opt_set(opts, "ipv6", "on");
-    } else if (qemu_isdigit(str[0])) {
-        /* IPv4 addr */
-        if (2 != sscanf(str,"%64[0-9.]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: ipv4 parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
-        }
-        qemu_opt_set(opts, "ipv4", "on");
-    } else {
-        /* hostname */
-        if (2 != sscanf(str,"%64[^:]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: hostname parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
-        }
-    }
-    qemu_opt_set(opts, "host", addr);
-    qemu_opt_set(opts, "port", port);
-
-    /* parse options */
-    optstr = str + pos;
-    h = strstr(optstr, ",to=");
-    if (h)
-        qemu_opt_set(opts, "to", h+4);
-    if (strstr(optstr, ",ipv4"))
-        qemu_opt_set(opts, "ipv4", "on");
-    if (strstr(optstr, ",ipv6"))
-        qemu_opt_set(opts, "ipv6", "on");
-    return 0;
-}
-
-int inet_listen(const char *str, char *ostr, int olen,
-                int socktype, int port_offset, Error **errp)
-{
-    QemuOpts *opts;
-    char *optstr;
-    int sock = -1;
-
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-    if (inet_parse(opts, str) == 0) {
-        sock = inet_listen_opts(opts, port_offset, errp);
-        if (sock != -1 && ostr) {
-            optstr = strchr(str, ',');
-            if (qemu_opt_get_bool(opts, "ipv6", 0)) {
-                snprintf(ostr, olen, "[%s]:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
-                         optstr ? optstr : "");
-            } else {
-                snprintf(ostr, olen, "%s:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
-                         optstr ? optstr : "");
-            }
-        }
-    } else {
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-    }
-    qemu_opts_del(opts);
-    return sock;
-}
-
-int inet_connect(const char *str, bool block, bool *in_progress, Error **errp)
-{
-    QemuOpts *opts;
-    int sock = -1;
-
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-    if (inet_parse(opts, str) == 0) {
-        if (block) {
-            qemu_opt_set(opts, "block", "on");
-        }
-        sock = inet_connect_opts(opts, in_progress, errp);
-    } else {
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-    }
-    qemu_opts_del(opts);
-    return sock;
-}
-
-#ifndef _WIN32
-
-int unix_listen_opts(QemuOpts *opts)
-{
-    struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
-    int sock, fd;
-
-    sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (sock < 0) {
-        perror("socket(unix)");
-        return -1;
-    }
-
-    memset(&un, 0, sizeof(un));
-    un.sun_family = AF_UNIX;
-    if (path && strlen(path)) {
-        snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
-    } else {
-        char *tmpdir = getenv("TMPDIR");
-        snprintf(un.sun_path, sizeof(un.sun_path), "%s/qemu-socket-XXXXXX",
-                 tmpdir ? tmpdir : "/tmp");
-        /*
-         * This dummy fd usage silences the mktemp() unsecure warning.
-         * Using mkstemp() doesn't make things more secure here
-         * though.  bind() complains about existing files, so we have
-         * to unlink first and thus re-open the race window.  The
-         * worst case possible is bind() failing, i.e. a DoS attack.
-         */
-        fd = mkstemp(un.sun_path); close(fd);
-        qemu_opt_set(opts, "path", un.sun_path);
-    }
-
-    unlink(un.sun_path);
-    if (bind(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        fprintf(stderr, "bind(unix:%s): %s\n", un.sun_path, strerror(errno));
-        goto err;
-    }
-    if (listen(sock, 1) < 0) {
-        fprintf(stderr, "listen(unix:%s): %s\n", un.sun_path, strerror(errno));
-        goto err;
-    }
-
-    return sock;
-
-err:
-    closesocket(sock);
-    return -1;
-}
-
-int unix_connect_opts(QemuOpts *opts)
-{
-    struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
-    int sock;
-
-    if (NULL == path) {
-        fprintf(stderr, "unix connect: no path specified\n");
-        return -1;
-    }
-
-    sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (sock < 0) {
-        perror("socket(unix)");
-        return -1;
-    }
-
-    memset(&un, 0, sizeof(un));
-    un.sun_family = AF_UNIX;
-    snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
-    if (connect(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        fprintf(stderr, "connect(unix:%s): %s\n", path, strerror(errno));
-        close(sock);
-	return -1;
-    }
-
-    return sock;
-}
-
-/* compatibility wrapper */
-int unix_listen(const char *str, char *ostr, int olen)
-{
-    QemuOpts *opts;
-    char *path, *optstr;
-    int sock, len;
-
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-
-    optstr = strchr(str, ',');
-    if (optstr) {
-        len = optstr - str;
-        if (len) {
-            path = g_malloc(len+1);
-            snprintf(path, len+1, "%.*s", len, str);
-            qemu_opt_set(opts, "path", path);
-            g_free(path);
-        }
-    } else {
-        qemu_opt_set(opts, "path", str);
-    }
-
-    sock = unix_listen_opts(opts);
-
-    if (sock != -1 && ostr)
-        snprintf(ostr, olen, "%s%s", qemu_opt_get(opts, "path"), optstr ? optstr : "");
-    qemu_opts_del(opts);
-    return sock;
-}
-
-int unix_connect(const char *path)
-{
-    QemuOpts *opts;
-    int sock;
-
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-    qemu_opt_set(opts, "path", path);
-    sock = unix_connect_opts(opts);
-    qemu_opts_del(opts);
-    return sock;
-}
-
-#else
-
-int unix_listen_opts(QemuOpts *opts)
-{
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
-}
-
-int unix_connect_opts(QemuOpts *opts)
-{
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
-}
-
-int unix_listen(const char *path, char *ostr, int olen)
-{
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
-}
-
-int unix_connect(const char *path)
-{
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
-}
-
-#endif
-
-#ifdef _WIN32
-static void socket_cleanup(void)
-{
-    WSACleanup();
-}
-#endif
-
-int socket_init(void)
-{
-#ifdef _WIN32
-    WSADATA Data;
-    int ret, err;
-
-    ret = WSAStartup(MAKEWORD(2,2), &Data);
-    if (ret != 0) {
-        err = WSAGetLastError();
-        fprintf(stderr, "WSAStartup: %d\n", err);
-        return -1;
-    }
-    atexit(socket_cleanup);
-#endif
-    return 0;
-}
diff --git a/qemu-tech.texi b/qemu-tech.texi
index d73dda8e35a..8aefa743a8b 100644
--- a/qemu-tech.texi
+++ b/qemu-tech.texi
@@ -262,16 +262,16 @@ Current QEMU limitations:
 
 @item Core Xtensa ISA emulation, including most options: code density,
 loop, extended L32R, 16- and 32-bit multiplication, 32-bit division,
-MAC16, miscellaneous operations, boolean, multiprocessor synchronization,
+MAC16, miscellaneous operations, boolean, FP coprocessor, coprocessor
+context, debug, multiprocessor synchronization,
 conditional store, exceptions, relocatable vectors, unaligned exception,
 interrupts (including high priority and timer), hardware alignment,
 region protection, region translation, MMU, windowed registers, thread
 pointer, processor ID.
 
-@item Not implemented options: FP coprocessor, coprocessor context,
-data/instruction cache (including cache prefetch and locking), XLMI,
-processor interface, debug. Also options not covered by the core ISA
-(e.g. FLIX, wide branches) are not implemented.
+@item Not implemented options: data/instruction cache (including cache
+prefetch and locking), XLMI, processor interface. Also options not
+covered by the core ISA (e.g. FLIX, wide branches) are not implemented.
 
 @item Can run most Xtensa Linux binaries.
 
diff --git a/qemu-timer.c b/qemu-timer.c
index c7a1551a360..8fb5c75df73 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -22,14 +22,16 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
 
 #include "hw/hw.h"
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
+#ifdef CONFIG_POSIX
+#include <pthread.h>
+#endif
 
 #ifdef _WIN32
 #include <mmsystem.h>
@@ -372,21 +374,20 @@ bool qemu_timer_expired(QEMUTimer *timer_head, int64_t current_time)
 
 void qemu_run_timers(QEMUClock *clock)
 {
-    QEMUTimer **ptimer_head, *ts;
+    QEMUTimer *ts;
     int64_t current_time;
    
     if (!clock->enabled)
         return;
 
     current_time = qemu_get_clock_ns(clock);
-    ptimer_head = &clock->active_timers;
     for(;;) {
-        ts = *ptimer_head;
+        ts = clock->active_timers;
         if (!qemu_timer_expired_ns(ts, current_time)) {
             break;
         }
         /* remove timer from the list before calling the callback */
-        *ptimer_head = ts->next;
+        clock->active_timers = ts->next;
         ts->next = NULL;
 
         /* run the callback (the timer list can be modified) */
@@ -431,9 +432,11 @@ void qemu_unregister_clock_reset_notifier(QEMUClock *clock, Notifier *notifier)
 
 void init_clocks(void)
 {
-    rt_clock = qemu_new_clock(QEMU_CLOCK_REALTIME);
-    vm_clock = qemu_new_clock(QEMU_CLOCK_VIRTUAL);
-    host_clock = qemu_new_clock(QEMU_CLOCK_HOST);
+    if (!rt_clock) {
+        rt_clock = qemu_new_clock(QEMU_CLOCK_REALTIME);
+        vm_clock = qemu_new_clock(QEMU_CLOCK_VIRTUAL);
+        host_clock = qemu_new_clock(QEMU_CLOCK_HOST);
+    }
 }
 
 uint64_t qemu_timer_expire_time_ns(QEMUTimer *ts)
@@ -474,7 +477,7 @@ static void host_alarm_handler(int host_signum)
 
 #if defined(__linux__)
 
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 
 static int dynticks_start_timer(struct qemu_alarm_timer *t)
 {
@@ -495,12 +498,12 @@ static int dynticks_start_timer(struct qemu_alarm_timer *t)
     memset(&ev, 0, sizeof(ev));
     ev.sigev_value.sival_int = 0;
     ev.sigev_notify = SIGEV_SIGNAL;
-#ifdef SIGEV_THREAD_ID
+#ifdef CONFIG_SIGEV_THREAD_ID
     if (qemu_signalfd_available()) {
         ev.sigev_notify = SIGEV_THREAD_ID;
         ev._sigev_un._tid = qemu_get_thread_id();
     }
-#endif /* SIGEV_THREAD_ID */
+#endif /* CONFIG_SIGEV_THREAD_ID */
     ev.sigev_signo = SIGALRM;
 
     if (timer_create(CLOCK_REALTIME, &ev, &host_timer)) {
@@ -741,11 +744,28 @@ static void quit_timers(void)
     t->stop(t);
 }
 
+#ifdef CONFIG_POSIX
+static void reinit_timers(void)
+{
+    struct qemu_alarm_timer *t = alarm_timer;
+    t->stop(t);
+    if (t->start(t)) {
+        fprintf(stderr, "Internal timer error: aborting\n");
+        exit(1);
+    }
+    qemu_rearm_alarm_timer(t);
+}
+#endif /* CONFIG_POSIX */
+
 int init_timer_alarm(void)
 {
     struct qemu_alarm_timer *t = NULL;
     int i, err = -1;
 
+    if (alarm_timer) {
+        return 0;
+    }
+
     for (i = 0; alarm_timers[i].name; i++) {
         t = &alarm_timers[i];
 
@@ -760,6 +780,9 @@ int init_timer_alarm(void)
     }
 
     atexit(quit_timers);
+#ifdef CONFIG_POSIX
+    pthread_atfork(NULL, NULL, reinit_timers);
+#endif
     alarm_timer = t;
     return 0;
 
diff --git a/qemu-tool.c b/qemu-tool.c
deleted file mode 100644
index 18205bababe..00000000000
--- a/qemu-tool.c
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Compatibility for qemu-img/qemu-nbd
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "monitor.h"
-#include "qemu-timer.h"
-#include "qemu-log.h"
-#include "migration.h"
-#include "main-loop.h"
-#include "qemu_socket.h"
-#include "slirp/libslirp.h"
-
-#include <sys/time.h>
-
-struct QEMUBH
-{
-    QEMUBHFunc *cb;
-    void *opaque;
-};
-
-const char *qemu_get_vm_name(void)
-{
-    return NULL;
-}
-
-Monitor *cur_mon;
-
-int monitor_cur_is_qmp(void)
-{
-    return 0;
-}
-
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-}
-
-void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
-{
-}
-
-void monitor_printf(Monitor *mon, const char *fmt, ...)
-{
-}
-
-void monitor_print_filename(Monitor *mon, const char *filename)
-{
-}
-
-void monitor_protocol_event(MonitorEvent event, QObject *data)
-{
-}
-
-int monitor_fdset_get_fd(int64_t fdset_id, int flags)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_remove(int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_find(int dup_fd)
-{
-    return -1;
-}
-
-int64_t cpu_get_clock(void)
-{
-    return qemu_get_clock_ns(rt_clock);
-}
-
-int64_t cpu_get_icount(void)
-{
-    abort();
-}
-
-void qemu_mutex_lock_iothread(void)
-{
-}
-
-void qemu_mutex_unlock_iothread(void)
-{
-}
-
-int use_icount;
-
-void qemu_clock_warp(QEMUClock *clock)
-{
-}
-
-int qemu_init_main_loop(void)
-{
-    init_clocks();
-    init_timer_alarm();
-    return main_loop_init();
-}
-
-void slirp_update_timeout(uint32_t *timeout)
-{
-}
-
-void slirp_select_fill(int *pnfds, fd_set *readfds,
-                       fd_set *writefds, fd_set *xfds)
-{
-}
-
-void slirp_select_poll(fd_set *readfds, fd_set *writefds,
-                       fd_set *xfds, int select_error)
-{
-}
-
-void migrate_add_blocker(Error *reason)
-{
-}
-
-void migrate_del_blocker(Error *reason)
-{
-}
diff --git a/qemu-user.c b/qemu-user.c
deleted file mode 100644
index 13fb9ae77be..00000000000
--- a/qemu-user.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Stubs for QEMU user emulation
- *
- * Copyright (c) 2012 SUSE LINUX Products GmbH
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- */
-
-#include "qemu-common.h"
-#include "monitor.h"
-
-Monitor *cur_mon;
-
-int monitor_cur_is_qmp(void)
-{
-    return 0;
-}
-
-void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
-{
-}
-
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-}
-
-int monitor_fdset_get_fd(int64_t fdset_id, int flags)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_remove(int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_find(int dup_fd)
-{
-    return -1;
-}
diff --git a/qemu_socket.h b/qemu_socket.h
deleted file mode 100644
index 30ae6af8b8c..00000000000
--- a/qemu_socket.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/* headers to use the BSD sockets */
-#ifndef QEMU_SOCKET_H
-#define QEMU_SOCKET_H
-
-#ifdef _WIN32
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-
-#define socket_error() WSAGetLastError()
-
-int inet_aton(const char *cp, struct in_addr *ia);
-
-#else
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <sys/un.h>
-
-#define socket_error() errno
-#define closesocket(s) close(s)
-
-#endif /* !_WIN32 */
-
-#include "qemu-option.h"
-#include "error.h"
-#include "qerror.h"
-
-/* misc helpers */
-int qemu_socket(int domain, int type, int protocol);
-int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
-int socket_set_cork(int fd, int v);
-void socket_set_block(int fd);
-void socket_set_nonblock(int fd);
-int send_all(int fd, const void *buf, int len1);
-
-/* New, ipv6-ready socket helper functions, see qemu-sockets.c */
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
-int inet_listen(const char *str, char *ostr, int olen,
-                int socktype, int port_offset, Error **errp);
-int inet_connect_opts(QemuOpts *opts, bool *in_progress, Error **errp);
-int inet_connect(const char *str, bool block, bool *in_progress, Error **errp);
-int inet_dgram_opts(QemuOpts *opts);
-const char *inet_strfamily(int family);
-
-int unix_listen_opts(QemuOpts *opts);
-int unix_listen(const char *path, char *ostr, int olen);
-int unix_connect_opts(QemuOpts *opts);
-int unix_connect(const char *path);
-
-/* Old, ipv4 only bits.  Don't use for new code. */
-int parse_host_port(struct sockaddr_in *saddr, const char *str);
-int socket_init(void);
-
-#endif /* QEMU_SOCKET_H */
diff --git a/qga/Makefile.objs b/qga/Makefile.objs
index cd3e13516c0..b8d7cd0a43e 100644
--- a/qga/Makefile.objs
+++ b/qga/Makefile.objs
@@ -1,4 +1,4 @@
-qga-obj-y = commands.o guest-agent-command-state.o
+qga-obj-y = commands.o guest-agent-command-state.o main.o
 qga-obj-$(CONFIG_POSIX) += commands-posix.o channel-posix.o
 qga-obj-$(CONFIG_WIN32) += commands-win32.o channel-win32.o service-win32.o
 qga-obj-y += qapi-generated/qga-qapi-types.o qapi-generated/qga-qapi-visit.o
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index 57eea06c476..e65dda3822c 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -1,6 +1,12 @@
 #include <glib.h>
 #include <termios.h>
-#include "qemu_socket.h"
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/sockets.h"
 #include "qga/channel.h"
 
 #ifdef CONFIG_SOLARIS
@@ -40,6 +46,7 @@ static gboolean ga_channel_listen_accept(GIOChannel *channel,
     ret = ga_channel_client_add(c, client_fd);
     if (ret) {
         g_warning("error setting up connection");
+        close(client_fd);
         goto out;
     }
     accepted = true;
@@ -134,19 +141,21 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
                            );
         if (fd == -1) {
             g_critical("error opening channel: %s", strerror(errno));
-            exit(EXIT_FAILURE);
+            return false;
         }
 #ifdef CONFIG_SOLARIS
         ret = ioctl(fd, I_SETSIG, S_OUTPUT | S_INPUT | S_HIPRI);
         if (ret == -1) {
             g_critical("error setting event mask for channel: %s",
                        strerror(errno));
-            exit(EXIT_FAILURE);
+            close(fd);
+            return false;
         }
 #endif
         ret = ga_channel_client_add(c, fd);
         if (ret) {
             g_critical("error adding channel to main loop");
+            close(fd);
             return false;
         }
         break;
@@ -156,7 +165,7 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
         int fd = qemu_open(path, O_RDWR | O_NOCTTY | O_NONBLOCK);
         if (fd == -1) {
             g_critical("error opening channel: %s", strerror(errno));
-            exit(EXIT_FAILURE);
+            return false;
         }
         tcgetattr(fd, &tio);
         /* set up serial port for non-canonical, dumb byte streaming */
@@ -176,14 +185,18 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
         tcsetattr(fd, TCSANOW, &tio);
         ret = ga_channel_client_add(c, fd);
         if (ret) {
-            g_error("error adding channel to main loop");
+            g_critical("error adding channel to main loop");
+            close(fd);
+            return false;
         }
         break;
     }
     case GA_CHANNEL_UNIX_LISTEN: {
-        int fd = unix_listen(path, NULL, strlen(path));
-        if (fd == -1) {
-            g_critical("error opening path: %s", strerror(errno));
+        Error *local_err = NULL;
+        int fd = unix_listen(path, NULL, strlen(path), &local_err);
+        if (local_err != NULL) {
+            g_critical("%s", error_get_pretty(local_err));
+            error_free(local_err);
             return false;
         }
         ga_channel_listen_add(c, fd, true);
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index ce9042123cd..7a0202eb2a2 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -17,9 +17,9 @@
 #include <sys/wait.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
-#include "qemu-queue.h"
-#include "host-utils.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/queue.h"
+#include "qemu/host-utils.h"
 
 #ifndef CONFIG_HAS_ENVIRON
 #ifdef __APPLE__
@@ -46,10 +46,29 @@ extern char **environ;
 #endif
 #endif
 
+static void ga_wait_child(pid_t pid, int *status, Error **err)
+{
+    pid_t rpid;
+
+    *status = 0;
+
+    do {
+        rpid = waitpid(pid, status, 0);
+    } while (rpid == -1 && errno == EINTR);
+
+    if (rpid == -1) {
+        error_setg_errno(err, errno, "failed to wait for child (pid: %d)", pid);
+        return;
+    }
+
+    g_assert(rpid == pid);
+}
+
 void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
 {
     const char *shutdown_flag;
-    pid_t rpid, pid;
+    Error *local_err = NULL;
+    pid_t pid;
     int status;
 
     slog("guest-shutdown called, mode: %s", mode);
@@ -60,8 +79,8 @@ void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
     } else if (strcmp(mode, "reboot") == 0) {
         shutdown_flag = "-r";
     } else {
-        error_set(err, QERR_INVALID_PARAMETER_VALUE, "mode",
-                  "halt|powerdown|reboot");
+        error_setg(err,
+                   "mode is invalid (valid values are: halt|powerdown|reboot");
         return;
     }
 
@@ -77,18 +96,27 @@ void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
                "hypervisor initiated shutdown", (char*)NULL, environ);
         _exit(EXIT_FAILURE);
     } else if (pid < 0) {
-        goto exit_err;
+        error_setg_errno(err, errno, "failed to create child process");
+        return;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status) && !WEXITSTATUS(status)) {
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
         return;
     }
 
-exit_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+    if (WEXITSTATUS(status)) {
+        error_setg(err, "child process has failed to shutdown");
+        return;
+    }
+
+    /* succeded */
 }
 
 typedef struct GuestFileHandle {
@@ -111,7 +139,7 @@ static void guest_file_handle_add(FILE *fh)
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
 }
 
-static GuestFileHandle *guest_file_handle_find(int64_t id)
+static GuestFileHandle *guest_file_handle_find(int64_t id, Error **err)
 {
     GuestFileHandle *gfh;
 
@@ -122,6 +150,7 @@ static GuestFileHandle *guest_file_handle_find(int64_t id)
         }
     }
 
+    error_setg(err, "handle '%" PRId64 "' has not been found", id);
     return NULL;
 }
 
@@ -137,7 +166,8 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
     slog("guest-file-open called, filepath: %s, mode: %s", path, mode);
     fh = fopen(path, mode);
     if (!fh) {
-        error_set(err, QERR_OPEN_FILE_FAILED, path);
+        error_setg_errno(err, errno, "failed to open file '%s' (mode: '%s')",
+                         path, mode);
         return -1;
     }
 
@@ -148,7 +178,8 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
     ret = fcntl(fd, F_GETFL);
     ret = fcntl(fd, F_SETFL, ret | O_NONBLOCK);
     if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fcntl() failed");
+        error_setg_errno(err, errno, "failed to make file '%s' non-blocking",
+                         path);
         fclose(fh);
         return -1;
     }
@@ -160,18 +191,17 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
 
 void qmp_guest_file_close(int64_t handle, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     int ret;
 
     slog("guest-file-close called, handle: %ld", handle);
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return;
     }
 
     ret = fclose(gfh->fh);
-    if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fclose() failed");
+    if (ret == EOF) {
+        error_setg_errno(err, errno, "failed to close handle");
         return;
     }
 
@@ -182,21 +212,21 @@ void qmp_guest_file_close(int64_t handle, Error **err)
 struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
                                           int64_t count, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     GuestFileRead *read_data = NULL;
     guchar *buf;
     FILE *fh;
     size_t read_count;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
     if (!has_count) {
         count = QGA_READ_COUNT_DEFAULT;
     } else if (count < 0) {
-        error_set(err, QERR_INVALID_PARAMETER, "count");
+        error_setg(err, "value '%" PRId64 "' is invalid for argument count",
+                   count);
         return NULL;
     }
 
@@ -204,8 +234,8 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
     buf = g_malloc0(count+1);
     read_count = fread(buf, 1, count, fh);
     if (ferror(fh)) {
+        error_setg_errno(err, errno, "failed to read file");
         slog("guest-file-read failed, handle: %ld", handle);
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fread() failed");
     } else {
         buf[read_count] = 0;
         read_data = g_malloc0(sizeof(GuestFileRead));
@@ -228,11 +258,10 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     guchar *buf;
     gsize buf_len;
     int write_count;
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     FILE *fh;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
@@ -242,15 +271,16 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     if (!has_count) {
         count = buf_len;
     } else if (count < 0 || count > buf_len) {
+        error_setg(err, "value '%" PRId64 "' is invalid for argument count",
+                   count);
         g_free(buf);
-        error_set(err, QERR_INVALID_PARAMETER, "count");
         return NULL;
     }
 
     write_count = fwrite(buf, 1, count, fh);
     if (ferror(fh)) {
+        error_setg_errno(err, errno, "failed to write to file");
         slog("guest-file-write failed, handle: %ld", handle);
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fwrite() error");
     } else {
         write_data = g_malloc0(sizeof(GuestFileWrite));
         write_data->count = write_count;
@@ -265,20 +295,19 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
 struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
                                           int64_t whence, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     GuestFileSeek *seek_data = NULL;
     FILE *fh;
     int ret;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
     fh = gfh->fh;
     ret = fseek(fh, offset, whence);
     if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, strerror(errno));
+        error_setg_errno(err, errno, "failed to seek file");
     } else {
         seek_data = g_malloc0(sizeof(GuestFileRead));
         seek_data->position = ftell(fh);
@@ -291,19 +320,18 @@ struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
 
 void qmp_guest_file_flush(int64_t handle, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     FILE *fh;
     int ret;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return;
     }
 
     fh = gfh->fh;
     ret = fflush(fh);
     if (ret == EOF) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, strerror(errno));
+        error_setg_errno(err, errno, "failed to flush file");
     }
 }
 
@@ -343,7 +371,7 @@ static void free_fs_mount_list(FsMountList *mounts)
 /*
  * Walk the mount table and build a list of local file systems
  */
-static int build_fs_mount_list(FsMountList *mounts)
+static void build_fs_mount_list(FsMountList *mounts, Error **err)
 {
     struct mntent *ment;
     FsMount *mount;
@@ -352,8 +380,8 @@ static int build_fs_mount_list(FsMountList *mounts)
 
     fp = setmntent(mtab, "r");
     if (!fp) {
-        g_warning("fsfreeze: unable to read mtab");
-        return -1;
+        error_setg(err, "failed to open mtab file: '%s'", mtab);
+        return;
     }
 
     while ((ment = getmntent(fp))) {
@@ -377,13 +405,71 @@ static int build_fs_mount_list(FsMountList *mounts)
     }
 
     endmntent(fp);
-
-    return 0;
 }
 #endif
 
 #if defined(CONFIG_FSFREEZE)
 
+typedef enum {
+    FSFREEZE_HOOK_THAW = 0,
+    FSFREEZE_HOOK_FREEZE,
+} FsfreezeHookArg;
+
+const char *fsfreeze_hook_arg_string[] = {
+    "thaw",
+    "freeze",
+};
+
+static void execute_fsfreeze_hook(FsfreezeHookArg arg, Error **err)
+{
+    int status;
+    pid_t pid;
+    const char *hook;
+    const char *arg_str = fsfreeze_hook_arg_string[arg];
+    Error *local_err = NULL;
+
+    hook = ga_fsfreeze_hook(ga_state);
+    if (!hook) {
+        return;
+    }
+    if (access(hook, X_OK) != 0) {
+        error_setg_errno(err, errno, "can't access fsfreeze hook '%s'", hook);
+        return;
+    }
+
+    slog("executing fsfreeze hook with arg '%s'", arg_str);
+    pid = fork();
+    if (pid == 0) {
+        setsid();
+        reopen_fd_to_null(0);
+        reopen_fd_to_null(1);
+        reopen_fd_to_null(2);
+
+        execle(hook, hook, arg_str, NULL, environ);
+        _exit(EXIT_FAILURE);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        return;
+    }
+
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    if (!WIFEXITED(status)) {
+        error_setg(err, "fsfreeze hook has terminated abnormally");
+        return;
+    }
+
+    status = WEXITSTATUS(status);
+    if (status) {
+        error_setg(err, "fsfreeze hook has failed with status %d", status);
+        return;
+    }
+}
+
 /*
  * Return status of freeze/thaw
  */
@@ -405,15 +491,22 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
     int ret = 0, i = 0;
     FsMountList mounts;
     struct FsMount *mount;
+    Error *local_err = NULL;
     int fd;
-    char err_msg[512];
 
     slog("guest-fsfreeze called");
 
+    execute_fsfreeze_hook(FSFREEZE_HOOK_FREEZE, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return -1;
+    }
+
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret < 0) {
-        return ret;
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return -1;
     }
 
     /* cannot risk guest agent blocking itself on a write in this state */
@@ -422,9 +515,7 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
     QTAILQ_FOREACH(mount, &mounts, next) {
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
-            sprintf(err_msg, "failed to open %s, %s", mount->dirname,
-                    strerror(errno));
-            error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+            error_setg_errno(err, errno, "failed to open %s", mount->dirname);
             goto error;
         }
 
@@ -440,9 +531,8 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
         ret = ioctl(fd, FIFREEZE);
         if (ret == -1) {
             if (errno != EOPNOTSUPP) {
-                sprintf(err_msg, "failed to freeze %s, %s",
-                        mount->dirname, strerror(errno));
-                error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(err, errno, "failed to freeze %s",
+                                 mount->dirname);
                 close(fd);
                 goto error;
             }
@@ -470,12 +560,12 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
     FsMountList mounts;
     FsMount *mount;
     int fd, i = 0, logged;
+    Error *local_err = NULL;
 
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret) {
-        error_set(err, QERR_QGA_COMMAND_FAILED,
-                  "failed to enumerate filesystems");
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
         return 0;
     }
 
@@ -513,18 +603,22 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 
     ga_unset_frozen(ga_state);
     free_fs_mount_list(&mounts);
+
+    execute_fsfreeze_hook(FSFREEZE_HOOK_THAW, err);
+
     return i;
 }
 
 static void guest_fsfreeze_cleanup(void)
 {
-    int64_t ret;
     Error *err = NULL;
 
     if (ga_is_frozen(ga_state) == GUEST_FSFREEZE_STATUS_FROZEN) {
-        ret = qmp_guest_fsfreeze_thaw(&err);
-        if (ret < 0 || err) {
-            slog("failed to clean up frozen filesystems");
+        qmp_guest_fsfreeze_thaw(&err);
+        if (err) {
+            slog("failed to clean up frozen filesystems: %s",
+                 error_get_pretty(err));
+            error_free(err);
         }
     }
 }
@@ -540,7 +634,7 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
     FsMountList mounts;
     struct FsMount *mount;
     int fd;
-    char err_msg[512];
+    Error *local_err = NULL;
     struct fstrim_range r = {
         .start = 0,
         .len = -1,
@@ -550,17 +644,16 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
     slog("guest-fstrim called");
 
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret < 0) {
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
         return;
     }
 
     QTAILQ_FOREACH(mount, &mounts, next) {
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
-            sprintf(err_msg, "failed to open %s, %s", mount->dirname,
-                    strerror(errno));
-            error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+            error_setg_errno(err, errno, "failed to open %s", mount->dirname);
             goto error;
         }
 
@@ -573,9 +666,8 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
         ret = ioctl(fd, FITRIM, &r);
         if (ret == -1) {
             if (errno != ENOTTY && errno != EOPNOTSUPP) {
-                sprintf(err_msg, "failed to trim %s, %s",
-                        mount->dirname, strerror(errno));
-                error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(err, errno, "failed to trim %s",
+                                 mount->dirname);
                 close(fd);
                 goto error;
             }
@@ -596,8 +688,9 @@ error:
 static void bios_supports_mode(const char *pmutils_bin, const char *pmutils_arg,
                                const char *sysfile_str, Error **err)
 {
+    Error *local_err = NULL;
     char *pmutils_path;
-    pid_t pid, rpid;
+    pid_t pid;
     int status;
 
     pmutils_path = g_find_program_in_path(pmutils_bin);
@@ -642,38 +735,46 @@ static void bios_supports_mode(const char *pmutils_bin, const char *pmutils_arg,
         }
 
         _exit(SUSPEND_NOT_SUPPORTED);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        goto out;
     }
 
-    g_free(pmutils_path);
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        goto out;
+    }
 
-    if (pid < 0) {
-        goto undef_err;
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
+        goto out;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status)) {
-        switch (WEXITSTATUS(status)) {
-        case SUSPEND_SUPPORTED:
-            return;
-        case SUSPEND_NOT_SUPPORTED:
-            error_set(err, QERR_UNSUPPORTED);
-            return;
-        default:
-            goto undef_err;
-        }
+    switch (WEXITSTATUS(status)) {
+    case SUSPEND_SUPPORTED:
+        goto out;
+    case SUSPEND_NOT_SUPPORTED:
+        error_setg(err,
+                   "the requested suspend mode is not supported by the guest");
+        goto out;
+    default:
+        error_setg(err,
+                   "the helper program '%s' returned an unexpected exit status"
+                   " code (%d)", pmutils_path, WEXITSTATUS(status));
+        goto out;
     }
 
-undef_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+out:
+    g_free(pmutils_path);
 }
 
 static void guest_suspend(const char *pmutils_bin, const char *sysfile_str,
                           Error **err)
 {
+    Error *local_err = NULL;
     char *pmutils_path;
-    pid_t rpid, pid;
+    pid_t pid;
     int status;
 
     pmutils_path = g_find_program_in_path(pmutils_bin);
@@ -711,23 +812,29 @@ static void guest_suspend(const char *pmutils_bin, const char *sysfile_str,
         }
 
         _exit(EXIT_SUCCESS);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        goto out;
     }
 
-    g_free(pmutils_path);
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        goto out;
+    }
 
-    if (pid < 0) {
-        goto exit_err;
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
+        goto out;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status) && !WEXITSTATUS(status)) {
-        return;
+    if (WEXITSTATUS(status)) {
+        error_setg(err, "child process has failed to suspend");
+        goto out;
     }
 
-exit_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+out:
+    g_free(pmutils_path);
 }
 
 void qmp_guest_suspend_disk(Error **err)
@@ -780,12 +887,9 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
 {
     GuestNetworkInterfaceList *head = NULL, *cur_item = NULL;
     struct ifaddrs *ifap, *ifa;
-    char err_msg[512];
 
     if (getifaddrs(&ifap) < 0) {
-        snprintf(err_msg, sizeof(err_msg),
-                 "getifaddrs failed: %s", strerror(errno));
-        error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+        error_setg_errno(errp, errno, "getifaddrs failed");
         goto error;
     }
 
@@ -821,53 +925,43 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
             /* we haven't obtained HW address yet */
             sock = socket(PF_INET, SOCK_STREAM, 0);
             if (sock == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to create socket: %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "failed to create socket");
                 goto error;
             }
 
             memset(&ifr, 0, sizeof(ifr));
-            strncpy(ifr.ifr_name,  info->value->name, IF_NAMESIZE);
+            pstrcpy(ifr.ifr_name, IF_NAMESIZE, info->value->name);
             if (ioctl(sock, SIOCGIFHWADDR, &ifr) == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to get MAC address of %s: %s",
-                         ifa->ifa_name,
-                         strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno,
+                                 "failed to get MAC address of %s",
+                                 ifa->ifa_name);
+                close(sock);
                 goto error;
             }
 
+            close(sock);
             mac_addr = (unsigned char *) &ifr.ifr_hwaddr.sa_data;
 
-            if (asprintf(&info->value->hardware_address,
-                         "%02x:%02x:%02x:%02x:%02x:%02x",
-                         (int) mac_addr[0], (int) mac_addr[1],
-                         (int) mac_addr[2], (int) mac_addr[3],
-                         (int) mac_addr[4], (int) mac_addr[5]) == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to format MAC: %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
-                goto error;
-            }
+            info->value->hardware_address =
+                g_strdup_printf("%02x:%02x:%02x:%02x:%02x:%02x",
+                                (int) mac_addr[0], (int) mac_addr[1],
+                                (int) mac_addr[2], (int) mac_addr[3],
+                                (int) mac_addr[4], (int) mac_addr[5]);
 
             info->value->has_hardware_address = true;
-            close(sock);
         }
 
         if (ifa->ifa_addr &&
             ifa->ifa_addr->sa_family == AF_INET) {
             /* interface with IPv4 address */
-            address_item = g_malloc0(sizeof(*address_item));
-            address_item->value = g_malloc0(sizeof(*address_item->value));
             p = &((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
             if (!inet_ntop(AF_INET, p, addr4, sizeof(addr4))) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "inet_ntop failed : %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "inet_ntop failed");
                 goto error;
             }
 
+            address_item = g_malloc0(sizeof(*address_item));
+            address_item->value = g_malloc0(sizeof(*address_item->value));
             address_item->value->ip_address = g_strdup(addr4);
             address_item->value->ip_address_type = GUEST_IP_ADDRESS_TYPE_IPV4;
 
@@ -880,16 +974,14 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
         } else if (ifa->ifa_addr &&
                    ifa->ifa_addr->sa_family == AF_INET6) {
             /* interface with IPv6 address */
-            address_item = g_malloc0(sizeof(*address_item));
-            address_item->value = g_malloc0(sizeof(*address_item->value));
             p = &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
             if (!inet_ntop(AF_INET6, p, addr6, sizeof(addr6))) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "inet_ntop failed : %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "inet_ntop failed");
                 goto error;
             }
 
+            address_item = g_malloc0(sizeof(*address_item));
+            address_item->value = g_malloc0(sizeof(*address_item->value));
             address_item->value->ip_address = g_strdup(addr6);
             address_item->value->ip_address_type = GUEST_IP_ADDRESS_TYPE_IPV6;
 
@@ -988,8 +1080,6 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
 {
     error_set(err, QERR_UNSUPPORTED);
-
-    return;
 }
 #endif
 
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index 54bc5462e2f..7e8ecb3b40e 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -16,7 +16,7 @@
 #include <powrprof.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 #ifndef SHTDN_REASON_FLAG_PLANNED
 #define SHTDN_REASON_FLAG_PLANNED 0x80000000
@@ -180,8 +180,6 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
 {
     error_set(err, QERR_UNSUPPORTED);
-
-    return;
 }
 
 typedef enum {
diff --git a/qga/commands.c b/qga/commands.c
index 46b0b083bc4..7ffb35e4af8 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -13,7 +13,7 @@
 #include <glib.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 /* Note: in some situations, like with the fsfreeze, logging may be
  * temporarilly disabled. if it is necessary that a command be able
diff --git a/qga/guest-agent-core.h b/qga/guest-agent-core.h
index 49a7abee955..33545983625 100644
--- a/qga/guest-agent-core.h
+++ b/qga/guest-agent-core.h
@@ -10,7 +10,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include "qapi/qmp-core.h"
+#include "qapi/qmp/dispatch.h"
 #include "qemu-common.h"
 
 #define QGA_READ_COUNT_DEFAULT 4096
@@ -34,6 +34,7 @@ void ga_set_response_delimited(GAState *s);
 bool ga_is_frozen(GAState *s);
 void ga_set_frozen(GAState *s);
 void ga_unset_frozen(GAState *s);
+const char *ga_fsfreeze_hook(GAState *s);
 
 #ifndef _WIN32
 void reopen_fd_to_null(int fd);
diff --git a/qemu-ga.c b/qga/main.c
index 76230798878..db281a508be 100644
--- a/qemu-ga.c
+++ b/qga/main.c
@@ -20,28 +20,37 @@
 #include <sys/wait.h>
 #include <sys/stat.h>
 #endif
-#include "json-streamer.h"
-#include "json-parser.h"
-#include "qint.h"
-#include "qjson.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qjson.h"
 #include "qga/guest-agent-core.h"
-#include "module.h"
+#include "qemu/module.h"
 #include "signal.h"
-#include "qerror.h"
-#include "qapi/qmp-core.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/dispatch.h"
 #include "qga/channel.h"
 #ifdef _WIN32
 #include "qga/service-win32.h"
 #include <windows.h>
 #endif
+#ifdef __linux__
+#include <linux/fs.h>
+#ifdef FIFREEZE
+#define CONFIG_FSFREEZE
+#endif
+#endif
 
 #ifndef _WIN32
 #define QGA_VIRTIO_PATH_DEFAULT "/dev/virtio-ports/org.qemu.guest_agent.0"
 #else
 #define QGA_VIRTIO_PATH_DEFAULT "\\\\.\\Global\\org.qemu.guest_agent.0"
 #endif
-#define QGA_PIDFILE_DEFAULT "/var/run/qemu-ga.pid"
-#define QGA_STATEDIR_DEFAULT "/tmp"
+#define QGA_STATEDIR_DEFAULT CONFIG_QEMU_LOCALSTATEDIR "/run"
+#define QGA_PIDFILE_DEFAULT QGA_STATEDIR_DEFAULT "/qemu-ga.pid"
+#ifdef CONFIG_FSFREEZE
+#define QGA_FSFREEZE_HOOK_DEFAULT CONFIG_QEMU_CONFDIR "/fsfreeze-hook"
+#endif
 #define QGA_SENTINEL_BYTE 0xFF
 
 struct GAState {
@@ -64,6 +73,9 @@ struct GAState {
         const char *log_filepath;
         const char *pid_filepath;
     } deferred_options;
+#ifdef CONFIG_FSFREEZE
+    const char *fsfreeze_hook;
+#endif
 };
 
 struct GAState *ga_state;
@@ -114,12 +126,10 @@ static gboolean register_signal_handlers(void)
     ret = sigaction(SIGINT, &sigact, NULL);
     if (ret == -1) {
         g_error("error configuring signal handler: %s", strerror(errno));
-        return false;
     }
     ret = sigaction(SIGTERM, &sigact, NULL);
     if (ret == -1) {
         g_error("error configuring signal handler: %s", strerror(errno));
-        return false;
     }
 
     return true;
@@ -155,6 +165,16 @@ static void usage(const char *cmd)
 "                    %s)\n"
 "  -l, --logfile     set logfile path, logs to stderr by default\n"
 "  -f, --pidfile     specify pidfile (default is %s)\n"
+#ifdef CONFIG_FSFREEZE
+"  -F, --fsfreeze-hook\n"
+"                    enable fsfreeze hook. Accepts an optional argument that\n"
+"                    specifies script to run on freeze/thaw. Script will be\n"
+"                    called with 'freeze'/'thaw' arguments accordingly.\n"
+"                    (default is %s)\n"
+"                    If using -F with an argument, do not follow -F with a\n"
+"                    space.\n"
+"                    (for example: -F/var/run/fsfreezehook.sh)\n"
+#endif
 "  -t, --statedir    specify dir to store state information (absolute paths\n"
 "                    only, default is %s)\n"
 "  -v, --verbose     log extra debugging information\n"
@@ -169,6 +189,9 @@ static void usage(const char *cmd)
 "\n"
 "Report bugs to <mdroth@linux.vnet.ibm.com>\n"
     , cmd, QEMU_VERSION, QGA_VIRTIO_PATH_DEFAULT, QGA_PIDFILE_DEFAULT,
+#ifdef CONFIG_FSFREEZE
+    QGA_FSFREEZE_HOOK_DEFAULT,
+#endif
     QGA_STATEDIR_DEFAULT);
 }
 
@@ -238,13 +261,26 @@ void ga_set_response_delimited(GAState *s)
     s->delimit_response = true;
 }
 
+static FILE *ga_open_logfile(const char *logfile)
+{
+    FILE *f;
+
+    f = fopen(logfile, "a");
+    if (!f) {
+        return NULL;
+    }
+
+    qemu_set_cloexec(fileno(f));
+    return f;
+}
+
 #ifndef _WIN32
 static bool ga_open_pidfile(const char *pidfile)
 {
     int pidfd;
     char pidstr[32];
 
-    pidfd = open(pidfile, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
+    pidfd = qemu_open(pidfile, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
     if (pidfd == -1 || lockf(pidfd, F_TLOCK, 0)) {
         g_critical("Cannot lock pid file, %s", strerror(errno));
         if (pidfd != -1) {
@@ -253,20 +289,22 @@ static bool ga_open_pidfile(const char *pidfile)
         return false;
     }
 
-    if (ftruncate(pidfd, 0) || lseek(pidfd, 0, SEEK_SET)) {
+    if (ftruncate(pidfd, 0)) {
         g_critical("Failed to truncate pid file");
         goto fail;
     }
-    sprintf(pidstr, "%d", getpid());
+    snprintf(pidstr, sizeof(pidstr), "%d\n", getpid());
     if (write(pidfd, pidstr, strlen(pidstr)) != strlen(pidstr)) {
         g_critical("Failed to write pid file");
         goto fail;
     }
 
+    /* keep pidfile open & locked forever */
     return true;
 
 fail:
     unlink(pidfile);
+    close(pidfd);
     return false;
 }
 #else /* _WIN32 */
@@ -379,7 +417,7 @@ void ga_unset_frozen(GAState *s)
      * in a frozen state at start up, do it now
      */
     if (s->deferred_options.log_filepath) {
-        s->log_file = fopen(s->deferred_options.log_filepath, "a");
+        s->log_file = ga_open_logfile(s->deferred_options.log_filepath);
         if (!s->log_file) {
             s->log_file = stderr;
         }
@@ -403,6 +441,13 @@ void ga_unset_frozen(GAState *s)
     }
 }
 
+#ifdef CONFIG_FSFREEZE
+const char *ga_fsfreeze_hook(GAState *s)
+{
+    return s->fsfreeze_hook;
+}
+#endif
+
 static void become_daemon(const char *pidfile)
 {
 #ifndef _WIN32
@@ -575,6 +620,7 @@ static gboolean channel_event_cb(GIOCondition condition, gpointer data)
         if (!s->virtio) {
             return false;
         }
+        /* fall through */
     case G_IO_STATUS_AGAIN:
         /* virtio causes us to spin here when no process is attached to
          * host-side chardev. sleep a bit to mitigate this
@@ -680,10 +726,13 @@ VOID WINAPI service_main(DWORD argc, TCHAR *argv[])
 
 int main(int argc, char **argv)
 {
-    const char *sopt = "hVvdm:p:l:f:b:s:t:";
+    const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
     const char *method = NULL, *path = NULL;
     const char *log_filepath = NULL;
     const char *pid_filepath = QGA_PIDFILE_DEFAULT;
+#ifdef CONFIG_FSFREEZE
+    const char *fsfreeze_hook = NULL;
+#endif
     const char *state_dir = QGA_STATEDIR_DEFAULT;
 #ifdef _WIN32
     const char *service = NULL;
@@ -693,6 +742,9 @@ int main(int argc, char **argv)
         { "version", 0, NULL, 'V' },
         { "logfile", 1, NULL, 'l' },
         { "pidfile", 1, NULL, 'f' },
+#ifdef CONFIG_FSFREEZE
+        { "fsfreeze-hook", 2, NULL, 'F' },
+#endif
         { "verbose", 0, NULL, 'v' },
         { "method", 1, NULL, 'm' },
         { "path", 1, NULL, 'p' },
@@ -725,6 +777,11 @@ int main(int argc, char **argv)
         case 'f':
             pid_filepath = optarg;
             break;
+#ifdef CONFIG_FSFREEZE
+        case 'F':
+            fsfreeze_hook = optarg ? optarg : QGA_FSFREEZE_HOOK_DEFAULT;
+            break;
+#endif
         case 't':
              state_dir = optarg;
              break;
@@ -788,6 +845,9 @@ int main(int argc, char **argv)
     s = g_malloc0(sizeof(GAState));
     s->log_level = log_level;
     s->log_file = stderr;
+#ifdef CONFIG_FSFREEZE
+    s->fsfreeze_hook = fsfreeze_hook;
+#endif
     g_log_set_default_handler(ga_log, s);
     g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
     ga_enable_logging(s);
@@ -840,7 +900,7 @@ int main(int argc, char **argv)
             become_daemon(pid_filepath);
         }
         if (log_filepath) {
-            FILE *log_file = fopen(log_filepath, "a");
+            FILE *log_file = ga_open_logfile(log_filepath);
             if (!log_file) {
                 g_critical("unable to open specified log file: %s",
                            strerror(errno));
diff --git a/qapi-schema-guest.json b/qga/qapi-schema.json
index d955cf11fb3..d91d903256d 100644
--- a/qapi-schema-guest.json
+++ b/qga/qapi-schema.json
@@ -31,7 +31,7 @@
 #
 # Since: 1.1
 # ##
-{ 'command': 'guest-sync-delimited'
+{ 'command': 'guest-sync-delimited',
   'data':    { 'id': 'int' },
   'returns': 'int' }
 
@@ -69,7 +69,7 @@
 #
 # Since: 0.15.0
 ##
-{ 'command': 'guest-sync'
+{ 'command': 'guest-sync',
   'data':    { 'id': 'int' },
   'returns': 'int' }
 
@@ -293,7 +293,7 @@
 ##
 # @GuestFsFreezeStatus
 #
-# An enumation of filesystem freeze states
+# An enumeration of filesystem freeze states
 #
 # @thawed: filesystems thawed/unfrozen
 #
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 3745a211999..f58a8411eaf 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -146,10 +146,7 @@ EQMP
     {
         .name       = "screendump",
         .args_type  = "filename:F",
-        .params     = "filename",
-        .help       = "save screen into PPM image 'filename'",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_screen_dump,
+        .mhandler.cmd_new = qmp_marshal_input_screendump,
     },
 
 SQMP
@@ -335,6 +332,34 @@ Example:
 EQMP
 
     {
+        .name       = "send-key",
+        .args_type  = "keys:O,hold-time:i?",
+        .mhandler.cmd_new = qmp_marshal_input_send_key,
+    },
+
+SQMP
+send-key
+----------
+
+Send keys to VM.
+
+Arguments:
+
+keys array:
+    - "key": key sequence (a json-array of key enum values)
+
+- hold-time: time to delay key up events, milliseconds. Defaults to 100
+             (json-int, optional)
+
+Example:
+
+-> { "execute": "send-key",
+     "arguments": { 'keys': [ 'ctrl', 'alt', 'delete' ] } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "cpu",
         .args_type  = "index:i",
         .mhandler.cmd_new = qmp_marshal_input_cpu,
@@ -441,6 +466,72 @@ Note: inject-nmi fails when the guest doesn't support injecting.
 EQMP
 
     {
+        .name       = "memchar-write",
+        .args_type  = "device:s,size:i,data:s,format:s?",
+        .mhandler.cmd_new = qmp_marshal_input_memchar_write,
+    },
+
+SQMP
+memchar-write
+-------------
+
+Provide writing interface for CirMemCharDriver. Write data to memory
+char device.
+
+Arguments:
+
+- "device": the name of the char device, must be unique (json-string)
+- "size": the memory size, in bytes, should be power of 2 (json-int)
+- "data": the source data write to memory (json-string)
+- "format": the data format write to memory, default is
+            utf8. (json-string, optional)
+          - Possible values: "utf8", "base64"
+
+Example:
+
+-> { "execute": "memchar-write",
+                "arguments": { "device": foo,
+                               "size": 8,
+                               "data": "abcdefgh",
+                               "format": "utf8" } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "memchar-read",
+        .args_type  = "device:s,size:i,format:s?",
+        .mhandler.cmd_new = qmp_marshal_input_memchar_read,
+    },
+
+SQMP
+memchar-read
+-------------
+
+Provide read interface for CirMemCharDriver. Read from the char
+device memory and return the data with size.
+
+Arguments:
+
+- "device": the name of the char device, must be unique (json-string)
+- "size": the memory size wanted to read in bytes (refer to unencoded
+          size of the raw data), would adjust to the init size of the
+          memchar if the requested size is larger than it. (json-int)
+- "format": the data format write to memchardev, default is
+            utf8. (json-string, optional)
+          - Possible values: "utf8", "base64"
+
+Example:
+
+-> { "execute": "memchar-read",
+                "arguments": { "device": foo,
+                               "size": 1000,
+                               "format": "utf8" } }
+<- { "return": { "data": "data string...", "count": 1000 } }
+
+EQMP
+
+    {
         .name       = "xen-save-devices-state",
         .args_type  = "filename:F",
     .mhandler.cmd_new = qmp_marshal_input_xen_save_devices_state,
@@ -468,6 +559,30 @@ Example:
 EQMP
 
     {
+        .name       = "xen-set-global-dirty-log",
+        .args_type  = "enable:b",
+        .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log,
+    },
+
+SQMP
+xen-set-global-dirty-log
+-------
+
+Enable or disable the global dirty log mode.
+
+Arguments:
+
+- "enable": Enable it or disable it.
+
+Example:
+
+-> { "execute": "xen-set-global-dirty-log",
+     "arguments": { "enable": true } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
         .mhandler.cmd_new = qmp_marshal_input_migrate,
@@ -762,11 +877,17 @@ EQMP
 
     {
         .name       = "block-stream",
-        .args_type  = "device:B,base:s?,speed:o?",
+        .args_type  = "device:B,base:s?,speed:o?,on-error:s?",
         .mhandler.cmd_new = qmp_marshal_input_block_stream,
     },
 
     {
+        .name       = "block-commit",
+        .args_type  = "device:B,base:s?,top:s,speed:o?",
+        .mhandler.cmd_new = qmp_marshal_input_block_commit,
+    },
+
+    {
         .name       = "block-job-set-speed",
         .args_type  = "device:B,speed:o",
         .mhandler.cmd_new = qmp_marshal_input_block_job_set_speed,
@@ -774,10 +895,25 @@ EQMP
 
     {
         .name       = "block-job-cancel",
-        .args_type  = "device:B",
+        .args_type  = "device:B,force:b?",
         .mhandler.cmd_new = qmp_marshal_input_block_job_cancel,
     },
     {
+        .name       = "block-job-pause",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_pause,
+    },
+    {
+        .name       = "block-job-resume",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_resume,
+    },
+    {
+        .name       = "block-job-complete",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_complete,
+    },
+    {
         .name       = "transaction",
         .args_type  = "actions:q",
         .mhandler.cmd_new = qmp_marshal_input_transaction,
@@ -866,6 +1002,62 @@ Example:
 EQMP
 
     {
+        .name       = "drive-mirror",
+        .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
+                      "on-source-error:s?,on-target-error:s?,"
+                      "granularity:i?,buf-size:i?",
+        .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
+    },
+
+SQMP
+drive-mirror
+------------
+
+Start mirroring a block device's writes to a new destination. target
+specifies the target of the new image. If the file exists, or if it is
+a device, it will be used as the new destination for writes. If it does not
+exist, a new file will be created. format specifies the format of the
+mirror image, default is to probe if mode='existing', else the format
+of the source.
+
+Arguments:
+
+- "device": device name to operate on (json-string)
+- "target": name of new image file (json-string)
+- "format": format of new image (json-string, optional)
+- "mode": how an image file should be created into the target
+  file/device (NewImageMode, optional, default 'absolute-paths')
+- "speed": maximum speed of the streaming job, in bytes per second
+  (json-int)
+- "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
+- "buf_size": maximum amount of data in flight from source to target, in bytes
+  (json-int, default 10M)
+- "sync": what parts of the disk image should be copied to the destination;
+  possibilities include "full" for all the disk, "top" for only the sectors
+  allocated in the topmost image, or "none" to only replicate new I/O
+  (MirrorSyncMode).
+- "on-source-error": the action to take on an error on the source
+  (BlockdevOnError, default 'report')
+- "on-target-error": the action to take on an error on the target
+  (BlockdevOnError, default 'report')
+
+The default value of the granularity is the image cluster size clamped
+between 4096 and 65536, if the image format defines one.  If the format
+does not define a cluster size, the default value of the granularity
+is 65536.
+
+
+Example:
+
+-> { "execute": "drive-mirror", "arguments": { "device": "ide-hd0",
+                                               "target": "/some/place/my-image",
+                                               "sync": "full",
+                                               "format": "qcow2" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "balloon",
         .args_type  = "value:M",
         .mhandler.cmd_new = qmp_marshal_input_balloon,
@@ -1206,10 +1398,7 @@ EQMP
     {
         .name       = "add_client",
         .args_type  = "protocol:s,fdname:s,skipauth:b?,tls:b?",
-        .params     = "protocol fdname skipauth tls",
-        .help       = "add a graphics client",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = add_graphics_client,
+        .mhandler.cmd_new = qmp_marshal_input_add_client,
     },
 
 SQMP
@@ -2242,6 +2431,11 @@ The main json-object contains the following:
 - "total-time": total amount of ms since migration started.  If
                 migration has ended, it returns the total migration
 		 time (json-int)
+- "downtime": only present when migration has finished correctly
+              total amount in ms for downtime that happened (json-int)
+- "expected-downtime": only present while migration is active
+                total amount in ms for downtime that was calculated on
+		the last bitmap round (json-int)
 - "ram": only present if "status" is "active", it is a json-object with the
   following RAM information (in bytes):
          - "transferred": amount transferred (json-int)
@@ -2279,6 +2473,7 @@ Examples:
           "remaining":123,
           "total":246,
           "total-time":12345,
+          "downtime":12345,
           "duplicate":123,
           "normal":123,
           "normal-bytes":123456
@@ -2302,6 +2497,7 @@ Examples:
             "remaining":123,
             "total":246,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":123,
             "normal":123,
             "normal-bytes":123456
@@ -2320,6 +2516,7 @@ Examples:
             "remaining":1053304,
             "transferred":3720,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":123,
             "normal":123,
             "normal-bytes":123456
@@ -2344,6 +2541,7 @@ Examples:
             "remaining":1053304,
             "transferred":3720,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":10,
             "normal":3333,
             "normal-bytes":3412992
@@ -2425,13 +2623,6 @@ Make an asynchronous request for balloon info. When the request completes a
 json-object will be returned containing the following data:
 
 - "actual": current balloon value in bytes (json-int)
-- "mem_swapped_in": Amount of memory swapped in bytes (json-int, optional)
-- "mem_swapped_out": Amount of memory swapped out in bytes (json-int, optional)
-- "major_page_faults": Number of major faults (json-int, optional)
-- "minor_page_faults": Number of minor faults (json-int, optional)
-- "free_mem": Total amount of free and unused memory in
-              bytes (json-int, optional)
-- "total_mem": Total amount of available memory in bytes (json-int, optional)
 
 Example:
 
@@ -2439,12 +2630,6 @@ Example:
 <- {
       "return":{
          "actual":1073741824,
-         "mem_swapped_in":0,
-         "mem_swapped_out":0,
-         "major_page_faults":142,
-         "minor_page_faults":239245,
-         "free_mem":1014185984,
-         "total_mem":1044668416
       }
    }
 
@@ -2481,6 +2666,22 @@ EQMP
     },
 
     {
+        .name       = "nbd-server-start",
+        .args_type  = "addr:q",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_start,
+    },
+    {
+        .name       = "nbd-server-add",
+        .args_type  = "device:B,writable:b?",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_add,
+    },
+    {
+        .name       = "nbd-server-stop",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_stop,
+    },
+
+    {
         .name       = "change-vnc-password",
         .args_type  = "password:s",
         .mhandler.cmd_new = qmp_marshal_input_change_vnc_password,
@@ -2514,3 +2715,64 @@ EQMP
         .args_type  = "",
         .mhandler.cmd_new = qmp_marshal_input_query_target,
     },
+
+    {
+        .name       = "chardev-add",
+        .args_type  = "id:s,backend:q",
+        .mhandler.cmd_new = qmp_marshal_input_chardev_add,
+    },
+
+SQMP
+chardev-add
+----------------
+
+Add a chardev.
+
+Arguments:
+
+- "id": the chardev's ID, must be unique (json-string)
+- "backend": chardev backend type + parameters
+
+Examples:
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "foo",
+                     "backend" : { "type" : "null", "data" : {} } } }
+<- { "return": {} }
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "bar",
+                     "backend" : { "type" : "file",
+                                   "data" : { "out" : "/tmp/bar.log" } } } }
+<- { "return": {} }
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "baz",
+                     "backend" : { "type" : "pty", "data" : {} } } }
+<- { "return": { "pty" : "/dev/pty/42" } }
+
+EQMP
+
+    {
+        .name       = "chardev-remove",
+        .args_type  = "id:s",
+        .mhandler.cmd_new = qmp_marshal_input_chardev_remove,
+    },
+
+
+SQMP
+chardev-remove
+--------------
+
+Remove a chardev.
+
+Arguments:
+
+- "id": the chardev's ID, must exist and not be in use (json-string)
+
+Example:
+
+-> { "execute": "chardev-remove", "arguments": { "id" : "foo" } }
+<- { "return": {} }
+
+EQMP
diff --git a/qmp.c b/qmp.c
index 84639220d03..55b056b558b 100644
--- a/qmp.c
+++ b/qmp.c
@@ -14,15 +14,16 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qmp-commands.h"
+#include "char/char.h"
 #include "ui/qemu-spice.h"
 #include "ui/vnc.h"
-#include "kvm.h"
-#include "arch_init.h"
+#include "sysemu/kvm.h"
+#include "sysemu/arch_init.h"
 #include "hw/qdev.h"
-#include "blockdev.h"
-#include "qemu/qom-qobject.h"
+#include "sysemu/blockdev.h"
+#include "qom/qom-qobject.h"
 
 NameInfo *qmp_query_name(Error **errp)
 {
@@ -85,7 +86,11 @@ void qmp_quit(Error **err)
 
 void qmp_stop(Error **errp)
 {
-    vm_stop(RUN_STATE_PAUSED);
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        autostart = 0;
+    } else {
+        vm_stop(RUN_STATE_PAUSED);
+    }
 }
 
 void qmp_system_reset(Error **errp)
@@ -144,10 +149,7 @@ void qmp_cont(Error **errp)
 {
     Error *local_err = NULL;
 
-    if (runstate_check(RUN_STATE_INMIGRATE)) {
-        error_set(errp, QERR_MIGRATION_EXPECTED);
-        return;
-    } else if (runstate_check(RUN_STATE_INTERNAL_ERROR) ||
+    if (runstate_check(RUN_STATE_INTERNAL_ERROR) ||
                runstate_check(RUN_STATE_SHUTDOWN)) {
         error_set(errp, QERR_RESET_REQUIRED);
         return;
@@ -162,7 +164,11 @@ void qmp_cont(Error **errp)
         return;
     }
 
-    vm_start();
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        autostart = 1;
+    } else {
+        vm_start();
+    }
 }
 
 void qmp_system_wakeup(Error **errp)
@@ -349,11 +355,9 @@ void qmp_change_vnc_password(const char *password, Error **errp)
     }
 }
 
-static void qmp_change_vnc_listen(const char *target, Error **err)
+static void qmp_change_vnc_listen(const char *target, Error **errp)
 {
-    if (vnc_display_open(NULL, target) < 0) {
-        error_set(err, QERR_VNC_SERVER_FAILED, target);
-    }
+    vnc_display_open(NULL, target, errp);
 }
 
 static void qmp_change_vnc(const char *target, bool has_arg, const char *arg,
@@ -468,14 +472,51 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
     return prop_list;
 }
 
-CpuDefinitionInfoList GCC_WEAK *arch_query_cpu_definitions(Error **errp)
-{
-    error_set(errp, QERR_NOT_SUPPORTED);
-    return NULL;
-}
-
 CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp)
 {
     return arch_query_cpu_definitions(errp);
 }
 
+void qmp_add_client(const char *protocol, const char *fdname,
+                    bool has_skipauth, bool skipauth, bool has_tls, bool tls,
+                    Error **errp)
+{
+    CharDriverState *s;
+    int fd;
+
+    fd = monitor_get_fd(cur_mon, fdname, errp);
+    if (fd < 0) {
+        return;
+    }
+
+    if (strcmp(protocol, "spice") == 0) {
+        if (!using_spice) {
+            error_set(errp, QERR_DEVICE_NOT_ACTIVE, "spice");
+            close(fd);
+            return;
+        }
+        skipauth = has_skipauth ? skipauth : false;
+        tls = has_tls ? tls : false;
+        if (qemu_spice_display_add_client(fd, skipauth, tls) < 0) {
+            error_setg(errp, "spice failed to add client");
+            close(fd);
+        }
+        return;
+#ifdef CONFIG_VNC
+    } else if (strcmp(protocol, "vnc") == 0) {
+        skipauth = has_skipauth ? skipauth : false;
+        vnc_display_add_client(NULL, fd, skipauth);
+        return;
+#endif
+    } else if ((s = qemu_chr_find(protocol)) != NULL) {
+        if (qemu_chr_add_client(s, fd) < 0) {
+            error_setg(errp, "failed to add client");
+            close(fd);
+            return;
+        }
+        return;
+    }
+
+    error_setg(errp, "protocol '%s' is invalid", protocol);
+    close(fd);
+}
diff --git a/qobject/Makefile.objs b/qobject/Makefile.objs
new file mode 100644
index 00000000000..c9ff59c6cc5
--- /dev/null
+++ b/qobject/Makefile.objs
@@ -0,0 +1,3 @@
+util-obj-y = qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
+util-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
+util-obj-y += qerror.o
diff --git a/json-lexer.c b/qobject/json-lexer.c
index 3cd3285825d..440df60392b 100644
--- a/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qstring.h"
-#include "qlist.h"
-#include "qdict.h"
-#include "qint.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qint.h"
 #include "qemu-common.h"
-#include "json-lexer.h"
+#include "qapi/qmp/json-lexer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
diff --git a/json-parser.c b/qobject/json-parser.c
index 457291b161a..05279c11eb5 100644
--- a/json-parser.c
+++ b/qobject/json-parser.c
@@ -14,15 +14,15 @@
 #include <stdarg.h>
 
 #include "qemu-common.h"
-#include "qstring.h"
-#include "qint.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "json-parser.h"
-#include "json-lexer.h"
-#include "qerror.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/qerror.h"
 
 typedef struct JSONParserContext
 {
diff --git a/json-streamer.c b/qobject/json-streamer.c
index c255c7818f3..1b2f9b1d107 100644
--- a/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qlist.h"
-#include "qint.h"
-#include "qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu-common.h"
-#include "json-lexer.h"
-#include "json-streamer.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 #define MAX_NESTING (1ULL << 10)
diff --git a/qbool.c b/qobject/qbool.c
index 590cd716ead..a3d2afa827f 100644
--- a/qbool.c
+++ b/qobject/qbool.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qbool.h"
-#include "qobject.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qbool_destroy_obj(QObject *obj);
diff --git a/qdict.c b/qobject/qdict.c
index 4bf308b61c6..7543ccc10ff 100644
--- a/qdict.c
+++ b/qobject/qdict.c
@@ -10,13 +10,13 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qint.h"
-#include "qfloat.h"
-#include "qdict.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qobject.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
 
 static void qdict_destroy_obj(QObject *obj);
diff --git a/qerror.c b/qobject/qerror.c
index 08185047b4b..3aee1cf6a69 100644
--- a/qerror.c
+++ b/qobject/qerror.c
@@ -10,9 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "monitor.h"
-#include "qjson.h"
-#include "qerror.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu-common.h"
 
 static void qerror_destroy_obj(QObject *obj);
diff --git a/qfloat.c b/qobject/qfloat.c
index 98338f3b71a..7de0992dba7 100644
--- a/qfloat.c
+++ b/qobject/qfloat.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qfloat.h"
-#include "qobject.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qfloat_destroy_obj(QObject *obj);
diff --git a/qint.c b/qobject/qint.c
index ee51804fbe9..86b9b04f0b6 100644
--- a/qint.c
+++ b/qobject/qint.c
@@ -10,8 +10,8 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qint.h"
-#include "qobject.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qint_destroy_obj(QObject *obj);
diff --git a/qjson.c b/qobject/qjson.c
index f9c8e775161..83a6b4f7c1d 100644
--- a/qjson.c
+++ b/qobject/qjson.c
@@ -11,15 +11,15 @@
  *
  */
 
-#include "json-lexer.h"
-#include "json-parser.h"
-#include "json-streamer.h"
-#include "qjson.h"
-#include "qint.h"
-#include "qlist.h"
-#include "qbool.h"
-#include "qfloat.h"
-#include "qdict.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
 
 typedef struct JSONParsingState
 {
diff --git a/qlist.c b/qobject/qlist.c
index b48ec5b9142..1ced0de58e2 100644
--- a/qlist.c
+++ b/qobject/qlist.c
@@ -10,9 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qlist.h"
-#include "qobject.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
 
 static void qlist_destroy_obj(QObject *obj);
diff --git a/qstring.c b/qobject/qstring.c
index b7e12e40158..5f7376c336f 100644
--- a/qstring.c
+++ b/qobject/qstring.c
@@ -10,8 +10,8 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qobject.h"
-#include "qstring.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 static void qstring_destroy_obj(QObject *obj);
diff --git a/qom/Makefile.objs b/qom/Makefile.objs
index 5ef060a4012..6a93ac7398c 100644
--- a/qom/Makefile.objs
+++ b/qom/Makefile.objs
@@ -1,4 +1,2 @@
-qom-obj-y = object.o container.o qom-qobject.o
-qom-obj-twice-y = cpu.o
-common-obj-y = $(qom-obj-twice-y)
-user-obj-y = $(qom-obj-twice-y)
+common-obj-y = object.o container.o qom-qobject.o
+common-obj-y += cpu.o
diff --git a/qom/container.c b/qom/container.c
index 4ca8b5cba32..62b1648add8 100644
--- a/qom/container.c
+++ b/qom/container.c
@@ -10,11 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/object.h"
-#include "module.h"
+#include "qom/object.h"
+#include "qemu/module.h"
 #include <assert.h>
 
-static TypeInfo container_info = {
+static const TypeInfo container_info = {
     .name          = "container",
     .instance_size = sizeof(Object),
     .parent        = TYPE_OBJECT,
diff --git a/qom/cpu.c b/qom/cpu.c
index 5b360469c5a..8fb538bf3b5 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -18,7 +18,7 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "qemu-common.h"
 
 void cpu_reset(CPUState *cpu)
@@ -34,16 +34,31 @@ static void cpu_common_reset(CPUState *cpu)
 {
 }
 
+ObjectClass *cpu_class_by_name(const char *typename, const char *cpu_model)
+{
+    CPUClass *cc = CPU_CLASS(object_class_by_name(typename));
+
+    return cc->class_by_name(cpu_model);
+}
+
+static ObjectClass *cpu_common_class_by_name(const char *cpu_model)
+{
+    return NULL;
+}
+
 static void cpu_class_init(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     CPUClass *k = CPU_CLASS(klass);
 
+    k->class_by_name = cpu_common_class_by_name;
     k->reset = cpu_common_reset;
+    dc->no_user = 1;
 }
 
-static TypeInfo cpu_type_info = {
+static const TypeInfo cpu_type_info = {
     .name = TYPE_CPU,
-    .parent = TYPE_OBJECT,
+    .parent = TYPE_DEVICE,
     .instance_size = sizeof(CPUState),
     .abstract = true,
     .class_size = sizeof(CPUClass),
diff --git a/qom/object.c b/qom/object.c
index e3e9242638d..e2002821725 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -10,19 +10,20 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/object.h"
+#include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
 #include "qapi/string-input-visitor.h"
 #include "qapi/string-output-visitor.h"
+#include "qapi/qmp/qerror.h"
 
 /* TODO: replace QObject with a simpler visitor to avoid a dependency
  * of the QOM core on QObject?  */
-#include "qemu/qom-qobject.h"
-#include "qobject.h"
-#include "qbool.h"
-#include "qint.h"
-#include "qstring.h"
+#include "qom/qom-qobject.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
 
 #define MAX_INTERFACES 32
 
@@ -307,6 +308,7 @@ void object_initialize_with_type(void *data, TypeImpl *type)
 
     memset(obj, 0, type->instance_size);
     obj->class = type->class;
+    object_ref(obj);
     QTAILQ_INIT(&obj->properties);
     object_init_with_type(obj, type);
 }
@@ -362,6 +364,9 @@ void object_unparent(Object *obj)
     if (obj->parent) {
         object_property_del_child(obj->parent, obj, NULL);
     }
+    if (obj->class->unparent) {
+        (obj->class->unparent)(obj);
+    }
 }
 
 static void object_deinit(Object *obj, TypeImpl *type)
@@ -375,7 +380,7 @@ static void object_deinit(Object *obj, TypeImpl *type)
     }
 }
 
-void object_finalize(void *data)
+static void object_finalize(void *data)
 {
     Object *obj = data;
     TypeImpl *ti = obj->class->type;
@@ -384,6 +389,9 @@ void object_finalize(void *data)
     object_property_del_all(obj);
 
     g_assert(obj->ref == 0);
+    if (obj->free) {
+        obj->free(obj);
+    }
 }
 
 Object *object_new_with_type(Type type)
@@ -395,7 +403,7 @@ Object *object_new_with_type(Type type)
 
     obj = g_malloc(type->instance_size);
     object_initialize_with_type(obj, type);
-    object_ref(obj);
+    obj->free = g_free;
 
     return obj;
 }
@@ -412,12 +420,11 @@ void object_delete(Object *obj)
     object_unparent(obj);
     g_assert(obj->ref == 1);
     object_unref(obj);
-    g_free(obj);
 }
 
 Object *object_dynamic_cast(Object *obj, const char *typename)
 {
-    if (object_class_dynamic_cast(object_get_class(obj), typename)) {
+    if (obj && object_class_dynamic_cast(object_get_class(obj), typename)) {
         return obj;
     }
 
@@ -430,7 +437,7 @@ Object *object_dynamic_cast_assert(Object *obj, const char *typename)
 
     inst = object_dynamic_cast(obj, typename);
 
-    if (!inst) {
+    if (!inst && obj) {
         fprintf(stderr, "Object %p is not an instance of type %s\n",
                 obj, typename);
         abort();
@@ -494,6 +501,11 @@ ObjectClass *object_get_class(Object *obj)
     return obj->class;
 }
 
+bool object_class_is_abstract(ObjectClass *klass)
+{
+    return klass->type->abstract;
+}
+
 const char *object_class_get_name(ObjectClass *klass)
 {
     return klass->type->name;
@@ -1010,7 +1022,7 @@ gchar *object_get_canonical_path(Object *obj)
     return newpath;
 }
 
-Object *object_resolve_path_component(Object *parent, gchar *part)
+Object *object_resolve_path_component(Object *parent, const gchar *part)
 {
     ObjectProperty *prop = object_property_find(parent, part, NULL);
     if (prop == NULL) {
@@ -1183,6 +1195,62 @@ void object_property_add_str(Object *obj, const char *name,
                         prop, errp);
 }
 
+typedef struct BoolProperty
+{
+    bool (*get)(Object *, Error **);
+    void (*set)(Object *, bool, Error **);
+} BoolProperty;
+
+static void property_get_bool(Object *obj, Visitor *v, void *opaque,
+                              const char *name, Error **errp)
+{
+    BoolProperty *prop = opaque;
+    bool value;
+
+    value = prop->get(obj, errp);
+    visit_type_bool(v, &value, name, errp);
+}
+
+static void property_set_bool(Object *obj, Visitor *v, void *opaque,
+                              const char *name, Error **errp)
+{
+    BoolProperty *prop = opaque;
+    bool value;
+    Error *local_err = NULL;
+
+    visit_type_bool(v, &value, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    prop->set(obj, value, errp);
+}
+
+static void property_release_bool(Object *obj, const char *name,
+                                  void *opaque)
+{
+    BoolProperty *prop = opaque;
+    g_free(prop);
+}
+
+void object_property_add_bool(Object *obj, const char *name,
+                              bool (*get)(Object *, Error **),
+                              void (*set)(Object *, bool, Error **),
+                              Error **errp)
+{
+    BoolProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_property_add(obj, name, "bool",
+                        get ? property_get_bool : NULL,
+                        set ? property_set_bool : NULL,
+                        property_release_bool,
+                        prop, errp);
+}
+
 static char *qdev_get_type(Object *obj, Error **errp)
 {
     return g_strdup(object_get_typename(obj));
diff --git a/qom/qom-qobject.c b/qom/qom-qobject.c
index 0689914e15d..6384b8e98ce 100644
--- a/qom/qom-qobject.c
+++ b/qom/qom-qobject.c
@@ -10,9 +10,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu/object.h"
-#include "qemu/qom-qobject.h"
-#include "qapi/qapi-visit-core.h"
+#include "qom/object.h"
+#include "qom/qom-qobject.h"
+#include "qapi/visitor.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 
diff --git a/qtest.c b/qtest.c
index fbfab4e1a77..c9b58ceb8b1 100644
--- a/qtest.c
+++ b/qtest.c
@@ -11,14 +11,14 @@
  *
  */
 
-#include "qtest.h"
+#include "sysemu/qtest.h"
 #include "hw/qdev.h"
-#include "qemu-char.h"
-#include "ioport.h"
-#include "memory.h"
+#include "char/char.h"
+#include "exec/ioport.h"
+#include "exec/memory.h"
 #include "hw/irq.h"
-#include "sysemu.h"
-#include "cpus.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/cpus.h"
 
 #define MAX_IRQ 256
 
diff --git a/readline.c b/readline.c
index 540cd8a025d..a0c9638e4d9 100644
--- a/readline.c
+++ b/readline.c
@@ -21,8 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "readline.h"
-#include "monitor.h"
+#include "monitor/readline.h"
+#include "monitor/monitor.h"
 
 #define IS_NORM 0
 #define IS_ESC  1
@@ -248,8 +248,8 @@ static void readline_hist_add(ReadLineState *rs, const char *cmdline)
     if (idx == READLINE_MAX_CMDS) {
 	/* Need to get one free slot */
 	free(rs->history[0]);
-	memcpy(rs->history, &rs->history[1],
-	       (READLINE_MAX_CMDS - 1) * sizeof(char *));
+	memmove(rs->history, &rs->history[1],
+	        (READLINE_MAX_CMDS - 1) * sizeof(char *));
 	rs->history[READLINE_MAX_CMDS - 1] = NULL;
 	idx = READLINE_MAX_CMDS - 1;
     }
diff --git a/roms/Makefile b/roms/Makefile
index feb9c2b145d..5e645bc7d10 100644
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -12,6 +12,7 @@ bios: config.seabios
 	sh configure-seabios.sh $<
 	make -C seabios out/bios.bin
 	cp seabios/out/bios.bin ../pc-bios/bios.bin
+	cp seabios/out/*dsdt.aml ../pc-bios/
 
 seavgabios: $(patsubst %,seavgabios-%,$(vgabios_variants))
 
diff --git a/roms/SLOF b/roms/SLOF
-Subproject f21f7a3f46b557eb5923f899ce8b4401b3cc6d9
+Subproject 0ad10f26c94a86a0c9c3970e53f9a9f6a744055
diff --git a/roms/seabios b/roms/seabios
-Subproject 5a023065388287e261ae9212452ff541f9fa9cd
+Subproject 4bd8aebf3534e10d9aa21e820903f2cf9120708
diff --git a/rules.mak b/rules.mak
index 1b173aa9812..6d82c0d5a0a 100644
--- a/rules.mak
+++ b/rules.mak
@@ -14,24 +14,42 @@ MAKEFLAGS += -rR
 # Flags for dependency generation
 QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
 
+# Same as -I$(SRC_PATH) -I., but for the nested source/object directories
+QEMU_CFLAGS += -I$(<D) -I$(@D)
+
 %.o: %.c
 	$(call quiet-command,$(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  CC    $(TARGET_DIR)$@")
 
 ifeq ($(LIBTOOL),)
-%.lo: %.c
-	@echo "missing libtool. please install and rerun configure"; exit 1
+LIBTOOL = /bin/false
+LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
+       $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(version-obj-y) \
+       $(LIBS),"  LINK  $(TARGET_DIR)$@")
 else
+LIBTOOL += $(if $(V),,--quiet)
 %.lo: %.c
-	$(call quiet-command,$(LIBTOOL) --mode=compile --quiet --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  lt CC $@")
+	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  lt CC $@")
+%.lo: %.dtrace
+	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, " lt GEN $(TARGET_DIR)$@")
+
+LINK = $(call quiet-command,\
+       $(if $(filter %.lo %.la,$^),$(LIBTOOL) --mode=link --tag=CC \
+       )$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
+       $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(version-obj-y) \
+       $(LIBS),$(if $(filter %.lo %.la,$^),"lt LINK ", "  LINK  ")"$(TARGET_DIR)$@")
 endif
 
-%.o: %.S
-	$(call quiet-command,$(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  AS    $(TARGET_DIR)$@")
+%.asm: %.S
+	$(call quiet-command,$(CPP) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -o $@ $<,"  CPP   $(TARGET_DIR)$@")
+
+%.o: %.asm
+	$(call quiet-command,$(AS) $(ASFLAGS) -o $@ $<,"  AS    $(TARGET_DIR)$@")
 
 %.o: %.m
 	$(call quiet-command,$(OBJCC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  OBJC  $(TARGET_DIR)$@")
 
-LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $(sort $(1)) $(LIBS),"  LINK  $(TARGET_DIR)$@")
+%.o: %.dtrace
+	$(call quiet-command,dtrace -o $@ -G -s $<, "  GEN   $(TARGET_DIR)$@")
 
 %$(EXESUF): %.o
 	$(call LINK,$^)
@@ -68,7 +86,7 @@ TRACETOOL=$(PYTHON) $(SRC_PATH)/scripts/tracetool.py
 	@test -f $@ || cp $< $@
 
 %.h-timestamp: %.mak
-	$(call quiet-command, sh $(SRC_PATH)/scripts/create_config < $< > $@, "  GEN   $*.h")
+	$(call quiet-command, sh $(SRC_PATH)/scripts/create_config < $< > $@, "  GEN   $(TARGET_DIR)$*.h")
 	@cmp $@ $*.h >/dev/null 2>&1 || cp $@ $*.h
 
 # will delete the target of a rule if commands exit with a nonzero exit status
diff --git a/savevm.c b/savevm.c
index c7fe2831456..304d1effe54 100644
--- a/savevm.c
+++ b/savevm.c
@@ -21,71 +21,24 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-#include <zlib.h>
-
-/* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
-
-#ifndef _WIN32
-#include <sys/times.h>
-#include <sys/wait.h>
-#include <termios.h>
-#include <sys/mman.h>
-#include <sys/ioctl.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <net/if.h>
-#include <arpa/inet.h>
-#include <dirent.h>
-#include <netdb.h>
-#include <sys/select.h>
-#ifdef CONFIG_BSD
-#include <sys/stat.h>
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
-#include <libutil.h>
-#else
-#include <util.h>
-#endif
-#ifdef __linux__
-#include <pty.h>
-#include <malloc.h>
-#include <linux/rtc.h>
-#endif
-#endif
-#endif
-
-#ifdef _WIN32
-#include <windows.h>
-#include <malloc.h>
-#include <sys/timeb.h>
-#include <mmsystem.h>
-#define getopt_long_only getopt_long
-#define memalign(align, size) malloc(size)
-#endif
 
+#include "config-host.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
-#include "net.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "net/net.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 #include "audio/audio.h"
-#include "migration.h"
-#include "qemu_socket.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
-#include "cpus.h"
-#include "memory.h"
+#include "migration/migration.h"
+#include "qemu/sockets.h"
+#include "qemu/queue.h"
+#include "sysemu/cpus.h"
+#include "exec/memory.h"
 #include "qmp-commands.h"
 #include "trace.h"
+#include "qemu/bitops.h"
 
 #define SELF_ANNOUNCE_ROUNDS 5
 
@@ -162,12 +115,7 @@ void qemu_announce_self(void)
 #define IO_BUF_SIZE 32768
 
 struct QEMUFile {
-    QEMUFilePutBufferFunc *put_buffer;
-    QEMUFileGetBufferFunc *get_buffer;
-    QEMUFileCloseFunc *close;
-    QEMUFileRateLimit *rate_limit;
-    QEMUFileSetRateLimit *set_rate_limit;
-    QEMUFileGetRateLimit *get_rate_limit;
+    const QEMUFileOps *ops;
     void *opaque;
     int is_write;
 
@@ -192,28 +140,52 @@ typedef struct QEMUFileSocket
     QEMUFile *file;
 } QEMUFileSocket;
 
+static int socket_get_fd(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+
+    return s->fd;
+}
+
 static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
 
-    do {
+    for (;;) {
         len = qemu_recv(s->fd, buf, size, 0);
-    } while (len == -1 && socket_error() == EINTR);
+        if (len != -1) {
+            break;
+        }
+        if (socket_error() == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (socket_error() != EINTR) {
+            break;
+        }
+    }
 
-    if (len == -1)
+    if (len == -1) {
         len = -socket_error();
-
+    }
     return len;
 }
 
 static int socket_close(void *opaque)
 {
     QEMUFileSocket *s = opaque;
+    closesocket(s->fd);
     g_free(s);
     return 0;
 }
 
+static int stdio_get_fd(void *opaque)
+{
+    QEMUFileStdio *s = opaque;
+
+    return fileno(s->stdio_file);
+}
+
 static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
 {
     QEMUFileStdio *s = opaque;
@@ -226,10 +198,19 @@ static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     FILE *fp = s->stdio_file;
     int bytes;
 
-    do {
+    for (;;) {
         clearerr(fp);
         bytes = fread(buf, 1, size, fp);
-    } while ((bytes == 0) && ferror(fp) && (errno == EINTR));
+        if (bytes != 0 || !ferror(fp)) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
     return bytes;
 }
 
@@ -256,6 +237,18 @@ static int stdio_fclose(void *opaque)
     return ret;
 }
 
+static const QEMUFileOps stdio_pipe_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_pclose
+};
+
+static const QEMUFileOps stdio_pipe_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_pclose
+};
+
 QEMUFile *qemu_popen(FILE *stdio_file, const char *mode)
 {
     QEMUFileStdio *s;
@@ -270,11 +263,9 @@ QEMUFile *qemu_popen(FILE *stdio_file, const char *mode)
     s->stdio_file = stdio_file;
 
     if(mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, NULL, stdio_get_buffer, stdio_pclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_pipe_read_ops);
     } else {
-        s->file = qemu_fopen_ops(s, stdio_put_buffer, NULL, stdio_pclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_pipe_write_ops);
     }
     return s->file;
 }
@@ -291,16 +282,17 @@ QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
     return qemu_popen(popen_file, mode);
 }
 
-int qemu_stdio_fd(QEMUFile *f)
-{
-    QEMUFileStdio *p;
-    int fd;
-
-    p = (QEMUFileStdio *)f->opaque;
-    fd = fileno(p->stdio_file);
+static const QEMUFileOps stdio_file_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_fclose
+};
 
-    return fd;
-}
+static const QEMUFileOps stdio_file_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_fclose
+};
 
 QEMUFile *qemu_fdopen(int fd, const char *mode)
 {
@@ -319,11 +311,9 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
         goto fail;
 
     if(mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, NULL, stdio_get_buffer, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
     } else {
-        s->file = qemu_fopen_ops(s, stdio_put_buffer, NULL, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
     }
     return s->file;
 
@@ -332,31 +322,21 @@ fail:
     return NULL;
 }
 
+static const QEMUFileOps socket_read_ops = {
+    .get_fd =     socket_get_fd,
+    .get_buffer = socket_get_buffer,
+    .close =      socket_close
+};
+
 QEMUFile *qemu_fopen_socket(int fd)
 {
     QEMUFileSocket *s = g_malloc0(sizeof(QEMUFileSocket));
 
     s->fd = fd;
-    s->file = qemu_fopen_ops(s, NULL, socket_get_buffer, socket_close, 
-			     NULL, NULL, NULL);
+    s->file = qemu_fopen_ops(s, &socket_read_ops);
     return s->file;
 }
 
-static int file_put_buffer(void *opaque, const uint8_t *buf,
-                            int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    fseek(s->stdio_file, pos, SEEK_SET);
-    return fwrite(buf, 1, size, s->stdio_file);
-}
-
-static int file_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    fseek(s->stdio_file, pos, SEEK_SET);
-    return fread(buf, 1, size, s->stdio_file);
-}
-
 QEMUFile *qemu_fopen(const char *filename, const char *mode)
 {
     QEMUFileStdio *s;
@@ -375,11 +355,9 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode)
         goto fail;
     
     if(mode[0] == 'w') {
-        s->file = qemu_fopen_ops(s, file_put_buffer, NULL, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
     } else {
-        s->file = qemu_fopen_ops(s, NULL, file_get_buffer, stdio_fclose, 
-			       NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
     }
     return s->file;
 fail:
@@ -404,32 +382,31 @@ static int bdrv_fclose(void *opaque)
     return bdrv_flush(opaque);
 }
 
+static const QEMUFileOps bdrv_read_ops = {
+    .get_buffer = block_get_buffer,
+    .close =      bdrv_fclose
+};
+
+static const QEMUFileOps bdrv_write_ops = {
+    .put_buffer = block_put_buffer,
+    .close =      bdrv_fclose
+};
+
 static QEMUFile *qemu_fopen_bdrv(BlockDriverState *bs, int is_writable)
 {
     if (is_writable)
-        return qemu_fopen_ops(bs, block_put_buffer, NULL, bdrv_fclose, 
-			      NULL, NULL, NULL);
-    return qemu_fopen_ops(bs, NULL, block_get_buffer, bdrv_fclose, NULL, NULL, NULL);
+        return qemu_fopen_ops(bs, &bdrv_write_ops);
+    return qemu_fopen_ops(bs, &bdrv_read_ops);
 }
 
-QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
-                         QEMUFileGetBufferFunc *get_buffer,
-                         QEMUFileCloseFunc *close,
-                         QEMUFileRateLimit *rate_limit,
-                         QEMUFileSetRateLimit *set_rate_limit,
-                         QEMUFileGetRateLimit *get_rate_limit)
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
 {
     QEMUFile *f;
 
     f = g_malloc0(sizeof(QEMUFile));
 
     f->opaque = opaque;
-    f->put_buffer = put_buffer;
-    f->get_buffer = get_buffer;
-    f->close = close;
-    f->rate_limit = rate_limit;
-    f->set_rate_limit = set_rate_limit;
-    f->get_rate_limit = get_rate_limit;
+    f->ops = ops;
     f->is_write = 0;
 
     return f;
@@ -440,42 +417,31 @@ int qemu_file_get_error(QEMUFile *f)
     return f->last_error;
 }
 
-void qemu_file_set_error(QEMUFile *f, int ret)
+static void qemu_file_set_error(QEMUFile *f, int ret)
 {
-    f->last_error = ret;
-}
-
-/** Sets last_error conditionally
- *
- * Sets last_error only if ret is negative _and_ no error
- * was set before.
- */
-static void qemu_file_set_if_error(QEMUFile *f, int ret)
-{
-    if (ret < 0 && !f->last_error) {
-        qemu_file_set_error(f, ret);
+    if (f->last_error == 0) {
+        f->last_error = ret;
     }
 }
 
 /** Flushes QEMUFile buffer
  *
- * In case of error, last_error is set.
  */
-void qemu_fflush(QEMUFile *f)
+static int qemu_fflush(QEMUFile *f)
 {
-    if (!f->put_buffer)
-        return;
+    int ret = 0;
 
-    if (f->is_write && f->buf_index > 0) {
-        int len;
+    if (!f->ops->put_buffer)
+        return 0;
 
-        len = f->put_buffer(f->opaque, f->buf, f->buf_offset, f->buf_index);
-        if (len > 0)
+    if (f->is_write && f->buf_index > 0) {
+        ret = f->ops->put_buffer(f->opaque, f->buf, f->buf_offset, f->buf_index);
+        if (ret >= 0) {
             f->buf_offset += f->buf_index;
-        else
-            qemu_file_set_error(f, -EINVAL);
+        }
         f->buf_index = 0;
     }
+    return ret;
 }
 
 static void qemu_fill_buffer(QEMUFile *f)
@@ -483,7 +449,7 @@ static void qemu_fill_buffer(QEMUFile *f)
     int len;
     int pending;
 
-    if (!f->get_buffer)
+    if (!f->ops->get_buffer)
         return;
 
     if (f->is_write)
@@ -496,31 +462,23 @@ static void qemu_fill_buffer(QEMUFile *f)
     f->buf_index = 0;
     f->buf_size = pending;
 
-    len = f->get_buffer(f->opaque, f->buf + pending, f->buf_offset,
+    len = f->ops->get_buffer(f->opaque, f->buf + pending, f->buf_offset,
                         IO_BUF_SIZE - pending);
     if (len > 0) {
         f->buf_size += len;
         f->buf_offset += len;
     } else if (len == 0) {
-        f->last_error = -EIO;
+        qemu_file_set_error(f, -EIO);
     } else if (len != -EAGAIN)
         qemu_file_set_error(f, len);
 }
 
-/** Calls close function and set last_error if needed
- *
- * Internal function. qemu_fflush() must be called before this.
- *
- * Returns f->close() return value, or 0 if close function is not set.
- */
-static int qemu_fclose_internal(QEMUFile *f)
+int qemu_get_fd(QEMUFile *f)
 {
-    int ret = 0;
-    if (f->close) {
-        ret = f->close(f->opaque);
-        qemu_file_set_if_error(f, ret);
+    if (f->ops->get_fd) {
+        return f->ops->get_fd(f->opaque);
     }
-    return ret;
+    return -1;
 }
 
 /** Closes the file
@@ -534,8 +492,14 @@ static int qemu_fclose_internal(QEMUFile *f)
 int qemu_fclose(QEMUFile *f)
 {
     int ret;
-    qemu_fflush(f);
-    ret = qemu_fclose_internal(f);
+    ret = qemu_fflush(f);
+
+    if (f->ops->close) {
+        int ret2 = f->ops->close(f->opaque);
+        if (ret >= 0) {
+            ret = ret2;
+        }
+    }
     /* If any error was spotted before closing, we should report it
      * instead of the close() return value.
      */
@@ -546,22 +510,21 @@ int qemu_fclose(QEMUFile *f)
     return ret;
 }
 
-void qemu_file_put_notify(QEMUFile *f)
-{
-    f->put_buffer(f->opaque, NULL, 0, 0);
-}
-
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
 {
     int l;
 
-    if (!f->last_error && f->is_write == 0 && f->buf_index > 0) {
+    if (f->last_error) {
+        return;
+    }
+
+    if (f->is_write == 0 && f->buf_index > 0) {
         fprintf(stderr,
                 "Attempted to write to buffer while read buffer is not empty\n");
         abort();
     }
 
-    while (!f->last_error && size > 0) {
+    while (size > 0) {
         l = IO_BUF_SIZE - f->buf_index;
         if (l > size)
             l = size;
@@ -570,14 +533,23 @@ void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
         f->buf_index += l;
         buf += l;
         size -= l;
-        if (f->buf_index >= IO_BUF_SIZE)
-            qemu_fflush(f);
+        if (f->buf_index >= IO_BUF_SIZE) {
+            int ret = qemu_fflush(f);
+            if (ret < 0) {
+                qemu_file_set_error(f, ret);
+                break;
+            }
+        }
     }
 }
 
 void qemu_put_byte(QEMUFile *f, int v)
 {
-    if (!f->last_error && f->is_write == 0 && f->buf_index > 0) {
+    if (f->last_error) {
+        return;
+    }
+
+    if (f->is_write == 0 && f->buf_index > 0) {
         fprintf(stderr,
                 "Attempted to write to buffer while read buffer is not empty\n");
         abort();
@@ -585,8 +557,12 @@ void qemu_put_byte(QEMUFile *f, int v)
 
     f->buf[f->buf_index++] = v;
     f->is_write = 1;
-    if (f->buf_index >= IO_BUF_SIZE)
-        qemu_fflush(f);
+    if (f->buf_index >= IO_BUF_SIZE) {
+        int ret = qemu_fflush(f);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+    }
 }
 
 static void qemu_file_skip(QEMUFile *f, int size)
@@ -671,44 +647,23 @@ int qemu_get_byte(QEMUFile *f)
     return result;
 }
 
-int64_t qemu_ftell(QEMUFile *f)
+static int64_t qemu_ftell(QEMUFile *f)
 {
     return f->buf_offset - f->buf_size + f->buf_index;
 }
 
-int64_t qemu_fseek(QEMUFile *f, int64_t pos, int whence)
-{
-    if (whence == SEEK_SET) {
-        /* nothing to do */
-    } else if (whence == SEEK_CUR) {
-        pos += qemu_ftell(f);
-    } else {
-        /* SEEK_END not supported */
-        return -1;
-    }
-    if (f->put_buffer) {
-        qemu_fflush(f);
-        f->buf_offset = pos;
-    } else {
-        f->buf_offset = pos;
-        f->buf_index = 0;
-        f->buf_size = 0;
-    }
-    return pos;
-}
-
 int qemu_file_rate_limit(QEMUFile *f)
 {
-    if (f->rate_limit)
-        return f->rate_limit(f->opaque);
+    if (f->ops->rate_limit)
+        return f->ops->rate_limit(f->opaque);
 
     return 0;
 }
 
 int64_t qemu_file_get_rate_limit(QEMUFile *f)
 {
-    if (f->get_rate_limit)
-        return f->get_rate_limit(f->opaque);
+    if (f->ops->get_rate_limit)
+        return f->ops->get_rate_limit(f->opaque);
 
     return 0;
 }
@@ -717,8 +672,8 @@ int64_t qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate)
 {
     /* any failed or completed migration keeps its state to allow probing of
      * migration data, but has no associated file anymore */
-    if (f && f->set_rate_limit)
-        return f->set_rate_limit(f->opaque, new_rate);
+    if (f && f->ops->set_rate_limit)
+        return f->ops->set_rate_limit(f->opaque, new_rate);
 
     return 0;
 }
@@ -1159,6 +1114,46 @@ const VMStateInfo vmstate_info_unused_buffer = {
     .put  = put_unused_buffer,
 };
 
+/* bitmaps (as defined by bitmap.h). Note that size here is the size
+ * of the bitmap in bits. The on-the-wire format of a bitmap is 64
+ * bit words with the bits in big endian order. The in-memory format
+ * is an array of 'unsigned long', which may be either 32 or 64 bits.
+ */
+/* This is the number of 64 bit words sent over the wire */
+#define BITS_TO_U64S(nr) DIV_ROUND_UP(nr, 64)
+static int get_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = qemu_get_be64(f);
+        bmp[idx++] = w;
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            bmp[idx++] = w >> 32;
+        }
+    }
+    return 0;
+}
+
+static void put_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = bmp[idx++];
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            w |= ((uint64_t)bmp[idx++]) << 32;
+        }
+        qemu_put_be64(f, w);
+    }
+}
+
+const VMStateInfo vmstate_info_bitmap = {
+    .name = "bitmap",
+    .get = get_bitmap,
+    .put = put_bitmap,
+};
+
 typedef struct CompatEntry {
     char idstr[256];
     int instance_id;
@@ -1595,13 +1590,13 @@ int qemu_savevm_state_begin(QEMUFile *f,
 
         ret = se->ops->save_live_setup(f, se->opaque);
         if (ret < 0) {
-            qemu_savevm_state_cancel(f);
+            qemu_savevm_state_cancel();
             return ret;
         }
     }
     ret = qemu_file_get_error(f);
     if (ret != 0) {
-        qemu_savevm_state_cancel(f);
+        qemu_savevm_state_cancel();
     }
 
     return ret;
@@ -1652,7 +1647,7 @@ int qemu_savevm_state_iterate(QEMUFile *f)
     }
     ret = qemu_file_get_error(f);
     if (ret != 0) {
-        qemu_savevm_state_cancel(f);
+        qemu_savevm_state_cancel();
     }
     return ret;
 }
@@ -1713,7 +1708,26 @@ int qemu_savevm_state_complete(QEMUFile *f)
     return qemu_file_get_error(f);
 }
 
-void qemu_savevm_state_cancel(QEMUFile *f)
+uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
+{
+    SaveStateEntry *se;
+    uint64_t ret = 0;
+
+    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
+        if (!se->ops || !se->ops->save_live_pending) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        ret += se->ops->save_live_pending(f, se->opaque, max_size);
+    }
+    return ret;
+}
+
+void qemu_savevm_state_cancel(void)
 {
     SaveStateEntry *se;
 
@@ -2072,13 +2086,8 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     QEMUFile *f;
     int saved_vm_running;
     uint64_t vm_state_size;
-#ifdef _WIN32
-    struct _timeb tb;
-    struct tm *ptm;
-#else
-    struct timeval tv;
+    qemu_timeval tv;
     struct tm tm;
-#endif
     const char *name = qdict_get_try_str(qdict, "name");
 
     /* Verify if there is a device that doesn't support snapshots and is writable */
@@ -2108,15 +2117,9 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     memset(sn, 0, sizeof(*sn));
 
     /* fill auxiliary fields */
-#ifdef _WIN32
-    _ftime(&tb);
-    sn->date_sec = tb.time;
-    sn->date_nsec = tb.millitm * 1000000;
-#else
-    gettimeofday(&tv, NULL);
+    qemu_gettimeofday(&tv);
     sn->date_sec = tv.tv_sec;
     sn->date_nsec = tv.tv_usec * 1000;
-#endif
     sn->vm_clock_nsec = qemu_get_clock_ns(vm_clock);
 
     if (name) {
@@ -2128,15 +2131,9 @@ void do_savevm(Monitor *mon, const QDict *qdict)
             pstrcpy(sn->name, sizeof(sn->name), name);
         }
     } else {
-#ifdef _WIN32
-        time_t t = tb.time;
-        ptm = localtime(&t);
-        strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", ptm);
-#else
         /* cast below needed for OpenBSD where tv_sec is still 'long' */
         localtime_r((const time_t *)&tv.tv_sec, &tm);
         strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
-#endif
     }
 
     /* Delete old snapshots of the same name */
@@ -2201,7 +2198,6 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
  the_end:
     if (saved_vm_running)
         vm_start();
-    return;
 }
 
 int load_vmstate(const char *name)
@@ -2313,7 +2309,7 @@ void do_delvm(Monitor *mon, const QDict *qdict)
     }
 }
 
-void do_info_snapshots(Monitor *mon)
+void do_info_snapshots(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
     QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s;
diff --git a/scripts/feature_to_c.sh b/scripts/feature_to_c.sh
index b62da8a0bd7..888548e58b9 100644
--- a/scripts/feature_to_c.sh
+++ b/scripts/feature_to_c.sh
@@ -38,7 +38,7 @@ for input; do
   ${AWK:-awk} 'BEGIN { n = 0
       printf "#include \"config.h\"\n"
       printf "#include \"qemu-common.h\"\n"
-      printf "#include \"gdbstub.h\"\n"
+      printf "#include \"exec/gdbstub.h\"\n"
       print "static const char '$arrayname'[] = {"
       for (i = 0; i < 255; i++)
         _ord_[sprintf("%c", i)] = i
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index d9c48e0982b..bf5342a08df 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -83,6 +83,8 @@ push(@signature_tags, "Signed-off-by:");
 push(@signature_tags, "Reviewed-by:");
 push(@signature_tags, "Acked-by:");
 
+my $signature_pattern = "\(" . join("|", @signature_tags) . "\)";
+
 # rfc822 email address - preloaded methods go here.
 my $rfc822_lwsp = "(?:(?:\\r\\n)?[ \\t])";
 my $rfc822_char = '[\\000-\\377]';
@@ -95,7 +97,7 @@ my %VCS_cmds_git = (
     "execute_cmd" => \&git_execute_cmd,
     "available" => '(which("git") ne "") && (-d ".git")',
     "find_signers_cmd" =>
-	"git log --no-color --since=\$email_git_since " .
+	"git log --no-color --follow --since=\$email_git_since " .
 	    '--format="GitCommit: %H%n' .
 		      'GitAuthor: %an <%ae>%n' .
 		      'GitDate: %aD%n' .
@@ -328,7 +330,8 @@ sub read_mailmap {
 	# name1 <mail1> <mail2>
 	# name1 <mail1> name2 <mail2>
 	# (see man git-shortlog)
-	if (/^(.+)<(.+)>$/) {
+
+	if (/^([^<]+)<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $address = $2;
 
@@ -336,13 +339,13 @@ sub read_mailmap {
 	    ($real_name, $address) = parse_email("$real_name <$address>");
 	    $mailmap->{names}->{$address} = $real_name;
 
-	} elsif (/^<([^\s]+)>\s*<([^\s]+)>$/) {
+	} elsif (/^<([^>]+)>\s*<([^>]+)>$/) {
 	    my $real_address = $1;
 	    my $wrong_address = $2;
 
 	    $mailmap->{addresses}->{$wrong_address} = $real_address;
 
-	} elsif (/^(.+)<([^\s]+)>\s*<([^\s]+)>$/) {
+	} elsif (/^(.+)<([^>]+)>\s*<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $real_address = $2;
 	    my $wrong_address = $3;
@@ -353,7 +356,7 @@ sub read_mailmap {
 	    $mailmap->{names}->{$wrong_address} = $real_name;
 	    $mailmap->{addresses}->{$wrong_address} = $real_address;
 
-	} elsif (/^(.+)<([^\s]+)>\s*([^\s].*)<([^\s]+)>$/) {
+	} elsif (/^(.+)<([^>]+)>\s*(.+)\s*<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $real_address = $2;
 	    my $wrong_name = $3;
@@ -472,7 +475,6 @@ my @subsystem = ();
 my @status = ();
 my %deduplicate_name_hash = ();
 my %deduplicate_address_hash = ();
-my $signature_pattern;
 
 my @maintainers = get_maintainers();
 
@@ -920,7 +922,7 @@ sub get_maintainer_role {
     my $start = find_starting_index($index);
     my $end = find_ending_index($index);
 
-    my $role;
+    my $role = "unknown";
     my $subsystem = $typevalue[$start];
     if (length($subsystem) > 20) {
 	$subsystem = substr($subsystem, 0, 17);
@@ -1016,8 +1018,13 @@ sub add_categories {
 		    if ($email_list) {
 			if (!$hash_list_to{lc($list_address)}) {
 			    $hash_list_to{lc($list_address)} = 1;
-			    push(@list_to, [$list_address,
-					    "open list${list_role}"]);
+			    if ($list_additional =~ m/moderated/) {
+				push(@list_to, [$list_address,
+						"moderated list${list_role}"]);
+			    } else {
+				push(@list_to, [$list_address,
+						"open list${list_role}"]);
+			    }
 			}
 		    }
 		}
diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat
index e8d68f05ca2..762544b197b 100755
--- a/scripts/kvm/kvm_stat
+++ b/scripts/kvm/kvm_stat
@@ -170,6 +170,12 @@ vendor_exit_reasons = {
     'IBM/S390': s390_exit_reasons,
 }
 
+syscall_numbers = {
+    'IBM/S390': 331,
+}
+
+sc_perf_evt_open = 298
+
 exit_reasons = None
 
 for line in file('/proc/cpuinfo').readlines():
@@ -177,7 +183,8 @@ for line in file('/proc/cpuinfo').readlines():
         for flag in line.split():
             if flag in vendor_exit_reasons:
                 exit_reasons = vendor_exit_reasons[flag]
-
+            if flag in syscall_numbers:
+                sc_perf_evt_open = syscall_numbers[flag]
 filters = {
     'kvm_exit': ('exit_reason', exit_reasons)
 }
@@ -206,7 +213,7 @@ class perf_event_attr(ctypes.Structure):
                 ('bp_len', ctypes.c_uint64),
                 ]
 def _perf_event_open(attr, pid, cpu, group_fd, flags):
-    return syscall(298, ctypes.pointer(attr), ctypes.c_int(pid),
+    return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid),
                    ctypes.c_int(cpu), ctypes.c_int(group_fd),
                    ctypes.c_long(flags))
 
diff --git a/scripts/kvm/vmxcap b/scripts/kvm/vmxcap
index cbe6440ba30..0b23f7795a3 100755
--- a/scripts/kvm/vmxcap
+++ b/scripts/kvm/vmxcap
@@ -147,6 +147,7 @@ controls = [
             5: 'Enable VPID',
             6: 'WBINVD exiting',
             7: 'Unrestricted guest',
+            9: 'Virtual interrupt delivery',
             10: 'PAUSE-loop exiting',
             11: 'RDRAND exiting',
             12: 'Enable INVPCID',
diff --git a/scripts/make_device_config.sh b/scripts/make_device_config.sh
index 5d14885dfc2..0778fe2a420 100644
--- a/scripts/make_device_config.sh
+++ b/scripts/make_device_config.sh
@@ -25,4 +25,4 @@ done
 process_includes $src > $dest
 
 cat $src $all_includes | grep -v '^include' > $dest
-echo "$1: $all_includes" > $dep
+echo "`basename $1`: $all_includes" > $dep
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index 3c4678dbf1e..e06332bd552 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -342,8 +342,8 @@ def gen_command_decl_prologue(header, guard, prefix=""):
 #define %(guard)s
 
 #include "%(prefix)sqapi-types.h"
-#include "qdict.h"
-#include "error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
 
 ''',
                  header=basename(header), guard=guardname(header), prefix=prefix)
@@ -366,12 +366,15 @@ def gen_command_def_prologue(prefix="", proxy=False):
  *
  */
 
-#include "qemu-objects.h"
-#include "qapi/qmp-core.h"
-#include "qapi/qapi-visit-core.h"
+#include "qemu-common.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/dispatch.h"
+#include "qapi/visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp-input-visitor.h"
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
 
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index cf601ae2d23..9e19920970c 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -28,6 +28,16 @@ typedef struct %(name)sList
 ''',
                  name=name)
 
+def generate_fwd_enum_struct(name, members):
+    return mcgen('''
+typedef struct %(name)sList
+{
+    %(name)s value;
+    struct %(name)sList *next;
+} %(name)sList;
+''',
+                 name=name)
+
 def generate_struct(structname, fieldname, members):
     ret = mcgen('''
 struct %(name)s
@@ -81,9 +91,9 @@ const char *%(name)s_lookup[] = {
 
 def generate_enum_name(name):
     if name.isupper():
-        return c_fun(name)
+        return c_fun(name, False)
     new_name = ''
-    for c in c_fun(name):
+    for c in c_fun(name, False):
         if c.isupper():
             new_name += '_'
         new_name += c
@@ -238,7 +248,7 @@ fdef.write(mcgen('''
  *
  */
 
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
 
@@ -263,7 +273,8 @@ fdecl.write(mcgen('''
 #ifndef %(guard)s
 #define %(guard)s
 
-#include "qemu-common.h"
+#include <stdbool.h>
+#include <stdint.h>
 
 ''',
                   guard=guardname(h_file)))
@@ -276,7 +287,8 @@ for expr in exprs:
     if expr.has_key('type'):
         ret += generate_fwd_struct(expr['type'], expr['data'])
     elif expr.has_key('enum'):
-        ret += generate_enum(expr['enum'], expr['data'])
+        ret += generate_enum(expr['enum'], expr['data']) + "\n"
+        ret += generate_fwd_enum_struct(expr['enum'], expr['data'])
         fdef.write(generate_enum_lookup(expr['enum'], expr['data']))
     elif expr.has_key('union'):
         ret += generate_fwd_struct(expr['union'], expr['data']) + "\n"
@@ -300,6 +312,9 @@ for expr in exprs:
         fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n")
         ret += generate_type_cleanup_decl(expr['union'])
         fdef.write(generate_type_cleanup(expr['union']) + "\n")
+    elif expr.has_key('enum'):
+        ret += generate_type_cleanup_decl(expr['enum'] + "List")
+        fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n")
     else:
         continue
     fdecl.write(ret)
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 04ef7c41ab4..a276540a184 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -157,7 +157,7 @@ void visit_type_%(name)s(Visitor *m, %(name)s ** obj, const char *name, Error **
     if (!error_is_set(errp)) {
         visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
         if (!err) {
-            if (!obj || *obj) {
+            if (obj && *obj) {
                 visit_type_%(name)sKind(m, &(*obj)->kind, "type", &err);
                 if (!err) {
                     switch ((*obj)->kind) {
@@ -173,7 +173,7 @@ void visit_type_%(name)s(Visitor *m, %(name)s ** obj, const char *name, Error **
                 break;
 ''',
                 abbrev = de_camel_case(name).upper(),
-                enum = c_fun(de_camel_case(key)).upper(),
+                enum = c_fun(de_camel_case(key),False).upper(),
                 c_type=members[key],
                 c_name=c_fun(key))
 
@@ -217,6 +217,16 @@ void visit_type_%(name)sList(Visitor *m, %(name)sList ** obj, const char *name,
 
     return ret
 
+def generate_enum_declaration(name, members, genlist=True):
+    ret = ""
+    if genlist:
+        ret += mcgen('''
+void visit_type_%(name)sList(Visitor *m, %(name)sList ** obj, const char *name, Error **errp);
+''',
+                     name=name)
+
+    return ret
+
 def generate_decl_enum(name, members, genlist=True):
     return mcgen('''
 
@@ -288,6 +298,7 @@ fdef.write(mcgen('''
  *
  */
 
+#include "qemu-common.h"
 #include "%(header)s"
 ''',
                  header=basename(h_file)))
@@ -311,7 +322,7 @@ fdecl.write(mcgen('''
 #ifndef %(guard)s
 #define %(guard)s
 
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
 #include "%(prefix)sqapi-types.h"
 ''',
                   prefix=prefix, guard=guardname(h_file)))
@@ -335,10 +346,12 @@ for expr in exprs:
         ret += generate_declaration(expr['union'], expr['data'])
         fdecl.write(ret)
     elif expr.has_key('enum'):
-        ret = generate_visit_enum(expr['enum'], expr['data'])
+        ret = generate_visit_list(expr['enum'], expr['data'])
+        ret += generate_visit_enum(expr['enum'], expr['data'])
         fdef.write(ret)
 
         ret = generate_decl_enum(expr['enum'], expr['data'])
+        ret += generate_enum_declaration(expr['enum'], expr['data'])
         fdecl.write(ret)
 
 fdecl.write('''
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 122b4cb6d13..afc5f32aebe 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -141,7 +141,7 @@ def camel_case(name):
             new_name += ch.lower()
     return new_name
 
-def c_var(name):
+def c_var(name, protect=True):
     # ANSI X3J11/88-090, 3.1.1
     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
                      'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
@@ -156,12 +156,14 @@ def c_var(name):
     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
     # excluding _.*
     gcc_words = set(['asm', 'typeof'])
-    if name in c89_words | c99_words | c11_words | gcc_words:
+    # namespace pollution:
+    polluted_words = set(['unix'])
+    if protect and (name in c89_words | c99_words | c11_words | gcc_words | polluted_words):
         return "q_" + name
     return name.replace('-', '_').lstrip("*")
 
-def c_fun(name):
-    return c_var(name).replace('.', '_')
+def c_fun(name, protect=True):
+    return c_var(name, protect).replace('.', '_')
 
 def c_list_type(name):
     return '%sList' % name
diff --git a/scripts/qemu-guest-agent/fsfreeze-hook b/scripts/qemu-guest-agent/fsfreeze-hook
new file mode 100755
index 00000000000..c27b29f282d
--- /dev/null
+++ b/scripts/qemu-guest-agent/fsfreeze-hook
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script is executed when a guest agent receives fsfreeze-freeze and
+# fsfreeze-thaw command, if it is specified in --fsfreeze-hook (-F)
+# option of qemu-ga or placed in default path (/etc/qemu/fsfreeze-hook).
+# When the agent receives fsfreeze-freeze request, this script is issued with
+# "freeze" argument before the filesystem is frozen. And for fsfreeze-thaw
+# request, it is issued with "thaw" argument after filesystem is thawed.
+
+LOGFILE=/var/log/qga-fsfreeze-hook.log
+FSFREEZE_D=$(dirname -- "$0")/fsfreeze-hook.d
+
+# Check whether file $1 is a backup or rpm-generated file and should be ignored
+is_ignored_file() {
+    case "$1" in
+        *~ | *.bak | *.orig | *.rpmnew | *.rpmorig | *.rpmsave | *.sample)
+            return 0 ;;
+    esac
+    return 1
+}
+
+# Iterate executables in directory "fsfreeze-hook.d" with the specified args
+[ ! -d "$FSFREEZE_D" ] && exit 0
+for file in "$FSFREEZE_D"/* ; do
+    is_ignored_file "$file" && continue
+    [ -x "$file" ] || continue
+    printf "$(date): execute $file $@\n" >>$LOGFILE
+    "$file" "$@" >>$LOGFILE 2>&1
+    STATUS=$?
+    printf "$(date): $file finished with status=$STATUS\n" >>$LOGFILE
+done
+
+exit 0
diff --git a/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample b/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample
new file mode 100755
index 00000000000..2b4fa3aeb1a
--- /dev/null
+++ b/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Flush MySQL tables to the disk before the filesystem is frozen.
+# At the same time, this keeps a read lock in order to avoid write accesses
+# from the other clients until the filesystem is thawed.
+
+MYSQL="/usr/bin/mysql"
+MYSQL_OPTS="-uroot" #"-prootpassword"
+FIFO=/var/run/mysql-flush.fifo
+
+# Check mysql is installed and the server running
+[ -x "$MYSQL" ] && "$MYSQL" $MYSQL_OPTS < /dev/null || exit 0
+
+flush_and_wait() {
+    printf "FLUSH TABLES WITH READ LOCK \\G\n"
+    trap 'printf "$(date): $0 is killed\n">&2' HUP INT QUIT ALRM TERM
+    read < $FIFO
+    printf "UNLOCK TABLES \\G\n"
+    rm -f $FIFO
+}
+
+case "$1" in
+    freeze)
+        mkfifo $FIFO || exit 1
+        flush_and_wait | "$MYSQL" $MYSQL_OPTS &
+        # wait until every block is flushed
+        while [ "$(echo 'SHOW STATUS LIKE "Key_blocks_not_flushed"' |\
+                 "$MYSQL" $MYSQL_OPTS | tail -1 | cut -f 2)" -gt 0 ]; do
+            sleep 1
+        done
+        # for InnoDB, wait until every log is flushed
+        INNODB_STATUS=$(mktemp /tmp/mysql-flush.XXXXXX)
+        [ $? -ne 0 ] && exit 2
+        trap "rm -f $INNODB_STATUS; exit 1" HUP INT QUIT ALRM TERM
+        while :; do
+            printf "SHOW ENGINE INNODB STATUS \\G" |\
+                "$MYSQL" $MYSQL_OPTS > $INNODB_STATUS
+            LOG_CURRENT=$(grep 'Log sequence number' $INNODB_STATUS |\
+                          tr -s ' ' | cut -d' ' -f4)
+            LOG_FLUSHED=$(grep 'Log flushed up to' $INNODB_STATUS |\
+                          tr -s ' ' | cut -d' ' -f5)
+            [ "$LOG_CURRENT" = "$LOG_FLUSHED" ] && break
+            sleep 1
+        done
+        rm -f $INNODB_STATUS
+        ;;
+
+    thaw)
+        [ ! -p $FIFO ] && exit 1
+        echo > $FIFO
+        ;;
+
+    *)
+        exit 1
+        ;;
+esac
diff --git a/scripts/tracetool/backend/dtrace.py b/scripts/tracetool/backend/dtrace.py
index 9cab75cde8d..ad5eb3b0ab2 100644
--- a/scripts/tracetool/backend/dtrace.py
+++ b/scripts/tracetool/backend/dtrace.py
@@ -37,7 +37,7 @@ def c(events):
 
 
 def h(events):
-    out('#include "trace-dtrace.h"',
+    out('#include "trace/generated-tracers-dtrace.h"',
         '')
 
     for e in events:
@@ -73,6 +73,15 @@ def d(events):
         '};')
 
 
+# Technically 'self' is not used by systemtap yet, but
+# they recommended we keep it in the reserved list anyway
+RESERVED_WORDS = (
+    'break', 'catch', 'continue', 'delete', 'else', 'for',
+    'foreach', 'function', 'global', 'if', 'in', 'limit',
+    'long', 'next', 'probe', 'return', 'self', 'string',
+    'try', 'while'
+    )
+
 def stap(events):
     for e in events:
         # Define prototype for probe arguments
@@ -87,7 +96,7 @@ def stap(events):
         if len(e.args) > 0:
             for name in e.args.names():
                 # Append underscore to reserved keywords
-                if name in ('limit', 'in', 'next', 'self'):
+                if name in RESERVED_WORDS:
                     name += '_'
                 out('  %s = $arg%d;' % (name, i))
                 i += 1
diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py
index 6ffb3c2f4e0..9a58de1331f 100644
--- a/scripts/tracetool/format/h.py
+++ b/scripts/tracetool/format/h.py
@@ -19,8 +19,8 @@ from tracetool import out
 def begin(events):
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
-        '#ifndef TRACE_H',
-        '#define TRACE_H',
+        '#ifndef TRACE__GENERATED_TRACERS_H',
+        '#define TRACE__GENERATED_TRACERS_H',
         '',
         '#include "qemu-common.h"')
 
@@ -32,7 +32,7 @@ def end(events):
             enabled = 1
         out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled))
     out('',
-        '#endif /* TRACE_H */')
+        '#endif /* TRACE__GENERATED_TRACERS_H */')
 
 def nop(events):
     for e in events:
diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh
index a639c5bff0d..120a694313f 100755
--- a/scripts/update-linux-headers.sh
+++ b/scripts/update-linux-headers.sh
@@ -28,7 +28,22 @@ if [ -z "$output" ]; then
     output="$PWD"
 fi
 
-for arch in x86 powerpc s390; do
+# This will pick up non-directories too (eg "Kconfig") but we will
+# ignore them in the next loop.
+ARCHLIST=$(cd "$linux/arch" && echo *)
+
+for arch in $ARCHLIST; do
+    # Discard anything which isn't a KVM-supporting architecture
+    if ! [ -e "$linux/arch/$arch/include/asm/kvm.h" ] &&
+        ! [ -e "$linux/arch/$arch/include/uapi/asm/kvm.h" ] ; then
+        continue
+    fi
+
+    # Blacklist architectures which have KVM headers but are actually dead
+    if [ "$arch" = "ia64" ]; then
+        continue
+    fi
+
     make -C "$linux" INSTALL_HDR_PATH="$tmpdir" SRCARCH=$arch headers_install
 
     rm -rf "$output/linux-headers/asm-$arch"
@@ -39,11 +54,14 @@ for arch in x86 powerpc s390; do
     if [ $arch = x86 ]; then
         cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86"
     fi
+    if [ $arch = powerpc ]; then
+        cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/"
+    fi
 done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
-for header in kvm.h kvm_para.h vhost.h virtio_config.h virtio_ring.h; do
+for header in kvm.h kvm_para.h vfio.h vhost.h virtio_config.h virtio_ring.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done
 rm -rf "$output/linux-headers/asm-generic"
diff --git a/slirp/Makefile.objs b/slirp/Makefile.objs
index bb43d3c08c4..2daa9dc58db 100644
--- a/slirp/Makefile.objs
+++ b/slirp/Makefile.objs
@@ -1,3 +1,3 @@
-common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o
+common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o dnssearch.o
 common-obj-y += slirp.o mbuf.o misc.o sbuf.o socket.o tcp_input.o tcp_output.o
 common-obj-y += tcp_subr.o tcp_timer.o udp.o bootp.o tftp.o arp_table.o
diff --git a/slirp/arp_table.c b/slirp/arp_table.c
index 5d7b8acd1d6..bf698c1ac57 100644
--- a/slirp/arp_table.c
+++ b/slirp/arp_table.c
@@ -38,7 +38,9 @@ void arp_table_add(Slirp *slirp, uint32_t ip_addr, uint8_t ethaddr[ETH_ALEN])
                 ethaddr[3], ethaddr[4], ethaddr[5]));
 
     /* Check 0.0.0.0/8 invalid source-only addresses */
-    assert((ip_addr & htonl(~(0xf << 28))) != 0);
+    if ((ip_addr & htonl(~(0xf << 28))) == 0) {
+        return;
+    }
 
     if (ip_addr == 0xffffffff || ip_addr == broadcast_addr) {
         /* Do not register broadcast addresses */
diff --git a/slirp/bootp.c b/slirp/bootp.c
index 64eac7d1018..b7db9fa3353 100644
--- a/slirp/bootp.c
+++ b/slirp/bootp.c
@@ -287,6 +287,18 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
             memcpy(q, slirp->client_hostname, val);
             q += val;
         }
+
+        if (slirp->vdnssearch) {
+            size_t spaceleft = sizeof(rbp->bp_vend) - (q - rbp->bp_vend);
+            val = slirp->vdnssearch_len;
+            if (val + 1 > spaceleft) {
+                g_warning("DHCP packet size exceeded, "
+                    "omitting domain-search option.");
+            } else {
+                memcpy(q, slirp->vdnssearch, val);
+                q += val;
+            }
+        }
     } else {
         static const char nak_msg[] = "requested address not available";
 
diff --git a/slirp/bootp.h b/slirp/bootp.h
index 30c30ab863b..ec3b68704c3 100644
--- a/slirp/bootp.h
+++ b/slirp/bootp.h
@@ -1,4 +1,6 @@
 /* bootp/dhcp defines */
+#ifndef SLIRP_BOOTP_H
+#define SLIRP_BOOTP_H 1
 
 #define BOOTP_SERVER	67
 #define BOOTP_CLIENT	68
@@ -120,3 +122,5 @@ typedef struct {
 #define NB_BOOTP_CLIENTS 16
 
 void bootp_input(struct mbuf *m);
+
+#endif
diff --git a/slirp/dnssearch.c b/slirp/dnssearch.c
new file mode 100644
index 00000000000..4c9064ecb68
--- /dev/null
+++ b/slirp/dnssearch.c
@@ -0,0 +1,314 @@
+/*
+ * Domain search option for DHCP (RFC 3397)
+ *
+ * Copyright (c) 2012 Klaus Stengel
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <glib.h>
+#include "slirp.h"
+
+static const uint8_t RFC3397_OPT_DOMAIN_SEARCH = 119;
+static const uint8_t MAX_OPT_LEN = 255;
+static const uint8_t OPT_HEADER_LEN = 2;
+static const uint8_t REFERENCE_LEN = 2;
+
+struct compact_domain;
+
+typedef struct compact_domain {
+    struct compact_domain *self;
+    struct compact_domain *refdom;
+    uint8_t *labels;
+    size_t len;
+    size_t common_octets;
+} CompactDomain;
+
+static size_t
+domain_suffix_diffoff(const CompactDomain *a, const CompactDomain *b)
+{
+    size_t la = a->len, lb = b->len;
+    uint8_t *da = a->labels + la, *db = b->labels + lb;
+    size_t i, lm = (la < lb) ? la : lb;
+
+    for (i = 0; i < lm; i++) {
+        da--; db--;
+        if (*da != *db) {
+            break;
+        }
+    }
+    return i;
+}
+
+static int domain_suffix_ord(const void *cva, const void *cvb)
+{
+    const CompactDomain *a = cva, *b = cvb;
+    size_t la = a->len, lb = b->len;
+    size_t doff = domain_suffix_diffoff(a, b);
+    uint8_t ca = a->labels[la - doff];
+    uint8_t cb = b->labels[lb - doff];
+
+    if (ca < cb) {
+        return -1;
+    }
+    if (ca > cb) {
+        return 1;
+    }
+    if (la < lb) {
+        return -1;
+    }
+    if (la > lb) {
+        return 1;
+    }
+    return 0;
+}
+
+static size_t domain_common_label(CompactDomain *a, CompactDomain *b)
+{
+    size_t res, doff = domain_suffix_diffoff(a, b);
+    uint8_t *first_eq_pos = a->labels + (a->len - doff);
+    uint8_t *label = a->labels;
+
+    while (*label && label < first_eq_pos) {
+        label += *label + 1;
+    }
+    res = a->len - (label - a->labels);
+    /* only report if it can help to reduce the packet size */
+    return (res > REFERENCE_LEN) ? res : 0;
+}
+
+static void domain_fixup_order(CompactDomain *cd, size_t n)
+{
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        CompactDomain *cur = cd + i, *next = cd[i].self;
+
+        while (!cur->common_octets) {
+            CompactDomain *tmp = next->self; /* backup target value */
+
+            next->self = cur;
+            cur->common_octets++;
+
+            cur = next;
+            next = tmp;
+        }
+    }
+}
+
+static void domain_mklabels(CompactDomain *cd, const char *input)
+{
+    uint8_t *len_marker = cd->labels;
+    uint8_t *output = len_marker; /* pre-incremented */
+    const char *in = input;
+    char cur_chr;
+    size_t len = 0;
+
+    if (cd->len == 0) {
+        goto fail;
+    }
+    cd->len++;
+
+    do {
+        cur_chr = *in++;
+        if (cur_chr == '.' || cur_chr == '\0') {
+            len = output - len_marker;
+            if ((len == 0 && cur_chr == '.') || len >= 64) {
+                goto fail;
+            }
+            *len_marker = len;
+
+            output++;
+            len_marker = output;
+        } else {
+            output++;
+            *output = cur_chr;
+        }
+    } while (cur_chr != '\0');
+
+    /* ensure proper zero-termination */
+    if (len != 0) {
+        *len_marker = 0;
+        cd->len++;
+    }
+    return;
+
+fail:
+    g_warning("failed to parse domain name '%s'\n", input);
+    cd->len = 0;
+}
+
+static void
+domain_mkxrefs(CompactDomain *doms, CompactDomain *last, size_t depth)
+{
+    CompactDomain *i = doms, *target = doms;
+
+    do {
+        if (i->labels < target->labels) {
+            target = i;
+        }
+    } while (i++ != last);
+
+    for (i = doms; i != last; i++) {
+        CompactDomain *group_last;
+        size_t next_depth;
+
+        if (i->common_octets == depth) {
+            continue;
+        }
+
+        next_depth = -1;
+        for (group_last = i; group_last != last; group_last++) {
+            size_t co = group_last->common_octets;
+            if (co <= depth) {
+                break;
+            }
+            if (co < next_depth) {
+                next_depth = co;
+            }
+        }
+        domain_mkxrefs(i, group_last, next_depth);
+
+        i = group_last;
+        if (i == last) {
+            break;
+        }
+    }
+
+    if (depth == 0) {
+        return;
+    }
+
+    i = doms;
+    do {
+        if (i != target && i->refdom == NULL) {
+            i->refdom = target;
+            i->common_octets = depth;
+        }
+    } while (i++ != last);
+}
+
+static size_t domain_compactify(CompactDomain *domains, size_t n)
+{
+    uint8_t *start = domains->self->labels, *outptr = start;
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        CompactDomain *cd = domains[i].self;
+        CompactDomain *rd = cd->refdom;
+
+        if (rd != NULL) {
+            size_t moff = (rd->labels - start)
+                    + (rd->len - cd->common_octets);
+            if (moff < 0x3FFFu) {
+                cd->len -= cd->common_octets - 2;
+                cd->labels[cd->len - 1] = moff & 0xFFu;
+                cd->labels[cd->len - 2] = 0xC0u | (moff >> 8);
+            }
+        }
+
+        if (cd->labels != outptr) {
+            memmove(outptr, cd->labels, cd->len);
+            cd->labels = outptr;
+        }
+        outptr += cd->len;
+    }
+    return outptr - start;
+}
+
+int translate_dnssearch(Slirp *s, const char **names)
+{
+    size_t blocks, bsrc_start, bsrc_end, bdst_start;
+    size_t i, num_domains, memreq = 0;
+    uint8_t *result = NULL, *outptr;
+    CompactDomain *domains = NULL;
+    const char **nameptr = names;
+
+    while (*nameptr != NULL) {
+        nameptr++;
+    }
+
+    num_domains = nameptr - names;
+    if (num_domains == 0) {
+        return -2;
+    }
+
+    domains = g_malloc(num_domains * sizeof(*domains));
+
+    for (i = 0; i < num_domains; i++) {
+        size_t nlen = strlen(names[i]);
+        memreq += nlen + 2; /* 1 zero octet + 1 label length octet */
+        domains[i].self = domains + i;
+        domains[i].len = nlen;
+        domains[i].common_octets = 0;
+        domains[i].refdom = NULL;
+    }
+
+    /* reserve extra 2 header bytes for each 255 bytes of output */
+    memreq += ((memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN) * OPT_HEADER_LEN;
+    result = g_malloc(memreq * sizeof(*result));
+
+    outptr = result;
+    for (i = 0; i < num_domains; i++) {
+        domains[i].labels = outptr;
+        domain_mklabels(domains + i, names[i]);
+        outptr += domains[i].len;
+    }
+
+    if (outptr == result) {
+        g_free(domains);
+        g_free(result);
+        return -1;
+    }
+
+    qsort(domains, num_domains, sizeof(*domains), domain_suffix_ord);
+    domain_fixup_order(domains, num_domains);
+
+    for (i = 1; i < num_domains; i++) {
+        size_t cl = domain_common_label(domains + i - 1, domains + i);
+        domains[i - 1].common_octets = cl;
+    }
+
+    domain_mkxrefs(domains, domains + num_domains - 1, 0);
+    memreq = domain_compactify(domains, num_domains);
+
+    blocks = (memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN;
+    bsrc_end = memreq;
+    bsrc_start = (blocks - 1) * MAX_OPT_LEN;
+    bdst_start = bsrc_start + blocks * OPT_HEADER_LEN;
+    memreq += blocks * OPT_HEADER_LEN;
+
+    while (blocks--) {
+        size_t len = bsrc_end - bsrc_start;
+        memmove(result + bdst_start, result + bsrc_start, len);
+        result[bdst_start - 2] = RFC3397_OPT_DOMAIN_SEARCH;
+        result[bdst_start - 1] = len;
+        bsrc_end = bsrc_start;
+        bsrc_start -= MAX_OPT_LEN;
+        bdst_start -= MAX_OPT_LEN + OPT_HEADER_LEN;
+    }
+
+    g_free(domains);
+    s->vdnssearch = result;
+    s->vdnssearch_len = memreq;
+    return 0;
+}
diff --git a/slirp/if.c b/slirp/if.c
index 533295dd075..dcd5fafe5d2 100644
--- a/slirp/if.c
+++ b/slirp/if.c
@@ -6,7 +6,7 @@
  */
 
 #include <slirp.h>
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 static void
 ifs_insque(struct mbuf *ifm, struct mbuf *ifmhead)
diff --git a/slirp/ip_icmp.c b/slirp/ip_icmp.c
index d571fd0b767..9f1cb08a18c 100644
--- a/slirp/ip_icmp.c
+++ b/slirp/ip_icmp.c
@@ -352,7 +352,7 @@ icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
 
   ip->ip_ttl = MAXTTL;
   ip->ip_p = IPPROTO_ICMP;
-  ip->ip_dst = ip->ip_src;    /* ip adresses */
+  ip->ip_dst = ip->ip_src;    /* ip addresses */
   ip->ip_src = m->slirp->vhost_addr;
 
   (void ) ip_output((struct socket *)NULL, m);
diff --git a/slirp/ip_icmp.h b/slirp/ip_icmp.h
index 1a1af91e1e4..be4426b8e78 100644
--- a/slirp/ip_icmp.h
+++ b/slirp/ip_icmp.h
@@ -92,8 +92,8 @@ struct icmp {
 
 /*
  * Lower bounds on packet lengths for various types.
- * For the error advice packets must first insure that the
- * packet is large enought to contain the returned ip header.
+ * For the error advice packets must first ensure that the
+ * packet is large enough to contain the returned ip header.
  * Only then can we do the check to see if 64 bits of packet
  * data have been returned, since we need to check the returned
  * ip header length.
diff --git a/slirp/ip_input.c b/slirp/ip_input.c
index ce24faf165f..880bdfd3ccf 100644
--- a/slirp/ip_input.c
+++ b/slirp/ip_input.c
@@ -39,7 +39,7 @@
  */
 
 #include <slirp.h>
-#include <osdep.h>
+#include <qemu/osdep.h>
 #include "ip_icmp.h"
 
 static struct ip *ip_reass(Slirp *slirp, struct ip *ip, struct ipq *fp);
@@ -213,7 +213,6 @@ ip_input(struct mbuf *m)
 	return;
 bad:
 	m_free(m);
-	return;
 }
 
 #define iptofrag(P) ((struct ipasfrag *)(((char*)(P)) - sizeof(struct qlink)))
diff --git a/slirp/libslirp.h b/slirp/libslirp.h
index 9b471b50538..49609c2ad7b 100644
--- a/slirp/libslirp.h
+++ b/slirp/libslirp.h
@@ -12,7 +12,8 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
                   const char *vhostname, const char *tftp_path,
                   const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, void *opaque);
+                  struct in_addr vnameserver, const char **vdnssearch,
+                  void *opaque);
 void slirp_cleanup(Slirp *slirp);
 
 void slirp_update_timeout(uint32_t *timeout);
diff --git a/slirp/main.h b/slirp/main.h
index 1f3b84de923..66e4f9252fe 100644
--- a/slirp/main.h
+++ b/slirp/main.h
@@ -4,6 +4,8 @@
  * Please read the file COPYRIGHT for the
  * terms and conditions of the copyright.
  */
+#ifndef SLIRP_MAIN_H
+#define SLIRP_MAIN_H 1
 
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
@@ -45,3 +47,5 @@ extern int tcp_keepintvl;
 
 int if_encap(Slirp *slirp, struct mbuf *ifm);
 ssize_t slirp_send(struct socket *so, const void *buf, size_t len, int flags);
+
+#endif
diff --git a/slirp/misc.c b/slirp/misc.c
index 0bee8643371..d4df972d133 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -8,7 +8,7 @@
 #include <slirp.h>
 #include <libslirp.h>
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #ifdef DEBUG
 int slirp_debug = DBG_CALL|DBG_MISC|DBG_ERROR;
@@ -242,7 +242,7 @@ strdup(str)
 }
 #endif
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 void lprint(const char *format, ...)
 {
@@ -253,20 +253,6 @@ void lprint(const char *format, ...)
     va_end(args);
 }
 
-void
-u_sleep(int usec)
-{
-	struct timeval t;
-	fd_set fdset;
-
-	FD_ZERO(&fdset);
-
-	t.tv_sec = 0;
-	t.tv_usec = usec * 1000;
-
-	select(0, &fdset, &fdset, &fdset, &t);
-}
-
 void slirp_connection_info(Slirp *slirp, Monitor *mon)
 {
     const char * const tcpstates[] = {
diff --git a/slirp/misc.h b/slirp/misc.h
index ed40a103c5f..cc36aeb9592 100644
--- a/slirp/misc.h
+++ b/slirp/misc.h
@@ -64,7 +64,6 @@ void snooze_hup(int);
 void snooze(void);
 void relay(int);
 void add_emu(char *);
-void u_sleep(int);
 void fd_nonblock(int);
 void fd_block(int);
 int rsh_exec(struct socket *, struct socket *, char *, char *, char *);
diff --git a/slirp/sbuf.c b/slirp/sbuf.c
index 637f8fea3d1..08ec2b4f44d 100644
--- a/slirp/sbuf.c
+++ b/slirp/sbuf.c
@@ -6,7 +6,7 @@
  */
 
 #include <slirp.h>
-#include <main-loop.h>
+#include <qemu/main-loop.h>
 
 static void sbappendsb(struct sbuf *sb, struct mbuf *m);
 
diff --git a/slirp/slirp.c b/slirp/slirp.c
index 38e0a2193ae..e93b5788327 100644
--- a/slirp/slirp.c
+++ b/slirp/slirp.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "slirp.h"
 #include "hw/hw.h"
 
@@ -203,7 +203,8 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
                   const char *vhostname, const char *tftp_path,
                   const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, void *opaque)
+                  struct in_addr vnameserver, const char **vdnssearch,
+                  void *opaque)
 {
     Slirp *slirp = g_malloc0(sizeof(Slirp));
 
@@ -233,6 +234,10 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
     slirp->vdhcp_startaddr = vdhcp_start;
     slirp->vnameserver_addr = vnameserver;
 
+    if (vdnssearch) {
+        translate_dnssearch(slirp, vdnssearch);
+    }
+
     slirp->opaque = opaque;
 
     register_savevm(NULL, "slirp", 0, 3,
@@ -252,6 +257,7 @@ void slirp_cleanup(Slirp *slirp)
     ip_cleanup(slirp);
     m_cleanup(slirp);
 
+    g_free(slirp->vdnssearch);
     g_free(slirp->tftp_prefix);
     g_free(slirp->bootp_filename);
     g_free(slirp);
diff --git a/slirp/slirp.h b/slirp/slirp.h
index f2c5eca892e..fe0e65d0eee 100644
--- a/slirp/slirp.h
+++ b/slirp/slirp.h
@@ -133,8 +133,8 @@ void free(void *ptr);
 
 #include "debug.h"
 
-#include "qemu-queue.h"
-#include "qemu_socket.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
 
 #include "libslirp.h"
 #include "ip.h"
@@ -215,7 +215,6 @@ struct Slirp {
     char client_hostname[33];
 
     int restricted;
-    struct timeval tt;
     struct ex_list *exec_list;
 
     /* mbuf states */
@@ -235,6 +234,8 @@ struct Slirp {
     /* bootp/dhcp states */
     BOOTPClient bootp_clients[NB_BOOTP_CLIENTS];
     char *bootp_filename;
+    size_t vdnssearch_len;
+    uint8_t *vdnssearch;
 
     /* tcp states */
     struct socket tcb;
@@ -294,6 +295,9 @@ void lprint(const char *, ...) GCC_FMT_ATTR(1, 2);
 #define SO_OPTIONS DO_KEEPALIVE
 #define TCP_MAXIDLE (TCPTV_KEEPCNT * TCPTV_KEEPINTVL)
 
+/* dnssearch.c */
+int translate_dnssearch(Slirp *s, const char ** names);
+
 /* cksum.c */
 int cksum(struct mbuf *m, int len);
 
diff --git a/slirp/tcp_input.c b/slirp/tcp_input.c
index 942aaf44f1b..6440eae7faf 100644
--- a/slirp/tcp_input.c
+++ b/slirp/tcp_input.c
@@ -1281,8 +1281,6 @@ drop:
 	 * Drop space held by incoming segment and return.
 	 */
 	m_free(m);
-
-	return;
 }
 
 static void
diff --git a/slirp/tcp_subr.c b/slirp/tcp_subr.c
index 025b3743678..1542e436192 100644
--- a/slirp/tcp_subr.c
+++ b/slirp/tcp_subr.c
@@ -114,9 +114,9 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 	int win = 0;
 
 	DEBUG_CALL("tcp_respond");
-	DEBUG_ARG("tp = %lx", (long)tp);
-	DEBUG_ARG("ti = %lx", (long)ti);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("tp = %p", tp);
+	DEBUG_ARG("ti = %p", ti);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("ack = %u", ack);
 	DEBUG_ARG("seq = %u", seq);
 	DEBUG_ARG("flags = %x", flags);
@@ -124,7 +124,7 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 	if (tp)
 		win = sbspace(&tp->t_socket->so_rcv);
         if (m == NULL) {
-		if ((m = m_get(tp->t_socket->slirp)) == NULL)
+		if (!tp || (m = m_get(tp->t_socket->slirp)) == NULL)
 			return;
 		tlen = 0;
 		m->m_data += IF_MAXLINKHDR;
diff --git a/slirp/tftp.c b/slirp/tftp.c
index b78765f3af1..1a79c45cfb6 100644
--- a/slirp/tftp.c
+++ b/slirp/tftp.c
@@ -37,6 +37,10 @@ static inline void tftp_session_update(struct tftp_session *spt)
 
 static void tftp_session_terminate(struct tftp_session *spt)
 {
+    if (spt->fd >= 0) {
+        close(spt->fd);
+        spt->fd = -1;
+    }
     g_free(spt->filename);
     spt->slirp = NULL;
 }
@@ -54,7 +58,7 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
 
     /* sessions time out after 5 inactive seconds */
     if ((int)(curtime - spt->timestamp) > 5000) {
-        g_free(spt->filename);
+        tftp_session_terminate(spt);
         goto found;
     }
   }
@@ -64,6 +68,7 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
  found:
   memset(spt, 0, sizeof(*spt));
   memcpy(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip));
+  spt->fd = -1;
   spt->client_port = tp->udp.uh_sport;
   spt->slirp = slirp;
 
@@ -92,37 +97,36 @@ static int tftp_session_find(Slirp *slirp, struct tftp_t *tp)
   return -1;
 }
 
-static int tftp_read_data(struct tftp_session *spt, uint16_t block_nr,
+static int tftp_read_data(struct tftp_session *spt, uint32_t block_nr,
                           uint8_t *buf, int len)
 {
-  int fd;
-  int bytes_read = 0;
+    int bytes_read = 0;
 
-  fd = open(spt->filename, O_RDONLY | O_BINARY);
+    if (spt->fd < 0) {
+        spt->fd = open(spt->filename, O_RDONLY | O_BINARY);
+    }
 
-  if (fd < 0) {
-    return -1;
-  }
+    if (spt->fd < 0) {
+        return -1;
+    }
 
-  if (len) {
-    lseek(fd, block_nr * 512, SEEK_SET);
+    if (len) {
+        lseek(spt->fd, block_nr * 512, SEEK_SET);
 
-    bytes_read = read(fd, buf, len);
-  }
-
-  close(fd);
+        bytes_read = read(spt->fd, buf, len);
+    }
 
-  return bytes_read;
+    return bytes_read;
 }
 
 static int tftp_send_oack(struct tftp_session *spt,
-                          const char *key, uint32_t value,
+                          const char *keys[], uint32_t values[], int nb,
                           struct tftp_t *recv_tp)
 {
     struct sockaddr_in saddr, daddr;
     struct mbuf *m;
     struct tftp_t *tp;
-    int n = 0;
+    int i, n = 0;
 
     m = m_get(spt->slirp);
 
@@ -136,10 +140,12 @@ static int tftp_send_oack(struct tftp_session *spt,
     m->m_data += sizeof(struct udpiphdr);
 
     tp->tp_op = htons(TFTP_OACK);
-    n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%s",
-                  key) + 1;
-    n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%u",
-                  value) + 1;
+    for (i = 0; i < nb; i++) {
+        n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%s",
+                      keys[i]) + 1;
+        n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%u",
+                      values[i]) + 1;
+    }
 
     saddr.sin_addr = recv_tp->ip.ip_dst;
     saddr.sin_port = recv_tp->udp.uh_dport;
@@ -193,23 +199,18 @@ out:
   tftp_session_terminate(spt);
 }
 
-static int tftp_send_data(struct tftp_session *spt,
-                          uint16_t block_nr,
-			  struct tftp_t *recv_tp)
+static void tftp_send_next_block(struct tftp_session *spt,
+                                 struct tftp_t *recv_tp)
 {
   struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
   int nobytes;
 
-  if (block_nr < 1) {
-    return -1;
-  }
-
   m = m_get(spt->slirp);
 
   if (!m) {
-    return -1;
+    return;
   }
 
   memset(m->m_data, 0, m->m_size);
@@ -219,7 +220,7 @@ static int tftp_send_data(struct tftp_session *spt,
   m->m_data += sizeof(struct udpiphdr);
 
   tp->tp_op = htons(TFTP_DATA);
-  tp->x.tp_data.tp_block_nr = htons(block_nr);
+  tp->x.tp_data.tp_block_nr = htons((spt->block_nr + 1) & 0xffff);
 
   saddr.sin_addr = recv_tp->ip.ip_dst;
   saddr.sin_port = recv_tp->udp.uh_dport;
@@ -227,7 +228,7 @@ static int tftp_send_data(struct tftp_session *spt,
   daddr.sin_addr = spt->client_ip;
   daddr.sin_port = spt->client_port;
 
-  nobytes = tftp_read_data(spt, block_nr - 1, tp->x.tp_data.tp_buf, 512);
+  nobytes = tftp_read_data(spt, spt->block_nr, tp->x.tp_data.tp_buf, 512);
 
   if (nobytes < 0) {
     m_free(m);
@@ -236,7 +237,7 @@ static int tftp_send_data(struct tftp_session *spt,
 
     tftp_send_error(spt, 1, "File not found", tp);
 
-    return -1;
+    return;
   }
 
   m->m_len = sizeof(struct tftp_t) - (512 - nobytes) -
@@ -251,7 +252,7 @@ static int tftp_send_data(struct tftp_session *spt,
     tftp_session_terminate(spt);
   }
 
-  return 0;
+  spt->block_nr++;
 }
 
 static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
@@ -260,6 +261,9 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
   int s, k;
   size_t prefix_len;
   char *req_fname;
+  const char *option_name[2];
+  uint32_t option_value[2];
+  int nb_options = 0;
 
   /* check if a session already exists and if so terminate it */
   s = tftp_session_find(slirp, tp);
@@ -337,7 +341,7 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
       return;
   }
 
-  while (k < pktlen) {
+  while (k < pktlen && nb_options < ARRAY_SIZE(option_name)) {
       const char *key, *value;
 
       key = &tp->x.tp_buf[k];
@@ -364,12 +368,32 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
 	      }
 	  }
 
-	  tftp_send_oack(spt, "tsize", tsize, tp);
-	  return;
+          option_name[nb_options] = "tsize";
+          option_value[nb_options] = tsize;
+          nb_options++;
+      } else if (strcasecmp(key, "blksize") == 0) {
+          int blksize = atoi(value);
+
+          /* If blksize option is bigger than what we will
+           * emit, accept the option with our packet size.
+           * Otherwise, simply do as we didn't see the option.
+           */
+          if (blksize >= 512) {
+              option_name[nb_options] = "blksize";
+              option_value[nb_options] = 512;
+              nb_options++;
+          }
       }
   }
 
-  tftp_send_data(spt, 1, tp);
+  if (nb_options > 0) {
+      assert(nb_options <= ARRAY_SIZE(option_name));
+      tftp_send_oack(spt, option_name, option_value, nb_options, tp);
+      return;
+  }
+
+  spt->block_nr = 0;
+  tftp_send_next_block(spt, tp);
 }
 
 static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
@@ -382,11 +406,7 @@ static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
     return;
   }
 
-  if (tftp_send_data(&slirp->tftp_sessions[s],
-		     ntohs(tp->x.tp_data.tp_block_nr) + 1,
-		     tp) < 0) {
-    return;
-  }
+  tftp_send_next_block(&slirp->tftp_sessions[s], tp);
 }
 
 static void tftp_handle_error(Slirp *slirp, struct tftp_t *tp, int pktlen)
diff --git a/slirp/tftp.h b/slirp/tftp.h
index 72e5e91bef6..87adeb53335 100644
--- a/slirp/tftp.h
+++ b/slirp/tftp.h
@@ -1,4 +1,6 @@
 /* tftp defines */
+#ifndef SLIRP_TFTP_H
+#define SLIRP_TFTP_H 1
 
 #define TFTP_SESSIONS_MAX 3
 
@@ -33,11 +35,15 @@ struct tftp_t {
 struct tftp_session {
     Slirp *slirp;
     char *filename;
+    int fd;
 
     struct in_addr client_ip;
     uint16_t client_port;
+    uint32_t block_nr;
 
     int timestamp;
 };
 
 void tftp_input(struct mbuf *m);
+
+#endif
diff --git a/slirp/udp.c b/slirp/udp.c
index ced509656d6..9286cb7d313 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -231,7 +231,6 @@ udp_input(register struct mbuf *m, int iphlen)
 	return;
 bad:
 	m_free(m);
-	return;
 }
 
 int udp_output2(struct socket *so, struct mbuf *m,
diff --git a/spice-qemu-char.c b/spice-qemu-char.c
index 09aa22d5668..a4d7de8c4f0 100644
--- a/spice-qemu-char.c
+++ b/spice-qemu-char.c
@@ -1,10 +1,12 @@
 #include "config-host.h"
 #include "trace.h"
 #include "ui/qemu-spice.h"
+#include "char/char.h"
 #include <spice.h>
 #include <spice-experimental.h>
+#include <spice/protocol.h>
 
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 #define dprintf(_scd, _level, _fmt, ...)                                \
     do {                                                                \
@@ -14,8 +16,6 @@
         }                                                               \
     } while (0)
 
-#define VMC_MAX_HOST_WRITE    2048
-
 typedef struct SpiceCharDriver {
     CharDriverState*      chr;
     SpiceCharDeviceInstance     sin;
@@ -25,8 +25,12 @@ typedef struct SpiceCharDriver {
     uint8_t               *datapos;
     ssize_t               bufsize, datalen;
     uint32_t              debug;
+    QLIST_ENTRY(SpiceCharDriver) next;
 } SpiceCharDriver;
 
+static QLIST_HEAD(, SpiceCharDriver) spice_chars =
+    QLIST_HEAD_INITIALIZER(spice_chars);
+
 static int vmc_write(SpiceCharDeviceInstance *sin, const uint8_t *buf, int len)
 {
     SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
@@ -35,8 +39,8 @@ static int vmc_write(SpiceCharDeviceInstance *sin, const uint8_t *buf, int len)
     uint8_t* p = (uint8_t*)buf;
 
     while (len > 0) {
-        last_out = MIN(len, VMC_MAX_HOST_WRITE);
-        if (qemu_chr_be_can_write(scd->chr) < last_out) {
+        last_out = MIN(len, qemu_chr_be_can_write(scd->chr));
+        if (last_out <= 0) {
             break;
         }
         qemu_chr_be_write(scd->chr, p, last_out);
@@ -69,6 +73,27 @@ static int vmc_read(SpiceCharDeviceInstance *sin, uint8_t *buf, int len)
     return bytes;
 }
 
+#if SPICE_SERVER_VERSION >= 0x000c02
+static void vmc_event(SpiceCharDeviceInstance *sin, uint8_t event)
+{
+    SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
+    int chr_event;
+
+    switch (event) {
+    case SPICE_PORT_EVENT_BREAK:
+        chr_event = CHR_EVENT_BREAK;
+        break;
+    default:
+        dprintf(scd, 2, "%s: unknown %d\n", __func__, event);
+        return;
+    }
+
+    dprintf(scd, 2, "%s: %d\n", __func__, event);
+    trace_spice_vmc_event(chr_event);
+    qemu_chr_be_event(scd->chr, chr_event);
+}
+#endif
+
 static void vmc_state(SpiceCharDeviceInstance *sin, int connected)
 {
     SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
@@ -105,6 +130,9 @@ static SpiceCharDeviceInterface vmc_interface = {
     .state              = vmc_state,
     .write              = vmc_write,
     .read               = vmc_read,
+#if SPICE_SERVER_VERSION >= 0x000c02
+    .event              = vmc_event,
+#endif
 };
 
 
@@ -156,6 +184,7 @@ static void spice_chr_close(struct CharDriverState *chr)
 
     printf("%s\n", __func__);
     vmc_unregister_interface(s);
+    QLIST_REMOVE(s, next);
     g_free(s);
 }
 
@@ -188,13 +217,34 @@ static void print_allowed_subtypes(void)
     fprintf(stderr, "\n");
 }
 
-CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
+static CharDriverState *chr_open(QemuOpts *opts, const char *subtype)
 {
     CharDriverState *chr;
     SpiceCharDriver *s;
-    const char* name = qemu_opt_get(opts, "name");
     uint32_t debug = qemu_opt_get_number(opts, "debug", 0);
-    const char** psubtype = spice_server_char_device_recognized_subtypes();
+
+    chr = g_malloc0(sizeof(CharDriverState));
+    s = g_malloc0(sizeof(SpiceCharDriver));
+    s->chr = chr;
+    s->debug = debug;
+    s->active = false;
+    s->sin.subtype = subtype;
+    chr->opaque = s;
+    chr->chr_write = spice_chr_write;
+    chr->chr_close = spice_chr_close;
+    chr->chr_guest_open = spice_chr_guest_open;
+    chr->chr_guest_close = spice_chr_guest_close;
+
+    QLIST_INSERT_HEAD(&spice_chars, s, next);
+
+    return chr;
+}
+
+CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    const char *name = qemu_opt_get(opts, "name");
+    const char **psubtype = spice_server_char_device_recognized_subtypes();
     const char *subtype = NULL;
 
     if (name == NULL) {
@@ -214,17 +264,7 @@ CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
         return NULL;
     }
 
-    chr = g_malloc0(sizeof(CharDriverState));
-    s = g_malloc0(sizeof(SpiceCharDriver));
-    s->chr = chr;
-    s->debug = debug;
-    s->active = false;
-    s->sin.subtype = subtype;
-    chr->opaque = s;
-    chr->chr_write = spice_chr_write;
-    chr->chr_close = spice_chr_close;
-    chr->chr_guest_open = spice_chr_guest_open;
-    chr->chr_guest_close = spice_chr_guest_close;
+    chr = chr_open(opts, subtype);
 
 #if SPICE_SERVER_VERSION < 0x000901
     /* See comment in vmc_state() */
@@ -235,3 +275,35 @@ CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
 
     return chr;
 }
+
+#if SPICE_SERVER_VERSION >= 0x000c02
+CharDriverState *qemu_chr_open_spice_port(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    SpiceCharDriver *s;
+    const char *name = qemu_opt_get(opts, "name");
+
+    if (name == NULL) {
+        fprintf(stderr, "spice-qemu-char: missing name parameter\n");
+        return NULL;
+    }
+
+    chr = chr_open(opts, "port");
+    s = chr->opaque;
+    s->sin.portname = name;
+
+    return chr;
+}
+
+void qemu_spice_register_ports(void)
+{
+    SpiceCharDriver *s;
+
+    QLIST_FOREACH(s, &spice_chars, next) {
+        if (s->sin.portname == NULL) {
+            continue;
+        }
+        vmc_register_interface(s);
+    }
+}
+#endif
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
new file mode 100644
index 00000000000..a2603947dbc
--- /dev/null
+++ b/stubs/Makefile.objs
@@ -0,0 +1,24 @@
+stub-obj-y += arch-query-cpu-def.o
+stub-obj-y += clock-warp.o
+stub-obj-y += cpu-get-clock.o
+stub-obj-y += cpu-get-icount.o
+stub-obj-y += fdset-add-fd.o
+stub-obj-y += fdset-find-fd.o
+stub-obj-y += fdset-get-fd.o
+stub-obj-y += fdset-remove-fd.o
+stub-obj-y += get-fd.o
+stub-obj-y += get-vm-name.o
+stub-obj-y += iothread-lock.o
+stub-obj-y += migr-blocker.o
+stub-obj-y += mon-is-qmp.o
+stub-obj-y += mon-printf.o
+stub-obj-y += mon-print-filename.o
+stub-obj-y += mon-protocol-event.o
+stub-obj-y += mon-set-error.o
+stub-obj-y += reset.o
+stub-obj-y += set-fd-handler.o
+stub-obj-y += slirp.o
+stub-obj-y += sysbus.o
+stub-obj-y += vm-stop.o
+stub-obj-y += vmstate.o
+stub-obj-$(CONFIG_WIN32) += fd-register.o
diff --git a/stubs/arch-query-cpu-def.c b/stubs/arch-query-cpu-def.c
new file mode 100644
index 00000000000..fa6789598ac
--- /dev/null
+++ b/stubs/arch-query-cpu-def.c
@@ -0,0 +1,9 @@
+#include "qemu-common.h"
+#include "sysemu/arch_init.h"
+#include "qapi/qmp/qerror.h"
+
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
+{
+    error_set(errp, QERR_NOT_SUPPORTED);
+    return NULL;
+}
diff --git a/stubs/clock-warp.c b/stubs/clock-warp.c
new file mode 100644
index 00000000000..b64c462e736
--- /dev/null
+++ b/stubs/clock-warp.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+void qemu_clock_warp(QEMUClock *clock)
+{
+}
+
diff --git a/stubs/cpu-get-clock.c b/stubs/cpu-get-clock.c
new file mode 100644
index 00000000000..5b34c976d9d
--- /dev/null
+++ b/stubs/cpu-get-clock.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+int64_t cpu_get_clock(void)
+{
+    return get_clock_realtime();
+}
diff --git a/stubs/cpu-get-icount.c b/stubs/cpu-get-icount.c
new file mode 100644
index 00000000000..d68585965fd
--- /dev/null
+++ b/stubs/cpu-get-icount.c
@@ -0,0 +1,9 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+int use_icount;
+
+int64_t cpu_get_icount(void)
+{
+    abort();
+}
diff --git a/stubs/fd-register.c b/stubs/fd-register.c
new file mode 100644
index 00000000000..d0c34fd2a3c
--- /dev/null
+++ b/stubs/fd-register.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+void qemu_fd_register(int fd)
+{
+}
diff --git a/stubs/fdset-add-fd.c b/stubs/fdset-add-fd.c
new file mode 100644
index 00000000000..ee1643708c2
--- /dev/null
+++ b/stubs/fdset-add-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
+{
+    return -1;
+}
diff --git a/stubs/fdset-find-fd.c b/stubs/fdset-find-fd.c
new file mode 100644
index 00000000000..4f18344bad9
--- /dev/null
+++ b/stubs/fdset-find-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_fdset_dup_fd_find(int dup_fd)
+{
+    return -1;
+}
diff --git a/stubs/fdset-get-fd.c b/stubs/fdset-get-fd.c
new file mode 100644
index 00000000000..7112c155e39
--- /dev/null
+++ b/stubs/fdset-get-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_fdset_get_fd(int64_t fdset_id, int flags)
+{
+    return -1;
+}
diff --git a/stubs/fdset-remove-fd.c b/stubs/fdset-remove-fd.c
new file mode 100644
index 00000000000..b3886d9f45c
--- /dev/null
+++ b/stubs/fdset-remove-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_fdset_dup_fd_remove(int dupfd)
+{
+    return -1;
+}
diff --git a/stubs/get-fd.c b/stubs/get-fd.c
new file mode 100644
index 00000000000..9f2c65cf0a5
--- /dev/null
+++ b/stubs/get-fd.c
@@ -0,0 +1,8 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_get_fd(Monitor *mon, const char *name, Error **errp)
+{
+    error_setg(errp, "only QEMU supports file descriptor passing");
+    return -1;
+}
diff --git a/stubs/get-vm-name.c b/stubs/get-vm-name.c
new file mode 100644
index 00000000000..e5f619ffab5
--- /dev/null
+++ b/stubs/get-vm-name.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+
+const char *qemu_get_vm_name(void)
+{
+    return NULL;
+}
+
diff --git a/stubs/iothread-lock.c b/stubs/iothread-lock.c
new file mode 100644
index 00000000000..5d8aca1b37d
--- /dev/null
+++ b/stubs/iothread-lock.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+void qemu_mutex_lock_iothread(void)
+{
+}
+
+void qemu_mutex_unlock_iothread(void)
+{
+}
diff --git a/stubs/migr-blocker.c b/stubs/migr-blocker.c
new file mode 100644
index 00000000000..300df6e205a
--- /dev/null
+++ b/stubs/migr-blocker.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "migration/migration.h"
+
+void migrate_add_blocker(Error *reason)
+{
+}
+
+void migrate_del_blocker(Error *reason)
+{
+}
diff --git a/stubs/mon-is-qmp.c b/stubs/mon-is-qmp.c
new file mode 100644
index 00000000000..1f0a8fd98a7
--- /dev/null
+++ b/stubs/mon-is-qmp.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_cur_is_qmp(void)
+{
+    return 0;
+}
diff --git a/stubs/mon-print-filename.c b/stubs/mon-print-filename.c
new file mode 100644
index 00000000000..9c939641ff2
--- /dev/null
+++ b/stubs/mon-print-filename.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_print_filename(Monitor *mon, const char *filename)
+{
+}
diff --git a/stubs/mon-printf.c b/stubs/mon-printf.c
new file mode 100644
index 00000000000..0ce2ca69253
--- /dev/null
+++ b/stubs/mon-printf.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_printf(Monitor *mon, const char *fmt, ...)
+{
+}
+
+void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+{
+}
diff --git a/stubs/mon-protocol-event.c b/stubs/mon-protocol-event.c
new file mode 100644
index 00000000000..0946e947246
--- /dev/null
+++ b/stubs/mon-protocol-event.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_protocol_event(MonitorEvent event, QObject *data)
+{
+}
diff --git a/stubs/mon-set-error.c b/stubs/mon-set-error.c
new file mode 100644
index 00000000000..d0411f97fa2
--- /dev/null
+++ b/stubs/mon-set-error.c
@@ -0,0 +1,8 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+Monitor *cur_mon;
+
+void monitor_set_error(Monitor *mon, QError *qerror)
+{
+}
diff --git a/stubs/reset.c b/stubs/reset.c
new file mode 100644
index 00000000000..ad287251ed4
--- /dev/null
+++ b/stubs/reset.c
@@ -0,0 +1,13 @@
+#include "hw/hw.h"
+
+/* Stub functions for binaries that never call qemu_devices_reset(),
+ * and don't need to keep track of the reset handler list.
+ */
+
+void qemu_register_reset(QEMUResetHandler *func, void *opaque)
+{
+}
+
+void qemu_unregister_reset(QEMUResetHandler *func, void *opaque)
+{
+}
diff --git a/stubs/set-fd-handler.c b/stubs/set-fd-handler.c
new file mode 100644
index 00000000000..fc874d33fe7
--- /dev/null
+++ b/stubs/set-fd-handler.c
@@ -0,0 +1,11 @@
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+int qemu_set_fd_handler2(int fd,
+                         IOCanReadHandler *fd_read_poll,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque)
+{
+    abort();
+}
diff --git a/stubs/slirp.c b/stubs/slirp.c
new file mode 100644
index 00000000000..9a3309a2b9b
--- /dev/null
+++ b/stubs/slirp.c
@@ -0,0 +1,17 @@
+#include "qemu-common.h"
+#include "slirp/slirp.h"
+
+void slirp_update_timeout(uint32_t *timeout)
+{
+}
+
+void slirp_select_fill(int *pnfds, fd_set *readfds,
+                       fd_set *writefds, fd_set *xfds)
+{
+}
+
+void slirp_select_poll(fd_set *readfds, fd_set *writefds,
+                       fd_set *xfds, int select_error)
+{
+}
+
diff --git a/stubs/sysbus.c b/stubs/sysbus.c
new file mode 100644
index 00000000000..e13496582b3
--- /dev/null
+++ b/stubs/sysbus.c
@@ -0,0 +1,6 @@
+#include "hw/qdev-core.h"
+
+BusState *sysbus_get_default(void)
+{
+    return NULL;
+}
diff --git a/stubs/vm-stop.c b/stubs/vm-stop.c
new file mode 100644
index 00000000000..45689354f66
--- /dev/null
+++ b/stubs/vm-stop.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "sysemu/sysemu.h"
+
+void vm_stop(RunState state)
+{
+    abort();
+}
diff --git a/stubs/vmstate.c b/stubs/vmstate.c
new file mode 100644
index 00000000000..3682af599eb
--- /dev/null
+++ b/stubs/vmstate.c
@@ -0,0 +1,17 @@
+#include "qemu-common.h"
+#include "migration/vmstate.h"
+
+int vmstate_register_with_alias_id(DeviceState *dev,
+                                   int instance_id,
+                                   const VMStateDescription *vmsd,
+                                   void *base, int alias_id,
+                                   int required_for_version)
+{
+    return 0;
+}
+
+void vmstate_unregister(DeviceState *dev,
+                        const VMStateDescription *vmsd,
+                        void *opaque)
+{
+}
diff --git a/sysconfigs/target/cpus-x86_64.conf b/sysconfigs/target/cpus-x86_64.conf
deleted file mode 100644
index cee0ea9e555..00000000000
--- a/sysconfigs/target/cpus-x86_64.conf
+++ /dev/null
@@ -1,128 +0,0 @@
-# x86 CPU MODELS
-
-[cpudef]
-   name = "Conroe"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "ssse3 sse3"
-   extfeature_edx = "i64 xd syscall"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Celeron_4x0 (Conroe/Merom Class Core 2)"
-
-[cpudef]
-   name = "Penryn"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 xd syscall"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Core 2 Duo P9xxx (Penryn Class Core 2)"
-
-[cpudef]
-   name = "Nehalem"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "popcnt sse4.2 sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 syscall xd"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Core i7 9xx (Nehalem Class Core i7)"
-
-[cpudef]
-   name = "Westmere"
-   level = "11"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "44"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "aes popcnt sse4.2 sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 syscall xd"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Westmere E56xx/L56xx/X56xx (Nehalem-C)"
-
-[cpudef]
-   name = "SandyBridge"
-   level = "0xd"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "42"
-   stepping = "1"
-   feature_edx = " sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "avx xsave aes tsc-deadline popcnt x2apic sse4.2 sse4.1 cx16 ssse3 pclmulqdq sse3"
-   extfeature_edx = "i64 rdtscp nx syscall "
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Xeon E312xx (Sandy Bridge)"
-
-[cpudef]
-   name = "Opteron_G1"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "sse3"
-   extfeature_edx = "lm fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = " "
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 240 (Gen 1 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G2"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "cx16 sse3"
-   extfeature_edx = "lm rdtscp fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = "svm lahf_lm"
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 22xx (Gen 2 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G3"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "popcnt cx16 monitor sse3"
-   extfeature_edx = "lm rdtscp fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = "misalignsse sse4a abm svm lahf_lm"
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 23xx (Gen 3 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G4"
-   level = "0xd"
-   vendor = "AuthenticAMD"
-   family = "21"
-   model = "1"
-   stepping = "2"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "avx xsave aes popcnt sse4.2 sse4.1 cx16 ssse3 pclmulqdq sse3"
-   extfeature_edx = "lm rdtscp pdpe1gb fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = " fma4 xop 3dnowprefetch misalignsse sse4a abm svm lahf_lm"
-   xlevel = "0x8000001A"
-   model_id = "AMD Opteron 62xx class CPU"
-
diff --git a/target-alpha/cpu-qom.h b/target-alpha/cpu-qom.h
index 6b4ca6d1d19..16367d2ea4a 100644
--- a/target-alpha/cpu-qom.h
+++ b/target-alpha/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_ALPHA_CPU_QOM_H
 #define QEMU_ALPHA_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_ALPHA_CPU "alpha-cpu"
@@ -58,6 +58,9 @@ typedef struct AlphaCPU {
     /*< public >*/
 
     CPUAlphaState env;
+
+    /* This alarm doesn't exist in real hardware; we wish it did.  */
+    struct QEMUTimer *alarm_timer;
 } AlphaCPU;
 
 static inline AlphaCPU *alpha_env_get_cpu(CPUAlphaState *env)
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index 62d2a669a99..0ad69f08597 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -19,10 +19,215 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
-#include "cpu-qom.h"
+#include "cpu.h"
 #include "qemu-common.h"
+#include "qapi/error.h"
 
 
+static void alpha_cpu_realize(Object *obj, Error **errp)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+
+    qemu_init_vcpu(&cpu->env);
+}
+
+/* Sort alphabetically by type name. */
+static gint alpha_cpu_list_compare(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *class_a = (ObjectClass *)a;
+    ObjectClass *class_b = (ObjectClass *)b;
+    const char *name_a, *name_b;
+
+    name_a = object_class_get_name(class_a);
+    name_b = object_class_get_name(class_b);
+    return strcmp(name_a, name_b);
+}
+
+static void alpha_cpu_list_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CPUListState *s = user_data;
+
+    (*s->cpu_fprintf)(s->file, "  %s\n",
+                      object_class_get_name(oc));
+}
+
+void alpha_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+    CPUListState s = {
+        .file = f,
+        .cpu_fprintf = cpu_fprintf,
+    };
+    GSList *list;
+
+    list = object_class_get_list(TYPE_ALPHA_CPU, false);
+    list = g_slist_sort(list, alpha_cpu_list_compare);
+    (*cpu_fprintf)(f, "Available CPUs:\n");
+    g_slist_foreach(list, alpha_cpu_list_entry, &s);
+    g_slist_free(list);
+}
+
+/* Models */
+
+#define TYPE(model) model "-" TYPE_ALPHA_CPU
+
+typedef struct AlphaCPUAlias {
+    const char *alias;
+    const char *typename;
+} AlphaCPUAlias;
+
+static const AlphaCPUAlias alpha_cpu_aliases[] = {
+    { "21064",   TYPE("ev4") },
+    { "21164",   TYPE("ev5") },
+    { "21164a",  TYPE("ev56") },
+    { "21164pc", TYPE("pca56") },
+    { "21264",   TYPE("ev6") },
+    { "21264a",  TYPE("ev67") },
+};
+
+static ObjectClass *alpha_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc = NULL;
+    char *typename;
+    int i;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && object_class_dynamic_cast(oc, TYPE_ALPHA_CPU) != NULL &&
+        !object_class_is_abstract(oc)) {
+        return oc;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(alpha_cpu_aliases); i++) {
+        if (strcmp(cpu_model, alpha_cpu_aliases[i].alias) == 0) {
+            oc = object_class_by_name(alpha_cpu_aliases[i].typename);
+            assert(oc != NULL && !object_class_is_abstract(oc));
+            return oc;
+        }
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_ALPHA_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (oc != NULL && object_class_is_abstract(oc)) {
+        oc = NULL;
+    }
+    return oc;
+}
+
+AlphaCPU *cpu_alpha_init(const char *cpu_model)
+{
+    AlphaCPU *cpu;
+    CPUAlphaState *env;
+    ObjectClass *cpu_class;
+
+    cpu_class = alpha_cpu_class_by_name(cpu_model);
+    if (cpu_class == NULL) {
+        /* Default to ev67; no reason not to emulate insns by default.  */
+        cpu_class = object_class_by_name(TYPE("ev67"));
+    }
+    cpu = ALPHA_CPU(object_new(object_class_get_name(cpu_class)));
+    env = &cpu->env;
+
+    env->cpu_model_str = cpu_model;
+
+    alpha_cpu_realize(OBJECT(cpu), NULL);
+    return cpu;
+}
+
+static void ev4_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_2106x;
+}
+
+static const TypeInfo ev4_cpu_type_info = {
+    .name = TYPE("ev4"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev4_cpu_initfn,
+};
+
+static void ev5_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_21164;
+}
+
+static const TypeInfo ev5_cpu_type_info = {
+    .name = TYPE("ev5"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev5_cpu_initfn,
+};
+
+static void ev56_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_BWX;
+}
+
+static const TypeInfo ev56_cpu_type_info = {
+    .name = TYPE("ev56"),
+    .parent = TYPE("ev5"),
+    .instance_init = ev56_cpu_initfn,
+};
+
+static void pca56_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_MVI;
+}
+
+static const TypeInfo pca56_cpu_type_info = {
+    .name = TYPE("pca56"),
+    .parent = TYPE("ev56"),
+    .instance_init = pca56_cpu_initfn,
+};
+
+static void ev6_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_21264;
+    env->amask = AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP;
+}
+
+static const TypeInfo ev6_cpu_type_info = {
+    .name = TYPE("ev6"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev6_cpu_initfn,
+};
+
+static void ev67_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_CIX | AMASK_PREFETCH;
+}
+
+static const TypeInfo ev67_cpu_type_info = {
+    .name = TYPE("ev67"),
+    .parent = TYPE("ev6"),
+    .instance_init = ev67_cpu_initfn,
+};
+
+static const TypeInfo ev68_cpu_type_info = {
+    .name = TYPE("ev68"),
+    .parent = TYPE("ev67"),
+};
+
 static void alpha_cpu_initfn(Object *obj)
 {
     AlphaCPU *cpu = ALPHA_CPU(obj);
@@ -31,6 +236,8 @@ static void alpha_cpu_initfn(Object *obj)
     cpu_exec_init(env);
     tlb_flush(env, 1);
 
+    alpha_translate_init();
+
 #if defined(CONFIG_USER_ONLY)
     env->ps = PS_USER_MODE;
     cpu_alpha_store_fpcr(env, (FPCR_INVD | FPCR_DZED | FPCR_OVFD
@@ -41,18 +248,33 @@ static void alpha_cpu_initfn(Object *obj)
     env->fen = 1;
 }
 
+static void alpha_cpu_class_init(ObjectClass *oc, void *data)
+{
+    CPUClass *cc = CPU_CLASS(oc);
+
+    cc->class_by_name = alpha_cpu_class_by_name;
+}
+
 static const TypeInfo alpha_cpu_type_info = {
     .name = TYPE_ALPHA_CPU,
     .parent = TYPE_CPU,
     .instance_size = sizeof(AlphaCPU),
     .instance_init = alpha_cpu_initfn,
-    .abstract = false,
+    .abstract = true,
     .class_size = sizeof(AlphaCPUClass),
+    .class_init = alpha_cpu_class_init,
 };
 
 static void alpha_cpu_register_types(void)
 {
     type_register_static(&alpha_cpu_type_info);
+    type_register_static(&ev4_cpu_type_info);
+    type_register_static(&ev5_cpu_type_info);
+    type_register_static(&ev56_cpu_type_info);
+    type_register_static(&pca56_cpu_type_info);
+    type_register_static(&ev6_cpu_type_info);
+    type_register_static(&ev67_cpu_type_info);
+    type_register_static(&ev68_cpu_type_info);
 }
 
 type_init(alpha_cpu_register_types)
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 5689760cefd..f1db6518cee 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -27,9 +27,9 @@
 
 #define CPUArchState struct CPUAlphaState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -277,16 +277,8 @@ struct CPUAlphaState {
 #endif
 
     /* This alarm doesn't exist in real hardware; we wish it did.  */
-    struct QEMUTimer *alarm_timer;
     uint64_t alarm_expire;
 
-#if TARGET_LONG_BITS > HOST_LONG_BITS
-    /* temporary fixed-point registers
-     * used to emulate 64 bits target on 32 bits hosts
-     */
-    target_ulong t0, t1;
-#endif
-
     /* Those resources are used only in QEMU core */
     CPU_COMMON
 
@@ -297,12 +289,12 @@ struct CPUAlphaState {
     int implver;
 };
 
-#define cpu_init cpu_alpha_init
+#define cpu_list alpha_cpu_list
 #define cpu_exec cpu_alpha_exec
 #define cpu_gen_code cpu_alpha_gen_code
 #define cpu_signal_handler cpu_alpha_signal_handler
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "cpu-qom.h"
 
 enum {
@@ -434,7 +426,20 @@ enum {
     IR_ZERO = 31,
 };
 
-CPUAlphaState * cpu_alpha_init (const char *cpu_model);
+void alpha_translate_init(void);
+
+AlphaCPU *cpu_alpha_init(const char *cpu_model);
+
+static inline CPUAlphaState *cpu_init(const char *cpu_model)
+{
+    AlphaCPU *cpu = cpu_alpha_init(cpu_model);
+    if (cpu == NULL) {
+        return NULL;
+    }
+    return &cpu->env;
+}
+
+void alpha_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 int cpu_alpha_exec(CPUAlphaState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -454,7 +459,7 @@ void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val);
 #ifndef CONFIG_USER_ONLY
 void swap_shadow_regs(CPUAlphaState *env);
 QEMU_NORETURN void cpu_unassigned_access(CPUAlphaState *env1,
-                                         target_phys_addr_t addr, int is_write,
+                                         hwaddr addr, int is_write,
                                          int is_exec, int unused, int size);
 #endif
 
@@ -510,8 +515,10 @@ static inline void cpu_set_tls(CPUAlphaState *env, target_ulong newtls)
 }
 #endif
 
-static inline bool cpu_has_work(CPUAlphaState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUAlphaState *env = &ALPHA_CPU(cpu)->env;
+
     /* Here we are checking to see if the CPU should wake up from HALT.
        We will have gotten into this state only for WTINT from PALmode.  */
     /* ??? I'm not sure how the IPL state works with WTINT to keep a CPU
@@ -525,7 +532,7 @@ static inline bool cpu_has_work(CPUAlphaState *env)
                                      | CPU_INTERRUPT_MCHK);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUAlphaState *env, TranslationBlock *tb)
 {
diff --git a/target-alpha/fpu_helper.c b/target-alpha/fpu_helper.c
index fe988ec459a..fad35755490 100644
--- a/target-alpha/fpu_helper.c
+++ b/target-alpha/fpu_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define FP_STATUS (env->fp_status)
 
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index 81d4763b492..22c9c6ec8c2 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -22,7 +22,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "helper.h"
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env)
@@ -315,7 +315,7 @@ static int get_physical_address(CPUAlphaState *env, target_ulong addr,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUAlphaState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUAlphaState *env, target_ulong addr)
 {
     target_ulong phys;
     int prot, fail;
@@ -494,16 +494,6 @@ void cpu_dump_state (CPUAlphaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "\n");
 }
 
-void do_restore_state(CPUAlphaState *env, uintptr_t retaddr)
-{
-    if (retaddr) {
-        TranslationBlock *tb = tb_find_pc(retaddr);
-        if (tb) {
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
 /* This should only be called from translate, via gen_excp.
    We expect that ENV->PC has already been updated.  */
 void QEMU_NORETURN helper_excp(CPUAlphaState *env, int excp, int error)
@@ -519,7 +509,9 @@ void QEMU_NORETURN dynamic_excp(CPUAlphaState *env, uintptr_t retaddr,
 {
     env->exception_index = excp;
     env->error_code = error;
-    do_restore_state(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     cpu_loop_exit(env);
 }
 
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index a184def3e9a..eac3041b879 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -1,102 +1,102 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_3(excp, noreturn, env, int, int)
-DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_CONST | TCG_CALL_PURE, i64, env)
-
-DEF_HELPER_3(addqv, i64, env, i64, i64)
-DEF_HELPER_3(addlv, i64, env, i64, i64)
-DEF_HELPER_3(subqv, i64, env, i64, i64)
-DEF_HELPER_3(sublv, i64, env, i64, i64)
-DEF_HELPER_3(mullv, i64, env, i64, i64)
-DEF_HELPER_3(mulqv, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(umulh, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_1(ctpop, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(ctlz, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(cttz, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-
-DEF_HELPER_FLAGS_2(zap, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_2(minub8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minsb8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minuw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minsw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxub8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxsb8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxuw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxsw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(perr, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_1(pklb, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(pkwb, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(unpkbl, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(unpkbw, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-
-DEF_HELPER_FLAGS_1(load_fpcr, TCG_CALL_CONST | TCG_CALL_PURE, i64, env)
-DEF_HELPER_FLAGS_2(store_fpcr, TCG_CALL_CONST, void, env, i64)
-
-DEF_HELPER_FLAGS_1(f_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64)
-DEF_HELPER_FLAGS_1(memory_to_f, TCG_CALL_CONST | TCG_CALL_PURE, i64, i32)
-DEF_HELPER_FLAGS_3(addf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mulf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtf, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_1(g_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(memory_to_g, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_3(addg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mulg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtg, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_1(s_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64)
-DEF_HELPER_FLAGS_1(memory_to_s, TCG_CALL_CONST | TCG_CALL_PURE, i64, i32)
-DEF_HELPER_FLAGS_3(adds, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subs, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(muls, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divs, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrts, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_3(addt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mult, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtt, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_3(cmptun, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpteq, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmptle, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmptlt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpgeq, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpgle, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpglt, TCG_CALL_CONST, i64, env, i64, i64)
-
-DEF_HELPER_FLAGS_2(cvtts, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtst, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqs, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqt, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqf, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtgf, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtgq, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqg, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_2(cvttq, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvttq_c, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvttq_svic, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_2(setroundmode, TCG_CALL_CONST, void, env, i32)
-DEF_HELPER_FLAGS_2(setflushzero, TCG_CALL_CONST, void, env, i32)
-DEF_HELPER_FLAGS_1(fp_exc_clear, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_1(fp_exc_get, TCG_CALL_CONST | TCG_CALL_PURE, i32, env)
-DEF_HELPER_3(fp_exc_raise, void, env, i32, i32)
-DEF_HELPER_3(fp_exc_raise_s, void, env, i32, i32)
-
-DEF_HELPER_2(ieee_input, void, env, i64)
-DEF_HELPER_2(ieee_input_cmp, void, env, i64)
+DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_NO_RWG_SE, i64, env)
+
+DEF_HELPER_FLAGS_3(addqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(addlv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(sublv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mullv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(umulh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_1(ctpop, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(ctlz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(cttz, TCG_CALL_NO_RWG_SE, i64, i64)
+
+DEF_HELPER_FLAGS_2(zap, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(minub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minsb8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minuw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minsw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxsb8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxuw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxsw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(perr, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_1(pklb, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pkwb, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(unpkbl, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(unpkbw, TCG_CALL_NO_RWG_SE, i64, i64)
+
+DEF_HELPER_FLAGS_1(load_fpcr, TCG_CALL_NO_RWG_SE, i64, env)
+DEF_HELPER_FLAGS_2(store_fpcr, TCG_CALL_NO_RWG, void, env, i64)
+
+DEF_HELPER_FLAGS_1(f_to_memory, TCG_CALL_NO_RWG_SE, i32, i64)
+DEF_HELPER_FLAGS_1(memory_to_f, TCG_CALL_NO_RWG_SE, i64, i32)
+DEF_HELPER_FLAGS_3(addf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtf, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_1(g_to_memory, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(memory_to_g, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_3(addg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtg, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_1(s_to_memory, TCG_CALL_NO_RWG_SE, i32, i64)
+DEF_HELPER_FLAGS_1(memory_to_s, TCG_CALL_NO_RWG_SE, i64, i32)
+DEF_HELPER_FLAGS_3(adds, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subs, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(muls, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrts, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_3(addt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mult, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtt, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_3(cmptun, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpteq, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmptle, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmptlt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpgeq, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpgle, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpglt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+
+DEF_HELPER_FLAGS_2(cvtts, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtst, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqs, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqt, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqf, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtgf, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtgq, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqg, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_2(cvttq, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvttq_c, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvttq_svic, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_2(setroundmode, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_2(setflushzero, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_1(fp_exc_clear, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_1(fp_exc_get, TCG_CALL_NO_RWG_SE, i32, env)
+DEF_HELPER_FLAGS_3(fp_exc_raise, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(fp_exc_raise_s, TCG_CALL_NO_WG, void, env, i32, i32)
+
+DEF_HELPER_FLAGS_2(ieee_input, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_2(ieee_input_cmp, TCG_CALL_NO_WG, void, env, i64)
 
 #if !defined (CONFIG_USER_ONLY)
 DEF_HELPER_2(hw_ret, void, env, i64)
@@ -110,13 +110,13 @@ DEF_HELPER_2(stq_phys, void, i64, i64)
 DEF_HELPER_3(stl_c_phys, i64, env, i64, i64)
 DEF_HELPER_3(stq_c_phys, i64, env, i64, i64)
 
-DEF_HELPER_FLAGS_1(tbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(tbis, TCG_CALL_CONST, void, env, i64)
+DEF_HELPER_FLAGS_1(tbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(tbis, TCG_CALL_NO_RWG, void, env, i64)
 
 DEF_HELPER_1(halt, void, i64);
 
-DEF_HELPER_FLAGS_0(get_time, TCG_CALL_CONST, i64)
-DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_CONST, void, env, i64)
+DEF_HELPER_FLAGS_0(get_time, TCG_CALL_NO_RWG, i64)
+DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_NO_RWG, void, env, i64)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-alpha/int_helper.c b/target-alpha/int_helper.c
index 1d832f0b576..c9b42b6ed42 100644
--- a/target-alpha/int_helper.c
+++ b/target-alpha/int_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 
 uint64_t helper_umulh(uint64_t op1, uint64_t op2)
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index 87cada48f93..3d2cd61358e 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -94,7 +94,9 @@ static void do_unaligned_access(CPUAlphaState *env, target_ulong addr,
     uint64_t pc;
     uint32_t insn;
 
-    do_restore_state(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
 
     pc = env->pc;
     insn = cpu_ldl_code(env, pc);
@@ -107,7 +109,7 @@ static void do_unaligned_access(CPUAlphaState *env, target_ulong addr,
     cpu_loop_exit(env);
 }
 
-void cpu_unassigned_access(CPUAlphaState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUAlphaState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size)
 {
     env->trap_arg0 = addr;
@@ -115,22 +117,22 @@ void cpu_unassigned_access(CPUAlphaState *env, target_phys_addr_t addr,
     dynamic_excp(env, 0, EXCP_MCHK, 0);
 }
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -143,7 +145,9 @@ void tlb_fill(CPUAlphaState *env, target_ulong addr, int is_write,
 
     ret = cpu_alpha_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret != 0)) {
-        do_restore_state(env, retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         /* Exception index and error code are already set */
         cpu_loop_exit(env);
     }
diff --git a/target-alpha/sys_helper.c b/target-alpha/sys_helper.c
index 40ca49c8837..339501af90f 100644
--- a/target-alpha/sys_helper.c
+++ b/target-alpha/sys_helper.c
@@ -19,8 +19,8 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 
 
 uint64_t helper_load_pcc(CPUAlphaState *env)
@@ -77,11 +77,13 @@ uint64_t helper_get_time(void)
 
 void helper_set_alarm(CPUAlphaState *env, uint64_t expire)
 {
+    AlphaCPU *cpu = alpha_env_get_cpu(env);
+
     if (expire) {
         env->alarm_expire = expire;
-        qemu_mod_timer(env->alarm_timer, expire);
+        qemu_mod_timer(cpu->alarm_timer, expire);
     } else {
-        qemu_del_timer(env->alarm_timer);
+        qemu_del_timer(cpu->alarm_timer);
     }
 }
 #endif /* CONFIG_USER_ONLY */
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 12de6a3fb65..f687b95c639 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -18,8 +18,8 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
-#include "host-utils.h"
+#include "disas/disas.h"
+#include "qemu/host-utils.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -88,9 +88,9 @@ static TCGv cpu_usp;
 /* register names */
 static char cpu_reg_names[10*4+21*5 + 10*5+21*6];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
-static void alpha_translate_init(void)
+void alpha_translate_init(void)
 {
     int i;
     char *p;
@@ -426,27 +426,15 @@ static ExitStatus gen_bcond_internal(DisasContext *ctx, TCGCond cond,
 
         return EXIT_GOTO_TB;
     } else {
-        int lab_over = gen_new_label();
-
-        /* ??? Consider using either
-             movi pc, next
-             addi tmp, pc, disp
-             movcond pc, cond, 0, tmp, pc
-           or
-             setcond tmp, cond, 0
-             movi pc, next
-             neg tmp, tmp
-             andi tmp, tmp, disp
-             add pc, pc, tmp
-           The current diamond subgraph surely isn't efficient.  */
+        TCGv_i64 z = tcg_const_i64(0);
+        TCGv_i64 d = tcg_const_i64(dest);
+        TCGv_i64 p = tcg_const_i64(ctx->pc);
 
-        tcg_gen_brcondi_i64(cond, cmp, 0, lab_true);
-        tcg_gen_movi_i64(cpu_pc, ctx->pc);
-        tcg_gen_br(lab_over);
-        gen_set_label(lab_true);
-        tcg_gen_movi_i64(cpu_pc, dest);
-        gen_set_label(lab_over);
+        tcg_gen_movcond_i64(cond, cpu_pc, cmp, z, d, p);
 
+        tcg_temp_free_i64(z);
+        tcg_temp_free_i64(d);
+        tcg_temp_free_i64(p);
         return EXIT_PC_UPDATED;
     }
 }
@@ -521,61 +509,67 @@ static ExitStatus gen_fbcond(DisasContext *ctx, TCGCond cond, int ra,
 static void gen_cmov(TCGCond cond, int ra, int rb, int rc,
                      int islit, uint8_t lit, int mask)
 {
-    TCGCond inv_cond = tcg_invert_cond(cond);
-    int l1;
+    TCGv_i64 c1, z, v1;
 
-    if (unlikely(rc == 31))
+    if (unlikely(rc == 31)) {
         return;
+    }
 
-    l1 = gen_new_label();
-
-    if (ra != 31) {
-        if (mask) {
-            TCGv tmp = tcg_temp_new();
-            tcg_gen_andi_i64(tmp, cpu_ir[ra], 1);
-            tcg_gen_brcondi_i64(inv_cond, tmp, 0, l1);
-            tcg_temp_free(tmp);
-        } else
-            tcg_gen_brcondi_i64(inv_cond, cpu_ir[ra], 0, l1);
-    } else {
+    if (ra == 31) {
         /* Very uncommon case - Do not bother to optimize.  */
-        TCGv tmp = tcg_const_i64(0);
-        tcg_gen_brcondi_i64(inv_cond, tmp, 0, l1);
-        tcg_temp_free(tmp);
+        c1 = tcg_const_i64(0);
+    } else if (mask) {
+        c1 = tcg_const_i64(1);
+        tcg_gen_and_i64(c1, c1, cpu_ir[ra]);
+    } else {
+        c1 = cpu_ir[ra];
+    }
+    if (islit) {
+        v1 = tcg_const_i64(lit);
+    } else {
+        v1 = cpu_ir[rb];
     }
+    z = tcg_const_i64(0);
 
-    if (islit)
-        tcg_gen_movi_i64(cpu_ir[rc], lit);
-    else
-        tcg_gen_mov_i64(cpu_ir[rc], cpu_ir[rb]);
-    gen_set_label(l1);
+    tcg_gen_movcond_i64(cond, cpu_ir[rc], c1, z, v1, cpu_ir[rc]);
+
+    tcg_temp_free_i64(z);
+    if (ra == 31 || mask) {
+        tcg_temp_free_i64(c1);
+    }
+    if (islit) {
+        tcg_temp_free_i64(v1);
+    }
 }
 
 static void gen_fcmov(TCGCond cond, int ra, int rb, int rc)
 {
-    TCGv cmp_tmp;
-    int l1;
+    TCGv_i64 c1, z, v1;
 
     if (unlikely(rc == 31)) {
         return;
     }
 
-    cmp_tmp = tcg_temp_new();
+    c1 = tcg_temp_new_i64();
     if (unlikely(ra == 31)) {
-        tcg_gen_movi_i64(cmp_tmp, 0);
+        tcg_gen_movi_i64(c1, 0);
+    } else {
+        gen_fold_mzero(cond, c1, cpu_fir[ra]);
+    }
+    if (rb == 31) {
+        v1 = tcg_const_i64(0);
     } else {
-        gen_fold_mzero(cond, cmp_tmp, cpu_fir[ra]);
+        v1 = cpu_fir[rb];
     }
+    z = tcg_const_i64(0);
 
-    l1 = gen_new_label();
-    tcg_gen_brcondi_i64(tcg_invert_cond(cond), cmp_tmp, 0, l1);
-    tcg_temp_free(cmp_tmp);
+    tcg_gen_movcond_i64(cond, cpu_fir[rc], c1, z, v1, cpu_fir[rc]);
 
-    if (rb != 31)
-        tcg_gen_mov_i64(cpu_fir[rc], cpu_fir[rb]);
-    else
-        tcg_gen_movi_i64(cpu_fir[rc], 0);
-    gen_set_label(l1);
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(c1);
+    if (rb == 31) {
+        tcg_temp_free_i64(v1);
+    }
 }
 
 #define QUAL_RM_N       0x080   /* Round mode nearest even */
@@ -617,7 +611,7 @@ static void gen_qual_roundmode(DisasContext *ctx, int fn11)
     }
 
 #if defined(CONFIG_SOFTFLOAT_INLINE)
-    /* ??? The "softfloat.h" interface is to call set_float_rounding_mode.
+    /* ??? The "fpu/softfloat.h" interface is to call set_float_rounding_mode.
        With CONFIG_SOFTFLOAT that expands to an out-of-line call that just
        sets the one field.  */
     tcg_gen_st8_i32(tmp, cpu_env,
@@ -1585,7 +1579,7 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
         case 0x3C:
             /* WHAMI */
             tcg_gen_ld32s_i64(cpu_ir[IR_V0], cpu_env,
-                              offsetof(CPUAlphaState, cpu_index));
+                -offsetof(AlphaCPU, env) + offsetof(CPUState, cpu_index));
             break;
 
         default:
@@ -3379,7 +3373,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     ctx.tb = tb;
     ctx.env = env;
@@ -3412,22 +3406,22 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         insn = cpu_ldl_code(env, ctx.pc);
         num_insns++;
 
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(ctx.pc);
         }
 
@@ -3438,7 +3432,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
            or exhaust instruction count, stop generation.  */
         if (ret == NO_EXIT
             && ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0
-                || gen_opc_ptr >= gen_opc_end
+                || tcg_ctx.gen_opc_ptr >= gen_opc_end
                 || num_insns >= max_insns
                 || singlestep
                 || env->singlestep_enabled)) {
@@ -3469,12 +3463,12 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
@@ -3483,7 +3477,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.pc - pc_start, 1);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
@@ -3499,62 +3493,7 @@ void gen_intermediate_code_pc (CPUAlphaState *env, struct TranslationBlock *tb)
     gen_intermediate_code_internal(env, tb, 1);
 }
 
-struct cpu_def_t {
-    const char *name;
-    int implver, amask;
-};
-
-static const struct cpu_def_t cpu_defs[] = {
-    { "ev4",   IMPLVER_2106x, 0 },
-    { "ev5",   IMPLVER_21164, 0 },
-    { "ev56",  IMPLVER_21164, AMASK_BWX },
-    { "pca56", IMPLVER_21164, AMASK_BWX | AMASK_MVI },
-    { "ev6",   IMPLVER_21264, AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP },
-    { "ev67",  IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-			       | AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), },
-    { "ev68",  IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-			       | AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), },
-    { "21064", IMPLVER_2106x, 0 },
-    { "21164", IMPLVER_21164, 0 },
-    { "21164a", IMPLVER_21164, AMASK_BWX },
-    { "21164pc", IMPLVER_21164, AMASK_BWX | AMASK_MVI },
-    { "21264", IMPLVER_21264, AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP },
-    { "21264a", IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-				| AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), }
-};
-
-CPUAlphaState * cpu_alpha_init (const char *cpu_model)
-{
-    AlphaCPU *cpu;
-    CPUAlphaState *env;
-    int implver, amask, i, max;
-
-    cpu = ALPHA_CPU(object_new(TYPE_ALPHA_CPU));
-    env = &cpu->env;
-
-    alpha_translate_init();
-
-    /* Default to ev67; no reason not to emulate insns by default.  */
-    implver = IMPLVER_21264;
-    amask = (AMASK_BWX | AMASK_FIX | AMASK_CIX | AMASK_MVI
-	     | AMASK_TRAP | AMASK_PREFETCH);
-
-    max = ARRAY_SIZE(cpu_defs);
-    for (i = 0; i < max; i++) {
-        if (strcmp (cpu_model, cpu_defs[i].name) == 0) {
-            implver = cpu_defs[i].implver;
-            amask = cpu_defs[i].amask;
-            break;
-        }
-    }
-    env->implver = implver;
-    env->amask = amask;
-
-    qemu_init_vcpu(env);
-    return env;
-}
-
 void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index f447c4fdf2d..b6f1a9ebf6a 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -2,5 +2,3 @@ obj-y += arm-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-arm/arm-semi.c b/target-arm/arm-semi.c
index 73bde584abf..847318d1f42 100644
--- a/target-arm/arm-semi.c
+++ b/target-arm/arm-semi.c
@@ -33,7 +33,7 @@
 #define ARM_ANGEL_HEAP_SIZE (128 * 1024 * 1024)
 #else
 #include "qemu-common.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "hw/arm-misc.h"
 #endif
 
@@ -113,7 +113,7 @@ static inline uint32_t set_swi_errno(CPUARMState *env, uint32_t code)
     return code;
 }
 
-#include "softmmu-semi.h"
+#include "exec/softmmu-semi.h"
 #endif
 
 static target_ulong arm_semi_syscall_len;
@@ -166,17 +166,20 @@ static void arm_semi_flen_cb(CPUARMState *env, target_ulong ret, target_ulong er
 #endif
 }
 
-#define ARG(n)					\
-({						\
-    target_ulong __arg;				\
-    /* FIXME - handle get_user() failure */	\
-    get_user_ual(__arg, args + (n) * 4);	\
-    __arg;					\
-})
+/* Read the input value from the argument block; fail the semihosting
+ * call if the memory read fails.
+ */
+#define GET_ARG(n) do {                                 \
+    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
+        return (uint32_t)-1;                            \
+    }                                                   \
+} while (0)
+
 #define SET_ARG(n, val) put_user_ual(val, args + (n) * 4)
 uint32_t do_arm_semihosting(CPUARMState *env)
 {
     target_ulong args;
+    target_ulong arg0, arg1, arg2, arg3;
     char * s;
     int nr;
     uint32_t ret;
@@ -191,33 +194,39 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     args = env->regs[1];
     switch (nr) {
     case TARGET_SYS_OPEN:
-        if (!(s = lock_user_string(ARG(0))))
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        s = lock_user_string(arg0);
+        if (!s) {
             /* FIXME - should this error code be -TARGET_EFAULT ? */
             return (uint32_t)-1;
-        if (ARG(1) >= 12) {
-            unlock_user(s, ARG(0), 0);
+        }
+        if (arg1 >= 12) {
+            unlock_user(s, arg0, 0);
             return (uint32_t)-1;
         }
         if (strcmp(s, ":tt") == 0) {
-            int result_fileno = ARG(1) < 4 ? STDIN_FILENO : STDOUT_FILENO;
-            unlock_user(s, ARG(0), 0);
+            int result_fileno = arg1 < 4 ? STDIN_FILENO : STDOUT_FILENO;
+            unlock_user(s, arg0, 0);
             return result_fileno;
         }
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", ARG(0),
-			   (int)ARG(2)+1, gdb_open_modeflags[ARG(1)]);
+            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", arg0,
+                           (int)arg2+1, gdb_open_modeflags[arg1]);
             ret = env->regs[0];
         } else {
-            ret = set_swi_errno(ts, open(s, open_modeflags[ARG(1)], 0644));
+            ret = set_swi_errno(ts, open(s, open_modeflags[arg1], 0644));
         }
-        unlock_user(s, ARG(0), 0);
+        unlock_user(s, arg0, 0);
         return ret;
     case TARGET_SYS_CLOSE:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "close,%x", ARG(0));
+            gdb_do_syscall(arm_semi_cb, "close,%x", arg0);
             return env->regs[0];
         } else {
-            return set_swi_errno(ts, close(ARG(0)));
+            return set_swi_errno(ts, close(arg0));
         }
     case TARGET_SYS_WRITEC:
         {
@@ -248,35 +257,45 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         unlock_user(s, args, 0);
         return ret;
     case TARGET_SYS_WRITE:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", ARG(0), ARG(1), len);
+            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", arg0, arg1, len);
             return env->regs[0];
         } else {
-            if (!(s = lock_user(VERIFY_READ, ARG(1), len, 1)))
+            s = lock_user(VERIFY_READ, arg1, len, 1);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
-            ret = set_swi_errno(ts, write(ARG(0), s, len));
-            unlock_user(s, ARG(1), 0);
+            }
+            ret = set_swi_errno(ts, write(arg0, s, len));
+            unlock_user(s, arg1, 0);
             if (ret == (uint32_t)-1)
                 return -1;
             return len - ret;
         }
     case TARGET_SYS_READ:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", ARG(0), ARG(1), len);
+            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", arg0, arg1, len);
             return env->regs[0];
         } else {
-            if (!(s = lock_user(VERIFY_WRITE, ARG(1), len, 0)))
+            s = lock_user(VERIFY_WRITE, arg1, len, 0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
-            do
-              ret = set_swi_errno(ts, read(ARG(0), s, len));
-            while (ret == -1 && errno == EINTR);
-            unlock_user(s, ARG(1), len);
+            }
+            do {
+                ret = set_swi_errno(ts, read(arg0, s, len));
+            } while (ret == -1 && errno == EINTR);
+            unlock_user(s, arg1, len);
             if (ret == (uint32_t)-1)
                 return -1;
             return len - ret;
@@ -285,30 +304,34 @@ uint32_t do_arm_semihosting(CPUARMState *env)
        /* XXX: Read from debug console. Not implemented.  */
         return 0;
     case TARGET_SYS_ISTTY:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "isatty,%x", ARG(0));
+            gdb_do_syscall(arm_semi_cb, "isatty,%x", arg0);
             return env->regs[0];
         } else {
-            return isatty(ARG(0));
+            return isatty(arg0);
         }
     case TARGET_SYS_SEEK:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", ARG(0), ARG(1));
+            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", arg0, arg1);
             return env->regs[0];
         } else {
-            ret = set_swi_errno(ts, lseek(ARG(0), ARG(1), SEEK_SET));
+            ret = set_swi_errno(ts, lseek(arg0, arg1, SEEK_SET));
             if (ret == (uint32_t)-1)
               return -1;
             return 0;
         }
     case TARGET_SYS_FLEN:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(arm_semi_flen_cb, "fstat,%x,%x",
-			   ARG(0), env->regs[13]-64);
+                           arg0, env->regs[13]-64);
             return env->regs[0];
         } else {
             struct stat buf;
-            ret = set_swi_errno(ts, fstat(ARG(0), &buf));
+            ret = set_swi_errno(ts, fstat(arg0, &buf));
             if (ret == (uint32_t)-1)
                 return -1;
             return buf.st_size;
@@ -317,35 +340,43 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         /* XXX: Not implemented.  */
         return -1;
     case TARGET_SYS_REMOVE:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "unlink,%s", ARG(0), (int)ARG(1)+1);
+            gdb_do_syscall(arm_semi_cb, "unlink,%s", arg0, (int)arg1+1);
             ret = env->regs[0];
         } else {
-            if (!(s = lock_user_string(ARG(0))))
+            s = lock_user_string(arg0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ret =  set_swi_errno(ts, remove(s));
-            unlock_user(s, ARG(0), 0);
+            unlock_user(s, arg0, 0);
         }
         return ret;
     case TARGET_SYS_RENAME:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(arm_semi_cb, "rename,%s,%s",
-                           ARG(0), (int)ARG(1)+1, ARG(2), (int)ARG(3)+1);
+                           arg0, (int)arg1+1, arg2, (int)arg3+1);
             return env->regs[0];
         } else {
             char *s2;
-            s = lock_user_string(ARG(0));
-            s2 = lock_user_string(ARG(2));
+            s = lock_user_string(arg0);
+            s2 = lock_user_string(arg2);
             if (!s || !s2)
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 ret = (uint32_t)-1;
             else
                 ret = set_swi_errno(ts, rename(s, s2));
             if (s2)
-                unlock_user(s2, ARG(2), 0);
+                unlock_user(s2, arg2, 0);
             if (s)
-                unlock_user(s, ARG(0), 0);
+                unlock_user(s, arg0, 0);
             return ret;
         }
     case TARGET_SYS_CLOCK:
@@ -353,15 +384,19 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_TIME:
         return set_swi_errno(ts, time(NULL));
     case TARGET_SYS_SYSTEM:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "system,%s", ARG(0), (int)ARG(1)+1);
+            gdb_do_syscall(arm_semi_cb, "system,%s", arg0, (int)arg1+1);
             return env->regs[0];
         } else {
-            if (!(s = lock_user_string(ARG(0))))
+            s = lock_user_string(arg0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ret = set_swi_errno(ts, system(s));
-            unlock_user(s, ARG(0), 0);
+            unlock_user(s, arg0, 0);
             return ret;
         }
     case TARGET_SYS_ERRNO:
@@ -375,22 +410,24 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             /* Build a command-line from the original argv.
              *
              * The inputs are:
-             *     * ARG(0), pointer to a buffer of at least the size
-             *               specified in ARG(1).
-             *     * ARG(1), size of the buffer pointed to by ARG(0) in
+             *     * arg0, pointer to a buffer of at least the size
+             *               specified in arg1.
+             *     * arg1, size of the buffer pointed to by arg0 in
              *               bytes.
              *
              * The outputs are:
-             *     * ARG(0), pointer to null-terminated string of the
+             *     * arg0, pointer to null-terminated string of the
              *               command line.
-             *     * ARG(1), length of the string pointed to by ARG(0).
+             *     * arg1, length of the string pointed to by arg0.
              */
 
             char *output_buffer;
-            size_t input_size = ARG(1);
+            size_t input_size;
             size_t output_size;
             int status = 0;
-
+            GET_ARG(0);
+            GET_ARG(1);
+            input_size = arg1;
             /* Compute the size of the output string.  */
 #if !defined(CONFIG_USER_ONLY)
             output_size = strlen(ts->boot_info->kernel_filename)
@@ -414,10 +451,13 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             }
 
             /* Adjust the command-line length.  */
-            SET_ARG(1, output_size - 1);
+            if (SET_ARG(1, output_size - 1)) {
+                /* Couldn't write back to argument block */
+                return -1;
+            }
 
             /* Lock the buffer on the ARM side.  */
-            output_buffer = lock_user(VERIFY_WRITE, ARG(0), output_size, 0);
+            output_buffer = lock_user(VERIFY_WRITE, arg0, output_size, 0);
             if (!output_buffer) {
                 return -1;
             }
@@ -449,7 +489,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         out:
 #endif
             /* Unlock the buffer on the ARM side.  */
-            unlock_user(output_buffer, ARG(0), output_size);
+            unlock_user(output_buffer, arg0, output_size);
 
             return status;
         }
@@ -457,6 +497,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         {
             uint32_t *ptr;
             uint32_t limit;
+            GET_ARG(0);
 
 #ifdef CONFIG_USER_ONLY
             /* Some C libraries assume the heap immediately follows .bss, so
@@ -477,25 +518,29 @@ uint32_t do_arm_semihosting(CPUARMState *env)
                 ts->heap_limit = limit;
             }
 
-            if (!(ptr = lock_user(VERIFY_WRITE, ARG(0), 16, 0)))
+            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
+            if (!ptr) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ptr[0] = tswap32(ts->heap_base);
             ptr[1] = tswap32(ts->heap_limit);
             ptr[2] = tswap32(ts->stack_base);
             ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, ARG(0), 16);
+            unlock_user(ptr, arg0, 16);
 #else
             limit = ram_size;
-            if (!(ptr = lock_user(VERIFY_WRITE, ARG(0), 16, 0)))
+            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
+            if (!ptr) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             /* TODO: Make this use the limit of the loaded application.  */
             ptr[0] = tswap32(limit / 2);
             ptr[1] = tswap32(limit);
             ptr[2] = tswap32(limit); /* Stack base */
             ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, ARG(0), 16);
+            unlock_user(ptr, arg0, 16);
 #endif
             return 0;
         }
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index beabf9a0a9c..0f455c40ff4 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_ARM_CPU_QOM_H
 #define QEMU_ARM_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_ARM_CPU "arm-cpu"
 
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index b00f5fa547a..d1a4c826801 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -23,7 +23,7 @@
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/loader.h"
 #endif
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
 {
@@ -64,7 +64,7 @@ static void arm_cpu_reset(CPUState *s)
     CPUARMState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -201,6 +201,22 @@ void arm_cpu_realize(ARMCPU *cpu)
 
 /* CPU models */
 
+static ObjectClass *arm_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+
+    if (!cpu_model) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (!oc || !object_class_dynamic_cast(oc, TYPE_ARM_CPU) ||
+        object_class_is_abstract(oc)) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void arm926_initfn(Object *obj)
 {
     ARMCPU *cpu = ARM_CPU(obj);
@@ -766,6 +782,8 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
 
     acc->parent_reset = cc->reset;
     cc->reset = arm_cpu_reset;
+
+    cc->class_by_name = arm_cpu_class_by_name;
 }
 
 static void cpu_register(const ARMCPUInfo *info)
@@ -778,7 +796,7 @@ static void cpu_register(const ARMCPUInfo *info)
         .class_size = sizeof(ARMCPUClass),
     };
 
-    type_register_static(&type_info);
+    type_register(&type_info);
 }
 
 static const TypeInfo arm_cpu_type_info = {
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index d7f93d98f0e..ffddfcbc0d1 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -27,9 +27,9 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -423,8 +423,6 @@ void armv7m_nvic_complete_irq(void *opaque, int irq);
     (((cp) << 16) | ((is64) << 15) | ((crn) << 11) |    \
      ((crm) << 7) | ((opc1) << 3) | (opc2))
 
-#define DECODE_CPREG_CRN(enc) (((enc) >> 7) & 0xf)
-
 /* ARMCPRegInfo type field bits. If the SPECIAL bit is set this is a
  * special-behaviour cp reg and bits [15..8] indicate what behaviour
  * it has. Otherwise it is a simple cp reg, where CONST indicates that
@@ -661,7 +659,7 @@ static inline void cpu_clone_regs(CPUARMState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Bit usage in the TB flags field: */
 #define ARM_TBFLAG_THUMB_SHIFT      0
@@ -720,13 +718,15 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
     }
 }
 
-static inline bool cpu_has_work(CPUARMState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUARMState *env = &ARM_CPU(cpu)->env;
+
     return env->interrupt_request &
         (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
 {
@@ -734,9 +734,10 @@ static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
 }
 
 /* Load an instruction and return it in the standard little-endian order */
-static inline uint32_t arm_ldl_code(uint32_t addr, bool do_swap)
+static inline uint32_t arm_ldl_code(CPUARMState *env, uint32_t addr,
+                                    bool do_swap)
 {
-    uint32_t insn = ldl_code(addr);
+    uint32_t insn = cpu_ldl_code(env, addr);
     if (do_swap) {
         return bswap32(insn);
     }
@@ -744,9 +745,10 @@ static inline uint32_t arm_ldl_code(uint32_t addr, bool do_swap)
 }
 
 /* Ditto, for a halfword (Thumb) instruction */
-static inline uint16_t arm_lduw_code(uint32_t addr, bool do_swap)
+static inline uint16_t arm_lduw_code(CPUARMState *env, uint32_t addr,
+                                     bool do_swap)
 {
-    uint16_t insn = lduw_code(addr);
+    uint16_t insn = cpu_lduw_code(env, addr);
     if (do_swap) {
         return bswap16(insn);
     }
diff --git a/target-arm/helper.c b/target-arm/helper.c
index dceaa95c802..7a10fddf25d 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -1,14 +1,14 @@
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "helper.h"
-#include "host-utils.h"
-#include "sysemu.h"
-#include "bitops.h"
+#include "qemu/host-utils.h"
+#include "sysemu/sysemu.h"
+#include "qemu/bitops.h"
 
 #ifndef CONFIG_USER_ONLY
 static inline int get_phys_addr(CPUARMState *env, uint32_t address,
                                 int access_type, int is_user,
-                                target_phys_addr_t *phys_ptr, int *prot,
+                                hwaddr *phys_ptr, int *prot,
                                 target_ulong *page_size);
 #endif
 
@@ -517,7 +517,7 @@ static inline bool extended_addresses_enabled(CPUARMState *env)
 
 static int ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret, is_user = ri->opc2 & 2;
@@ -645,7 +645,7 @@ static int pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri,
 static int arm946_prbs_read(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t *value)
 {
-    if (ri->crm > 8) {
+    if (ri->crm >= 8) {
         return EXCP_UDEF;
     }
     *value = env->cp15.c6_region[ri->crm];
@@ -655,7 +655,7 @@ static int arm946_prbs_read(CPUARMState *env, const ARMCPRegInfo *ri,
 static int arm946_prbs_write(CPUARMState *env, const ARMCPRegInfo *ri,
                              uint64_t value)
 {
-    if (ri->crm > 8) {
+    if (ri->crm >= 8) {
         return EXCP_UDEF;
     }
     env->cp15.c6_region[ri->crm] = value;
@@ -902,7 +902,8 @@ static const ARMCPRegInfo strongarm_cp_reginfo[] = {
 static int mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri,
                       uint64_t *value)
 {
-    uint32_t mpidr = env->cpu_index;
+    CPUState *cs = CPU(arm_env_get_cpu(env));
+    uint32_t mpidr = cs->cpu_index;
     /* We don't support setting cluster ID ([8..11])
      * so these bits always RAZ.
      */
@@ -1261,12 +1262,14 @@ ARMCPU *cpu_arm_init(const char *cpu_model)
 {
     ARMCPU *cpu;
     CPUARMState *env;
+    ObjectClass *oc;
     static int inited = 0;
 
-    if (!object_class_by_name(cpu_model)) {
+    oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
+    if (!oc) {
         return NULL;
     }
-    cpu = ARM_CPU(object_new(cpu_model));
+    cpu = ARM_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
     env->cpu_model_str = cpu_model;
     arm_cpu_realize(cpu);
@@ -1291,11 +1294,6 @@ ARMCPU *cpu_arm_init(const char *cpu_model)
     return cpu;
 }
 
-typedef struct ARMCPUListState {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} ARMCPUListState;
-
 /* Sort alphabetically by type name, except for "any". */
 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -1317,7 +1315,7 @@ static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *oc = data;
-    ARMCPUListState *s = user_data;
+    CPUListState *s = user_data;
 
     (*s->cpu_fprintf)(s->file, "  %s\n",
                       object_class_get_name(oc));
@@ -1325,7 +1323,7 @@ static void arm_cpu_list_entry(gpointer data, gpointer user_data)
 
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    ARMCPUListState s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
@@ -1562,11 +1560,6 @@ uint32_t HELPER(rbit)(uint32_t x)
     return x;
 }
 
-uint32_t HELPER(abs)(uint32_t x)
-{
-    return ((int32_t)x < 0) ? -x : x;
-}
-
 #if defined(CONFIG_USER_ONLY)
 
 void do_interrupt (CPUARMState *env)
@@ -1746,7 +1739,7 @@ static void do_interrupt_v7m(CPUARMState *env)
         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
         return;
     case EXCP_SWI:
-        env->regs[15] += 2;
+        /* The PC already points to the next instruction.  */
         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
         return;
     case EXCP_PREFETCH_ABORT:
@@ -1756,7 +1749,7 @@ static void do_interrupt_v7m(CPUARMState *env)
     case EXCP_BKPT:
         if (semihosting_enabled) {
             int nr;
-            nr = arm_lduw_code(env->regs[15], env->bswap_code) & 0xff;
+            nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
             if (nr == 0xab) {
                 env->regs[15] += 2;
                 env->regs[0] = do_arm_semihosting(env);
@@ -1828,9 +1821,10 @@ void do_interrupt(CPUARMState *env)
         if (semihosting_enabled) {
             /* Check for semihosting interrupt.  */
             if (env->thumb) {
-                mask = arm_lduw_code(env->regs[15] - 2, env->bswap_code) & 0xff;
+                mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
+                    & 0xff;
             } else {
-                mask = arm_ldl_code(env->regs[15] - 4, env->bswap_code)
+                mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
                     & 0xffffff;
             }
             /* Only intercept calls from privileged modes, to provide some
@@ -1851,7 +1845,7 @@ void do_interrupt(CPUARMState *env)
     case EXCP_BKPT:
         /* See if this is a semihosting syscall.  */
         if (env->thumb && semihosting_enabled) {
-            mask = arm_lduw_code(env->regs[15], env->bswap_code) & 0xff;
+            mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
             if (mask == 0xab
                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
                 env->regs[15] += 2;
@@ -1979,7 +1973,7 @@ static uint32_t get_level1_table_address(CPUARMState *env, uint32_t address)
 }
 
 static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, target_phys_addr_t *phys_ptr,
+                            int is_user, hwaddr *phys_ptr,
                             int *prot, target_ulong *page_size)
 {
     int code;
@@ -1989,7 +1983,7 @@ static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
     int ap;
     int domain;
     int domain_prot;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
@@ -2074,7 +2068,7 @@ do_fault:
 }
 
 static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, target_phys_addr_t *phys_ptr,
+                            int is_user, hwaddr *phys_ptr,
                             int *prot, target_ulong *page_size)
 {
     int code;
@@ -2086,7 +2080,7 @@ static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
     int ap;
     int domain = 0;
     int domain_prot;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
@@ -2196,7 +2190,7 @@ typedef enum {
 
 static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
                               int access_type, int is_user,
-                              target_phys_addr_t *phys_ptr, int *prot,
+                              hwaddr *phys_ptr, int *prot,
                               target_ulong *page_size_ptr)
 {
     /* Read an LPAE long-descriptor translation table. */
@@ -2207,7 +2201,7 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
     uint64_t ttbr;
     int ttbr_select;
     int n;
-    target_phys_addr_t descaddr;
+    hwaddr descaddr;
     uint32_t tableattrs;
     target_ulong page_size;
     uint32_t attrs;
@@ -2365,7 +2359,7 @@ do_fault:
 
 static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
                              int access_type, int is_user,
-                             target_phys_addr_t *phys_ptr, int *prot)
+                             hwaddr *phys_ptr, int *prot)
 {
     int n;
     uint32_t mask;
@@ -2449,7 +2443,7 @@ static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
  */
 static inline int get_phys_addr(CPUARMState *env, uint32_t address,
                                 int access_type, int is_user,
-                                target_phys_addr_t *phys_ptr, int *prot,
+                                hwaddr *phys_ptr, int *prot,
                                 target_ulong *page_size)
 {
     /* Fast Context Switch Extension.  */
@@ -2481,7 +2475,7 @@ static inline int get_phys_addr(CPUARMState *env, uint32_t address,
 int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
                               int access_type, int mmu_idx)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret, is_user;
@@ -2491,7 +2485,7 @@ int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
                         &page_size);
     if (ret == 0) {
         /* Map a single [sub]page.  */
-        phys_addr &= ~(target_phys_addr_t)0x3ff;
+        phys_addr &= ~(hwaddr)0x3ff;
         address &= ~(uint32_t)0x3ff;
         tlb_set_page (env, address, phys_addr, prot, mmu_idx, page_size);
         return 0;
@@ -2511,9 +2505,9 @@ int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
     return 1;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUARMState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUARMState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret;
diff --git a/target-arm/helper.h b/target-arm/helper.h
index 21e9cfe05fc..8544f82a944 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -1,19 +1,18 @@
-#include "def-helper.h"
-
-DEF_HELPER_1(clz, i32, i32)
-DEF_HELPER_1(sxtb16, i32, i32)
-DEF_HELPER_1(uxtb16, i32, i32)
-
-DEF_HELPER_2(add_setq, i32, i32, i32)
-DEF_HELPER_2(add_saturate, i32, i32, i32)
-DEF_HELPER_2(sub_saturate, i32, i32, i32)
-DEF_HELPER_2(add_usaturate, i32, i32, i32)
-DEF_HELPER_2(sub_usaturate, i32, i32, i32)
-DEF_HELPER_1(double_saturate, i32, s32)
-DEF_HELPER_2(sdiv, s32, s32, s32)
-DEF_HELPER_2(udiv, i32, i32, i32)
-DEF_HELPER_1(rbit, i32, i32)
-DEF_HELPER_1(abs, i32, i32)
+#include "exec/def-helper.h"
+
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(sxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(uxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
+
+DEF_HELPER_3(add_setq, i32, env, i32, i32)
+DEF_HELPER_3(add_saturate, i32, env, i32, i32)
+DEF_HELPER_3(sub_saturate, i32, env, i32, i32)
+DEF_HELPER_3(add_usaturate, i32, env, i32, i32)
+DEF_HELPER_3(sub_usaturate, i32, env, i32, i32)
+DEF_HELPER_2(double_saturate, i32, env, s32)
+DEF_HELPER_FLAGS_2(sdiv, TCG_CALL_NO_RWG_SE, s32, s32, s32)
+DEF_HELPER_FLAGS_2(udiv, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_1(rbit, TCG_CALL_NO_RWG_SE, i32, i32)
 
 #define PAS_OP(pfx)  \
     DEF_HELPER_3(pfx ## add8, i32, i32, i32, ptr) \
@@ -40,21 +39,22 @@ PAS_OP(uq)
 PAS_OP(uh)
 #undef PAS_OP
 
-DEF_HELPER_2(ssat, i32, i32, i32)
-DEF_HELPER_2(usat, i32, i32, i32)
-DEF_HELPER_2(ssat16, i32, i32, i32)
-DEF_HELPER_2(usat16, i32, i32, i32)
+DEF_HELPER_3(ssat, i32, env, i32, i32)
+DEF_HELPER_3(usat, i32, env, i32, i32)
+DEF_HELPER_3(ssat16, i32, env, i32, i32)
+DEF_HELPER_3(usat16, i32, env, i32, i32)
 
-DEF_HELPER_2(usad8, i32, i32, i32)
+DEF_HELPER_FLAGS_2(usad8, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 
 DEF_HELPER_1(logicq_cc, i32, i64)
 
-DEF_HELPER_3(sel_flags, i32, i32, i32, i32)
-DEF_HELPER_1(exception, void, i32)
-DEF_HELPER_0(wfi, void)
+DEF_HELPER_FLAGS_3(sel_flags, TCG_CALL_NO_RWG_SE,
+                   i32, i32, i32, i32)
+DEF_HELPER_2(exception, void, env, i32)
+DEF_HELPER_1(wfi, void, env)
 
-DEF_HELPER_2(cpsr_write, void, i32, i32)
-DEF_HELPER_0(cpsr_read, i32)
+DEF_HELPER_3(cpsr_write, void, env, i32, i32)
+DEF_HELPER_1(cpsr_read, i32, env)
 
 DEF_HELPER_3(v7m_msr, void, env, i32, i32)
 DEF_HELPER_2(v7m_mrs, i32, env, i32)
@@ -67,8 +67,8 @@ DEF_HELPER_2(get_cp_reg64, i64, env, ptr)
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
 DEF_HELPER_3(set_r13_banked, void, env, i32, i32)
 
-DEF_HELPER_1(get_user_reg, i32, i32)
-DEF_HELPER_2(set_user_reg, void, i32, i32)
+DEF_HELPER_2(get_user_reg, i32, env, i32)
+DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
 DEF_HELPER_1(vfp_get_fpscr, i32, env)
 DEF_HELPER_2(vfp_set_fpscr, void, env, i32)
@@ -140,20 +140,15 @@ DEF_HELPER_2(recpe_f32, f32, f32, env)
 DEF_HELPER_2(rsqrte_f32, f32, f32, env)
 DEF_HELPER_2(recpe_u32, i32, i32, env)
 DEF_HELPER_2(rsqrte_u32, i32, i32, env)
-DEF_HELPER_4(neon_tbl, i32, i32, i32, i32, i32)
+DEF_HELPER_5(neon_tbl, i32, env, i32, i32, i32, i32)
 
-DEF_HELPER_2(add_cc, i32, i32, i32)
-DEF_HELPER_2(adc_cc, i32, i32, i32)
-DEF_HELPER_2(sub_cc, i32, i32, i32)
-DEF_HELPER_2(sbc_cc, i32, i32, i32)
+DEF_HELPER_3(adc_cc, i32, env, i32, i32)
+DEF_HELPER_3(sbc_cc, i32, env, i32, i32)
 
-DEF_HELPER_2(shl, i32, i32, i32)
-DEF_HELPER_2(shr, i32, i32, i32)
-DEF_HELPER_2(sar, i32, i32, i32)
-DEF_HELPER_2(shl_cc, i32, i32, i32)
-DEF_HELPER_2(shr_cc, i32, i32, i32)
-DEF_HELPER_2(sar_cc, i32, i32, i32)
-DEF_HELPER_2(ror_cc, i32, i32, i32)
+DEF_HELPER_3(shl_cc, i32, env, i32, i32)
+DEF_HELPER_3(shr_cc, i32, env, i32, i32)
+DEF_HELPER_3(sar_cc, i32, env, i32, i32)
+DEF_HELPER_3(ror_cc, i32, env, i32, i32)
 
 /* neon_helper.c */
 DEF_HELPER_3(neon_qadd_u8, i32, env, i32, i32)
@@ -343,7 +338,6 @@ DEF_HELPER_2(neon_mull_s16, i64, i32, i32)
 
 DEF_HELPER_1(neon_negl_u16, i64, i64)
 DEF_HELPER_1(neon_negl_u32, i64, i64)
-DEF_HELPER_1(neon_negl_u64, i64, i64)
 
 DEF_HELPER_2(neon_qabs_s8, i32, env, i32)
 DEF_HELPER_2(neon_qabs_s16, i32, env, i32)
@@ -469,4 +463,4 @@ DEF_HELPER_3(neon_qzip8, void, env, i32, i32)
 DEF_HELPER_3(neon_qzip16, void, env, i32, i32)
 DEF_HELPER_3(neon_qzip32, void, env, i32, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
index 1dd8d1a3af4..7953b53f7e0 100644
--- a/target-arm/iwmmxt_helper.c
+++ b/target-arm/iwmmxt_helper.c
@@ -23,7 +23,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "helper.h"
 
 /* iwMMXt macros extracted from GNU gdb.  */
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index 8bb5129d6a5..b028cc2c93b 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -10,7 +10,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
@@ -788,7 +788,6 @@ uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shift
     return helper_neon_qshl_u64(env, valop, shiftop);
 }
 
-/* FIXME: This is wrong.  */
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
@@ -1665,12 +1664,6 @@ uint64_t HELPER(neon_negl_u32)(uint64_t x)
     return low | ((uint64_t)high << 32);
 }
 
-/* FIXME:  There should be a native op for this.  */
-uint64_t HELPER(neon_negl_u64)(uint64_t x)
-{
-    return -x;
-}
-
 /* Saturating sign manipulation.  */
 /* ??? Make these use NEON_VOP1 */
 #define DO_QABS8(x) do { \
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index d77bfab771a..99610d77348 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -17,19 +17,18 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
-static void raise_exception(int tt)
+static void raise_exception(CPUARMState *env, int tt)
 {
     env->exception_index = tt;
     cpu_loop_exit(env);
 }
 
-uint32_t HELPER(neon_tbl)(uint32_t ireg, uint32_t def,
+uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
                           uint32_t rn, uint32_t maxindex)
 {
     uint32_t val;
@@ -53,55 +52,42 @@ uint32_t HELPER(neon_tbl)(uint32_t ireg, uint32_t def,
 
 #if !defined(CONFIG_USER_ONLY)
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUARMState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUARMState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUARMState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_arm_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
-        raise_exception(env->exception_index);
+        raise_exception(env, env->exception_index);
     }
-    env = saved_env;
 }
 #endif
 
-/* FIXME: Pass an explicit pointer to QF to CPUARMState, and move saturating
-   instructions into helper.c  */
-uint32_t HELPER(add_setq)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
@@ -109,7 +95,7 @@ uint32_t HELPER(add_setq)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(add_saturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
@@ -119,7 +105,7 @@ uint32_t HELPER(add_saturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(sub_saturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a - b;
     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
@@ -129,7 +115,7 @@ uint32_t HELPER(sub_saturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(double_saturate)(int32_t val)
+uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 {
     uint32_t res;
     if (val >= 0x40000000) {
@@ -144,7 +130,7 @@ uint32_t HELPER(double_saturate)(int32_t val)
     return res;
 }
 
-uint32_t HELPER(add_usaturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (res < a) {
@@ -154,7 +140,7 @@ uint32_t HELPER(add_usaturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(sub_usaturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a - b;
     if (res > a) {
@@ -165,7 +151,7 @@ uint32_t HELPER(sub_usaturate)(uint32_t a, uint32_t b)
 }
 
 /* Signed saturation.  */
-static inline uint32_t do_ssat(int32_t val, int shift)
+static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 {
     int32_t top;
     uint32_t mask;
@@ -183,7 +169,7 @@ static inline uint32_t do_ssat(int32_t val, int shift)
 }
 
 /* Unsigned saturation.  */
-static inline uint32_t do_usat(int32_t val, int shift)
+static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 {
     uint32_t max;
 
@@ -199,62 +185,62 @@ static inline uint32_t do_usat(int32_t val, int shift)
 }
 
 /* Signed saturate.  */
-uint32_t HELPER(ssat)(uint32_t x, uint32_t shift)
+uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
-    return do_ssat(x, shift);
+    return do_ssat(env, x, shift);
 }
 
 /* Dual halfword signed saturate.  */
-uint32_t HELPER(ssat16)(uint32_t x, uint32_t shift)
+uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
     uint32_t res;
 
-    res = (uint16_t)do_ssat((int16_t)x, shift);
-    res |= do_ssat(((int32_t)x) >> 16, shift) << 16;
+    res = (uint16_t)do_ssat(env, (int16_t)x, shift);
+    res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
     return res;
 }
 
 /* Unsigned saturate.  */
-uint32_t HELPER(usat)(uint32_t x, uint32_t shift)
+uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
-    return do_usat(x, shift);
+    return do_usat(env, x, shift);
 }
 
 /* Dual halfword unsigned saturate.  */
-uint32_t HELPER(usat16)(uint32_t x, uint32_t shift)
+uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
     uint32_t res;
 
-    res = (uint16_t)do_usat((int16_t)x, shift);
-    res |= do_usat(((int32_t)x) >> 16, shift) << 16;
+    res = (uint16_t)do_usat(env, (int16_t)x, shift);
+    res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
     return res;
 }
 
-void HELPER(wfi)(void)
+void HELPER(wfi)(CPUARMState *env)
 {
     env->exception_index = EXCP_HLT;
     env->halted = 1;
     cpu_loop_exit(env);
 }
 
-void HELPER(exception)(uint32_t excp)
+void HELPER(exception)(CPUARMState *env, uint32_t excp)
 {
     env->exception_index = excp;
     cpu_loop_exit(env);
 }
 
-uint32_t HELPER(cpsr_read)(void)
+uint32_t HELPER(cpsr_read)(CPUARMState *env)
 {
     return cpsr_read(env) & ~CPSR_EXEC;
 }
 
-void HELPER(cpsr_write)(uint32_t val, uint32_t mask)
+void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 {
     cpsr_write(env, val, mask);
 }
 
 /* Access to user mode registers from privileged modes.  */
-uint32_t HELPER(get_user_reg)(uint32_t regno)
+uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 {
     uint32_t val;
 
@@ -271,7 +257,7 @@ uint32_t HELPER(get_user_reg)(uint32_t regno)
     return val;
 }
 
-void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
+void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 {
     if (regno == 13) {
         env->banked_r13[0] = val;
@@ -290,7 +276,7 @@ void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
     const ARMCPRegInfo *ri = rip;
     int excp = ri->writefn(env, ri, value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
 }
 
@@ -300,7 +286,7 @@ uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
     uint64_t value;
     int excp = ri->readfn(env, ri, &value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
     return value;
 }
@@ -310,7 +296,7 @@ void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
     const ARMCPRegInfo *ri = rip;
     int excp = ri->writefn(env, ri, value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
 }
 
@@ -320,7 +306,7 @@ uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
     uint64_t value;
     int excp = ri->readfn(env, ri, &value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
     return value;
 }
@@ -329,17 +315,7 @@ uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
    The only way to do that in TCG is a conditional branch, which clobbers
    all our temporaries.  For now implement these as helper functions.  */
 
-uint32_t HELPER (add_cc)(uint32_t a, uint32_t b)
-{
-    uint32_t result;
-    result = a + b;
-    env->NF = env->ZF = result;
-    env->CF = result < a;
-    env->VF = (a ^ b ^ -1) & (a ^ result);
-    return result;
-}
-
-uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(adc_cc)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -354,17 +330,7 @@ uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
-{
-    uint32_t result;
-    result = a - b;
-    env->NF = env->ZF = result;
-    env->CF = a >= b;
-    env->VF = (a ^ b) & (a ^ result);
-    return result;
-}
-
-uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sbc_cc)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -381,31 +347,7 @@ uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
 
 /* Similarly for variable shift instructions.  */
 
-uint32_t HELPER(shl)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        return 0;
-    return x << shift;
-}
-
-uint32_t HELPER(shr)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        return 0;
-    return (uint32_t)x >> shift;
-}
-
-uint32_t HELPER(sar)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        shift = 31;
-    return (int32_t)x >> shift;
-}
-
-uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -421,7 +363,7 @@ uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -437,7 +379,7 @@ uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -450,7 +392,7 @@ uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift1, shift;
     shift1 = i & 0xff;
diff --git a/target-arm/translate.c b/target-arm/translate.c
index edef79a2cff..724e00f7cf2 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -25,9 +25,9 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -85,6 +85,7 @@ static TCGv_ptr cpu_env;
 /* We reuse the same 64-bit temporaries for efficiency.  */
 static TCGv_i64 cpu_V0, cpu_V1, cpu_M0;
 static TCGv_i32 cpu_R[16];
+static TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
 static TCGv_i32 cpu_exclusive_addr;
 static TCGv_i32 cpu_exclusive_val;
 static TCGv_i32 cpu_exclusive_high;
@@ -97,7 +98,7 @@ static TCGv_i32 cpu_exclusive_info;
 static TCGv cpu_F0s, cpu_F1s;
 static TCGv_i64 cpu_F0d, cpu_F1d;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static const char *regnames[] =
     { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
@@ -115,6 +116,11 @@ void arm_translate_init(void)
                                           offsetof(CPUARMState, regs[i]),
                                           regnames[i]);
     }
+    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
+    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
+    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
+    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
+
     cpu_exclusive_addr = tcg_global_mem_new_i32(TCG_AREG0,
         offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
     cpu_exclusive_val = tcg_global_mem_new_i32(TCG_AREG0,
@@ -199,7 +205,7 @@ static void store_reg(DisasContext *s, int reg, TCGv var)
 static inline void gen_set_cpsr(TCGv var, uint32_t mask)
 {
     TCGv tmp_mask = tcg_const_i32(mask);
-    gen_helper_cpsr_write(var, tmp_mask);
+    gen_helper_cpsr_write(cpu_env, var, tmp_mask);
     tcg_temp_free_i32(tmp_mask);
 }
 /* Set NZCV flags from the high 4 bits of var.  */
@@ -209,7 +215,7 @@ static void gen_exception(int excp)
 {
     TCGv tmp = tcg_temp_new_i32();
     tcg_gen_movi_i32(tmp, excp);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
 }
 
@@ -271,15 +277,6 @@ static void gen_sbfx(TCGv var, int shift, int width)
     }
 }
 
-/* Bitfield insertion.  Insert val into base.  Clobbers base and val.  */
-static void gen_bfi(TCGv dest, TCGv base, TCGv val, int shift, uint32_t mask)
-{
-    tcg_gen_andi_i32(val, val, mask);
-    tcg_gen_shli_i32(val, val, shift);
-    tcg_gen_andi_i32(base, base, ~(mask << shift));
-    tcg_gen_or_i32(dest, base, val);
-}
-
 /* Return (b << 32) + a. Mark inputs as dead */
 static TCGv_i64 gen_addq_msw(TCGv_i64 a, TCGv b)
 {
@@ -369,70 +366,122 @@ static void gen_add16(TCGv t0, TCGv t1)
     tcg_temp_free_i32(t1);
 }
 
-#define gen_set_CF(var) tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, CF))
-
 /* Set CF to the top bit of var.  */
 static void gen_set_CF_bit31(TCGv var)
 {
-    TCGv tmp = tcg_temp_new_i32();
-    tcg_gen_shri_i32(tmp, var, 31);
-    gen_set_CF(tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_shri_i32(cpu_CF, var, 31);
 }
 
 /* Set N and Z flags from var.  */
 static inline void gen_logic_CC(TCGv var)
 {
-    tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, NF));
-    tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, ZF));
+    tcg_gen_mov_i32(cpu_NF, var);
+    tcg_gen_mov_i32(cpu_ZF, var);
 }
 
 /* T0 += T1 + CF.  */
 static void gen_adc(TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_add_i32(t0, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(t0, t0, tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_add_i32(t0, t0, cpu_CF);
 }
 
 /* dest = T0 + T1 + CF. */
 static void gen_add_carry(TCGv dest, TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_add_i32(dest, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(dest, dest, tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_add_i32(dest, dest, cpu_CF);
 }
 
 /* dest = T0 - T1 + CF - 1.  */
 static void gen_sub_carry(TCGv dest, TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_sub_i32(dest, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(dest, dest, tmp);
+    tcg_gen_add_i32(dest, dest, cpu_CF);
     tcg_gen_subi_i32(dest, dest, 1);
+}
+
+/* dest = T0 + T1. Compute C, N, V and Z flags */
+static void gen_add_CC(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp;
+    tcg_gen_add_i32(cpu_NF, t0, t1);
+    tcg_gen_mov_i32(cpu_ZF, cpu_NF);
+    tcg_gen_setcond_i32(TCG_COND_LTU, cpu_CF, cpu_NF, t0);
+    tcg_gen_xor_i32(cpu_VF, cpu_NF, t0);
+    tmp = tcg_temp_new_i32();
+    tcg_gen_xor_i32(tmp, t0, t1);
+    tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
     tcg_temp_free_i32(tmp);
+    tcg_gen_mov_i32(dest, cpu_NF);
 }
 
-/* FIXME:  Implement this natively.  */
-#define tcg_gen_abs_i32(t0, t1) gen_helper_abs(t0, t1)
+/* dest = T0 - T1. Compute C, N, V and Z flags */
+static void gen_sub_CC(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp;
+    tcg_gen_sub_i32(cpu_NF, t0, t1);
+    tcg_gen_mov_i32(cpu_ZF, cpu_NF);
+    tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0, t1);
+    tcg_gen_xor_i32(cpu_VF, cpu_NF, t0);
+    tmp = tcg_temp_new_i32();
+    tcg_gen_xor_i32(tmp, t0, t1);
+    tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
+    tcg_temp_free_i32(tmp);
+    tcg_gen_mov_i32(dest, cpu_NF);
+}
 
-static void shifter_out_im(TCGv var, int shift)
+#define GEN_SHIFT(name)                                               \
+static void gen_##name(TCGv dest, TCGv t0, TCGv t1)                   \
+{                                                                     \
+    TCGv tmp1, tmp2, tmp3;                                            \
+    tmp1 = tcg_temp_new_i32();                                        \
+    tcg_gen_andi_i32(tmp1, t1, 0xff);                                 \
+    tmp2 = tcg_const_i32(0);                                          \
+    tmp3 = tcg_const_i32(0x1f);                                       \
+    tcg_gen_movcond_i32(TCG_COND_GTU, tmp2, tmp1, tmp3, tmp2, t0);    \
+    tcg_temp_free_i32(tmp3);                                          \
+    tcg_gen_andi_i32(tmp1, tmp1, 0x1f);                               \
+    tcg_gen_##name##_i32(dest, tmp2, tmp1);                           \
+    tcg_temp_free_i32(tmp2);                                          \
+    tcg_temp_free_i32(tmp1);                                          \
+}
+GEN_SHIFT(shl)
+GEN_SHIFT(shr)
+#undef GEN_SHIFT
+
+static void gen_sar(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp1, tmp2;
+    tmp1 = tcg_temp_new_i32();
+    tcg_gen_andi_i32(tmp1, t1, 0xff);
+    tmp2 = tcg_const_i32(0x1f);
+    tcg_gen_movcond_i32(TCG_COND_GTU, tmp1, tmp1, tmp2, tmp2, tmp1);
+    tcg_temp_free_i32(tmp2);
+    tcg_gen_sar_i32(dest, t0, tmp1);
+    tcg_temp_free_i32(tmp1);
+}
+
+static void tcg_gen_abs_i32(TCGv dest, TCGv src)
 {
+    TCGv c0 = tcg_const_i32(0);
     TCGv tmp = tcg_temp_new_i32();
+    tcg_gen_neg_i32(tmp, src);
+    tcg_gen_movcond_i32(TCG_COND_GT, dest, src, c0, src, tmp);
+    tcg_temp_free_i32(c0);
+    tcg_temp_free_i32(tmp);
+}
+
+static void shifter_out_im(TCGv var, int shift)
+{
     if (shift == 0) {
-        tcg_gen_andi_i32(tmp, var, 1);
+        tcg_gen_andi_i32(cpu_CF, var, 1);
     } else {
-        tcg_gen_shri_i32(tmp, var, shift);
-        if (shift != 31)
-            tcg_gen_andi_i32(tmp, tmp, 1);
+        tcg_gen_shri_i32(cpu_CF, var, shift);
+        if (shift != 31) {
+            tcg_gen_andi_i32(cpu_CF, cpu_CF, 1);
+        }
     }
-    gen_set_CF(tmp);
-    tcg_temp_free_i32(tmp);
 }
 
 /* Shift by immediate.  Includes special handling for shift == 0.  */
@@ -449,8 +498,7 @@ static inline void gen_arm_shift_im(TCGv var, int shiftop, int shift, int flags)
     case 1: /* LSR */
         if (shift == 0) {
             if (flags) {
-                tcg_gen_shri_i32(var, var, 31);
-                gen_set_CF(var);
+                tcg_gen_shri_i32(cpu_CF, var, 31);
             }
             tcg_gen_movi_i32(var, 0);
         } else {
@@ -474,11 +522,11 @@ static inline void gen_arm_shift_im(TCGv var, int shiftop, int shift, int flags)
                 shifter_out_im(var, shift - 1);
             tcg_gen_rotri_i32(var, var, shift); break;
         } else {
-            TCGv tmp = load_cpu_field(CF);
+            TCGv tmp = tcg_temp_new_i32();
+            tcg_gen_shli_i32(tmp, cpu_CF, 31);
             if (flags)
                 shifter_out_im(var, 0);
             tcg_gen_shri_i32(var, var, 1);
-            tcg_gen_shli_i32(tmp, tmp, 31);
             tcg_gen_or_i32(var, var, tmp);
             tcg_temp_free_i32(tmp);
         }
@@ -490,16 +538,22 @@ static inline void gen_arm_shift_reg(TCGv var, int shiftop,
 {
     if (flags) {
         switch (shiftop) {
-        case 0: gen_helper_shl_cc(var, var, shift); break;
-        case 1: gen_helper_shr_cc(var, var, shift); break;
-        case 2: gen_helper_sar_cc(var, var, shift); break;
-        case 3: gen_helper_ror_cc(var, var, shift); break;
+        case 0: gen_helper_shl_cc(var, cpu_env, var, shift); break;
+        case 1: gen_helper_shr_cc(var, cpu_env, var, shift); break;
+        case 2: gen_helper_sar_cc(var, cpu_env, var, shift); break;
+        case 3: gen_helper_ror_cc(var, cpu_env, var, shift); break;
         }
     } else {
         switch (shiftop) {
-        case 0: gen_helper_shl(var, var, shift); break;
-        case 1: gen_helper_shr(var, var, shift); break;
-        case 2: gen_helper_sar(var, var, shift); break;
+        case 0:
+            gen_shl(var, var, shift);
+            break;
+        case 1:
+            gen_shr(var, var, shift);
+            break;
+        case 2:
+            gen_sar(var, var, shift);
+            break;
         case 3: tcg_gen_andi_i32(shift, shift, 0x1f);
                 tcg_gen_rotr_i32(var, var, shift); break;
         }
@@ -603,99 +657,75 @@ static void gen_thumb2_parallel_addsub(int op1, int op2, TCGv a, TCGv b)
 static void gen_test_cc(int cc, int label)
 {
     TCGv tmp;
-    TCGv tmp2;
     int inv;
 
     switch (cc) {
     case 0: /* eq: Z */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
         break;
     case 1: /* ne: !Z */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
         break;
     case 2: /* cs: C */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_CF, 0, label);
         break;
     case 3: /* cc: !C */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
         break;
     case 4: /* mi: N */
-        tmp = load_cpu_field(NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_NF, 0, label);
         break;
     case 5: /* pl: !N */
-        tmp = load_cpu_field(NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_NF, 0, label);
         break;
     case 6: /* vs: V */
-        tmp = load_cpu_field(VF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_VF, 0, label);
         break;
     case 7: /* vc: !V */
-        tmp = load_cpu_field(VF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_VF, 0, label);
         break;
     case 8: /* hi: C && !Z */
         inv = gen_new_label();
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, inv);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, inv);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
         gen_set_label(inv);
         break;
     case 9: /* ls: !C || Z */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
         break;
     case 10: /* ge: N == V -> N ^ V == 0 */
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     case 11: /* lt: N != V -> N ^ V != 0 */
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     case 12: /* gt: !Z && N == V */
         inv = gen_new_label();
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, inv);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, inv);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         gen_set_label(inv);
         break;
     case 13: /* le: Z || N != V */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     default:
         fprintf(stderr, "Bad condition code 0x%x\n", cc);
         abort();
     }
-    tcg_temp_free_i32(tmp);
 }
 
 static const uint8_t table_logic_cc[16] = {
@@ -2628,12 +2658,12 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                         switch (size) {
                         case 0:
                             tmp2 = neon_load_reg(rn, pass);
-                            gen_bfi(tmp, tmp2, tmp, offset, 0xff);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 8);
                             tcg_temp_free_i32(tmp2);
                             break;
                         case 1:
                             tmp2 = neon_load_reg(rn, pass);
-                            gen_bfi(tmp, tmp2, tmp, offset, 0xffff);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 16);
                             tcg_temp_free_i32(tmp2);
                             break;
                         case 2:
@@ -3989,7 +4019,8 @@ static int disas_neon_ls_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                     }
                     if (size != 2) {
                         tmp2 = neon_load_reg(rd, pass);
-                        gen_bfi(tmp, tmp2, tmp, shift, size ? 0xffff : 0xff);
+                        tcg_gen_deposit_i32(tmp, tmp2, tmp,
+                                            shift, size ? 16 : 8);
                         tcg_temp_free_i32(tmp2);
                     }
                     neon_store_reg(rd, pass, tmp);
@@ -4160,7 +4191,9 @@ static inline void gen_neon_negl(TCGv_i64 var, int size)
     switch (size) {
     case 0: gen_helper_neon_negl_u16(var, var); break;
     case 1: gen_helper_neon_negl_u32(var, var); break;
-    case 2: gen_helper_neon_negl_u64(var, var); break;
+    case 2:
+        tcg_gen_neg_i64(var, var);
+        break;
     default: abort();
     }
 }
@@ -6121,7 +6154,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                 tmp2 = neon_load_reg(rm, 0);
                 tmp4 = tcg_const_i32(rn);
                 tmp5 = tcg_const_i32(n);
-                gen_helper_neon_tbl(tmp2, tmp2, tmp, tmp4, tmp5);
+                gen_helper_neon_tbl(tmp2, cpu_env, tmp2, tmp, tmp4, tmp5);
                 tcg_temp_free_i32(tmp);
                 if (insn & (1 << 6)) {
                     tmp = neon_load_reg(rd, 1);
@@ -6130,7 +6163,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     tcg_gen_movi_i32(tmp, 0);
                 }
                 tmp3 = neon_load_reg(rm, 1);
-                gen_helper_neon_tbl(tmp3, tmp3, tmp, tmp4, tmp5);
+                gen_helper_neon_tbl(tmp3, cpu_env, tmp3, tmp, tmp4, tmp5);
                 tcg_temp_free_i32(tmp5);
                 tcg_temp_free_i32(tmp4);
                 neon_store_reg(rd, 0, tmp2);
@@ -6534,7 +6567,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
     TCGv addr;
     TCGv_i64 tmp64;
 
-    insn = arm_ldl_code(s->pc, s->bswap_code);
+    insn = arm_ldl_code(env, s->pc, s->bswap_code);
     s->pc += 4;
 
     /* M variants do not implement ARM mode.  */
@@ -6818,7 +6851,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                     tmp = load_cpu_field(spsr);
                 } else {
                     tmp = tcg_temp_new_i32();
-                    gen_helper_cpsr_read(tmp);
+                    gen_helper_cpsr_read(tmp, cpu_env);
                 }
                 store_reg(s, rd, tmp);
             }
@@ -6869,11 +6902,11 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             tmp = load_reg(s, rm);
             tmp2 = load_reg(s, rn);
             if (op1 & 2)
-                gen_helper_double_saturate(tmp2, tmp2);
+                gen_helper_double_saturate(tmp2, cpu_env, tmp2);
             if (op1 & 1)
-                gen_helper_sub_saturate(tmp, tmp, tmp2);
+                gen_helper_sub_saturate(tmp, cpu_env, tmp, tmp2);
             else
-                gen_helper_add_saturate(tmp, tmp, tmp2);
+                gen_helper_add_saturate(tmp, cpu_env, tmp, tmp2);
             tcg_temp_free_i32(tmp2);
             store_reg(s, rd, tmp);
             break;
@@ -6911,7 +6944,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 tcg_temp_free_i64(tmp64);
                 if ((sh & 2) == 0) {
                     tmp2 = load_reg(s, rn);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                 }
                 store_reg(s, rd, tmp);
@@ -6931,7 +6964,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 } else {
                     if (op1 == 0) {
                         tmp2 = load_reg(s, rn);
-                        gen_helper_add_setq(tmp, tmp, tmp2);
+                        gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                     }
                     store_reg(s, rd, tmp);
@@ -7005,11 +7038,11 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 if (IS_USER(s)) {
                     goto illegal_op;
                 }
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 gen_exception_return(s, tmp);
             } else {
                 if (set_cc) {
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
                 } else {
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 }
@@ -7018,7 +7051,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x03:
             if (set_cc) {
-                gen_helper_sub_cc(tmp, tmp2, tmp);
+                gen_sub_CC(tmp, tmp2, tmp);
             } else {
                 tcg_gen_sub_i32(tmp, tmp2, tmp);
             }
@@ -7026,7 +7059,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x04:
             if (set_cc) {
-                gen_helper_add_cc(tmp, tmp, tmp2);
+                gen_add_CC(tmp, tmp, tmp2);
             } else {
                 tcg_gen_add_i32(tmp, tmp, tmp2);
             }
@@ -7034,7 +7067,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x05:
             if (set_cc) {
-                gen_helper_adc_cc(tmp, tmp, tmp2);
+                gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 gen_add_carry(tmp, tmp, tmp2);
             }
@@ -7042,7 +7075,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x06:
             if (set_cc) {
-                gen_helper_sbc_cc(tmp, tmp, tmp2);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 gen_sub_carry(tmp, tmp, tmp2);
             }
@@ -7050,7 +7083,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x07:
             if (set_cc) {
-                gen_helper_sbc_cc(tmp, tmp2, tmp);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp2, tmp);
             } else {
                 gen_sub_carry(tmp, tmp2, tmp);
             }
@@ -7072,13 +7105,13 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x0a:
             if (set_cc) {
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp);
             break;
         case 0x0b:
             if (set_cc) {
-                gen_helper_add_cc(tmp, tmp, tmp2);
+                gen_add_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp);
             break;
@@ -7395,9 +7428,9 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         sh = (insn >> 16) & 0x1f;
                         tmp2 = tcg_const_i32(sh);
                         if (insn & (1 << 22))
-                          gen_helper_usat(tmp, tmp, tmp2);
+                          gen_helper_usat(tmp, cpu_env, tmp, tmp2);
                         else
-                          gen_helper_ssat(tmp, tmp, tmp2);
+                          gen_helper_ssat(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                         store_reg(s, rd, tmp);
                     } else if ((insn & 0x00300fe0) == 0x00200f20) {
@@ -7406,9 +7439,9 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         sh = (insn >> 16) & 0x1f;
                         tmp2 = tcg_const_i32(sh);
                         if (insn & (1 << 22))
-                          gen_helper_usat16(tmp, tmp, tmp2);
+                          gen_helper_usat16(tmp, cpu_env, tmp, tmp2);
                         else
-                          gen_helper_ssat16(tmp, tmp, tmp2);
+                          gen_helper_ssat16(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                         store_reg(s, rd, tmp);
                     } else if ((insn & 0x00700fe0) == 0x00000fa0) {
@@ -7518,7 +7551,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                              * however it may overflow considered as a signed
                              * operation, in which case we must set the Q flag.
                              */
-                            gen_helper_add_setq(tmp, tmp, tmp2);
+                            gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                         }
                         tcg_temp_free_i32(tmp2);
                         if (insn & (1 << 22)) {
@@ -7534,7 +7567,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             if (rd != 15)
                               {
                                 tmp2 = load_reg(s, rd);
-                                gen_helper_add_setq(tmp, tmp, tmp2);
+                                gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                                 tcg_temp_free_i32(tmp2);
                               }
                             store_reg(s, rn, tmp);
@@ -7593,7 +7626,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         }
                         if (i != 32) {
                             tmp2 = load_reg(s, rd);
-                            gen_bfi(tmp, tmp2, tmp, shift, (1u << i) - 1);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, shift, i);
                             tcg_temp_free_i32(tmp2);
                         }
                         store_reg(s, rd, tmp);
@@ -7719,7 +7752,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             tmp = gen_ld32(addr, IS_USER(s));
                             if (user) {
                                 tmp2 = tcg_const_i32(i);
-                                gen_helper_set_user_reg(tmp2, tmp);
+                                gen_helper_set_user_reg(cpu_env, tmp2, tmp);
                                 tcg_temp_free_i32(tmp2);
                                 tcg_temp_free_i32(tmp);
                             } else if (i == rn) {
@@ -7738,7 +7771,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             } else if (user) {
                                 tmp = tcg_temp_new_i32();
                                 tmp2 = tcg_const_i32(i);
-                                gen_helper_get_user_reg(tmp, tmp2);
+                                gen_helper_get_user_reg(tmp, cpu_env, tmp2);
                                 tcg_temp_free_i32(tmp2);
                             } else {
                                 tmp = load_reg(s, i);
@@ -7865,31 +7898,31 @@ gen_thumb2_data_op(DisasContext *s, int op, int conds, uint32_t shifter_out, TCG
         break;
     case 8: /* add */
         if (conds)
-            gen_helper_add_cc(t0, t0, t1);
+            gen_add_CC(t0, t0, t1);
         else
             tcg_gen_add_i32(t0, t0, t1);
         break;
     case 10: /* adc */
         if (conds)
-            gen_helper_adc_cc(t0, t0, t1);
+            gen_helper_adc_cc(t0, cpu_env, t0, t1);
         else
             gen_adc(t0, t1);
         break;
     case 11: /* sbc */
         if (conds)
-            gen_helper_sbc_cc(t0, t0, t1);
+            gen_helper_sbc_cc(t0, cpu_env, t0, t1);
         else
             gen_sub_carry(t0, t0, t1);
         break;
     case 13: /* sub */
         if (conds)
-            gen_helper_sub_cc(t0, t0, t1);
+            gen_sub_CC(t0, t0, t1);
         else
             tcg_gen_sub_i32(t0, t0, t1);
         break;
     case 14: /* rsb */
         if (conds)
-            gen_helper_sub_cc(t0, t1, t0);
+            gen_sub_CC(t0, t1, t0);
         else
             tcg_gen_sub_i32(t0, t1, t0);
         break;
@@ -7962,7 +7995,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
         /* Fall through to 32-bit decode.  */
     }
 
-    insn = arm_lduw_code(s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->bswap_code);
     s->pc += 2;
     insn |= (uint32_t)insn_hw1 << 16;
 
@@ -8111,7 +8144,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     gen_st32(tmp, addr, 0);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = tcg_temp_new_i32();
-                    gen_helper_cpsr_read(tmp);
+                    gen_helper_cpsr_read(tmp, cpu_env);
                     gen_st32(tmp, addr, 0);
                     if (insn & (1 << 21)) {
                         if ((insn & (1 << 24)) == 0) {
@@ -8293,11 +8326,11 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tmp = load_reg(s, rn);
                 tmp2 = load_reg(s, rm);
                 if (op & 1)
-                    gen_helper_double_saturate(tmp, tmp);
+                    gen_helper_double_saturate(tmp, cpu_env, tmp);
                 if (op & 2)
-                    gen_helper_sub_saturate(tmp, tmp2, tmp);
+                    gen_helper_sub_saturate(tmp, cpu_env, tmp2, tmp);
                 else
-                    gen_helper_add_saturate(tmp, tmp, tmp2);
+                    gen_helper_add_saturate(tmp, cpu_env, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
             } else {
                 tmp = load_reg(s, rn);
@@ -8353,7 +8386,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tcg_temp_free_i32(tmp2);
                 if (rs != 15) {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                 }
                 break;
@@ -8370,13 +8403,13 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                      * however it may overflow considered as a signed
                      * operation, in which case we must set the Q flag.
                      */
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                 }
                 tcg_temp_free_i32(tmp2);
                 if (rs != 15)
                   {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                   }
                 break;
@@ -8393,7 +8426,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 if (rs != 15)
                   {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                   }
                 break;
@@ -8632,7 +8665,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                             gen_helper_v7m_mrs(tmp, cpu_env, addr);
                             tcg_temp_free_i32(addr);
                         } else {
-                            gen_helper_cpsr_read(tmp);
+                            gen_helper_cpsr_read(tmp, cpu_env);
                         }
                         store_reg(s, rd, tmp);
                         break;
@@ -8704,7 +8737,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         imm = imm + 1 - shift;
                         if (imm != 32) {
                             tmp2 = load_reg(s, rd);
-                            gen_bfi(tmp, tmp2, tmp, shift, (1u << imm) - 1);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, shift, imm);
                             tcg_temp_free_i32(tmp2);
                         }
                         break;
@@ -8721,15 +8754,15 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         if (op & 4) {
                             /* Unsigned.  */
                             if ((op & 1) && shift == 0)
-                                gen_helper_usat16(tmp, tmp, tmp2);
+                                gen_helper_usat16(tmp, cpu_env, tmp, tmp2);
                             else
-                                gen_helper_usat(tmp, tmp, tmp2);
+                                gen_helper_usat(tmp, cpu_env, tmp, tmp2);
                         } else {
                             /* Signed.  */
                             if ((op & 1) && shift == 0)
-                                gen_helper_ssat16(tmp, tmp, tmp2);
+                                gen_helper_ssat16(tmp, cpu_env, tmp, tmp2);
                             else
-                                gen_helper_ssat(tmp, tmp, tmp2);
+                                gen_helper_ssat(tmp, cpu_env, tmp, tmp2);
                         }
                         tcg_temp_free_i32(tmp2);
                         break;
@@ -8992,7 +9025,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         }
     }
 
-    insn = arm_lduw_code(s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->bswap_code);
     s->pc += 2;
 
     switch (insn >> 12) {
@@ -9017,12 +9050,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
             } else {
                 if (s->condexec_mask)
                     tcg_gen_add_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_add_cc(tmp, tmp, tmp2);
+                    gen_add_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp2);
             store_reg(s, rd, tmp);
@@ -9053,7 +9086,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             tcg_gen_movi_i32(tmp2, insn & 0xff);
             switch (op) {
             case 1: /* cmp */
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp);
                 tcg_temp_free_i32(tmp2);
                 break;
@@ -9061,7 +9094,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_add_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_add_cc(tmp, tmp, tmp2);
+                    gen_add_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 store_reg(s, rd, tmp);
                 break;
@@ -9069,7 +9102,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 store_reg(s, rd, tmp);
                 break;
@@ -9105,7 +9138,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             case 1: /* cmp */
                 tmp = load_reg(s, rd);
                 tmp2 = load_reg(s, rm);
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 tcg_temp_free_i32(tmp);
                 break;
@@ -9166,25 +9199,25 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             break;
         case 0x2: /* lsl */
             if (s->condexec_mask) {
-                gen_helper_shl(tmp2, tmp2, tmp);
+                gen_shl(tmp2, tmp2, tmp);
             } else {
-                gen_helper_shl_cc(tmp2, tmp2, tmp);
+                gen_helper_shl_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
         case 0x3: /* lsr */
             if (s->condexec_mask) {
-                gen_helper_shr(tmp2, tmp2, tmp);
+                gen_shr(tmp2, tmp2, tmp);
             } else {
-                gen_helper_shr_cc(tmp2, tmp2, tmp);
+                gen_helper_shr_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
         case 0x4: /* asr */
             if (s->condexec_mask) {
-                gen_helper_sar(tmp2, tmp2, tmp);
+                gen_sar(tmp2, tmp2, tmp);
             } else {
-                gen_helper_sar_cc(tmp2, tmp2, tmp);
+                gen_helper_sar_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
@@ -9192,20 +9225,20 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             if (s->condexec_mask)
                 gen_adc(tmp, tmp2);
             else
-                gen_helper_adc_cc(tmp, tmp, tmp2);
+                gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
             break;
         case 0x6: /* sbc */
             if (s->condexec_mask)
                 gen_sub_carry(tmp, tmp, tmp2);
             else
-                gen_helper_sbc_cc(tmp, tmp, tmp2);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
             break;
         case 0x7: /* ror */
             if (s->condexec_mask) {
                 tcg_gen_andi_i32(tmp, tmp, 0x1f);
                 tcg_gen_rotr_i32(tmp2, tmp2, tmp);
             } else {
-                gen_helper_ror_cc(tmp2, tmp2, tmp);
+                gen_helper_ror_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
@@ -9218,14 +9251,14 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             if (s->condexec_mask)
                 tcg_gen_neg_i32(tmp, tmp2);
             else
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
             break;
         case 0xa: /* cmp */
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_sub_CC(tmp, tmp, tmp2);
             rd = 16;
             break;
         case 0xb: /* cmn */
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_add_CC(tmp, tmp, tmp2);
             rd = 16;
             break;
         case 0xc: /* orr */
@@ -9694,7 +9727,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -9801,22 +9834,22 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
             gen_opc_condexec_bits[lj] = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc->pc);
         }
 
@@ -9848,7 +9881,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
         num_insns ++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              dc->pc < next_page_start &&
@@ -9913,7 +9946,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
             /* nothing more to generate */
             break;
         case DISAS_WFI:
-            gen_helper_wfi();
+            gen_helper_wfi(cpu_env);
             break;
         case DISAS_SWI:
             gen_exception(EXCP_SWI);
@@ -9929,22 +9962,22 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
 
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start,
+        log_target_disas(env, pc_start, dc->pc - pc_start,
                          dc->thumb | (dc->bswap_code << 1));
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
         tb->icount = num_insns;
@@ -9970,19 +10003,6 @@ void cpu_dump_state(CPUARMState *env, FILE *f, fprintf_function cpu_fprintf,
                     int flags)
 {
     int i;
-#if 0
-    union {
-        uint32_t i;
-        float s;
-    } s0, s1;
-    CPU_DoubleU d;
-    /* ??? This assumes float64 and double have the same layout.
-       Oh well, it's only debug dumps.  */
-    union {
-        float64 f64;
-        double d;
-    } d0;
-#endif
     uint32_t psr;
 
     for(i=0;i<16;i++) {
@@ -10002,24 +10022,27 @@ void cpu_dump_state(CPUARMState *env, FILE *f, fprintf_function cpu_fprintf,
                 psr & CPSR_T ? 'T' : 'A',
                 cpu_mode_names[psr & 0xf], (psr & 0x10) ? 32 : 26);
 
-#if 0
-    for (i = 0; i < 16; i++) {
-        d.d = env->vfp.regs[i];
-        s0.i = d.l.lower;
-        s1.i = d.l.upper;
-        d0.f64 = d.d;
-        cpu_fprintf(f, "s%02d=%08x(%8g) s%02d=%08x(%8g) d%02d=%08x%08x(%8g)\n",
-                    i * 2, (int)s0.i, s0.s,
-                    i * 2 + 1, (int)s1.i, s1.s,
-                    i, (int)(uint32_t)d.l.upper, (int)(uint32_t)d.l.lower,
-                    d0.d);
+    if (flags & CPU_DUMP_FPU) {
+        int numvfpregs = 0;
+        if (arm_feature(env, ARM_FEATURE_VFP)) {
+            numvfpregs += 16;
+        }
+        if (arm_feature(env, ARM_FEATURE_VFP3)) {
+            numvfpregs += 16;
+        }
+        for (i = 0; i < numvfpregs; i++) {
+            uint64_t v = float64_val(env->vfp.regs[i]);
+            cpu_fprintf(f, "s%02d=%08x s%02d=%08x d%02d=%016" PRIx64 "\n",
+                        i * 2, (uint32_t)v,
+                        i * 2 + 1, (uint32_t)(v >> 32),
+                        i, v);
+        }
+        cpu_fprintf(f, "FPSCR: %08x\n", (int)env->vfp.xregs[ARM_VFP_FPSCR]);
     }
-    cpu_fprintf(f, "FPSCR: %08x\n", (int)env->vfp.xregs[ARM_VFP_FPSCR]);
-#endif
 }
 
 void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->regs[15] = gen_opc_pc[pc_pos];
+    env->regs[15] = tcg_ctx.gen_opc_pc[pc_pos];
     env->condexec_bits = gen_opc_condexec_bits[pc_pos];
 }
diff --git a/target-cris/Makefile.objs b/target-cris/Makefile.objs
index 4b09e8c6b59..afb87bcc801 100644
--- a/target-cris/Makefile.objs
+++ b/target-cris/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += mmu.o machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-cris/cpu-qom.h b/target-cris/cpu-qom.h
index d0e5f04f78b..41ab9b2fa5e 100644
--- a/target-cris/cpu-qom.h
+++ b/target-cris/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_CRIS_CPU_QOM_H
 #define QEMU_CRIS_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_CRIS_CPU "cris-cpu"
 
diff --git a/target-cris/cpu.c b/target-cris/cpu.c
index c596609bd44..3f64a5747e0 100644
--- a/target-cris/cpu.c
+++ b/target-cris/cpu.c
@@ -35,7 +35,7 @@ static void cris_cpu_reset(CPUState *s)
     uint32_t vr;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index 4f4df6d9b5d..257cb52be25 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -27,7 +27,7 @@
 
 #define CPUArchState struct CPUCRISState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -175,7 +175,6 @@ typedef struct CPUCRISState {
 
 CRISCPU *cpu_cris_init(const char *cpu_model);
 int cpu_cris_exec(CPUCRISState *s);
-void cpu_cris_close(CPUCRISState *s);
 void do_interrupt(CPUCRISState *env);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -270,7 +269,7 @@ static inline void cpu_set_tls(CPUCRISState *env, target_ulong newtls)
 #define SFR_RW_MM_TLB_LO   env->pregs[PR_SRS]][5
 #define SFR_RW_MM_TLB_HI   env->pregs[PR_SRS]][6
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
@@ -285,12 +284,14 @@ static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
 #define cpu_list cris_cpu_list
 void cris_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 
-static inline bool cpu_has_work(CPUCRISState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUCRISState *env = &CRIS_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUCRISState *env, TranslationBlock *tb)
 {
diff --git a/target-cris/crisv32-decode.h b/target-cris/crisv32-decode.h
index ed141de0b79..cdba3778175 100644
--- a/target-cris/crisv32-decode.h
+++ b/target-cris/crisv32-decode.h
@@ -17,6 +17,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef CRISV32_DECODE_H
+#define CRISV32_DECODE_H 1
 
 /* Convenient binary macros.  */
 #define HEX__(n) 0x##n##LU
@@ -126,3 +128,5 @@
 
 #define DEC_FTAG_FIDX_D_M {B8(10101011), B8(11111111)}
 #define DEC_FTAG_FIDX_I_M {B8(11010011), B8(11111111)}
+
+#endif
diff --git a/target-cris/helper.c b/target-cris/helper.c
index bfbc29ec6ac..6e75e9819e7 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -20,7 +20,7 @@
 
 #include "cpu.h"
 #include "mmu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 
 //#define CRIS_HELPER_DEBUG
@@ -28,7 +28,7 @@
 
 #ifdef CRIS_HELPER_DEBUG
 #define D(x) x
-#define D_LOG(...) qemu_log(__VA__ARGS__)
+#define D_LOG(...) qemu_log(__VA_ARGS__)
 #else
 #define D(x)
 #define D_LOG(...) do { } while (0)
@@ -151,7 +151,7 @@ static void do_interruptv10(CPUCRISState *env)
 	}
 
 	/* Now that we are in kernel mode, load the handlers address.  */
-	env->pc = ldl_code(env->pregs[PR_EBP] + ex_vec * 4);
+        env->pc = cpu_ldl_code(env, env->pregs[PR_EBP] + ex_vec * 4);
 	env->locked_irq = 1;
 	env->pregs[PR_CCS] |= F_FLAG_V10; /* set F.  */
 
@@ -233,7 +233,7 @@ void do_interrupt(CPUCRISState *env)
 	/* Now that we are in kernel mode, load the handlers address.
 	   This load may not fault, real hw leaves that behaviour as
 	   undefined.  */
-	env->pc = ldl_code(env->pregs[PR_EBP] + ex_vec * 4);
+        env->pc = cpu_ldl_code(env, env->pregs[PR_EBP] + ex_vec * 4);
 
 	/* Clear the excption_index to avoid spurios hw_aborts for recursive
 	   bus faults.  */
@@ -246,7 +246,7 @@ void do_interrupt(CPUCRISState *env)
 		   env->pregs[PR_ERP]);
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUCRISState * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUCRISState * env, target_ulong addr)
 {
 	uint32_t phy = addr;
 	struct cris_mmu_result res;
diff --git a/target-cris/helper.h b/target-cris/helper.h
index 093063a14b5..8e8365cf69c 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -1,26 +1,29 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_1(tlb_flush_pid, void, i32)
-DEF_HELPER_1(spc_write, void, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_2(tlb_flush_pid, void, env, i32)
+DEF_HELPER_2(spc_write, void, env, i32)
 DEF_HELPER_3(dump, void, i32, i32, i32)
-DEF_HELPER_0(rfe, void);
-DEF_HELPER_0(rfn, void);
+DEF_HELPER_1(rfe, void, env);
+DEF_HELPER_1(rfn, void, env);
 
-DEF_HELPER_2(movl_sreg_reg, void, i32, i32)
-DEF_HELPER_2(movl_reg_sreg, void, i32, i32)
+DEF_HELPER_3(movl_sreg_reg, void, env, i32, i32)
+DEF_HELPER_3(movl_reg_sreg, void, env, i32, i32)
 
-DEF_HELPER_FLAGS_1(lz, TCG_CALL_PURE, i32, i32);
-DEF_HELPER_FLAGS_3(btst, TCG_CALL_PURE, i32, i32, i32, i32);
+DEF_HELPER_FLAGS_1(lz, TCG_CALL_NO_SE, i32, i32);
+DEF_HELPER_FLAGS_4(btst, TCG_CALL_NO_SE, i32, env, i32, i32, i32);
 
-DEF_HELPER_FLAGS_3(evaluate_flags_muls, TCG_CALL_PURE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(evaluate_flags_mulu, TCG_CALL_PURE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_mcp, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_alu_4, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_sub_4, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_2(evaluate_flags_move_4, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(evaluate_flags_move_2, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_0(evaluate_flags, void)
-DEF_HELPER_0(top_evaluate_flags, void)
+DEF_HELPER_FLAGS_4(evaluate_flags_muls, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
+DEF_HELPER_FLAGS_4(evaluate_flags_mulu, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_mcp, TCG_CALL_NO_SE, i32, env,
+                                                      i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_alu_4, TCG_CALL_NO_SE, i32, env,
+                                                        i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_sub_4, TCG_CALL_NO_SE, i32, env,
+                                                        i32, i32, i32, i32)
+DEF_HELPER_FLAGS_3(evaluate_flags_move_4, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(evaluate_flags_move_2, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_1(evaluate_flags, void, env)
+DEF_HELPER_1(top_evaluate_flags, void, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-cris/op_helper.c b/target-cris/op_helper.c
index ac7c98c8ed6..0f6a1eeb0ac 100644
--- a/target-cris/op_helper.c
+++ b/target-cris/op_helper.c
@@ -19,83 +19,70 @@
  */
 
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "mmu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 //#define CRIS_OP_HELPER_DEBUG
 
 
 #ifdef CRIS_OP_HELPER_DEBUG
 #define D(x) x
-#define D_LOG(...) qemu_log(__VA__ARGS__)
+#define D_LOG(...) qemu_log(__VA_ARGS__)
 #else
 #define D(x)
 #define D_LOG(...) do { } while (0)
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUCRISState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUCRISState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUCRISState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     D_LOG("%s pc=%x tpc=%x ra=%p\n", __func__,
           env->pc, env->debug1, (void *)retaddr);
     ret = cpu_cris_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-
+            if (cpu_restore_state(env, retaddr)) {
 		/* Evaluate flags after retranslation.  */
-                helper_top_evaluate_flags();
+                helper_top_evaluate_flags(env);
             }
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
 #endif
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPUCRISState *env, uint32_t index)
 {
 	env->exception_index = index;
         cpu_loop_exit(env);
 }
 
-void helper_tlb_flush_pid(uint32_t pid)
+void helper_tlb_flush_pid(CPUCRISState *env, uint32_t pid)
 {
 #if !defined(CONFIG_USER_ONLY)
 	pid &= 0xff;
@@ -104,7 +91,7 @@ void helper_tlb_flush_pid(uint32_t pid)
 #endif
 }
 
-void helper_spc_write(uint32_t new_spc)
+void helper_spc_write(CPUCRISState *env, uint32_t new_spc)
 {
 #if !defined(CONFIG_USER_ONLY)
 	tlb_flush_page(env, env->pregs[PR_SPC]);
@@ -121,7 +108,7 @@ void helper_dump(uint32_t a0, uint32_t a1, uint32_t a2)
 #define EXTRACT_FIELD(src, start, end) \
 	    (((src) >> start) & ((1 << (end - start + 1)) - 1))
 
-void helper_movl_sreg_reg (uint32_t sreg, uint32_t reg)
+void helper_movl_sreg_reg(CPUCRISState *env, uint32_t sreg, uint32_t reg)
 {
 	uint32_t srs;
 	srs = env->pregs[PR_SRS];
@@ -171,7 +158,7 @@ void helper_movl_sreg_reg (uint32_t sreg, uint32_t reg)
 #endif
 }
 
-void helper_movl_reg_sreg (uint32_t reg, uint32_t sreg)
+void helper_movl_reg_sreg(CPUCRISState *env, uint32_t reg, uint32_t sreg)
 {
 	uint32_t srs;
 	env->pregs[PR_SRS] &= 3;
@@ -216,7 +203,7 @@ static void cris_ccs_rshift(CPUCRISState *env)
 	env->pregs[PR_CCS] = ccs;
 }
 
-void helper_rfe(void)
+void helper_rfe(CPUCRISState *env)
 {
 	int rflag = env->pregs[PR_CCS] & R_FLAG;
 
@@ -232,7 +219,7 @@ void helper_rfe(void)
 		env->pregs[PR_CCS] |= P_FLAG;
 }
 
-void helper_rfn(void)
+void helper_rfn(CPUCRISState *env)
 {
 	int rflag = env->pregs[PR_CCS] & R_FLAG;
 
@@ -256,7 +243,7 @@ uint32_t helper_lz(uint32_t t0)
 	return clz32(t0);
 }
 
-uint32_t helper_btst(uint32_t t0, uint32_t t1, uint32_t ccs)
+uint32_t helper_btst(CPUCRISState *env, uint32_t t0, uint32_t t1, uint32_t ccs)
 {
 	/* FIXME: clean this up.  */
 
@@ -284,7 +271,8 @@ uint32_t helper_btst(uint32_t t0, uint32_t t1, uint32_t ccs)
 	return ccs;
 }
 
-static inline uint32_t evaluate_flags_writeback(uint32_t flags, uint32_t ccs)
+static inline uint32_t evaluate_flags_writeback(CPUCRISState *env,
+                                                uint32_t flags, uint32_t ccs)
 {
 	unsigned int x, z, mask;
 
@@ -303,7 +291,8 @@ static inline uint32_t evaluate_flags_writeback(uint32_t flags, uint32_t ccs)
 	return ccs;
 }
 
-uint32_t helper_evaluate_flags_muls(uint32_t ccs, uint32_t res, uint32_t mof)
+uint32_t helper_evaluate_flags_muls(CPUCRISState *env,
+                                    uint32_t ccs, uint32_t res, uint32_t mof)
 {
 	uint32_t flags = 0;
 	int64_t tmp;
@@ -321,10 +310,11 @@ uint32_t helper_evaluate_flags_muls(uint32_t ccs, uint32_t res, uint32_t mof)
 	if ((dneg && mof != -1)
 	    || (!dneg && mof != 0))
 		flags |= V_FLAG;
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_mulu(uint32_t ccs, uint32_t res, uint32_t mof)
+uint32_t helper_evaluate_flags_mulu(CPUCRISState *env,
+                                    uint32_t ccs, uint32_t res, uint32_t mof)
 {
 	uint32_t flags = 0;
 	uint64_t tmp;
@@ -339,10 +329,10 @@ uint32_t helper_evaluate_flags_mulu(uint32_t ccs, uint32_t res, uint32_t mof)
 	if (mof)
 		flags |= V_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_mcp(uint32_t ccs,
+uint32_t helper_evaluate_flags_mcp(CPUCRISState *env, uint32_t ccs,
 				   uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -368,10 +358,10 @@ uint32_t helper_evaluate_flags_mcp(uint32_t ccs,
 			flags |= R_FLAG;
 	}
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_alu_4(uint32_t ccs,
+uint32_t helper_evaluate_flags_alu_4(CPUCRISState *env, uint32_t ccs,
 				     uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -397,10 +387,10 @@ uint32_t helper_evaluate_flags_alu_4(uint32_t ccs,
 			flags |= C_FLAG;
 	}
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_sub_4(uint32_t ccs,
+uint32_t helper_evaluate_flags_sub_4(CPUCRISState *env, uint32_t ccs,
 				     uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -427,10 +417,11 @@ uint32_t helper_evaluate_flags_sub_4(uint32_t ccs,
 	}
 
 	flags ^= C_FLAG;
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_move_4(uint32_t ccs, uint32_t res)
+uint32_t helper_evaluate_flags_move_4(CPUCRISState *env,
+                                      uint32_t ccs, uint32_t res)
 {
 	uint32_t flags = 0;
 
@@ -439,9 +430,10 @@ uint32_t helper_evaluate_flags_move_4(uint32_t ccs, uint32_t res)
 	else if (res == 0L)
 		flags |= Z_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
-uint32_t helper_evaluate_flags_move_2(uint32_t ccs, uint32_t res)
+uint32_t helper_evaluate_flags_move_2(CPUCRISState *env,
+                                      uint32_t ccs, uint32_t res)
 {
 	uint32_t flags = 0;
 
@@ -450,12 +442,12 @@ uint32_t helper_evaluate_flags_move_2(uint32_t ccs, uint32_t res)
 	else if (res == 0)
 		flags |= Z_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
 /* TODO: This is expensive. We could split things up and only evaluate part of
    CCR on a need to know basis. For now, we simply re-evaluate everything.  */
-void  helper_evaluate_flags(void)
+void helper_evaluate_flags(CPUCRISState *env)
 {
 	uint32_t src, dst, res;
 	uint32_t flags = 0;
@@ -571,25 +563,26 @@ void  helper_evaluate_flags(void)
 	if (env->cc_op == CC_OP_SUB || env->cc_op == CC_OP_CMP)
 		flags ^= C_FLAG;
 
-	env->pregs[PR_CCS] = evaluate_flags_writeback(flags, env->pregs[PR_CCS]);
+        env->pregs[PR_CCS] = evaluate_flags_writeback(env, flags,
+                                                      env->pregs[PR_CCS]);
 }
 
-void helper_top_evaluate_flags(void)
+void helper_top_evaluate_flags(CPUCRISState *env)
 {
 	switch (env->cc_op)
 	{
 		case CC_OP_MCP:
-			env->pregs[PR_CCS] = helper_evaluate_flags_mcp(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_mcp(env,
 					env->pregs[PR_CCS], env->cc_src,
 					env->cc_dest, env->cc_result);
 			break;
 		case CC_OP_MULS:
-			env->pregs[PR_CCS] = helper_evaluate_flags_muls(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_muls(env,
 					env->pregs[PR_CCS], env->cc_result,
 					env->pregs[PR_MOF]);
 			break;
 		case CC_OP_MULU:
-			env->pregs[PR_CCS] = helper_evaluate_flags_mulu(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_mulu(env,
 					env->pregs[PR_CCS], env->cc_result,
 					env->pregs[PR_MOF]);
 			break;
@@ -604,18 +597,18 @@ void helper_top_evaluate_flags(void)
 		{
 			case 4:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_move_4(
+                                        helper_evaluate_flags_move_4(env,
 							env->pregs[PR_CCS],
 							env->cc_result);
 				break;
 			case 2:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_move_2(
+                                        helper_evaluate_flags_move_2(env,
 							env->pregs[PR_CCS],
 							env->cc_result);
 				break;
 			default:
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 				break;
 		}
 		break;
@@ -626,12 +619,12 @@ void helper_top_evaluate_flags(void)
 		case CC_OP_CMP:
 			if (env->cc_size == 4)
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_sub_4(
+                                        helper_evaluate_flags_sub_4(env,
 						env->pregs[PR_CCS],
 						env->cc_src, env->cc_dest,
 						env->cc_result);
 			else
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 			break;
 		default:
 		{
@@ -639,13 +632,13 @@ void helper_top_evaluate_flags(void)
 			{
 			case 4:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_alu_4(
+                                        helper_evaluate_flags_alu_4(env,
 						env->pregs[PR_CCS],
 						env->cc_src, env->cc_dest,
 						env->cc_result);
 				break;
 			default:
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 				break;
 			}
 		}
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 1ad9ec788e5..09e6011ea47 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -24,7 +24,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "helper.h"
 #include "mmu.h"
@@ -70,93 +70,93 @@ static TCGv env_btaken;
 static TCGv env_btarget;
 static TCGv env_pc;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
-	CPUCRISState *env;
-	target_ulong pc, ppc;
-
-	/* Decoder.  */
-	unsigned int (*decoder)(struct DisasContext *dc);
-	uint32_t ir;
-	uint32_t opcode;
-	unsigned int op1;
-	unsigned int op2;
-	unsigned int zsize, zzsize;
-	unsigned int mode;
-	unsigned int postinc;
-
-	unsigned int size;
-	unsigned int src;
-	unsigned int dst;
-	unsigned int cond;
-
-	int update_cc;
-	int cc_op;
-	int cc_size;
-	uint32_t cc_mask;
-
-	int cc_size_uptodate; /* -1 invalid or last written value.  */
-
-	int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
-	int flags_uptodate; /* Wether or not $ccs is uptodate.  */
-	int flagx_known; /* Wether or not flags_x has the x flag known at
-			    translation time.  */
-	int flags_x;
-
-	int clear_x; /* Clear x after this insn?  */
-	int clear_prefix; /* Clear prefix after this insn?  */
-	int clear_locked_irq; /* Clear the irq lockout.  */
-	int cpustate_changed;
-	unsigned int tb_flags; /* tb dependent flags.  */
-	int is_jmp;
+    CPUCRISState *env;
+    target_ulong pc, ppc;
+
+    /* Decoder.  */
+        unsigned int (*decoder)(CPUCRISState *env, struct DisasContext *dc);
+    uint32_t ir;
+    uint32_t opcode;
+    unsigned int op1;
+    unsigned int op2;
+    unsigned int zsize, zzsize;
+    unsigned int mode;
+    unsigned int postinc;
+
+    unsigned int size;
+    unsigned int src;
+    unsigned int dst;
+    unsigned int cond;
+
+    int update_cc;
+    int cc_op;
+    int cc_size;
+    uint32_t cc_mask;
+
+    int cc_size_uptodate; /* -1 invalid or last written value.  */
+
+    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
+    int flags_uptodate; /* Wether or not $ccs is uptodate.  */
+    int flagx_known; /* Wether or not flags_x has the x flag known at
+                translation time.  */
+    int flags_x;
+
+    int clear_x; /* Clear x after this insn?  */
+    int clear_prefix; /* Clear prefix after this insn?  */
+    int clear_locked_irq; /* Clear the irq lockout.  */
+    int cpustate_changed;
+    unsigned int tb_flags; /* tb dependent flags.  */
+    int is_jmp;
 
 #define JMP_NOJMP     0
 #define JMP_DIRECT    1
 #define JMP_DIRECT_CC 2
 #define JMP_INDIRECT  3
-	int jmp; /* 0=nojmp, 1=direct, 2=indirect.  */ 
-	uint32_t jmp_pc;
+    int jmp; /* 0=nojmp, 1=direct, 2=indirect.  */
+    uint32_t jmp_pc;
 
-	int delayed_branch;
+    int delayed_branch;
 
-	struct TranslationBlock *tb;
-	int singlestep_enabled;
+    struct TranslationBlock *tb;
+    int singlestep_enabled;
 } DisasContext;
 
 static void gen_BUG(DisasContext *dc, const char *file, int line)
 {
-	printf ("BUG: pc=%x %s %d\n", dc->pc, file, line);
-	qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
-	cpu_abort(dc->env, "%s:%d\n", file, line);
+    printf("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    cpu_abort(dc->env, "%s:%d\n", file, line);
 }
 
 static const char *regnames[] =
 {
-	"$r0", "$r1", "$r2", "$r3",
-	"$r4", "$r5", "$r6", "$r7",
-	"$r8", "$r9", "$r10", "$r11",
-	"$r12", "$r13", "$sp", "$acr",
+    "$r0", "$r1", "$r2", "$r3",
+    "$r4", "$r5", "$r6", "$r7",
+    "$r8", "$r9", "$r10", "$r11",
+    "$r12", "$r13", "$sp", "$acr",
 };
 static const char *pregnames[] =
 {
-	"$bz", "$vr", "$pid", "$srs",
-	"$wz", "$exs", "$eda", "$mof",
-	"$dz", "$ebp", "$erp", "$srp",
-	"$nrp", "$ccs", "$usp", "$spc",
+    "$bz", "$vr", "$pid", "$srs",
+    "$wz", "$exs", "$eda", "$mof",
+    "$dz", "$ebp", "$erp", "$srp",
+    "$nrp", "$ccs", "$usp", "$spc",
 };
 
 /* We need this table to handle preg-moves with implicit width.  */
 static int preg_sizes[] = {
-	1, /* bz.  */
-	1, /* vr.  */
-	4, /* pid.  */
-	1, /* srs.  */
-	2, /* wz.  */
-	4, 4, 4,
-	4, 4, 4, 4,
-	4, 4, 4, 4,
+    1, /* bz.  */
+    1, /* vr.  */
+    4, /* pid.  */
+    1, /* srs.  */
+    2, /* wz.  */
+    4, 4, 4,
+    4, 4, 4, 4,
+    4, 4, 4, 4,
 };
 
 #define t_gen_mov_TN_env(tn, member) \
@@ -166,358 +166,368 @@ static int preg_sizes[] = {
 
 static inline void t_gen_mov_TN_reg(TCGv tn, int r)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register read $r%d\n", r);
-	tcg_gen_mov_tl(tn, cpu_R[r]);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register read $r%d\n", r);
+    }
+    tcg_gen_mov_tl(tn, cpu_R[r]);
 }
 static inline void t_gen_mov_reg_TN(int r, TCGv tn)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register write $r%d\n", r);
-	tcg_gen_mov_tl(cpu_R[r], tn);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register write $r%d\n", r);
+    }
+    tcg_gen_mov_tl(cpu_R[r], tn);
 }
 
 static inline void _t_gen_mov_TN_env(TCGv tn, int offset)
 {
-	if (offset > sizeof (CPUCRISState))
-		fprintf(stderr, "wrong load from env from off=%d\n", offset);
-	tcg_gen_ld_tl(tn, cpu_env, offset);
+    if (offset > sizeof(CPUCRISState)) {
+        fprintf(stderr, "wrong load from env from off=%d\n", offset);
+    }
+    tcg_gen_ld_tl(tn, cpu_env, offset);
 }
 static inline void _t_gen_mov_env_TN(int offset, TCGv tn)
 {
-	if (offset > sizeof (CPUCRISState))
-		fprintf(stderr, "wrong store to env at off=%d\n", offset);
-	tcg_gen_st_tl(tn, cpu_env, offset);
+    if (offset > sizeof(CPUCRISState)) {
+        fprintf(stderr, "wrong store to env at off=%d\n", offset);
+    }
+    tcg_gen_st_tl(tn, cpu_env, offset);
 }
 
 static inline void t_gen_mov_TN_preg(TCGv tn, int r)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register read $p%d\n", r);
-	if (r == PR_BZ || r == PR_WZ || r == PR_DZ)
-		tcg_gen_mov_tl(tn, tcg_const_tl(0));
-	else if (r == PR_VR)
-		tcg_gen_mov_tl(tn, tcg_const_tl(32));
-	else
-		tcg_gen_mov_tl(tn, cpu_PR[r]);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register read $p%d\n", r);
+    }
+    if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
+        tcg_gen_mov_tl(tn, tcg_const_tl(0));
+    } else if (r == PR_VR) {
+        tcg_gen_mov_tl(tn, tcg_const_tl(32));
+    } else {
+        tcg_gen_mov_tl(tn, cpu_PR[r]);
+    }
 }
 static inline void t_gen_mov_preg_TN(DisasContext *dc, int r, TCGv tn)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register write $p%d\n", r);
-	if (r == PR_BZ || r == PR_WZ || r == PR_DZ)
-		return;
-	else if (r == PR_SRS)
-		tcg_gen_andi_tl(cpu_PR[r], tn, 3);
-	else {
-		if (r == PR_PID) 
-			gen_helper_tlb_flush_pid(tn);
-		if (dc->tb_flags & S_FLAG && r == PR_SPC) 
-			gen_helper_spc_write(tn);
-		else if (r == PR_CCS)
-			dc->cpustate_changed = 1;
-		tcg_gen_mov_tl(cpu_PR[r], tn);
-	}
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register write $p%d\n", r);
+    }
+    if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
+        return;
+    } else if (r == PR_SRS) {
+        tcg_gen_andi_tl(cpu_PR[r], tn, 3);
+    } else {
+        if (r == PR_PID) {
+            gen_helper_tlb_flush_pid(cpu_env, tn);
+        }
+        if (dc->tb_flags & S_FLAG && r == PR_SPC) {
+            gen_helper_spc_write(cpu_env, tn);
+        } else if (r == PR_CCS) {
+            dc->cpustate_changed = 1;
+        }
+        tcg_gen_mov_tl(cpu_PR[r], tn);
+    }
 }
 
 /* Sign extend at translation time.  */
 static int sign_extend(unsigned int val, unsigned int width)
 {
-	int sval;
-
-	/* LSL.  */
-	val <<= 31 - width;
-	sval = val;
-	/* ASR.  */
-	sval >>= 31 - width;
-	return sval;
-}
-
-static int cris_fetch(DisasContext *dc, uint32_t addr,
-		      unsigned int size, unsigned int sign)
-{
-	int r;
-
-	switch (size) {
-		case 4:
-		{
-			r = ldl_code(addr);
-			break;
-		}
-		case 2:
-		{
-			if (sign) {
-				r = ldsw_code(addr);
-			} else {
-				r = lduw_code(addr);
-			}
-			break;
-		}
-		case 1:
-		{
-			if (sign) {
-				r = ldsb_code(addr);
-			} else {
-				r = ldub_code(addr);
-			}
-			break;
-		}
-		default:
-			cpu_abort(dc->env, "Invalid fetch size %d\n", size);
-			break;
-	}
-	return r;
+    int sval;
+
+    /* LSL.  */
+    val <<= 31 - width;
+    sval = val;
+    /* ASR.  */
+    sval >>= 31 - width;
+    return sval;
+}
+
+static int cris_fetch(CPUCRISState *env, DisasContext *dc, uint32_t addr,
+              unsigned int size, unsigned int sign)
+{
+    int r;
+
+    switch (size) {
+    case 4:
+    {
+        r = cpu_ldl_code(env, addr);
+        break;
+    }
+    case 2:
+    {
+        if (sign) {
+            r = cpu_ldsw_code(env, addr);
+        } else {
+            r = cpu_lduw_code(env, addr);
+        }
+        break;
+    }
+    case 1:
+    {
+        if (sign) {
+            r = cpu_ldsb_code(env, addr);
+        } else {
+            r = cpu_ldub_code(env, addr);
+        }
+        break;
+    }
+    default:
+        cpu_abort(dc->env, "Invalid fetch size %d\n", size);
+        break;
+    }
+    return r;
 }
 
 static void cris_lock_irq(DisasContext *dc)
 {
-	dc->clear_locked_irq = 0;
-	t_gen_mov_env_TN(locked_irq, tcg_const_tl(1));
+    dc->clear_locked_irq = 0;
+    t_gen_mov_env_TN(locked_irq, tcg_const_tl(1));
 }
 
 static inline void t_gen_raise_exception(uint32_t index)
 {
         TCGv_i32 tmp = tcg_const_i32(index);
-	gen_helper_raise_exception(tmp);
+        gen_helper_raise_exception(cpu_env, tmp);
         tcg_temp_free_i32(tmp);
 }
 
 static void t_gen_lsl(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_const_tl(31);
-	tcg_gen_shl_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_const_tl(31);
+    tcg_gen_shl_tl(d, a, b);
 
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_and_tl(t0, t0, d);
-	tcg_gen_xor_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_and_tl(t0, t0, d);
+    tcg_gen_xor_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 static void t_gen_lsr(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_temp_new();
-	tcg_gen_shr_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_temp_new();
+    tcg_gen_shr_tl(d, a, b);
 
-	tcg_gen_movi_tl(t_31, 31);
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_and_tl(t0, t0, d);
-	tcg_gen_xor_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_movi_tl(t_31, 31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_and_tl(t0, t0, d);
+    tcg_gen_xor_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 static void t_gen_asr(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_temp_new();
-	tcg_gen_sar_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_temp_new();
+    tcg_gen_sar_tl(d, a, b);
 
-	tcg_gen_movi_tl(t_31, 31);
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_or_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_movi_tl(t_31, 31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_or_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 /* 64-bit signed mul, lower result in d and upper in d2.  */
 static void t_gen_muls(TCGv d, TCGv d2, TCGv a, TCGv b)
 {
-	TCGv_i64 t0, t1;
+    TCGv_i64 t0, t1;
 
-	t0 = tcg_temp_new_i64();
-	t1 = tcg_temp_new_i64();
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
 
-	tcg_gen_ext_i32_i64(t0, a);
-	tcg_gen_ext_i32_i64(t1, b);
-	tcg_gen_mul_i64(t0, t0, t1);
+    tcg_gen_ext_i32_i64(t0, a);
+    tcg_gen_ext_i32_i64(t1, b);
+    tcg_gen_mul_i64(t0, t0, t1);
 
-	tcg_gen_trunc_i64_i32(d, t0);
-	tcg_gen_shri_i64(t0, t0, 32);
-	tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_shri_i64(t0, t0, 32);
+    tcg_gen_trunc_i64_i32(d2, t0);
 
-	tcg_temp_free_i64(t0);
-	tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
 }
 
 /* 64-bit unsigned muls, lower result in d and upper in d2.  */
 static void t_gen_mulu(TCGv d, TCGv d2, TCGv a, TCGv b)
 {
-	TCGv_i64 t0, t1;
+    TCGv_i64 t0, t1;
 
-	t0 = tcg_temp_new_i64();
-	t1 = tcg_temp_new_i64();
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
 
-	tcg_gen_extu_i32_i64(t0, a);
-	tcg_gen_extu_i32_i64(t1, b);
-	tcg_gen_mul_i64(t0, t0, t1);
+    tcg_gen_extu_i32_i64(t0, a);
+    tcg_gen_extu_i32_i64(t1, b);
+    tcg_gen_mul_i64(t0, t0, t1);
 
-	tcg_gen_trunc_i64_i32(d, t0);
-	tcg_gen_shri_i64(t0, t0, 32);
-	tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_shri_i64(t0, t0, 32);
+    tcg_gen_trunc_i64_i32(d2, t0);
 
-	tcg_temp_free_i64(t0);
-	tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
 }
 
 static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
 {
-	int l1;
+    int l1;
 
-	l1 = gen_new_label();
+    l1 = gen_new_label();
 
-	/* 
-	 * d <<= 1
-	 * if (d >= s)
-	 *    d -= s;
-	 */
-	tcg_gen_shli_tl(d, a, 1);
-	tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
-	tcg_gen_sub_tl(d, d, b);
-	gen_set_label(l1);
+    /*
+     * d <<= 1
+     * if (d >= s)
+     *    d -= s;
+     */
+    tcg_gen_shli_tl(d, a, 1);
+    tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
+    tcg_gen_sub_tl(d, d, b);
+    gen_set_label(l1);
 }
 
 static void t_gen_cris_mstep(TCGv d, TCGv a, TCGv b, TCGv ccs)
 {
-	TCGv t;
+    TCGv t;
 
-	/* 
-	 * d <<= 1
-	 * if (n)
-	 *    d += s;
-	 */
-	t = tcg_temp_new();
-	tcg_gen_shli_tl(d, a, 1);
-	tcg_gen_shli_tl(t, ccs, 31 - 3);
-	tcg_gen_sari_tl(t, t, 31);
-	tcg_gen_and_tl(t, t, b);
-	tcg_gen_add_tl(d, d, t);
-	tcg_temp_free(t);
+    /*
+     * d <<= 1
+     * if (n)
+     *    d += s;
+     */
+    t = tcg_temp_new();
+    tcg_gen_shli_tl(d, a, 1);
+    tcg_gen_shli_tl(t, ccs, 31 - 3);
+    tcg_gen_sari_tl(t, t, 31);
+    tcg_gen_and_tl(t, t, b);
+    tcg_gen_add_tl(d, d, t);
+    tcg_temp_free(t);
 }
 
 /* Extended arithmetics on CRIS.  */
 static inline void t_gen_add_flag(TCGv d, int flag)
 {
-	TCGv c;
+    TCGv c;
 
-	c = tcg_temp_new();
-	t_gen_mov_TN_preg(c, PR_CCS);
-	/* Propagate carry into d.  */
-	tcg_gen_andi_tl(c, c, 1 << flag);
-	if (flag)
-		tcg_gen_shri_tl(c, c, flag);
-	tcg_gen_add_tl(d, d, c);
-	tcg_temp_free(c);
+    c = tcg_temp_new();
+    t_gen_mov_TN_preg(c, PR_CCS);
+    /* Propagate carry into d.  */
+    tcg_gen_andi_tl(c, c, 1 << flag);
+    if (flag) {
+        tcg_gen_shri_tl(c, c, flag);
+    }
+    tcg_gen_add_tl(d, d, c);
+    tcg_temp_free(c);
 }
 
 static inline void t_gen_addx_carry(DisasContext *dc, TCGv d)
 {
-	if (dc->flagx_known) {
-		if (dc->flags_x) {
-			TCGv c;
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            TCGv c;
             
-			c = tcg_temp_new();
-			t_gen_mov_TN_preg(c, PR_CCS);
-			/* C flag is already at bit 0.  */
-			tcg_gen_andi_tl(c, c, C_FLAG);
-			tcg_gen_add_tl(d, d, c);
-			tcg_temp_free(c);
-		}
-	} else {
-		TCGv x, c;
-
-		x = tcg_temp_new();
-		c = tcg_temp_new();
-		t_gen_mov_TN_preg(x, PR_CCS);
-		tcg_gen_mov_tl(c, x);
-
-		/* Propagate carry into d if X is set. Branch free.  */
-		tcg_gen_andi_tl(c, c, C_FLAG);
-		tcg_gen_andi_tl(x, x, X_FLAG);
-		tcg_gen_shri_tl(x, x, 4);
-
-		tcg_gen_and_tl(x, x, c);
-		tcg_gen_add_tl(d, d, x);        
-		tcg_temp_free(x);
-		tcg_temp_free(c);
-	}
+            c = tcg_temp_new();
+            t_gen_mov_TN_preg(c, PR_CCS);
+            /* C flag is already at bit 0.  */
+            tcg_gen_andi_tl(c, c, C_FLAG);
+            tcg_gen_add_tl(d, d, c);
+            tcg_temp_free(c);
+        }
+    } else {
+        TCGv x, c;
+
+        x = tcg_temp_new();
+        c = tcg_temp_new();
+        t_gen_mov_TN_preg(x, PR_CCS);
+        tcg_gen_mov_tl(c, x);
+
+        /* Propagate carry into d if X is set. Branch free.  */
+        tcg_gen_andi_tl(c, c, C_FLAG);
+        tcg_gen_andi_tl(x, x, X_FLAG);
+        tcg_gen_shri_tl(x, x, 4);
+
+        tcg_gen_and_tl(x, x, c);
+        tcg_gen_add_tl(d, d, x);
+        tcg_temp_free(x);
+        tcg_temp_free(c);
+    }
 }
 
 static inline void t_gen_subx_carry(DisasContext *dc, TCGv d)
 {
-	if (dc->flagx_known) {
-		if (dc->flags_x) {
-			TCGv c;
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            TCGv c;
             
-			c = tcg_temp_new();
-			t_gen_mov_TN_preg(c, PR_CCS);
-			/* C flag is already at bit 0.  */
-			tcg_gen_andi_tl(c, c, C_FLAG);
-			tcg_gen_sub_tl(d, d, c);
-			tcg_temp_free(c);
-		}
-	} else {
-		TCGv x, c;
-
-		x = tcg_temp_new();
-		c = tcg_temp_new();
-		t_gen_mov_TN_preg(x, PR_CCS);
-		tcg_gen_mov_tl(c, x);
-
-		/* Propagate carry into d if X is set. Branch free.  */
-		tcg_gen_andi_tl(c, c, C_FLAG);
-		tcg_gen_andi_tl(x, x, X_FLAG);
-		tcg_gen_shri_tl(x, x, 4);
-
-		tcg_gen_and_tl(x, x, c);
-		tcg_gen_sub_tl(d, d, x);
-		tcg_temp_free(x);
-		tcg_temp_free(c);
-	}
+            c = tcg_temp_new();
+            t_gen_mov_TN_preg(c, PR_CCS);
+            /* C flag is already at bit 0.  */
+            tcg_gen_andi_tl(c, c, C_FLAG);
+            tcg_gen_sub_tl(d, d, c);
+            tcg_temp_free(c);
+        }
+    } else {
+        TCGv x, c;
+
+        x = tcg_temp_new();
+        c = tcg_temp_new();
+        t_gen_mov_TN_preg(x, PR_CCS);
+        tcg_gen_mov_tl(c, x);
+
+        /* Propagate carry into d if X is set. Branch free.  */
+        tcg_gen_andi_tl(c, c, C_FLAG);
+        tcg_gen_andi_tl(x, x, X_FLAG);
+        tcg_gen_shri_tl(x, x, 4);
+
+        tcg_gen_and_tl(x, x, c);
+        tcg_gen_sub_tl(d, d, x);
+        tcg_temp_free(x);
+        tcg_temp_free(c);
+    }
 }
 
 /* Swap the two bytes within each half word of the s operand.
    T0 = ((T0 << 8) & 0xff00ff00) | ((T0 >> 8) & 0x00ff00ff)  */
 static inline void t_gen_swapb(TCGv d, TCGv s)
 {
-	TCGv t, org_s;
+    TCGv t, org_s;
 
-	t = tcg_temp_new();
-	org_s = tcg_temp_new();
+    t = tcg_temp_new();
+    org_s = tcg_temp_new();
 
-	/* d and s may refer to the same object.  */
-	tcg_gen_mov_tl(org_s, s);
-	tcg_gen_shli_tl(t, org_s, 8);
-	tcg_gen_andi_tl(d, t, 0xff00ff00);
-	tcg_gen_shri_tl(t, org_s, 8);
-	tcg_gen_andi_tl(t, t, 0x00ff00ff);
-	tcg_gen_or_tl(d, d, t);
-	tcg_temp_free(t);
-	tcg_temp_free(org_s);
+    /* d and s may refer to the same object.  */
+    tcg_gen_mov_tl(org_s, s);
+    tcg_gen_shli_tl(t, org_s, 8);
+    tcg_gen_andi_tl(d, t, 0xff00ff00);
+    tcg_gen_shri_tl(t, org_s, 8);
+    tcg_gen_andi_tl(t, t, 0x00ff00ff);
+    tcg_gen_or_tl(d, d, t);
+    tcg_temp_free(t);
+    tcg_temp_free(org_s);
 }
 
 /* Swap the halfwords of the s operand.  */
 static inline void t_gen_swapw(TCGv d, TCGv s)
 {
-	TCGv t;
-	/* d and s refer the same object.  */
-	t = tcg_temp_new();
-	tcg_gen_mov_tl(t, s);
-	tcg_gen_shli_tl(d, t, 16);
-	tcg_gen_shri_tl(t, t, 16);
-	tcg_gen_or_tl(d, d, t);
-	tcg_temp_free(t);
+    TCGv t;
+    /* d and s refer the same object.  */
+    t = tcg_temp_new();
+    tcg_gen_mov_tl(t, s);
+    tcg_gen_shli_tl(d, t, 16);
+    tcg_gen_shri_tl(t, t, 16);
+    tcg_gen_or_tl(d, d, t);
+    tcg_temp_free(t);
 }
 
 /* Reverse the within each byte.
@@ -532,607 +542,611 @@ static inline void t_gen_swapw(TCGv d, TCGv s)
  */
 static inline void t_gen_swapr(TCGv d, TCGv s)
 {
-	struct {
-		int shift; /* LSL when positive, LSR when negative.  */
-		uint32_t mask;
-	} bitrev [] = {
-		{7, 0x80808080},
-		{5, 0x40404040},
-		{3, 0x20202020},
-		{1, 0x10101010},
-		{-1, 0x08080808},
-		{-3, 0x04040404},
-		{-5, 0x02020202},
-		{-7, 0x01010101}
-	};
-	int i;
-	TCGv t, org_s;
-
-	/* d and s refer the same object.  */
-	t = tcg_temp_new();
-	org_s = tcg_temp_new();
-	tcg_gen_mov_tl(org_s, s);
-
-	tcg_gen_shli_tl(t, org_s,  bitrev[0].shift);
-	tcg_gen_andi_tl(d, t,  bitrev[0].mask);
-	for (i = 1; i < ARRAY_SIZE(bitrev); i++) {
-		if (bitrev[i].shift >= 0) {
-			tcg_gen_shli_tl(t, org_s,  bitrev[i].shift);
-		} else {
-			tcg_gen_shri_tl(t, org_s,  -bitrev[i].shift);
-		}
-		tcg_gen_andi_tl(t, t,  bitrev[i].mask);
-		tcg_gen_or_tl(d, d, t);
-	}
-	tcg_temp_free(t);
-	tcg_temp_free(org_s);
+    struct {
+        int shift; /* LSL when positive, LSR when negative.  */
+        uint32_t mask;
+    } bitrev[] = {
+        {7, 0x80808080},
+        {5, 0x40404040},
+        {3, 0x20202020},
+        {1, 0x10101010},
+        {-1, 0x08080808},
+        {-3, 0x04040404},
+        {-5, 0x02020202},
+        {-7, 0x01010101}
+    };
+    int i;
+    TCGv t, org_s;
+
+    /* d and s refer the same object.  */
+    t = tcg_temp_new();
+    org_s = tcg_temp_new();
+    tcg_gen_mov_tl(org_s, s);
+
+    tcg_gen_shli_tl(t, org_s,  bitrev[0].shift);
+    tcg_gen_andi_tl(d, t,  bitrev[0].mask);
+    for (i = 1; i < ARRAY_SIZE(bitrev); i++) {
+        if (bitrev[i].shift >= 0) {
+            tcg_gen_shli_tl(t, org_s,  bitrev[i].shift);
+        } else {
+            tcg_gen_shri_tl(t, org_s,  -bitrev[i].shift);
+        }
+        tcg_gen_andi_tl(t, t,  bitrev[i].mask);
+        tcg_gen_or_tl(d, d, t);
+    }
+    tcg_temp_free(t);
+    tcg_temp_free(org_s);
 }
 
 static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false)
 {
-	int l1;
+    int l1;
 
-	l1 = gen_new_label();
+    l1 = gen_new_label();
 
-	/* Conditional jmp.  */
-	tcg_gen_mov_tl(env_pc, pc_false);
-	tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, l1);
-	tcg_gen_mov_tl(env_pc, pc_true);
-	gen_set_label(l1);
+    /* Conditional jmp.  */
+    tcg_gen_mov_tl(env_pc, pc_false);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, l1);
+    tcg_gen_mov_tl(env_pc, pc_true);
+    gen_set_label(l1);
 }
 
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-	TranslationBlock *tb;
-	tb = dc->tb;
-	if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
-		tcg_gen_goto_tb(n);
-		tcg_gen_movi_tl(env_pc, dest);
+    TranslationBlock *tb;
+    tb = dc->tb;
+    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+        tcg_gen_goto_tb(n);
+        tcg_gen_movi_tl(env_pc, dest);
                 tcg_gen_exit_tb((tcg_target_long)tb + n);
-	} else {
-		tcg_gen_movi_tl(env_pc, dest);
-		tcg_gen_exit_tb(0);
-	}
+    } else {
+        tcg_gen_movi_tl(env_pc, dest);
+        tcg_gen_exit_tb(0);
+    }
 }
 
 static inline void cris_clear_x_flag(DisasContext *dc)
 {
-	if (dc->flagx_known && dc->flags_x)
-		dc->flags_uptodate = 0;
+    if (dc->flagx_known && dc->flags_x) {
+        dc->flags_uptodate = 0;
+    }
 
-	dc->flagx_known = 1;
-	dc->flags_x = 0;
+    dc->flagx_known = 1;
+    dc->flags_x = 0;
 }
 
 static void cris_flush_cc_state(DisasContext *dc)
 {
-	if (dc->cc_size_uptodate != dc->cc_size) {
-		tcg_gen_movi_tl(cc_size, dc->cc_size);
-		dc->cc_size_uptodate = dc->cc_size;
-	}
-	tcg_gen_movi_tl(cc_op, dc->cc_op);
-	tcg_gen_movi_tl(cc_mask, dc->cc_mask);
+    if (dc->cc_size_uptodate != dc->cc_size) {
+        tcg_gen_movi_tl(cc_size, dc->cc_size);
+        dc->cc_size_uptodate = dc->cc_size;
+    }
+    tcg_gen_movi_tl(cc_op, dc->cc_op);
+    tcg_gen_movi_tl(cc_mask, dc->cc_mask);
 }
 
 static void cris_evaluate_flags(DisasContext *dc)
 {
-	if (dc->flags_uptodate)
-		return;
-
-	cris_flush_cc_state(dc);
-
-	switch (dc->cc_op)
-	{
-	case CC_OP_MCP:
-		gen_helper_evaluate_flags_mcp(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_src,
-					cc_dest, cc_result);
-		break;
-	case CC_OP_MULS:
-		gen_helper_evaluate_flags_muls(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_result,
-					cpu_PR[PR_MOF]);
-		break;
-	case CC_OP_MULU:
-		gen_helper_evaluate_flags_mulu(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_result,
-					cpu_PR[PR_MOF]);
-		break;
-	case CC_OP_MOVE:
-	case CC_OP_AND:
-	case CC_OP_OR:
-	case CC_OP_XOR:
-	case CC_OP_ASR:
-	case CC_OP_LSR:
-	case CC_OP_LSL:
-		switch (dc->cc_size)
-		{
-		case 4:
-			gen_helper_evaluate_flags_move_4(cpu_PR[PR_CCS],
-						cpu_PR[PR_CCS], cc_result);
-			break;
-		case 2:
-			gen_helper_evaluate_flags_move_2(cpu_PR[PR_CCS],
-						cpu_PR[PR_CCS], cc_result);
-			break;
-		default:
-			gen_helper_evaluate_flags();
-			break;
-		}
-		break;
-	case CC_OP_FLAGS:
-		/* live.  */
-		break;
-	case CC_OP_SUB:
-	case CC_OP_CMP:
-		if (dc->cc_size == 4)
-			gen_helper_evaluate_flags_sub_4(cpu_PR[PR_CCS],
-				cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
-		else
-			gen_helper_evaluate_flags();
-
-		break;
-	default:
-		switch (dc->cc_size)
-		{
-			case 4:
-			gen_helper_evaluate_flags_alu_4(cpu_PR[PR_CCS],
-				cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
-				break;
-			default:
-				gen_helper_evaluate_flags();
-				break;
-		}
-		break;
-	}
-
-	if (dc->flagx_known) {
-		if (dc->flags_x)
-			tcg_gen_ori_tl(cpu_PR[PR_CCS], 
-				       cpu_PR[PR_CCS], X_FLAG);
-		else if (dc->cc_op == CC_OP_FLAGS)
-			tcg_gen_andi_tl(cpu_PR[PR_CCS], 
-					cpu_PR[PR_CCS], ~X_FLAG);
+    if (dc->flags_uptodate) {
+        return;
+    }
+
+    cris_flush_cc_state(dc);
+
+    switch (dc->cc_op) {
+    case CC_OP_MCP:
+        gen_helper_evaluate_flags_mcp(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_src,
+                cc_dest, cc_result);
+        break;
+    case CC_OP_MULS:
+        gen_helper_evaluate_flags_muls(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_result,
+                cpu_PR[PR_MOF]);
+        break;
+    case CC_OP_MULU:
+        gen_helper_evaluate_flags_mulu(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_result,
+                cpu_PR[PR_MOF]);
+        break;
+    case CC_OP_MOVE:
+    case CC_OP_AND:
+    case CC_OP_OR:
+    case CC_OP_XOR:
+    case CC_OP_ASR:
+    case CC_OP_LSR:
+    case CC_OP_LSL:
+        switch (dc->cc_size) {
+        case 4:
+            gen_helper_evaluate_flags_move_4(cpu_PR[PR_CCS],
+                    cpu_env, cpu_PR[PR_CCS], cc_result);
+            break;
+        case 2:
+            gen_helper_evaluate_flags_move_2(cpu_PR[PR_CCS],
+                    cpu_env, cpu_PR[PR_CCS], cc_result);
+            break;
+        default:
+            gen_helper_evaluate_flags(cpu_env);
+            break;
         }
-	dc->flags_uptodate = 1;
+        break;
+    case CC_OP_FLAGS:
+        /* live.  */
+        break;
+    case CC_OP_SUB:
+    case CC_OP_CMP:
+        if (dc->cc_size == 4) {
+            gen_helper_evaluate_flags_sub_4(cpu_PR[PR_CCS], cpu_env,
+                    cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
+        } else {
+            gen_helper_evaluate_flags(cpu_env);
+        }
+
+        break;
+    default:
+        switch (dc->cc_size) {
+        case 4:
+            gen_helper_evaluate_flags_alu_4(cpu_PR[PR_CCS], cpu_env,
+                    cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
+            break;
+        default:
+            gen_helper_evaluate_flags(cpu_env);
+            break;
+        }
+        break;
+    }
+
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], X_FLAG);
+        } else if (dc->cc_op == CC_OP_FLAGS) {
+            tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~X_FLAG);
+        }
+    }
+    dc->flags_uptodate = 1;
 }
 
 static void cris_cc_mask(DisasContext *dc, unsigned int mask)
 {
-	uint32_t ovl;
+    uint32_t ovl;
 
-	if (!mask) {
-		dc->update_cc = 0;
-		return;
-	}	
+    if (!mask) {
+        dc->update_cc = 0;
+        return;
+    }
 
-	/* Check if we need to evaluate the condition codes due to 
-	   CC overlaying.  */
-	ovl = (dc->cc_mask ^ mask) & ~mask;
-	if (ovl) {
-		/* TODO: optimize this case. It trigs all the time.  */
-		cris_evaluate_flags (dc);
-	}
-	dc->cc_mask = mask;
-	dc->update_cc = 1;
+    /* Check if we need to evaluate the condition codes due to
+       CC overlaying.  */
+    ovl = (dc->cc_mask ^ mask) & ~mask;
+    if (ovl) {
+        /* TODO: optimize this case. It trigs all the time.  */
+        cris_evaluate_flags(dc);
+    }
+    dc->cc_mask = mask;
+    dc->update_cc = 1;
 }
 
 static void cris_update_cc_op(DisasContext *dc, int op, int size)
 {
-	dc->cc_op = op;
-	dc->cc_size = size;
-	dc->flags_uptodate = 0;
+    dc->cc_op = op;
+    dc->cc_size = size;
+    dc->flags_uptodate = 0;
 }
 
 static inline void cris_update_cc_x(DisasContext *dc)
 {
-	/* Save the x flag state at the time of the cc snapshot.  */
-	if (dc->flagx_known) {
-		if (dc->cc_x_uptodate == (2 | dc->flags_x))
-			return;
-		tcg_gen_movi_tl(cc_x, dc->flags_x);
-		dc->cc_x_uptodate = 2 | dc->flags_x;
-	}
-	else {
-		tcg_gen_andi_tl(cc_x, cpu_PR[PR_CCS], X_FLAG);
-		dc->cc_x_uptodate = 1;
-	}
+    /* Save the x flag state at the time of the cc snapshot.  */
+    if (dc->flagx_known) {
+        if (dc->cc_x_uptodate == (2 | dc->flags_x)) {
+            return;
+        }
+        tcg_gen_movi_tl(cc_x, dc->flags_x);
+        dc->cc_x_uptodate = 2 | dc->flags_x;
+    } else {
+        tcg_gen_andi_tl(cc_x, cpu_PR[PR_CCS], X_FLAG);
+        dc->cc_x_uptodate = 1;
+    }
 }
 
 /* Update cc prior to executing ALU op. Needs source operands untouched.  */
 static void cris_pre_alu_update_cc(DisasContext *dc, int op, 
-				   TCGv dst, TCGv src, int size)
-{
-	if (dc->update_cc) {
-		cris_update_cc_op(dc, op, size);
-		tcg_gen_mov_tl(cc_src, src);
-
-		if (op != CC_OP_MOVE
-		    && op != CC_OP_AND
-		    && op != CC_OP_OR
-		    && op != CC_OP_XOR
-		    && op != CC_OP_ASR
-		    && op != CC_OP_LSR
-		    && op != CC_OP_LSL)
-			tcg_gen_mov_tl(cc_dest, dst);
+                   TCGv dst, TCGv src, int size)
+{
+    if (dc->update_cc) {
+        cris_update_cc_op(dc, op, size);
+        tcg_gen_mov_tl(cc_src, src);
+
+        if (op != CC_OP_MOVE
+            && op != CC_OP_AND
+            && op != CC_OP_OR
+            && op != CC_OP_XOR
+            && op != CC_OP_ASR
+            && op != CC_OP_LSR
+            && op != CC_OP_LSL) {
+            tcg_gen_mov_tl(cc_dest, dst);
+        }
 
-		cris_update_cc_x(dc);
-	}
+        cris_update_cc_x(dc);
+    }
 }
 
 /* Update cc after executing ALU op. needs the result.  */
 static inline void cris_update_result(DisasContext *dc, TCGv res)
 {
-	if (dc->update_cc)
-		tcg_gen_mov_tl(cc_result, res);
+    if (dc->update_cc) {
+        tcg_gen_mov_tl(cc_result, res);
+    }
 }
 
 /* Returns one if the write back stage should execute.  */
 static void cris_alu_op_exec(DisasContext *dc, int op, 
-			       TCGv dst, TCGv a, TCGv b, int size)
-{
-	/* Emit the ALU insns.  */
-	switch (op)
-	{
-		case CC_OP_ADD:
-			tcg_gen_add_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_addx_carry(dc, dst);
-			break;
-		case CC_OP_ADDC:
-			tcg_gen_add_tl(dst, a, b);
-			t_gen_add_flag(dst, 0); /* C_FLAG.  */
-			break;
-		case CC_OP_MCP:
-			tcg_gen_add_tl(dst, a, b);
-			t_gen_add_flag(dst, 8); /* R_FLAG.  */
-			break;
-		case CC_OP_SUB:
-			tcg_gen_sub_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		case CC_OP_MOVE:
-			tcg_gen_mov_tl(dst, b);
-			break;
-		case CC_OP_OR:
-			tcg_gen_or_tl(dst, a, b);
-			break;
-		case CC_OP_AND:
-			tcg_gen_and_tl(dst, a, b);
-			break;
-		case CC_OP_XOR:
-			tcg_gen_xor_tl(dst, a, b);
-			break;
-		case CC_OP_LSL:
-			t_gen_lsl(dst, a, b);
-			break;
-		case CC_OP_LSR:
-			t_gen_lsr(dst, a, b);
-			break;
-		case CC_OP_ASR:
-			t_gen_asr(dst, a, b);
-			break;
-		case CC_OP_NEG:
-			tcg_gen_neg_tl(dst, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		case CC_OP_LZ:
-			gen_helper_lz(dst, b);
-			break;
-		case CC_OP_MULS:
-			t_gen_muls(dst, cpu_PR[PR_MOF], a, b);
-			break;
-		case CC_OP_MULU:
-			t_gen_mulu(dst, cpu_PR[PR_MOF], a, b);
-			break;
-		case CC_OP_DSTEP:
-			t_gen_cris_dstep(dst, a, b);
-			break;
-		case CC_OP_MSTEP:
-			t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
-			break;
-		case CC_OP_BOUND:
-		{
-			int l1;
-			l1 = gen_new_label();
-			tcg_gen_mov_tl(dst, a);
-			tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
-			tcg_gen_mov_tl(dst, b);
-			gen_set_label(l1);
-		}
-		break;
-		case CC_OP_CMP:
-			tcg_gen_sub_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		default:
-			qemu_log("illegal ALU op.\n");
-			BUG();
-			break;
-	}
-
-	if (size == 1)
-		tcg_gen_andi_tl(dst, dst, 0xff);
-	else if (size == 2)
-		tcg_gen_andi_tl(dst, dst, 0xffff);
+                   TCGv dst, TCGv a, TCGv b, int size)
+{
+    /* Emit the ALU insns.  */
+    switch (op) {
+    case CC_OP_ADD:
+        tcg_gen_add_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_addx_carry(dc, dst);
+        break;
+    case CC_OP_ADDC:
+        tcg_gen_add_tl(dst, a, b);
+        t_gen_add_flag(dst, 0); /* C_FLAG.  */
+        break;
+    case CC_OP_MCP:
+        tcg_gen_add_tl(dst, a, b);
+        t_gen_add_flag(dst, 8); /* R_FLAG.  */
+        break;
+    case CC_OP_SUB:
+        tcg_gen_sub_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    case CC_OP_MOVE:
+        tcg_gen_mov_tl(dst, b);
+        break;
+    case CC_OP_OR:
+        tcg_gen_or_tl(dst, a, b);
+        break;
+    case CC_OP_AND:
+        tcg_gen_and_tl(dst, a, b);
+        break;
+    case CC_OP_XOR:
+        tcg_gen_xor_tl(dst, a, b);
+        break;
+    case CC_OP_LSL:
+        t_gen_lsl(dst, a, b);
+        break;
+    case CC_OP_LSR:
+        t_gen_lsr(dst, a, b);
+        break;
+    case CC_OP_ASR:
+        t_gen_asr(dst, a, b);
+        break;
+    case CC_OP_NEG:
+        tcg_gen_neg_tl(dst, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    case CC_OP_LZ:
+        gen_helper_lz(dst, b);
+        break;
+    case CC_OP_MULS:
+        t_gen_muls(dst, cpu_PR[PR_MOF], a, b);
+        break;
+    case CC_OP_MULU:
+        t_gen_mulu(dst, cpu_PR[PR_MOF], a, b);
+        break;
+    case CC_OP_DSTEP:
+        t_gen_cris_dstep(dst, a, b);
+        break;
+    case CC_OP_MSTEP:
+        t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
+        break;
+    case CC_OP_BOUND:
+    {
+        int l1;
+        l1 = gen_new_label();
+        tcg_gen_mov_tl(dst, a);
+        tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
+        tcg_gen_mov_tl(dst, b);
+        gen_set_label(l1);
+    }
+        break;
+    case CC_OP_CMP:
+        tcg_gen_sub_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    default:
+        qemu_log("illegal ALU op.\n");
+        BUG();
+        break;
+    }
+
+    if (size == 1) {
+        tcg_gen_andi_tl(dst, dst, 0xff);
+    } else if (size == 2) {
+        tcg_gen_andi_tl(dst, dst, 0xffff);
+    }
 }
 
 static void cris_alu(DisasContext *dc, int op,
-			       TCGv d, TCGv op_a, TCGv op_b, int size)
+                   TCGv d, TCGv op_a, TCGv op_b, int size)
 {
-	TCGv tmp;
-	int writeback;
+    TCGv tmp;
+    int writeback;
 
-	writeback = 1;
+    writeback = 1;
 
-	if (op == CC_OP_CMP) {
-		tmp = tcg_temp_new();
-		writeback = 0;
-	} else if (size == 4) {
-		tmp = d;
-		writeback = 0;
-	} else
-		tmp = tcg_temp_new();
+    if (op == CC_OP_CMP) {
+        tmp = tcg_temp_new();
+        writeback = 0;
+    } else if (size == 4) {
+        tmp = d;
+        writeback = 0;
+    } else {
+        tmp = tcg_temp_new();
+    }
 
 
-	cris_pre_alu_update_cc(dc, op, op_a, op_b, size);
-	cris_alu_op_exec(dc, op, tmp, op_a, op_b, size);
-	cris_update_result(dc, tmp);
+    cris_pre_alu_update_cc(dc, op, op_a, op_b, size);
+    cris_alu_op_exec(dc, op, tmp, op_a, op_b, size);
+    cris_update_result(dc, tmp);
 
-	/* Writeback.  */
-	if (writeback) {
-		if (size == 1)
-			tcg_gen_andi_tl(d, d, ~0xff);
-		else
-			tcg_gen_andi_tl(d, d, ~0xffff);
-		tcg_gen_or_tl(d, d, tmp);
-	}
-	if (!TCGV_EQUAL(tmp, d))
-		tcg_temp_free(tmp);
+    /* Writeback.  */
+    if (writeback) {
+        if (size == 1) {
+            tcg_gen_andi_tl(d, d, ~0xff);
+        } else {
+            tcg_gen_andi_tl(d, d, ~0xffff);
+        }
+        tcg_gen_or_tl(d, d, tmp);
+    }
+    if (!TCGV_EQUAL(tmp, d)) {
+        tcg_temp_free(tmp);
+    }
 }
 
 static int arith_cc(DisasContext *dc)
 {
-	if (dc->update_cc) {
-		switch (dc->cc_op) {
-			case CC_OP_ADDC: return 1;
-			case CC_OP_ADD: return 1;
-			case CC_OP_SUB: return 1;
-			case CC_OP_DSTEP: return 1;
-			case CC_OP_LSL: return 1;
-			case CC_OP_LSR: return 1;
-			case CC_OP_ASR: return 1;
-			case CC_OP_CMP: return 1;
-			case CC_OP_NEG: return 1;
-			case CC_OP_OR: return 1;
-			case CC_OP_AND: return 1;
-			case CC_OP_XOR: return 1;
-			case CC_OP_MULU: return 1;
-			case CC_OP_MULS: return 1;
-			default:
-				return 0;
-		}
-	}
-	return 0;
+    if (dc->update_cc) {
+        switch (dc->cc_op) {
+        case CC_OP_ADDC: return 1;
+        case CC_OP_ADD: return 1;
+        case CC_OP_SUB: return 1;
+        case CC_OP_DSTEP: return 1;
+        case CC_OP_LSL: return 1;
+        case CC_OP_LSR: return 1;
+        case CC_OP_ASR: return 1;
+        case CC_OP_CMP: return 1;
+        case CC_OP_NEG: return 1;
+        case CC_OP_OR: return 1;
+        case CC_OP_AND: return 1;
+        case CC_OP_XOR: return 1;
+        case CC_OP_MULU: return 1;
+        case CC_OP_MULS: return 1;
+        default:
+            return 0;
+        }
+    }
+    return 0;
 }
 
 static void gen_tst_cc (DisasContext *dc, TCGv cc, int cond)
 {
-	int arith_opt, move_opt;
-
-	/* TODO: optimize more condition codes.  */
-
-	/*
-	 * If the flags are live, we've gotta look into the bits of CCS.
-	 * Otherwise, if we just did an arithmetic operation we try to
-	 * evaluate the condition code faster.
-	 *
-	 * When this function is done, T0 should be non-zero if the condition
-	 * code is true.
-	 */
-	arith_opt = arith_cc(dc) && !dc->flags_uptodate;
-	move_opt = (dc->cc_op == CC_OP_MOVE);
-	switch (cond) {
-		case CC_EQ:
-			if ((arith_opt || move_opt)
-			    && dc->cc_x_uptodate != (2 | X_FLAG)) {
-				tcg_gen_setcond_tl(TCG_COND_EQ, cc,
-						   cc_result, tcg_const_tl(0));
-			}
-			else {
-				cris_evaluate_flags(dc);
-				tcg_gen_andi_tl(cc, 
-						cpu_PR[PR_CCS], Z_FLAG);
-			}
-			break;
-		case CC_NE:
-			if ((arith_opt || move_opt)
-			    && dc->cc_x_uptodate != (2 | X_FLAG)) {
-				tcg_gen_mov_tl(cc, cc_result);
-	 		} else {
-				cris_evaluate_flags(dc);
-				tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-						Z_FLAG);
-				tcg_gen_andi_tl(cc, cc, Z_FLAG);
-			}
-			break;
-		case CC_CS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], C_FLAG);
-			break;
-		case CC_CC:
-			cris_evaluate_flags(dc);
-			tcg_gen_xori_tl(cc, cpu_PR[PR_CCS], C_FLAG);
-			tcg_gen_andi_tl(cc, cc, C_FLAG);
-			break;
-		case CC_VS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], V_FLAG);
-			break;
-		case CC_VC:
-			cris_evaluate_flags(dc);
-			tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-					V_FLAG);
-			tcg_gen_andi_tl(cc, cc, V_FLAG);
-			break;
-		case CC_PL:
-			if (arith_opt || move_opt) {
-				int bits = 31;
-
-				if (dc->cc_size == 1)
-					bits = 7;
-				else if (dc->cc_size == 2)
-					bits = 15;	
-
-				tcg_gen_shri_tl(cc, cc_result, bits);
-				tcg_gen_xori_tl(cc, cc, 1);
-			} else {
-				cris_evaluate_flags(dc);
-				tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-						N_FLAG);
-				tcg_gen_andi_tl(cc, cc, N_FLAG);
-			}
-			break;
-		case CC_MI:
-			if (arith_opt || move_opt) {
-				int bits = 31;
-
-				if (dc->cc_size == 1)
-					bits = 7;
-				else if (dc->cc_size == 2)
-					bits = 15;	
-
-				tcg_gen_shri_tl(cc, cc_result, bits);
-				tcg_gen_andi_tl(cc, cc, 1);
-			}
-			else {
-				cris_evaluate_flags(dc);
-				tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
-						N_FLAG);
-			}
-			break;
-		case CC_LS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
-					C_FLAG | Z_FLAG);
-			break;
-		case CC_HI:
-			cris_evaluate_flags(dc);
-			{
-				TCGv tmp;
-
-				tmp = tcg_temp_new();
-				tcg_gen_xori_tl(tmp, cpu_PR[PR_CCS],
-						C_FLAG | Z_FLAG);
-				/* Overlay the C flag on top of the Z.  */
-				tcg_gen_shli_tl(cc, tmp, 2);
-				tcg_gen_and_tl(cc, tmp, cc);
-				tcg_gen_andi_tl(cc, cc, Z_FLAG);
-
-				tcg_temp_free(tmp);
-			}
-			break;
-		case CC_GE:
-			cris_evaluate_flags(dc);
-			/* Overlay the V flag on top of the N.  */
-			tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
-			tcg_gen_xor_tl(cc,
-				       cpu_PR[PR_CCS], cc);
-			tcg_gen_andi_tl(cc, cc, N_FLAG);
-			tcg_gen_xori_tl(cc, cc, N_FLAG);
-			break;
-		case CC_LT:
-			cris_evaluate_flags(dc);
-			/* Overlay the V flag on top of the N.  */
-			tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
-			tcg_gen_xor_tl(cc,
-				       cpu_PR[PR_CCS], cc);
-			tcg_gen_andi_tl(cc, cc, N_FLAG);
-			break;
-		case CC_GT:
-			cris_evaluate_flags(dc);
-			{
-				TCGv n, z;
-
-				n = tcg_temp_new();
-				z = tcg_temp_new();
-
-				/* To avoid a shift we overlay everything on
-				   the V flag.  */
-				tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
-				tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
-				/* invert Z.  */
-				tcg_gen_xori_tl(z, z, 2);
-
-				tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
-				tcg_gen_xori_tl(n, n, 2);
-				tcg_gen_and_tl(cc, z, n);
-				tcg_gen_andi_tl(cc, cc, 2);
-
-				tcg_temp_free(n);
-				tcg_temp_free(z);
-			}
-			break;
-		case CC_LE:
-			cris_evaluate_flags(dc);
-			{
-				TCGv n, z;
-
-				n = tcg_temp_new();
-				z = tcg_temp_new();
-
-				/* To avoid a shift we overlay everything on
-				   the V flag.  */
-				tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
-				tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
-
-				tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
-				tcg_gen_or_tl(cc, z, n);
-				tcg_gen_andi_tl(cc, cc, 2);
-
-				tcg_temp_free(n);
-				tcg_temp_free(z);
-			}
-			break;
-		case CC_P:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], P_FLAG);
-			break;
-		case CC_A:
-			tcg_gen_movi_tl(cc, 1);
-			break;
-		default:
-			BUG();
-			break;
-	};
+    int arith_opt, move_opt;
+
+    /* TODO: optimize more condition codes.  */
+
+    /*
+     * If the flags are live, we've gotta look into the bits of CCS.
+     * Otherwise, if we just did an arithmetic operation we try to
+     * evaluate the condition code faster.
+     *
+     * When this function is done, T0 should be non-zero if the condition
+     * code is true.
+     */
+    arith_opt = arith_cc(dc) && !dc->flags_uptodate;
+    move_opt = (dc->cc_op == CC_OP_MOVE);
+    switch (cond) {
+    case CC_EQ:
+        if ((arith_opt || move_opt)
+                && dc->cc_x_uptodate != (2 | X_FLAG)) {
+            tcg_gen_setcond_tl(TCG_COND_EQ, cc,
+                    cc_result, tcg_const_tl(0));
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_andi_tl(cc,
+                    cpu_PR[PR_CCS], Z_FLAG);
+        }
+        break;
+    case CC_NE:
+        if ((arith_opt || move_opt)
+                && dc->cc_x_uptodate != (2 | X_FLAG)) {
+            tcg_gen_mov_tl(cc, cc_result);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                    Z_FLAG);
+            tcg_gen_andi_tl(cc, cc, Z_FLAG);
+        }
+        break;
+    case CC_CS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], C_FLAG);
+        break;
+    case CC_CC:
+        cris_evaluate_flags(dc);
+        tcg_gen_xori_tl(cc, cpu_PR[PR_CCS], C_FLAG);
+        tcg_gen_andi_tl(cc, cc, C_FLAG);
+        break;
+    case CC_VS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], V_FLAG);
+        break;
+    case CC_VC:
+        cris_evaluate_flags(dc);
+        tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                V_FLAG);
+        tcg_gen_andi_tl(cc, cc, V_FLAG);
+        break;
+    case CC_PL:
+        if (arith_opt || move_opt) {
+            int bits = 31;
+
+            if (dc->cc_size == 1) {
+                bits = 7;
+            } else if (dc->cc_size == 2) {
+                bits = 15;
+            }
+
+            tcg_gen_shri_tl(cc, cc_result, bits);
+            tcg_gen_xori_tl(cc, cc, 1);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                    N_FLAG);
+            tcg_gen_andi_tl(cc, cc, N_FLAG);
+        }
+        break;
+    case CC_MI:
+        if (arith_opt || move_opt) {
+            int bits = 31;
+
+            if (dc->cc_size == 1) {
+                bits = 7;
+            } else if (dc->cc_size == 2) {
+                bits = 15;
+            }
+
+            tcg_gen_shri_tl(cc, cc_result, bits);
+            tcg_gen_andi_tl(cc, cc, 1);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
+                    N_FLAG);
+        }
+        break;
+    case CC_LS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
+                C_FLAG | Z_FLAG);
+        break;
+    case CC_HI:
+        cris_evaluate_flags(dc);
+        {
+            TCGv tmp;
+
+            tmp = tcg_temp_new();
+            tcg_gen_xori_tl(tmp, cpu_PR[PR_CCS],
+                    C_FLAG | Z_FLAG);
+            /* Overlay the C flag on top of the Z.  */
+            tcg_gen_shli_tl(cc, tmp, 2);
+            tcg_gen_and_tl(cc, tmp, cc);
+            tcg_gen_andi_tl(cc, cc, Z_FLAG);
+
+            tcg_temp_free(tmp);
+        }
+        break;
+    case CC_GE:
+        cris_evaluate_flags(dc);
+        /* Overlay the V flag on top of the N.  */
+        tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
+        tcg_gen_xor_tl(cc,
+                cpu_PR[PR_CCS], cc);
+        tcg_gen_andi_tl(cc, cc, N_FLAG);
+        tcg_gen_xori_tl(cc, cc, N_FLAG);
+        break;
+    case CC_LT:
+        cris_evaluate_flags(dc);
+        /* Overlay the V flag on top of the N.  */
+        tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
+        tcg_gen_xor_tl(cc,
+                cpu_PR[PR_CCS], cc);
+        tcg_gen_andi_tl(cc, cc, N_FLAG);
+        break;
+    case CC_GT:
+        cris_evaluate_flags(dc);
+        {
+            TCGv n, z;
+
+            n = tcg_temp_new();
+            z = tcg_temp_new();
+
+            /* To avoid a shift we overlay everything on
+                   the V flag.  */
+            tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
+            tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
+            /* invert Z.  */
+            tcg_gen_xori_tl(z, z, 2);
+
+            tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
+            tcg_gen_xori_tl(n, n, 2);
+            tcg_gen_and_tl(cc, z, n);
+            tcg_gen_andi_tl(cc, cc, 2);
+
+            tcg_temp_free(n);
+            tcg_temp_free(z);
+        }
+        break;
+    case CC_LE:
+        cris_evaluate_flags(dc);
+        {
+            TCGv n, z;
+
+            n = tcg_temp_new();
+            z = tcg_temp_new();
+
+            /* To avoid a shift we overlay everything on
+                   the V flag.  */
+            tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
+            tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
+
+            tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
+            tcg_gen_or_tl(cc, z, n);
+            tcg_gen_andi_tl(cc, cc, 2);
+
+            tcg_temp_free(n);
+            tcg_temp_free(z);
+        }
+        break;
+    case CC_P:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], P_FLAG);
+        break;
+    case CC_A:
+        tcg_gen_movi_tl(cc, 1);
+        break;
+    default:
+        BUG();
+        break;
+    };
 }
 
 static void cris_store_direct_jmp(DisasContext *dc)
 {
-	/* Store the direct jmp state into the cpu-state.  */
-	if (dc->jmp == JMP_DIRECT || dc->jmp == JMP_DIRECT_CC) {
-		if (dc->jmp == JMP_DIRECT) {
-			tcg_gen_movi_tl(env_btaken, 1);
-		}
-		tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
-		dc->jmp = JMP_INDIRECT;
-	}
+    /* Store the direct jmp state into the cpu-state.  */
+    if (dc->jmp == JMP_DIRECT || dc->jmp == JMP_DIRECT_CC) {
+        if (dc->jmp == JMP_DIRECT) {
+            tcg_gen_movi_tl(env_btaken, 1);
+        }
+        tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
+        dc->jmp = JMP_INDIRECT;
+    }
 }
 
 static void cris_prepare_cc_branch (DisasContext *dc, 
-				    int offset, int cond)
+                    int offset, int cond)
 {
-	/* This helps us re-schedule the micro-code to insns in delay-slots
-	   before the actual jump.  */
-	dc->delayed_branch = 2;
-	dc->jmp = JMP_DIRECT_CC;
-	dc->jmp_pc = dc->pc + offset;
+    /* This helps us re-schedule the micro-code to insns in delay-slots
+       before the actual jump.  */
+    dc->delayed_branch = 2;
+    dc->jmp = JMP_DIRECT_CC;
+    dc->jmp_pc = dc->pc + offset;
 
-	gen_tst_cc (dc, env_btaken, cond);
-	tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
+    gen_tst_cc(dc, env_btaken, cond);
+    tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
 }
 
 
@@ -1140,1994 +1154,2013 @@ static void cris_prepare_cc_branch (DisasContext *dc,
    when the dest addr is constant to allow tb chaining.  */
 static inline void cris_prepare_jmp (DisasContext *dc, unsigned int type)
 {
-	/* This helps us re-schedule the micro-code to insns in delay-slots
-	   before the actual jump.  */
-	dc->delayed_branch = 2;
-	dc->jmp = type;
-	if (type == JMP_INDIRECT) {
-		tcg_gen_movi_tl(env_btaken, 1);
-	}
+    /* This helps us re-schedule the micro-code to insns in delay-slots
+       before the actual jump.  */
+    dc->delayed_branch = 2;
+    dc->jmp = type;
+    if (type == JMP_INDIRECT) {
+        tcg_gen_movi_tl(env_btaken, 1);
+    }
 }
 
 static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 {
-	int mem_index = cpu_mmu_index(dc->env);
+    int mem_index = cpu_mmu_index(dc->env);
 
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
-		cris_store_direct_jmp(dc);
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
 
-        tcg_gen_qemu_ld64(dst, addr, mem_index);
+    tcg_gen_qemu_ld64(dst, addr, mem_index);
 }
 
 static void gen_load(DisasContext *dc, TCGv dst, TCGv addr, 
-		     unsigned int size, int sign)
-{
-	int mem_index = cpu_mmu_index(dc->env);
-
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
-		cris_store_direct_jmp(dc);
-
-	if (size == 1) {
-		if (sign)
-			tcg_gen_qemu_ld8s(dst, addr, mem_index);
-		else
-			tcg_gen_qemu_ld8u(dst, addr, mem_index);
-	}
-	else if (size == 2) {
-		if (sign)
-			tcg_gen_qemu_ld16s(dst, addr, mem_index);
-		else
-			tcg_gen_qemu_ld16u(dst, addr, mem_index);
-	}
-	else if (size == 4) {
-		tcg_gen_qemu_ld32u(dst, addr, mem_index);
-	}
-	else {
-		abort();
-	}
+             unsigned int size, int sign)
+{
+    int mem_index = cpu_mmu_index(dc->env);
+
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
+
+    if (size == 1) {
+        if (sign) {
+            tcg_gen_qemu_ld8s(dst, addr, mem_index);
+        } else {
+            tcg_gen_qemu_ld8u(dst, addr, mem_index);
+        }
+    } else if (size == 2) {
+        if (sign) {
+            tcg_gen_qemu_ld16s(dst, addr, mem_index);
+        } else {
+            tcg_gen_qemu_ld16u(dst, addr, mem_index);
+        }
+    } else if (size == 4) {
+        tcg_gen_qemu_ld32u(dst, addr, mem_index);
+    } else {
+        abort();
+    }
 }
 
 static void gen_store (DisasContext *dc, TCGv addr, TCGv val,
-		       unsigned int size)
+               unsigned int size)
 {
-	int mem_index = cpu_mmu_index(dc->env);
+    int mem_index = cpu_mmu_index(dc->env);
 
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
- 		cris_store_direct_jmp(dc);
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
 
 
-	/* Conditional writes. We only support the kind were X and P are known
-	   at translation time.  */
-	if (dc->flagx_known && dc->flags_x && (dc->tb_flags & P_FLAG)) {
-		dc->postinc = 0;
-		cris_evaluate_flags(dc);
-		tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], C_FLAG);
-		return;
-	}
+    /* Conditional writes. We only support the kind were X and P are known
+       at translation time.  */
+    if (dc->flagx_known && dc->flags_x && (dc->tb_flags & P_FLAG)) {
+        dc->postinc = 0;
+        cris_evaluate_flags(dc);
+        tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], C_FLAG);
+        return;
+    }
 
-	if (size == 1)
-		tcg_gen_qemu_st8(val, addr, mem_index);
-	else if (size == 2)
-		tcg_gen_qemu_st16(val, addr, mem_index);
-	else
-		tcg_gen_qemu_st32(val, addr, mem_index);
+    if (size == 1) {
+        tcg_gen_qemu_st8(val, addr, mem_index);
+    } else if (size == 2) {
+        tcg_gen_qemu_st16(val, addr, mem_index);
+    } else {
+        tcg_gen_qemu_st32(val, addr, mem_index);
+    }
 
-	if (dc->flagx_known && dc->flags_x) {
-		cris_evaluate_flags(dc);
-		tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~C_FLAG);
-	}
+    if (dc->flagx_known && dc->flags_x) {
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~C_FLAG);
+    }
 }
 
 static inline void t_gen_sext(TCGv d, TCGv s, int size)
 {
-	if (size == 1)
-		tcg_gen_ext8s_i32(d, s);
-	else if (size == 2)
-		tcg_gen_ext16s_i32(d, s);
-	else if(!TCGV_EQUAL(d, s))
-		tcg_gen_mov_tl(d, s);
+    if (size == 1) {
+        tcg_gen_ext8s_i32(d, s);
+    } else if (size == 2) {
+        tcg_gen_ext16s_i32(d, s);
+    } else if (!TCGV_EQUAL(d, s)) {
+        tcg_gen_mov_tl(d, s);
+    }
 }
 
 static inline void t_gen_zext(TCGv d, TCGv s, int size)
 {
-	if (size == 1)
-		tcg_gen_ext8u_i32(d, s);
-	else if (size == 2)
-		tcg_gen_ext16u_i32(d, s);
-	else if (!TCGV_EQUAL(d, s))
-		tcg_gen_mov_tl(d, s);
+    if (size == 1) {
+        tcg_gen_ext8u_i32(d, s);
+    } else if (size == 2) {
+        tcg_gen_ext16u_i32(d, s);
+    } else if (!TCGV_EQUAL(d, s)) {
+        tcg_gen_mov_tl(d, s);
+    }
 }
 
 #if DISAS_CRIS
 static char memsize_char(int size)
 {
-	switch (size)
-	{
-		case 1: return 'b';  break;
-		case 2: return 'w';  break;
-		case 4: return 'd';  break;
-		default:
-			return 'x';
-			break;
-	}
+    switch (size) {
+    case 1: return 'b';  break;
+    case 2: return 'w';  break;
+    case 4: return 'd';  break;
+    default:
+        return 'x';
+        break;
+    }
 }
 #endif
 
 static inline unsigned int memsize_z(DisasContext *dc)
 {
-	return dc->zsize + 1;
+    return dc->zsize + 1;
 }
 
 static inline unsigned int memsize_zz(DisasContext *dc)
 {
-	switch (dc->zzsize)
-	{
-		case 0: return 1;
-		case 1: return 2;
-		default:
-			return 4;
-	}
+    switch (dc->zzsize) {
+    case 0: return 1;
+    case 1: return 2;
+    default:
+        return 4;
+    }
 }
 
 static inline void do_postinc (DisasContext *dc, int size)
 {
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], size);
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], size);
+    }
 }
 
 static inline void dec_prep_move_r(DisasContext *dc, int rs, int rd,
-				   int size, int s_ext, TCGv dst)
+                   int size, int s_ext, TCGv dst)
 {
-	if (s_ext)
-		t_gen_sext(dst, cpu_R[rs], size);
-	else
-		t_gen_zext(dst, cpu_R[rs], size);
+    if (s_ext) {
+        t_gen_sext(dst, cpu_R[rs], size);
+    } else {
+        t_gen_zext(dst, cpu_R[rs], size);
+    }
 }
 
 /* Prepare T0 and T1 for a register alu operation.
    s_ext decides if the operand1 should be sign-extended or zero-extended when
    needed.  */
 static void dec_prep_alu_r(DisasContext *dc, int rs, int rd,
-			  int size, int s_ext, TCGv dst, TCGv src)
+              int size, int s_ext, TCGv dst, TCGv src)
 {
-	dec_prep_move_r(dc, rs, rd, size, s_ext, src);
+    dec_prep_move_r(dc, rs, rd, size, s_ext, src);
 
-	if (s_ext)
-		t_gen_sext(dst, cpu_R[rd], size);
-	else
-		t_gen_zext(dst, cpu_R[rd], size);
+    if (s_ext) {
+        t_gen_sext(dst, cpu_R[rd], size);
+    } else {
+        t_gen_zext(dst, cpu_R[rd], size);
+    }
 }
 
-static int dec_prep_move_m(DisasContext *dc, int s_ext, int memsize,
-			   TCGv dst)
+static int dec_prep_move_m(CPUCRISState *env, DisasContext *dc,
+                           int s_ext, int memsize, TCGv dst)
 {
-	unsigned int rs;
-	uint32_t imm;
-	int is_imm;
-	int insn_len = 2;
+    unsigned int rs;
+    uint32_t imm;
+    int is_imm;
+    int insn_len = 2;
 
-	rs = dc->op1;
-	is_imm = rs == 15 && dc->postinc;
+    rs = dc->op1;
+    is_imm = rs == 15 && dc->postinc;
 
-	/* Load [$rs] onto T1.  */
-	if (is_imm) {
-		insn_len = 2 + memsize;
-		if (memsize == 1)
-			insn_len++;
+    /* Load [$rs] onto T1.  */
+    if (is_imm) {
+        insn_len = 2 + memsize;
+        if (memsize == 1) {
+            insn_len++;
+        }
 
-		imm = cris_fetch(dc, dc->pc + 2, memsize, s_ext);
-		tcg_gen_movi_tl(dst, imm);
-		dc->postinc = 0;
-	} else {
-		cris_flush_cc_state(dc);
-		gen_load(dc, dst, cpu_R[rs], memsize, 0);
-		if (s_ext)
-			t_gen_sext(dst, dst, memsize);
-		else
-			t_gen_zext(dst, dst, memsize);
-	}
-	return insn_len;
+        imm = cris_fetch(env, dc, dc->pc + 2, memsize, s_ext);
+        tcg_gen_movi_tl(dst, imm);
+        dc->postinc = 0;
+    } else {
+        cris_flush_cc_state(dc);
+        gen_load(dc, dst, cpu_R[rs], memsize, 0);
+        if (s_ext) {
+            t_gen_sext(dst, dst, memsize);
+        } else {
+            t_gen_zext(dst, dst, memsize);
+        }
+    }
+    return insn_len;
 }
 
 /* Prepare T0 and T1 for a memory + alu operation.
    s_ext decides if the operand1 should be sign-extended or zero-extended when
    needed.  */
-static int dec_prep_alu_m(DisasContext *dc, int s_ext, int memsize,
-			  TCGv dst, TCGv src)
+static int dec_prep_alu_m(CPUCRISState *env, DisasContext *dc,
+                          int s_ext, int memsize, TCGv dst, TCGv src)
 {
-	int insn_len;
+    int insn_len;
 
-	insn_len = dec_prep_move_m(dc, s_ext, memsize, src);
-	tcg_gen_mov_tl(dst, cpu_R[dc->op2]);
-	return insn_len;
+    insn_len = dec_prep_move_m(env, dc, s_ext, memsize, src);
+    tcg_gen_mov_tl(dst, cpu_R[dc->op2]);
+    return insn_len;
 }
 
 #if DISAS_CRIS
 static const char *cc_name(int cc)
 {
-	static const char *cc_names[16] = {
-		"cc", "cs", "ne", "eq", "vc", "vs", "pl", "mi",
-		"ls", "hi", "ge", "lt", "gt", "le", "a", "p"
-	};
-	assert(cc < 16);
-	return cc_names[cc];
+    static const char *cc_names[16] = {
+        "cc", "cs", "ne", "eq", "vc", "vs", "pl", "mi",
+        "ls", "hi", "ge", "lt", "gt", "le", "a", "p"
+    };
+    assert(cc < 16);
+    return cc_names[cc];
 }
 #endif
 
 /* Start of insn decoders.  */
 
-static int dec_bccq(DisasContext *dc)
+static int dec_bccq(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t offset;
-	int sign;
-	uint32_t cond = dc->op2;
+    int32_t offset;
+    int sign;
+    uint32_t cond = dc->op2;
 
-	offset = EXTRACT_FIELD (dc->ir, 1, 7);
-	sign = EXTRACT_FIELD(dc->ir, 0, 0);
+    offset = EXTRACT_FIELD(dc->ir, 1, 7);
+    sign = EXTRACT_FIELD(dc->ir, 0, 0);
 
-	offset *= 2;
-	offset |= sign << 8;
-	offset = sign_extend(offset, 8);
+    offset *= 2;
+    offset |= sign << 8;
+    offset = sign_extend(offset, 8);
 
-	LOG_DIS("b%s %x\n", cc_name(cond), dc->pc + offset);
+    LOG_DIS("b%s %x\n", cc_name(cond), dc->pc + offset);
 
-	/* op2 holds the condition-code.  */
-	cris_cc_mask(dc, 0);
-	cris_prepare_cc_branch (dc, offset, cond);
-	return 2;
+    /* op2 holds the condition-code.  */
+    cris_cc_mask(dc, 0);
+    cris_prepare_cc_branch(dc, offset, cond);
+    return 2;
 }
-static int dec_addoq(DisasContext *dc)
+static int dec_addoq(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t imm;
+    int32_t imm;
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 7);
-	imm = sign_extend(dc->op1, 7);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 7);
+    imm = sign_extend(dc->op1, 7);
 
-	LOG_DIS("addoq %d, $r%u\n", imm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Fetch register operand,  */
-	tcg_gen_addi_tl(cpu_R[R_ACR], cpu_R[dc->op2], imm);
+    LOG_DIS("addoq %d, $r%u\n", imm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Fetch register operand,  */
+    tcg_gen_addi_tl(cpu_R[R_ACR], cpu_R[dc->op2], imm);
 
-	return 2;
+    return 2;
 }
-static int dec_addq(DisasContext *dc)
+static int dec_addq(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("addq %u, $r%u\n", dc->op1, dc->op2);
+    LOG_DIS("addq %u, $r%u\n", dc->op1, dc->op2);
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_cc_mask(dc, CC_MASK_NZVC);
 
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
-	return 2;
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
+    return 2;
 }
-static int dec_moveq(DisasContext *dc)
+static int dec_moveq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
-	LOG_DIS("moveq %d, $r%u\n", imm, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
+    LOG_DIS("moveq %d, $r%u\n", imm, dc->op2);
 
-	tcg_gen_movi_tl(cpu_R[dc->op2], imm);
-	return 2;
+    tcg_gen_movi_tl(cpu_R[dc->op2], imm);
+    return 2;
 }
-static int dec_subq(DisasContext *dc)
+static int dec_subq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
 
-	LOG_DIS("subq %u, $r%u\n", dc->op1, dc->op2);
+    LOG_DIS("subq %u, $r%u\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
+    return 2;
 }
-static int dec_cmpq(DisasContext *dc)
+static int dec_cmpq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
 
-	LOG_DIS("cmpq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
+    LOG_DIS("cmpq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
 
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_andq(DisasContext *dc)
+static int dec_andq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
 
-	LOG_DIS("andq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    LOG_DIS("andq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu(dc, CC_OP_AND,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_AND,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_orq(DisasContext *dc)
+static int dec_orq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
-	LOG_DIS("orq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
+    LOG_DIS("orq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu(dc, CC_OP_OR,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_OR,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_btstq(DisasContext *dc)
+static int dec_btstq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("btstq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("btstq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_evaluate_flags(dc);
-	gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->op2],
-			tcg_const_tl(dc->op1), cpu_PR[PR_CCS]);
-	cris_alu(dc, CC_OP_MOVE,
-		 cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->flags_uptodate = 1;
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_evaluate_flags(dc);
+        gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->op2],
+            tcg_const_tl(dc->op1), cpu_PR[PR_CCS]);
+    cris_alu(dc, CC_OP_MOVE,
+         cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->flags_uptodate = 1;
+    return 2;
 }
-static int dec_asrq(DisasContext *dc)
+static int dec_asrq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("asrq %u, $r%d\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("asrq %u, $r%d\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_sari_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    tcg_gen_sari_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
-static int dec_lslq(DisasContext *dc)
+static int dec_lslq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("lslq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("lslq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_shli_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    tcg_gen_shli_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
 
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
-static int dec_lsrq(DisasContext *dc)
+static int dec_lsrq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("lsrq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("lsrq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_shri_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    tcg_gen_shri_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
 
-static int dec_move_r(DisasContext *dc)
+static int dec_move_r(CPUCRISState *env, DisasContext *dc)
 {
-	int size = memsize_zz(dc);
+    int size = memsize_zz(dc);
 
-	LOG_DIS("move.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("move.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	if (size == 4) {
-		dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, cpu_R[dc->op2]);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_update_cc_op(dc, CC_OP_MOVE, 4);
-		cris_update_cc_x(dc);
-		cris_update_result(dc, cpu_R[dc->op2]);
-	}
-	else {
-		TCGv t0;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    if (size == 4) {
+        dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, cpu_R[dc->op2]);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_update_cc_op(dc, CC_OP_MOVE, 4);
+        cris_update_cc_x(dc);
+        cris_update_result(dc, cpu_R[dc->op2]);
+    } else {
+        TCGv t0;
 
-		t0 = tcg_temp_new();
-		dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
-		cris_alu(dc, CC_OP_MOVE,
-			 cpu_R[dc->op2],
-			 cpu_R[dc->op2], t0, size);
-		tcg_temp_free(t0);
-	}
-	return 2;
+        t0 = tcg_temp_new();
+        dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
+        cris_alu(dc, CC_OP_MOVE,
+             cpu_R[dc->op2],
+             cpu_R[dc->op2], t0, size);
+        tcg_temp_free(t0);
+    }
+    return 2;
 }
 
-static int dec_scc_r(DisasContext *dc)
+static int dec_scc_r(CPUCRISState *env, DisasContext *dc)
 {
-	int cond = dc->op2;
+    int cond = dc->op2;
 
-	LOG_DIS("s%s $r%u\n",
-		    cc_name(cond), dc->op1);
+    LOG_DIS("s%s $r%u\n",
+            cc_name(cond), dc->op1);
 
-	if (cond != CC_A)
-	{
-		int l1;
+    if (cond != CC_A) {
+        int l1;
 
-		gen_tst_cc (dc, cpu_R[dc->op1], cond);
-		l1 = gen_new_label();
-		tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
-		tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-		gen_set_label(l1);
-	}
-	else
-		tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+        gen_tst_cc(dc, cpu_R[dc->op1], cond);
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
+        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+        gen_set_label(l1);
+    } else {
+        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+    }
 
-	cris_cc_mask(dc, 0);
-	return 2;
+    cris_cc_mask(dc, 0);
+    return 2;
 }
 
 static inline void cris_alu_alloc_temps(DisasContext *dc, int size, TCGv *t)
 {
-	if (size == 4) {
-		t[0] = cpu_R[dc->op2];
-		t[1] = cpu_R[dc->op1];
-	} else {
-		t[0] = tcg_temp_new();
-		t[1] = tcg_temp_new();
-	}
+    if (size == 4) {
+        t[0] = cpu_R[dc->op2];
+        t[1] = cpu_R[dc->op1];
+    } else {
+        t[0] = tcg_temp_new();
+        t[1] = tcg_temp_new();
+    }
 }
 
 static inline void cris_alu_free_temps(DisasContext *dc, int size, TCGv *t)
 {
-	if (size != 4) {
-		tcg_temp_free(t[0]);
-		tcg_temp_free(t[1]);
-	}
+    if (size != 4) {
+        tcg_temp_free(t[0]);
+        tcg_temp_free(t[1]);
+    }
 }
 
-static int dec_and_r(DisasContext *dc)
+static int dec_and_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("and.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("and.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_lz_r(DisasContext *dc)
+static int dec_lz_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	LOG_DIS("lz $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	dec_prep_alu_r(dc, dc->op1, dc->op2, 4, 0, cpu_R[dc->op2], t0);
-	cris_alu(dc, CC_OP_LZ, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    TCGv t0;
+    LOG_DIS("lz $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    dec_prep_alu_r(dc, dc->op1, dc->op2, 4, 0, cpu_R[dc->op2], t0);
+    cris_alu(dc, CC_OP_LZ, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
-static int dec_lsl_r(DisasContext *dc)
+static int dec_lsl_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("lsl.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("lsl.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_LSL, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_alloc_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_LSL, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_alloc_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_lsr_r(DisasContext *dc)
+static int dec_lsr_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("lsr.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("lsr.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_LSR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_LSR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_asr_r(DisasContext *dc)
+static int dec_asr_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("asr.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("asr.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_ASR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_ASR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_muls_r(DisasContext *dc)
+static int dec_muls_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("muls.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZV);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
+    LOG_DIS("muls.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZV);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_MULS, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_MULS, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_mulu_r(DisasContext *dc)
+static int dec_mulu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("mulu.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZV);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    LOG_DIS("mulu.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZV);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_MULU, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_alloc_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_MULU, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_alloc_temps(dc, size, t);
+    return 2;
 }
 
 
-static int dec_dstep_r(DisasContext *dc)
+static int dec_dstep_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("dstep $r%u, $r%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_DSTEP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
-	return 2;
+    LOG_DIS("dstep $r%u, $r%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_DSTEP,
+            cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
+    return 2;
 }
 
-static int dec_xor_r(DisasContext *dc)
+static int dec_xor_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("xor.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	BUG_ON(size != 4); /* xor is dword.  */
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("xor.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    BUG_ON(size != 4); /* xor is dword.  */
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_XOR, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_XOR, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_bound_r(DisasContext *dc)
+static int dec_bound_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv l0;
-	int size = memsize_zz(dc);
-	LOG_DIS("bound.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	l0 = tcg_temp_local_new();
-	dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, l0);
-	cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], cpu_R[dc->op2], l0, 4);
-	tcg_temp_free(l0);
-	return 2;
+    TCGv l0;
+    int size = memsize_zz(dc);
+    LOG_DIS("bound.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    l0 = tcg_temp_local_new();
+    dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, l0);
+    cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], cpu_R[dc->op2], l0, 4);
+    tcg_temp_free(l0);
+    return 2;
 }
 
-static int dec_cmp_r(DisasContext *dc)
+static int dec_cmp_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("cmp.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("cmp.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_abs_r(DisasContext *dc)
+static int dec_abs_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
+    TCGv t0;
 
-	LOG_DIS("abs $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    LOG_DIS("abs $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	t0 = tcg_temp_new();
-	tcg_gen_sari_tl(t0, cpu_R[dc->op1], 31);
-	tcg_gen_xor_tl(cpu_R[dc->op2], cpu_R[dc->op1], t0);
-	tcg_gen_sub_tl(cpu_R[dc->op2], cpu_R[dc->op2], t0);
-	tcg_temp_free(t0);
+    t0 = tcg_temp_new();
+    tcg_gen_sari_tl(t0, cpu_R[dc->op1], 31);
+    tcg_gen_xor_tl(cpu_R[dc->op2], cpu_R[dc->op1], t0);
+    tcg_gen_sub_tl(cpu_R[dc->op2], cpu_R[dc->op2], t0);
+    tcg_temp_free(t0);
 
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
 
-static int dec_add_r(DisasContext *dc)
+static int dec_add_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("add.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("add.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_addc_r(DisasContext *dc)
+static int dec_addc_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("addc $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_evaluate_flags(dc);
-	/* Set for this insn.  */
-	dc->flagx_known = 1;
-	dc->flags_x = X_FLAG;
+    LOG_DIS("addc $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_evaluate_flags(dc);
+    /* Set for this insn.  */
+    dc->flagx_known = 1;
+    dc->flags_x = X_FLAG;
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADDC,
-		 cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADDC,
+         cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
+    return 2;
 }
 
-static int dec_mcp_r(DisasContext *dc)
+static int dec_mcp_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("mcp $p%u, $r%u\n",
-		     dc->op2, dc->op1);
-	cris_evaluate_flags(dc);
-	cris_cc_mask(dc, CC_MASK_RNZV);
-	cris_alu(dc, CC_OP_MCP,
-		    cpu_R[dc->op1], cpu_R[dc->op1], cpu_PR[dc->op2], 4);
-	return 2;
+    LOG_DIS("mcp $p%u, $r%u\n",
+             dc->op2, dc->op1);
+    cris_evaluate_flags(dc);
+    cris_cc_mask(dc, CC_MASK_RNZV);
+    cris_alu(dc, CC_OP_MCP,
+            cpu_R[dc->op1], cpu_R[dc->op1], cpu_PR[dc->op2], 4);
+    return 2;
 }
 
 #if DISAS_CRIS
 static char * swapmode_name(int mode, char *modename) {
-	int i = 0;
-	if (mode & 8)
-		modename[i++] = 'n';
-	if (mode & 4)
-		modename[i++] = 'w';
-	if (mode & 2)
-		modename[i++] = 'b';
-	if (mode & 1)
-		modename[i++] = 'r';
-	modename[i++] = 0;
-	return modename;
+    int i = 0;
+    if (mode & 8) {
+        modename[i++] = 'n';
+    }
+    if (mode & 4) {
+        modename[i++] = 'w';
+    }
+    if (mode & 2) {
+        modename[i++] = 'b';
+    }
+    if (mode & 1) {
+        modename[i++] = 'r';
+    }
+    modename[i++] = 0;
+    return modename;
 }
 #endif
 
-static int dec_swap_r(DisasContext *dc)
+static int dec_swap_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
+    TCGv t0;
 #if DISAS_CRIS
-	char modename[4];
+    char modename[4];
 #endif
-	LOG_DIS("swap%s $r%u\n",
-		     swapmode_name(dc->op2, modename), dc->op1);
-
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	t_gen_mov_TN_reg(t0, dc->op1);
-	if (dc->op2 & 8)
-		tcg_gen_not_tl(t0, t0);
-	if (dc->op2 & 4)
-		t_gen_swapw(t0, t0);
-	if (dc->op2 & 2)
-		t_gen_swapb(t0, t0);
-	if (dc->op2 & 1)
-		t_gen_swapr(t0, t0);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op1], cpu_R[dc->op1], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_or_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("or.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_OR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
-}
-
-static int dec_addi_r(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("addi.%c $r%u, $r%u\n",
-		    memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	t0 = tcg_temp_new();
-	tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
-	tcg_gen_add_tl(cpu_R[dc->op1], cpu_R[dc->op1], t0);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_addi_acr(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("addi.%c $r%u, $r%u, $acr\n",
-		  memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	t0 = tcg_temp_new();
-	tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
-	tcg_gen_add_tl(cpu_R[R_ACR], cpu_R[dc->op1], t0);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_neg_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("neg.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-
-	cris_alu(dc, CC_OP_NEG, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
-}
-
-static int dec_btst_r(DisasContext *dc)
-{
-	LOG_DIS("btst $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_evaluate_flags(dc);
-	gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->op2],
-			cpu_R[dc->op1], cpu_PR[PR_CCS]);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2],
-		 cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->flags_uptodate = 1;
-	return 2;
-}
-
-static int dec_sub_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("sub.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    LOG_DIS("swap%s $r%u\n",
+             swapmode_name(dc->op2, modename), dc->op1);
+
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    t_gen_mov_TN_reg(t0, dc->op1);
+    if (dc->op2 & 8) {
+        tcg_gen_not_tl(t0, t0);
+    }
+    if (dc->op2 & 4) {
+        t_gen_swapw(t0, t0);
+    }
+    if (dc->op2 & 2) {
+        t_gen_swapb(t0, t0);
+    }
+    if (dc->op2 & 1) {
+        t_gen_swapr(t0, t0);
+    }
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op1], cpu_R[dc->op1], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_or_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("or.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_OR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
+}
+
+static int dec_addi_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("addi.%c $r%u, $r%u\n",
+            memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+    t0 = tcg_temp_new();
+    tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
+    tcg_gen_add_tl(cpu_R[dc->op1], cpu_R[dc->op1], t0);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_addi_acr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("addi.%c $r%u, $r%u, $acr\n",
+          memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+    t0 = tcg_temp_new();
+    tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
+    tcg_gen_add_tl(cpu_R[R_ACR], cpu_R[dc->op1], t0);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_neg_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("neg.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+
+    cris_alu(dc, CC_OP_NEG, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
+}
+
+static int dec_btst_r(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("btst $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_evaluate_flags(dc);
+        gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->op2],
+            cpu_R[dc->op1], cpu_PR[PR_CCS]);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2],
+         cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->flags_uptodate = 1;
+    return 2;
+}
+
+static int dec_sub_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("sub.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
 /* Zero extension. From size to dword.  */
-static int dec_movu_r(DisasContext *dc)
+static int dec_movu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("movu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("movu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_movs_r(DisasContext *dc)
+static int dec_movs_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("movs.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("movs.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op1], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op1], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* zero extension. From size to dword.  */
-static int dec_addu_r(DisasContext *dc)
+static int dec_addu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("addu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("addu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_zext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_zext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_adds_r(DisasContext *dc)
+static int dec_adds_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("adds.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("adds.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Zero extension. From size to dword.  */
-static int dec_subu_r(DisasContext *dc)
+static int dec_subu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("subu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("subu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_zext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_zext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_subs_r(DisasContext *dc)
-{
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("subs.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
-
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_setclrf(DisasContext *dc)
-{
-	uint32_t flags;
-	int set = (~dc->opcode >> 2) & 1;
-
-
-	flags = (EXTRACT_FIELD(dc->ir, 12, 15) << 4)
-		| EXTRACT_FIELD(dc->ir, 0, 3);
-	if (set && flags == 0) {
-		LOG_DIS("nop\n");
-		return 2;
-	} else if (!set && (flags & 0x20)) {
-		LOG_DIS("di\n");
-	}
-	else {
-		LOG_DIS("%sf %x\n",
-			     set ? "set" : "clr",
-			    flags);
-	}
-
-	/* User space is not allowed to touch these. Silently ignore.  */
-	if (dc->tb_flags & U_FLAG) {
-		flags &= ~(S_FLAG | I_FLAG | U_FLAG);
-	}
-
-	if (flags & X_FLAG) {
-		dc->flagx_known = 1;
-		if (set)
-			dc->flags_x = X_FLAG;
-		else
-			dc->flags_x = 0;
-	}
-
-	/* Break the TB if any of the SPI flag changes.  */
-	if (flags & (P_FLAG | S_FLAG)) {
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		dc->is_jmp = DISAS_UPDATE;
-		dc->cpustate_changed = 1;
-	}
-
-	/* For the I flag, only act on posedge.  */
-	if ((flags & I_FLAG)) {
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		dc->is_jmp = DISAS_UPDATE;
-		dc->cpustate_changed = 1;
-	}
-
-
-	/* Simply decode the flags.  */
-	cris_evaluate_flags (dc);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	cris_update_cc_x(dc);
-	tcg_gen_movi_tl(cc_op, dc->cc_op);
-
-	if (set) {
-		if (!(dc->tb_flags & U_FLAG) && (flags & U_FLAG)) {
-			/* Enter user mode.  */
-			t_gen_mov_env_TN(ksp, cpu_R[R_SP]);
-			tcg_gen_mov_tl(cpu_R[R_SP], cpu_PR[PR_USP]);
-			dc->cpustate_changed = 1;
-		}
-		tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], flags);
-	}
-	else
-		tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~flags);
-
-	dc->flags_uptodate = 1;
-	dc->clear_x = 0;
-	return 2;
-}
-
-static int dec_move_rs(DisasContext *dc)
-{
-	LOG_DIS("move $r%u, $s%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	gen_helper_movl_sreg_reg(tcg_const_tl(dc->op2), tcg_const_tl(dc->op1));
-	return 2;
-}
-static int dec_move_sr(DisasContext *dc)
-{
-	LOG_DIS("move $s%u, $r%u\n", dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	gen_helper_movl_reg_sreg(tcg_const_tl(dc->op1), tcg_const_tl(dc->op2));
-	return 2;
-}
-
-static int dec_move_rp(DisasContext *dc)
-{
-	TCGv t[2];
-	LOG_DIS("move $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-
-	t[0] = tcg_temp_new();
-	if (dc->op2 == PR_CCS) {
-		cris_evaluate_flags(dc);
-		t_gen_mov_TN_reg(t[0], dc->op1);
-		if (dc->tb_flags & U_FLAG) {
-			t[1] = tcg_temp_new();
-			/* User space is not allowed to touch all flags.  */
-			tcg_gen_andi_tl(t[0], t[0], 0x39f);
-			tcg_gen_andi_tl(t[1], cpu_PR[PR_CCS], ~0x39f);
-			tcg_gen_or_tl(t[0], t[1], t[0]);
-			tcg_temp_free(t[1]);
-		}
-	}
-	else
-		t_gen_mov_TN_reg(t[0], dc->op1);
-
-	t_gen_mov_preg_TN(dc, dc->op2, t[0]);
-	if (dc->op2 == PR_CCS) {
-		cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-		dc->flags_uptodate = 1;
-	}
-	tcg_temp_free(t[0]);
-	return 2;
-}
-static int dec_move_pr(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("move $p%u, $r%u\n", dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-
-        if (dc->op2 == PR_DZ) {
-		tcg_gen_movi_tl(cpu_R[dc->op1], 0);
+static int dec_subs_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("subs.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
+
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_setclrf(CPUCRISState *env, DisasContext *dc)
+{
+    uint32_t flags;
+    int set = (~dc->opcode >> 2) & 1;
+
+
+    flags = (EXTRACT_FIELD(dc->ir, 12, 15) << 4)
+        | EXTRACT_FIELD(dc->ir, 0, 3);
+    if (set && flags == 0) {
+        LOG_DIS("nop\n");
+        return 2;
+    } else if (!set && (flags & 0x20)) {
+        LOG_DIS("di\n");
+    } else {
+        LOG_DIS("%sf %x\n", set ? "set" : "clr", flags);
+    }
+
+    /* User space is not allowed to touch these. Silently ignore.  */
+    if (dc->tb_flags & U_FLAG) {
+        flags &= ~(S_FLAG | I_FLAG | U_FLAG);
+    }
+
+    if (flags & X_FLAG) {
+        dc->flagx_known = 1;
+        if (set) {
+            dc->flags_x = X_FLAG;
         } else {
-		t0 = tcg_temp_new();
-		t_gen_mov_TN_preg(t0, dc->op2);
-		cris_alu(dc, CC_OP_MOVE,
-			 cpu_R[dc->op1], cpu_R[dc->op1], t0,
-			 preg_sizes[dc->op2]);
-		tcg_temp_free(t0);
-	}
-	return 2;
-}
-
-static int dec_move_mr(DisasContext *dc)
-{
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("move.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	if (memsize == 4) {
-		insn_len = dec_prep_move_m(dc, 0, 4, cpu_R[dc->op2]);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_update_cc_op(dc, CC_OP_MOVE, 4);
-		cris_update_cc_x(dc);
-		cris_update_result(dc, cpu_R[dc->op2]);
-	}
-	else {
-		TCGv t0;
-
-		t0 = tcg_temp_new();
-		insn_len = dec_prep_move_m(dc, 0, memsize, t0);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_alu(dc, CC_OP_MOVE,
-			    cpu_R[dc->op2], cpu_R[dc->op2], t0, memsize);
-		tcg_temp_free(t0);
-	}
-	do_postinc(dc, memsize);
-	return insn_len;
+            dc->flags_x = 0;
+        }
+    }
+
+    /* Break the TB if any of the SPI flag changes.  */
+    if (flags & (P_FLAG | S_FLAG)) {
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        dc->is_jmp = DISAS_UPDATE;
+        dc->cpustate_changed = 1;
+    }
+
+    /* For the I flag, only act on posedge.  */
+    if ((flags & I_FLAG)) {
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        dc->is_jmp = DISAS_UPDATE;
+        dc->cpustate_changed = 1;
+    }
+
+
+    /* Simply decode the flags.  */
+    cris_evaluate_flags(dc);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    cris_update_cc_x(dc);
+    tcg_gen_movi_tl(cc_op, dc->cc_op);
+
+    if (set) {
+        if (!(dc->tb_flags & U_FLAG) && (flags & U_FLAG)) {
+            /* Enter user mode.  */
+            t_gen_mov_env_TN(ksp, cpu_R[R_SP]);
+            tcg_gen_mov_tl(cpu_R[R_SP], cpu_PR[PR_USP]);
+            dc->cpustate_changed = 1;
+        }
+        tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], flags);
+    } else {
+        tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~flags);
+    }
+
+    dc->flags_uptodate = 1;
+    dc->clear_x = 0;
+    return 2;
+}
+
+static int dec_move_rs(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("move $r%u, $s%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+        gen_helper_movl_sreg_reg(cpu_env, tcg_const_tl(dc->op2),
+                                 tcg_const_tl(dc->op1));
+    return 2;
+}
+static int dec_move_sr(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("move $s%u, $r%u\n", dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+        gen_helper_movl_reg_sreg(cpu_env, tcg_const_tl(dc->op1),
+                                 tcg_const_tl(dc->op2));
+    return 2;
+}
+
+static int dec_move_rp(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    LOG_DIS("move $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+
+    t[0] = tcg_temp_new();
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+        t_gen_mov_TN_reg(t[0], dc->op1);
+        if (dc->tb_flags & U_FLAG) {
+            t[1] = tcg_temp_new();
+            /* User space is not allowed to touch all flags.  */
+            tcg_gen_andi_tl(t[0], t[0], 0x39f);
+            tcg_gen_andi_tl(t[1], cpu_PR[PR_CCS], ~0x39f);
+            tcg_gen_or_tl(t[0], t[1], t[0]);
+            tcg_temp_free(t[1]);
+        }
+    } else {
+        t_gen_mov_TN_reg(t[0], dc->op1);
+    }
+
+    t_gen_mov_preg_TN(dc, dc->op2, t[0]);
+    if (dc->op2 == PR_CCS) {
+        cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+        dc->flags_uptodate = 1;
+    }
+    tcg_temp_free(t[0]);
+    return 2;
+}
+static int dec_move_pr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("move $p%u, $r%u\n", dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+
+    if (dc->op2 == PR_DZ) {
+        tcg_gen_movi_tl(cpu_R[dc->op1], 0);
+    } else {
+        t0 = tcg_temp_new();
+        t_gen_mov_TN_preg(t0, dc->op2);
+        cris_alu(dc, CC_OP_MOVE,
+                cpu_R[dc->op1], cpu_R[dc->op1], t0,
+                preg_sizes[dc->op2]);
+        tcg_temp_free(t0);
+    }
+    return 2;
+}
+
+static int dec_move_mr(CPUCRISState *env, DisasContext *dc)
+{
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("move.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+                    dc->op2);
+
+    if (memsize == 4) {
+        insn_len = dec_prep_move_m(env, dc, 0, 4, cpu_R[dc->op2]);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_update_cc_op(dc, CC_OP_MOVE, 4);
+        cris_update_cc_x(dc);
+        cris_update_result(dc, cpu_R[dc->op2]);
+    } else {
+        TCGv t0;
+
+        t0 = tcg_temp_new();
+        insn_len = dec_prep_move_m(env, dc, 0, memsize, t0);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_alu(dc, CC_OP_MOVE,
+                cpu_R[dc->op2], cpu_R[dc->op2], t0, memsize);
+        tcg_temp_free(t0);
+    }
+    do_postinc(dc, memsize);
+    return insn_len;
 }
 
 static inline void cris_alu_m_alloc_temps(TCGv *t)
 {
-	t[0] = tcg_temp_new();
-	t[1] = tcg_temp_new();
+    t[0] = tcg_temp_new();
+    t[1] = tcg_temp_new();
 }
 
 static inline void cris_alu_m_free_temps(TCGv *t)
 {
-	tcg_temp_free(t[0]);
-	tcg_temp_free(t[1]);
-}
-
-static int dec_movs_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("movs.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_addu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("addu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_adds_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("adds.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_subu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("subu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_subs_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("subs.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_movu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-
-	LOG_DIS("movu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmpu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("cmpu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmps_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("cmps.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1],
-		    memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmp_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("cmp.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1],
-		    memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_test_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("test.%c [$r%u%s] op2=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_evaluate_flags(dc);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
-
-	cris_alu(dc, CC_OP_CMP,
-		 cpu_R[dc->op2], t[1], tcg_const_tl(0), memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_and_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("and.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_add_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("add.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD,
-		 cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_addo_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("add.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, 0);
-	cris_alu(dc, CC_OP_ADD, cpu_R[R_ACR], t[0], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_bound_m(DisasContext *dc)
-{
-	TCGv l[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("bound.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	l[0] = tcg_temp_local_new();
-	l[1] = tcg_temp_local_new();
-	insn_len = dec_prep_alu_m(dc, 0, memsize, l[0], l[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], l[0], l[1], 4);
-	do_postinc(dc, memsize);
-	tcg_temp_free(l[0]);
-	tcg_temp_free(l[1]);
-	return insn_len;
-}
-
-static int dec_addc_mr(DisasContext *dc)
-{
-	TCGv t[2];
-	int insn_len = 2;
-	LOG_DIS("addc [$r%u%s, $r%u\n",
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_evaluate_flags(dc);
-
-	/* Set for this insn.  */
-	dc->flagx_known = 1;
-	dc->flags_x = X_FLAG;
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, 4, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADDC, cpu_R[dc->op2], t[0], t[1], 4);
-	do_postinc(dc, 4);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_sub_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("sub.%c [$r%u%s, $r%u ir=%x zz=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2, dc->ir, dc->zzsize);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], memsize);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_or_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("or.%c [$r%u%s, $r%u pc=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2, dc->pc);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_OR,
-		    cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_move_mp(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len = 2;
-
-	LOG_DIS("move.%c [$r%u%s, $p%u\n",
-		    memsize_char(memsize),
-		    dc->op1,
-		    dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, 0);
-	if (dc->op2 == PR_CCS) {
-		cris_evaluate_flags(dc);
-		if (dc->tb_flags & U_FLAG) {
-			/* User space is not allowed to touch all flags.  */
-			tcg_gen_andi_tl(t[1], t[1], 0x39f);
-			tcg_gen_andi_tl(t[0], cpu_PR[PR_CCS], ~0x39f);
-			tcg_gen_or_tl(t[1], t[0], t[1]);
-		}
-	}
-
-	t_gen_mov_preg_TN(dc, dc->op2, t[1]);
-
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_move_pm(DisasContext *dc)
-{
-	TCGv t0;
-	int memsize;
-
-	memsize = preg_sizes[dc->op2];
-
-	LOG_DIS("move.%c $p%u, [$r%u%s\n",
-		     memsize_char(memsize), 
-		     dc->op2, dc->op1, dc->postinc ? "+]" : "]");
-
-	/* prepare store. Address in T0, value in T1.  */
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-	t0 = tcg_temp_new();
-	t_gen_mov_TN_preg(t0, dc->op2);
-	cris_flush_cc_state(dc);
-	gen_store(dc, cpu_R[dc->op1], t0, memsize);
-	tcg_temp_free(t0);
-
-	cris_cc_mask(dc, 0);
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
-	return 2;
-}
-
-static int dec_movem_mr(DisasContext *dc)
-{
-	TCGv_i64 tmp[16];
-        TCGv tmp32;
-	TCGv addr;
-	int i;
-	int nr = dc->op2 + 1;
-
-	LOG_DIS("movem [$r%u%s, $r%u\n", dc->op1,
-		    dc->postinc ? "+]" : "]", dc->op2);
-
-	addr = tcg_temp_new();
-	/* There are probably better ways of doing this.  */
-	cris_flush_cc_state(dc);
-	for (i = 0; i < (nr >> 1); i++) {
-		tmp[i] = tcg_temp_new_i64();
-		tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
-		gen_load64(dc, tmp[i], addr);
-	}
-	if (nr & 1) {
-		tmp32 = tcg_temp_new_i32();
-		tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
-		gen_load(dc, tmp32, addr, 4, 0);
-	} else
-		TCGV_UNUSED(tmp32);
-	tcg_temp_free(addr);
-
-	for (i = 0; i < (nr >> 1); i++) {
-		tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
-		tcg_gen_shri_i64(tmp[i], tmp[i], 32);
-		tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
-		tcg_temp_free_i64(tmp[i]);
-	}
-	if (nr & 1) {
-		tcg_gen_mov_tl(cpu_R[dc->op2], tmp32);
-		tcg_temp_free(tmp32);
-	}
+    tcg_temp_free(t[0]);
+    tcg_temp_free(t[1]);
+}
+
+static int dec_movs_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("movs.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_addu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("addu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_adds_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("adds.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_subu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("subu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_subs_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("subs.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_movu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+
+    LOG_DIS("movu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmpu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("cmpu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmps_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("cmps.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1],
+            memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmp_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("cmp.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1],
+            memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_test_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("test.%c [$r%u%s] op2=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_evaluate_flags(dc);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
+
+    cris_alu(dc, CC_OP_CMP,
+         cpu_R[dc->op2], t[1], tcg_const_tl(0), memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_and_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("and.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_add_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("add.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD,
+         cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_addo_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("add.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, 0);
+    cris_alu(dc, CC_OP_ADD, cpu_R[R_ACR], t[0], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_bound_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv l[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("bound.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    l[0] = tcg_temp_local_new();
+    l[1] = tcg_temp_local_new();
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, l[0], l[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], l[0], l[1], 4);
+    do_postinc(dc, memsize);
+    tcg_temp_free(l[0]);
+    tcg_temp_free(l[1]);
+    return insn_len;
+}
+
+static int dec_addc_mr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int insn_len = 2;
+    LOG_DIS("addc [$r%u%s, $r%u\n",
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_evaluate_flags(dc);
+
+    /* Set for this insn.  */
+    dc->flagx_known = 1;
+    dc->flags_x = X_FLAG;
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, 4, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADDC, cpu_R[dc->op2], t[0], t[1], 4);
+    do_postinc(dc, 4);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_sub_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("sub.%c [$r%u%s, $r%u ir=%x zz=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2, dc->ir, dc->zzsize);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], memsize);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_or_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("or.%c [$r%u%s, $r%u pc=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2, dc->pc);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_OR,
+            cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_move_mp(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len = 2;
+
+    LOG_DIS("move.%c [$r%u%s, $p%u\n",
+            memsize_char(memsize),
+            dc->op1,
+            dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, 0);
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+        if (dc->tb_flags & U_FLAG) {
+            /* User space is not allowed to touch all flags.  */
+            tcg_gen_andi_tl(t[1], t[1], 0x39f);
+            tcg_gen_andi_tl(t[0], cpu_PR[PR_CCS], ~0x39f);
+            tcg_gen_or_tl(t[1], t[0], t[1]);
+        }
+    }
+
+    t_gen_mov_preg_TN(dc, dc->op2, t[1]);
+
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_move_pm(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    int memsize;
+
+    memsize = preg_sizes[dc->op2];
+
+    LOG_DIS("move.%c $p%u, [$r%u%s\n",
+            memsize_char(memsize),
+            dc->op2, dc->op1, dc->postinc ? "+]" : "]");
+
+    /* prepare store. Address in T0, value in T1.  */
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+    t0 = tcg_temp_new();
+    t_gen_mov_TN_preg(t0, dc->op2);
+    cris_flush_cc_state(dc);
+    gen_store(dc, cpu_R[dc->op1], t0, memsize);
+    tcg_temp_free(t0);
+
+    cris_cc_mask(dc, 0);
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
+    }
+    return 2;
+}
+
+static int dec_movem_mr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv_i64 tmp[16];
+    TCGv tmp32;
+    TCGv addr;
+    int i;
+    int nr = dc->op2 + 1;
+
+    LOG_DIS("movem [$r%u%s, $r%u\n", dc->op1,
+            dc->postinc ? "+]" : "]", dc->op2);
+
+    addr = tcg_temp_new();
+    /* There are probably better ways of doing this.  */
+    cris_flush_cc_state(dc);
+    for (i = 0; i < (nr >> 1); i++) {
+        tmp[i] = tcg_temp_new_i64();
+        tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
+        gen_load64(dc, tmp[i], addr);
+    }
+    if (nr & 1) {
+        tmp32 = tcg_temp_new_i32();
+        tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
+        gen_load(dc, tmp32, addr, 4, 0);
+    } else {
+        TCGV_UNUSED(tmp32);
+    }
+    tcg_temp_free(addr);
 
-	/* writeback the updated pointer value.  */
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], nr * 4);
+    for (i = 0; i < (nr >> 1); i++) {
+        tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
+        tcg_gen_shri_i64(tmp[i], tmp[i], 32);
+        tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
+        tcg_temp_free_i64(tmp[i]);
+    }
+    if (nr & 1) {
+        tcg_gen_mov_tl(cpu_R[dc->op2], tmp32);
+        tcg_temp_free(tmp32);
+    }
 
-	/* gen_load might want to evaluate the previous insns flags.  */
-	cris_cc_mask(dc, 0);
-	return 2;
-}
-
-static int dec_movem_rm(DisasContext *dc)
-{
-	TCGv tmp;
-	TCGv addr;
-	int i;
+    /* writeback the updated pointer value.  */
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], nr * 4);
+    }
 
-	LOG_DIS("movem $r%u, [$r%u%s\n", dc->op2, dc->op1,
-		     dc->postinc ? "+]" : "]");
+    /* gen_load might want to evaluate the previous insns flags.  */
+    cris_cc_mask(dc, 0);
+    return 2;
+}
 
-	cris_flush_cc_state(dc);
+static int dec_movem_rm(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv tmp;
+    TCGv addr;
+    int i;
+
+    LOG_DIS("movem $r%u, [$r%u%s\n", dc->op2, dc->op1,
+            dc->postinc ? "+]" : "]");
 
-	tmp = tcg_temp_new();
-	addr = tcg_temp_new();
-	tcg_gen_movi_tl(tmp, 4);
-	tcg_gen_mov_tl(addr, cpu_R[dc->op1]);
-	for (i = 0; i <= dc->op2; i++) {
-		/* Displace addr.  */
-		/* Perform the store.  */
-		gen_store(dc, addr, cpu_R[i], 4);
-		tcg_gen_add_tl(addr, addr, tmp);
-	}
-	if (dc->postinc)
-		tcg_gen_mov_tl(cpu_R[dc->op1], addr);
-	cris_cc_mask(dc, 0);
-	tcg_temp_free(tmp);
-	tcg_temp_free(addr);
-	return 2;
+    cris_flush_cc_state(dc);
+
+    tmp = tcg_temp_new();
+    addr = tcg_temp_new();
+    tcg_gen_movi_tl(tmp, 4);
+    tcg_gen_mov_tl(addr, cpu_R[dc->op1]);
+    for (i = 0; i <= dc->op2; i++) {
+        /* Displace addr.  */
+        /* Perform the store.  */
+        gen_store(dc, addr, cpu_R[i], 4);
+        tcg_gen_add_tl(addr, addr, tmp);
+    }
+    if (dc->postinc) {
+        tcg_gen_mov_tl(cpu_R[dc->op1], addr);
+    }
+    cris_cc_mask(dc, 0);
+    tcg_temp_free(tmp);
+    tcg_temp_free(addr);
+    return 2;
 }
 
-static int dec_move_rm(DisasContext *dc)
+static int dec_move_rm(CPUCRISState *env, DisasContext *dc)
 {
-	int memsize;
+    int memsize;
+
+    memsize = memsize_zz(dc);
 
-	memsize = memsize_zz(dc);
-
-	LOG_DIS("move.%c $r%u, [$r%u]\n",
-		     memsize_char(memsize), dc->op2, dc->op1);
+    LOG_DIS("move.%c $r%u, [$r%u]\n",
+            memsize_char(memsize), dc->op2, dc->op1);
 
-	/* prepare store.  */
-	cris_flush_cc_state(dc);
-	gen_store(dc, cpu_R[dc->op1], cpu_R[dc->op2], memsize);
+    /* prepare store.  */
+    cris_flush_cc_state(dc);
+    gen_store(dc, cpu_R[dc->op1], cpu_R[dc->op2], memsize);
 
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
-	cris_cc_mask(dc, 0);
-	return 2;
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
+    }
+    cris_cc_mask(dc, 0);
+    return 2;
 }
 
-static int dec_lapcq(DisasContext *dc)
+static int dec_lapcq(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("lapcq %x, $r%u\n",
-		    dc->pc + dc->op1*2, dc->op2);
-	cris_cc_mask(dc, 0);
-	tcg_gen_movi_tl(cpu_R[dc->op2], dc->pc + dc->op1 * 2);
-	return 2;
-}
+    LOG_DIS("lapcq %x, $r%u\n",
+            dc->pc + dc->op1*2, dc->op2);
+    cris_cc_mask(dc, 0);
+    tcg_gen_movi_tl(cpu_R[dc->op2], dc->pc + dc->op1 * 2);
+    return 2;
+}
 
-static int dec_lapc_im(DisasContext *dc)
+static int dec_lapc_im(CPUCRISState *env, DisasContext *dc)
 {
-	unsigned int rd;
-	int32_t imm;
-	int32_t pc;
+    unsigned int rd;
+    int32_t imm;
+    int32_t pc;
 
-	rd = dc->op2;
+    rd = dc->op2;
 
-	cris_cc_mask(dc, 0);
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
-	LOG_DIS("lapc 0x%x, $r%u\n", imm + dc->pc, dc->op2);
+    cris_cc_mask(dc, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
+    LOG_DIS("lapc 0x%x, $r%u\n", imm + dc->pc, dc->op2);
 
-	pc = dc->pc;
-	pc += imm;
-	tcg_gen_movi_tl(cpu_R[rd], pc);
-	return 6;
+    pc = dc->pc;
+    pc += imm;
+    tcg_gen_movi_tl(cpu_R[rd], pc);
+    return 6;
 }
 
 /* Jump to special reg.  */
-static int dec_jump_p(DisasContext *dc)
+static int dec_jump_p(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("jump $p%u\n", dc->op2);
+    LOG_DIS("jump $p%u\n", dc->op2);
 
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-	t_gen_mov_TN_preg(env_btarget, dc->op2);
-	/* rete will often have low bit set to indicate delayslot.  */
-	tcg_gen_andi_tl(env_btarget, env_btarget, ~1);
-	cris_cc_mask(dc, 0);
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+    t_gen_mov_TN_preg(env_btarget, dc->op2);
+    /* rete will often have low bit set to indicate delayslot.  */
+    tcg_gen_andi_tl(env_btarget, env_btarget, ~1);
+    cris_cc_mask(dc, 0);
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
 /* Jump and save.  */
-static int dec_jas_r(DisasContext *dc)
-{
-	LOG_DIS("jas $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
-	if (dc->op2 > 15)
-		abort();
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4));
+static int dec_jas_r(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("jas $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
+    if (dc->op2 > 15) {
+        abort();
+    }
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4));
 
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
-static int dec_jas_im(DisasContext *dc)
+static int dec_jas_im(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("jas 0x%x\n", imm);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
+    LOG_DIS("jas 0x%x\n", imm);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
 
-	dc->jmp_pc = imm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = imm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_jasc_im(DisasContext *dc)
+static int dec_jasc_im(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("jasc 0x%x\n", imm);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8 + 4));
+    LOG_DIS("jasc 0x%x\n", imm);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8 + 4));
 
-	dc->jmp_pc = imm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = imm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_jasc_r(DisasContext *dc)
+static int dec_jasc_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("jasc_r $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4 + 4));
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    LOG_DIS("jasc_r $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4 + 4));
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
-static int dec_bcc_im(DisasContext *dc)
+static int dec_bcc_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t offset;
-	uint32_t cond = dc->op2;
+    int32_t offset;
+    uint32_t cond = dc->op2;
 
-	offset = cris_fetch(dc, dc->pc + 2, 2, 1);
+    offset = cris_fetch(env, dc, dc->pc + 2, 2, 1);
 
-	LOG_DIS("b%s %d pc=%x dst=%x\n",
-		    cc_name(cond), offset,
-		    dc->pc, dc->pc + offset);
+    LOG_DIS("b%s %d pc=%x dst=%x\n",
+            cc_name(cond), offset,
+            dc->pc, dc->pc + offset);
 
-	cris_cc_mask(dc, 0);
-	/* op2 holds the condition-code.  */
-	cris_prepare_cc_branch (dc, offset, cond);
-	return 4;
+    cris_cc_mask(dc, 0);
+    /* op2 holds the condition-code.  */
+    cris_prepare_cc_branch(dc, offset, cond);
+    return 4;
 }
 
-static int dec_bas_im(DisasContext *dc)
+static int dec_bas_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t simm;
+    int32_t simm;
 
+    simm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	simm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    LOG_DIS("bas 0x%x, $p%u\n", dc->pc + simm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
 
-	LOG_DIS("bas 0x%x, $p%u\n", dc->pc + simm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
-
-	dc->jmp_pc = dc->pc + simm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = dc->pc + simm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_basc_im(DisasContext *dc)
+static int dec_basc_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t simm;
-	simm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    int32_t simm;
+    simm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("basc 0x%x, $p%u\n", dc->pc + simm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 12));
+    LOG_DIS("basc 0x%x, $p%u\n", dc->pc + simm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 12));
 
-	dc->jmp_pc = dc->pc + simm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = dc->pc + simm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_rfe_etc(DisasContext *dc)
+static int dec_rfe_etc(CPUCRISState *env, DisasContext *dc)
 {
-	cris_cc_mask(dc, 0);
-
-	if (dc->op2 == 15) {
-		t_gen_mov_env_TN(halted, tcg_const_tl(1));
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		t_gen_raise_exception(EXCP_HLT);
-		return 2;
-	}
+    cris_cc_mask(dc, 0);
 
-	switch (dc->op2 & 7) {
-		case 2:
-			/* rfe.  */
-			LOG_DIS("rfe\n");
-			cris_evaluate_flags(dc);
-			gen_helper_rfe();
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		case 5:
-			/* rfn.  */
-			LOG_DIS("rfn\n");
-			cris_evaluate_flags(dc);
-			gen_helper_rfn();
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		case 6:
-			LOG_DIS("break %d\n", dc->op1);
-			cris_evaluate_flags (dc);
-			/* break.  */
-			tcg_gen_movi_tl(env_pc, dc->pc + 2);
+    if (dc->op2 == 15) {
+        t_gen_mov_env_TN(halted, tcg_const_tl(1));
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        t_gen_raise_exception(EXCP_HLT);
+        return 2;
+    }
 
-			/* Breaks start at 16 in the exception vector.  */
-			t_gen_mov_env_TN(trap_vector, 
-					 tcg_const_tl(dc->op1 + 16));
-			t_gen_raise_exception(EXCP_BREAK);
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		default:
-			printf ("op2=%x\n", dc->op2);
-			BUG();
-			break;
+    switch (dc->op2 & 7) {
+    case 2:
+        /* rfe.  */
+        LOG_DIS("rfe\n");
+        cris_evaluate_flags(dc);
+        gen_helper_rfe(cpu_env);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    case 5:
+        /* rfn.  */
+        LOG_DIS("rfn\n");
+        cris_evaluate_flags(dc);
+        gen_helper_rfn(cpu_env);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    case 6:
+        LOG_DIS("break %d\n", dc->op1);
+        cris_evaluate_flags(dc);
+        /* break.  */
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+
+        /* Breaks start at 16 in the exception vector.  */
+        t_gen_mov_env_TN(trap_vector,
+                tcg_const_tl(dc->op1 + 16));
+        t_gen_raise_exception(EXCP_BREAK);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    default:
+        printf("op2=%x\n", dc->op2);
+        BUG();
+        break;
 
-	}
-	return 2;
+    }
+    return 2;
 }
 
-static int dec_ftag_fidx_d_m(DisasContext *dc)
+static int dec_ftag_fidx_d_m(CPUCRISState *env, DisasContext *dc)
 {
-	return 2;
+    return 2;
 }
 
-static int dec_ftag_fidx_i_m(DisasContext *dc)
+static int dec_ftag_fidx_i_m(CPUCRISState *env, DisasContext *dc)
 {
-	return 2;
+    return 2;
 }
 
-static int dec_null(DisasContext *dc)
+static int dec_null(CPUCRISState *env, DisasContext *dc)
 {
-	printf ("unknown insn pc=%x opc=%x op1=%x op2=%x\n",
-		dc->pc, dc->opcode, dc->op1, dc->op2);
-	fflush(NULL);
-	BUG();
-	return 2;
+    printf("unknown insn pc=%x opc=%x op1=%x op2=%x\n",
+        dc->pc, dc->opcode, dc->op1, dc->op2);
+    fflush(NULL);
+    BUG();
+    return 2;
 }
 
 static struct decoder_info {
-	struct {
-		uint32_t bits;
-		uint32_t mask;
-	};
-	int (*dec)(DisasContext *dc);
+    struct {
+        uint32_t bits;
+        uint32_t mask;
+    };
+    int (*dec)(CPUCRISState *env, DisasContext *dc);
 } decinfo[] = {
-	/* Order matters here.  */
-	{DEC_MOVEQ, dec_moveq},
-	{DEC_BTSTQ, dec_btstq},
-	{DEC_CMPQ, dec_cmpq},
-	{DEC_ADDOQ, dec_addoq},
-	{DEC_ADDQ, dec_addq},
-	{DEC_SUBQ, dec_subq},
-	{DEC_ANDQ, dec_andq},
-	{DEC_ORQ, dec_orq},
-	{DEC_ASRQ, dec_asrq},
-	{DEC_LSLQ, dec_lslq},
-	{DEC_LSRQ, dec_lsrq},
-	{DEC_BCCQ, dec_bccq},
-
-	{DEC_BCC_IM, dec_bcc_im},
-	{DEC_JAS_IM, dec_jas_im},
-	{DEC_JAS_R, dec_jas_r},
-	{DEC_JASC_IM, dec_jasc_im},
-	{DEC_JASC_R, dec_jasc_r},
-	{DEC_BAS_IM, dec_bas_im},
-	{DEC_BASC_IM, dec_basc_im},
-	{DEC_JUMP_P, dec_jump_p},
-	{DEC_LAPC_IM, dec_lapc_im},
-	{DEC_LAPCQ, dec_lapcq},
-
-	{DEC_RFE_ETC, dec_rfe_etc},
-	{DEC_ADDC_MR, dec_addc_mr},
-
-	{DEC_MOVE_MP, dec_move_mp},
-	{DEC_MOVE_PM, dec_move_pm},
-	{DEC_MOVEM_MR, dec_movem_mr},
-	{DEC_MOVEM_RM, dec_movem_rm},
-	{DEC_MOVE_PR, dec_move_pr},
-	{DEC_SCC_R, dec_scc_r},
-	{DEC_SETF, dec_setclrf},
-	{DEC_CLEARF, dec_setclrf},
-
-	{DEC_MOVE_SR, dec_move_sr},
-	{DEC_MOVE_RP, dec_move_rp},
-	{DEC_SWAP_R, dec_swap_r},
-	{DEC_ABS_R, dec_abs_r},
-	{DEC_LZ_R, dec_lz_r},
-	{DEC_MOVE_RS, dec_move_rs},
-	{DEC_BTST_R, dec_btst_r},
-	{DEC_ADDC_R, dec_addc_r},
-
-	{DEC_DSTEP_R, dec_dstep_r},
-	{DEC_XOR_R, dec_xor_r},
-	{DEC_MCP_R, dec_mcp_r},
-	{DEC_CMP_R, dec_cmp_r},
-
-	{DEC_ADDI_R, dec_addi_r},
-	{DEC_ADDI_ACR, dec_addi_acr},
-
-	{DEC_ADD_R, dec_add_r},
-	{DEC_SUB_R, dec_sub_r},
-
-	{DEC_ADDU_R, dec_addu_r},
-	{DEC_ADDS_R, dec_adds_r},
-	{DEC_SUBU_R, dec_subu_r},
-	{DEC_SUBS_R, dec_subs_r},
-	{DEC_LSL_R, dec_lsl_r},
-
-	{DEC_AND_R, dec_and_r},
-	{DEC_OR_R, dec_or_r},
-	{DEC_BOUND_R, dec_bound_r},
-	{DEC_ASR_R, dec_asr_r},
-	{DEC_LSR_R, dec_lsr_r},
-
-	{DEC_MOVU_R, dec_movu_r},
-	{DEC_MOVS_R, dec_movs_r},
-	{DEC_NEG_R, dec_neg_r},
-	{DEC_MOVE_R, dec_move_r},
-
-	{DEC_FTAG_FIDX_I_M, dec_ftag_fidx_i_m},
-	{DEC_FTAG_FIDX_D_M, dec_ftag_fidx_d_m},
-
-	{DEC_MULS_R, dec_muls_r},
-	{DEC_MULU_R, dec_mulu_r},
-
-	{DEC_ADDU_M, dec_addu_m},
-	{DEC_ADDS_M, dec_adds_m},
-	{DEC_SUBU_M, dec_subu_m},
-	{DEC_SUBS_M, dec_subs_m},
-
-	{DEC_CMPU_M, dec_cmpu_m},
-	{DEC_CMPS_M, dec_cmps_m},
-	{DEC_MOVU_M, dec_movu_m},
-	{DEC_MOVS_M, dec_movs_m},
-
-	{DEC_CMP_M, dec_cmp_m},
-	{DEC_ADDO_M, dec_addo_m},
-	{DEC_BOUND_M, dec_bound_m},
-	{DEC_ADD_M, dec_add_m},
-	{DEC_SUB_M, dec_sub_m},
-	{DEC_AND_M, dec_and_m},
-	{DEC_OR_M, dec_or_m},
-	{DEC_MOVE_RM, dec_move_rm},
-	{DEC_TEST_M, dec_test_m},
-	{DEC_MOVE_MR, dec_move_mr},
-
-	{{0, 0}, dec_null}
+    /* Order matters here.  */
+    {DEC_MOVEQ, dec_moveq},
+    {DEC_BTSTQ, dec_btstq},
+    {DEC_CMPQ, dec_cmpq},
+    {DEC_ADDOQ, dec_addoq},
+    {DEC_ADDQ, dec_addq},
+    {DEC_SUBQ, dec_subq},
+    {DEC_ANDQ, dec_andq},
+    {DEC_ORQ, dec_orq},
+    {DEC_ASRQ, dec_asrq},
+    {DEC_LSLQ, dec_lslq},
+    {DEC_LSRQ, dec_lsrq},
+    {DEC_BCCQ, dec_bccq},
+
+    {DEC_BCC_IM, dec_bcc_im},
+    {DEC_JAS_IM, dec_jas_im},
+    {DEC_JAS_R, dec_jas_r},
+    {DEC_JASC_IM, dec_jasc_im},
+    {DEC_JASC_R, dec_jasc_r},
+    {DEC_BAS_IM, dec_bas_im},
+    {DEC_BASC_IM, dec_basc_im},
+    {DEC_JUMP_P, dec_jump_p},
+    {DEC_LAPC_IM, dec_lapc_im},
+    {DEC_LAPCQ, dec_lapcq},
+
+    {DEC_RFE_ETC, dec_rfe_etc},
+    {DEC_ADDC_MR, dec_addc_mr},
+
+    {DEC_MOVE_MP, dec_move_mp},
+    {DEC_MOVE_PM, dec_move_pm},
+    {DEC_MOVEM_MR, dec_movem_mr},
+    {DEC_MOVEM_RM, dec_movem_rm},
+    {DEC_MOVE_PR, dec_move_pr},
+    {DEC_SCC_R, dec_scc_r},
+    {DEC_SETF, dec_setclrf},
+    {DEC_CLEARF, dec_setclrf},
+
+    {DEC_MOVE_SR, dec_move_sr},
+    {DEC_MOVE_RP, dec_move_rp},
+    {DEC_SWAP_R, dec_swap_r},
+    {DEC_ABS_R, dec_abs_r},
+    {DEC_LZ_R, dec_lz_r},
+    {DEC_MOVE_RS, dec_move_rs},
+    {DEC_BTST_R, dec_btst_r},
+    {DEC_ADDC_R, dec_addc_r},
+
+    {DEC_DSTEP_R, dec_dstep_r},
+    {DEC_XOR_R, dec_xor_r},
+    {DEC_MCP_R, dec_mcp_r},
+    {DEC_CMP_R, dec_cmp_r},
+
+    {DEC_ADDI_R, dec_addi_r},
+    {DEC_ADDI_ACR, dec_addi_acr},
+
+    {DEC_ADD_R, dec_add_r},
+    {DEC_SUB_R, dec_sub_r},
+
+    {DEC_ADDU_R, dec_addu_r},
+    {DEC_ADDS_R, dec_adds_r},
+    {DEC_SUBU_R, dec_subu_r},
+    {DEC_SUBS_R, dec_subs_r},
+    {DEC_LSL_R, dec_lsl_r},
+
+    {DEC_AND_R, dec_and_r},
+    {DEC_OR_R, dec_or_r},
+    {DEC_BOUND_R, dec_bound_r},
+    {DEC_ASR_R, dec_asr_r},
+    {DEC_LSR_R, dec_lsr_r},
+
+    {DEC_MOVU_R, dec_movu_r},
+    {DEC_MOVS_R, dec_movs_r},
+    {DEC_NEG_R, dec_neg_r},
+    {DEC_MOVE_R, dec_move_r},
+
+    {DEC_FTAG_FIDX_I_M, dec_ftag_fidx_i_m},
+    {DEC_FTAG_FIDX_D_M, dec_ftag_fidx_d_m},
+
+    {DEC_MULS_R, dec_muls_r},
+    {DEC_MULU_R, dec_mulu_r},
+
+    {DEC_ADDU_M, dec_addu_m},
+    {DEC_ADDS_M, dec_adds_m},
+    {DEC_SUBU_M, dec_subu_m},
+    {DEC_SUBS_M, dec_subs_m},
+
+    {DEC_CMPU_M, dec_cmpu_m},
+    {DEC_CMPS_M, dec_cmps_m},
+    {DEC_MOVU_M, dec_movu_m},
+    {DEC_MOVS_M, dec_movs_m},
+
+    {DEC_CMP_M, dec_cmp_m},
+    {DEC_ADDO_M, dec_addo_m},
+    {DEC_BOUND_M, dec_bound_m},
+    {DEC_ADD_M, dec_add_m},
+    {DEC_SUB_M, dec_sub_m},
+    {DEC_AND_M, dec_and_m},
+    {DEC_OR_M, dec_or_m},
+    {DEC_MOVE_RM, dec_move_rm},
+    {DEC_TEST_M, dec_test_m},
+    {DEC_MOVE_MR, dec_move_mr},
+
+    {{0, 0}, dec_null}
 };
 
-static unsigned int crisv32_decoder(DisasContext *dc)
+static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
 {
-	int insn_len = 2;
-	int i;
-
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
-		tcg_gen_debug_insn_start(dc->pc);
-
-	/* Load a halfword onto the instruction register.  */
-	dc->ir = cris_fetch(dc, dc->pc, 2, 0);
+    int insn_len = 2;
+    int i;
 
-	/* Now decode it.  */
-	dc->opcode   = EXTRACT_FIELD(dc->ir, 4, 11);
-	dc->op1      = EXTRACT_FIELD(dc->ir, 0, 3);
-	dc->op2      = EXTRACT_FIELD(dc->ir, 12, 15);
-	dc->zsize    = EXTRACT_FIELD(dc->ir, 4, 4);
-	dc->zzsize   = EXTRACT_FIELD(dc->ir, 4, 5);
-	dc->postinc  = EXTRACT_FIELD(dc->ir, 10, 10);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(dc->pc);
+        }
 
-	/* Large switch for all insns.  */
-	for (i = 0; i < ARRAY_SIZE(decinfo); i++) {
-		if ((dc->opcode & decinfo[i].mask) == decinfo[i].bits)
-		{
-			insn_len = decinfo[i].dec(dc);
-			break;
-		}
-	}
+    /* Load a halfword onto the instruction register.  */
+        dc->ir = cris_fetch(env, dc, dc->pc, 2, 0);
+
+    /* Now decode it.  */
+    dc->opcode   = EXTRACT_FIELD(dc->ir, 4, 11);
+    dc->op1      = EXTRACT_FIELD(dc->ir, 0, 3);
+    dc->op2      = EXTRACT_FIELD(dc->ir, 12, 15);
+    dc->zsize    = EXTRACT_FIELD(dc->ir, 4, 4);
+    dc->zzsize   = EXTRACT_FIELD(dc->ir, 4, 5);
+    dc->postinc  = EXTRACT_FIELD(dc->ir, 10, 10);
+
+    /* Large switch for all insns.  */
+    for (i = 0; i < ARRAY_SIZE(decinfo); i++) {
+        if ((dc->opcode & decinfo[i].mask) == decinfo[i].bits) {
+            insn_len = decinfo[i].dec(env, dc);
+            break;
+        }
+    }
 
 #if !defined(CONFIG_USER_ONLY)
-	/* Single-stepping ?  */
-	if (dc->tb_flags & S_FLAG) {
-		int l1;
-
-		l1 = gen_new_label();
-		tcg_gen_brcondi_tl(TCG_COND_NE, cpu_PR[PR_SPC], dc->pc, l1);
-		/* We treat SPC as a break with an odd trap vector.  */
-		cris_evaluate_flags (dc);
-		t_gen_mov_env_TN(trap_vector, tcg_const_tl(3));
-		tcg_gen_movi_tl(env_pc, dc->pc + insn_len);
-		tcg_gen_movi_tl(cpu_PR[PR_SPC], dc->pc + insn_len);
-		t_gen_raise_exception(EXCP_BREAK);
-		gen_set_label(l1);
-	}
+    /* Single-stepping ?  */
+    if (dc->tb_flags & S_FLAG) {
+        int l1;
+
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_NE, cpu_PR[PR_SPC], dc->pc, l1);
+        /* We treat SPC as a break with an odd trap vector.  */
+        cris_evaluate_flags(dc);
+        t_gen_mov_env_TN(trap_vector, tcg_const_tl(3));
+        tcg_gen_movi_tl(env_pc, dc->pc + insn_len);
+        tcg_gen_movi_tl(cpu_PR[PR_SPC], dc->pc + insn_len);
+        t_gen_raise_exception(EXCP_BREAK);
+        gen_set_label(l1);
+    }
 #endif
-	return insn_len;
+    return insn_len;
 }
 
 static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
 {
-	CPUBreakpoint *bp;
+    CPUBreakpoint *bp;
 
-	if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
-		QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
-			if (bp->pc == dc->pc) {
-				cris_evaluate_flags (dc);
-				tcg_gen_movi_tl(env_pc, dc->pc);
-				t_gen_raise_exception(EXCP_DEBUG);
-				dc->is_jmp = DISAS_UPDATE;
-			}
-		}
-	}
+    if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
+        QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
+            if (bp->pc == dc->pc) {
+                cris_evaluate_flags(dc);
+                tcg_gen_movi_tl(env_pc, dc->pc);
+                t_gen_raise_exception(EXCP_DEBUG);
+                dc->is_jmp = DISAS_UPDATE;
+            }
+        }
+    }
 }
 
 #include "translate_v10.c"
@@ -3171,250 +3204,256 @@ static void
 gen_intermediate_code_internal(CPUCRISState *env, TranslationBlock *tb,
                                int search_pc)
 {
-	uint16_t *gen_opc_end;
-   	uint32_t pc_start;
-	unsigned int insn_len;
-	int j, lj;
-	struct DisasContext ctx;
-	struct DisasContext *dc = &ctx;
-	uint32_t next_page_start;
-	target_ulong npc;
-        int num_insns;
-        int max_insns;
-
-	qemu_log_try_set_file(stderr);
-
-	if (env->pregs[PR_VR] == 32) {
-		dc->decoder = crisv32_decoder;
-		dc->clear_locked_irq = 0;
-	} else {
-		dc->decoder = crisv10_decoder;
-		dc->clear_locked_irq = 1;
-	}
-
-	/* Odd PC indicates that branch is rexecuting due to exception in the
-	 * delayslot, like in real hw.
-	 */
-	pc_start = tb->pc & ~1;
-	dc->env = env;
-	dc->tb = tb;
-
-	gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
-
-	dc->is_jmp = DISAS_NEXT;
-	dc->ppc = pc_start;
-	dc->pc = pc_start;
-	dc->singlestep_enabled = env->singlestep_enabled;
-	dc->flags_uptodate = 1;
-	dc->flagx_known = 1;
-	dc->flags_x = tb->flags & X_FLAG;
-	dc->cc_x_uptodate = 0;
-	dc->cc_mask = 0;
-	dc->update_cc = 0;
-	dc->clear_prefix = 0;
-
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->cc_size_uptodate = -1;
-
-	/* Decode TB flags.  */
-	dc->tb_flags = tb->flags & (S_FLAG | P_FLAG | U_FLAG \
-					| X_FLAG | PFIX_FLAG);
-	dc->delayed_branch = !!(tb->flags & 7);
-	if (dc->delayed_branch)
-		dc->jmp = JMP_INDIRECT;
-	else
-		dc->jmp = JMP_NOJMP;
-
-	dc->cpustate_changed = 0;
-
-	if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
-		qemu_log(
-                        "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
-			"pid=%x usp=%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n",
-			search_pc, dc->pc, dc->ppc,
-                        (uint64_t)tb->flags,
-			env->btarget, (unsigned)tb->flags & 7,
-			env->pregs[PR_CCS], 
-			env->pregs[PR_PID], env->pregs[PR_USP],
-			env->regs[0], env->regs[1], env->regs[2], env->regs[3],
-			env->regs[4], env->regs[5], env->regs[6], env->regs[7],
-			env->regs[8], env->regs[9],
-			env->regs[10], env->regs[11],
-			env->regs[12], env->regs[13],
-			env->regs[14], env->regs[15]);
-		qemu_log("--------------\n");
-		qemu_log("IN: %s\n", lookup_symbol(pc_start));
-	}
-
-	next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-	lj = -1;
-        num_insns = 0;
-        max_insns = tb->cflags & CF_COUNT_MASK;
-        if (max_insns == 0)
-            max_insns = CF_COUNT_MASK;
-
-        gen_icount_start();
-	do
-	{
-		check_breakpoint(env, dc);
-
-		if (search_pc) {
-			j = gen_opc_ptr - gen_opc_buf;
-			if (lj < j) {
-				lj++;
-				while (lj < j)
-					gen_opc_instr_start[lj++] = 0;
-			}
-			if (dc->delayed_branch == 1)
-				gen_opc_pc[lj] = dc->ppc | 1;
-			else
-				gen_opc_pc[lj] = dc->pc;
-			gen_opc_instr_start[lj] = 1;
-                        gen_opc_icount[lj] = num_insns;
-		}
-
-		/* Pretty disas.  */
-		LOG_DIS("%8.8x:\t", dc->pc);
-
-                if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
-                    gen_io_start();
-		dc->clear_x = 1;
-
-		insn_len = dc->decoder(dc);
-		dc->ppc = dc->pc;
-		dc->pc += insn_len;
-		if (dc->clear_x)
-			cris_clear_x_flag(dc);
-
-                num_insns++;
-		/* Check for delayed branches here. If we do it before
-		   actually generating any host code, the simulator will just
-		   loop doing nothing for on this program location.  */
-		if (dc->delayed_branch) {
-			dc->delayed_branch--;
-			if (dc->delayed_branch == 0)
-			{
-				if (tb->flags & 7)
-					t_gen_mov_env_TN(dslot, 
-						tcg_const_tl(0));
-				if (dc->cpustate_changed || !dc->flagx_known
-				    || (dc->flags_x != (tb->flags & X_FLAG))) {
-					cris_store_direct_jmp(dc);
-				}
-
-				if (dc->clear_locked_irq) {
-					dc->clear_locked_irq = 0;
-					t_gen_mov_env_TN(locked_irq,
-							 tcg_const_tl(0));
-				}
-
-				if (dc->jmp == JMP_DIRECT_CC) {
-					int l1;
-
-					l1 = gen_new_label();
-					cris_evaluate_flags(dc);
-
-					/* Conditional jmp.  */
-					tcg_gen_brcondi_tl(TCG_COND_EQ,
-							   env_btaken, 0, l1);
-					gen_goto_tb(dc, 1, dc->jmp_pc);
-					gen_set_label(l1);
-					gen_goto_tb(dc, 0, dc->pc);
-					dc->is_jmp = DISAS_TB_JUMP;
-					dc->jmp = JMP_NOJMP;
-				} else if (dc->jmp == JMP_DIRECT) {
-					cris_evaluate_flags(dc);
-					gen_goto_tb(dc, 0, dc->jmp_pc);
-					dc->is_jmp = DISAS_TB_JUMP;
-					dc->jmp = JMP_NOJMP;
-				} else {
-					t_gen_cc_jmp(env_btarget, 
-						     tcg_const_tl(dc->pc));
-					dc->is_jmp = DISAS_JUMP;
-				}
-				break;
-			}
-		}
-
-		/* If we are rexecuting a branch due to exceptions on
-		   delay slots dont break.  */
-		if (!(tb->pc & 1) && env->singlestep_enabled)
-			break;
-	} while (!dc->is_jmp && !dc->cpustate_changed
-		 && gen_opc_ptr < gen_opc_end
-                 && !singlestep
-		 && (dc->pc < next_page_start)
-                 && num_insns < max_insns);
-
-	if (dc->clear_locked_irq)
-		t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
-
-	npc = dc->pc;
+    uint16_t *gen_opc_end;
+    uint32_t pc_start;
+    unsigned int insn_len;
+    int j, lj;
+    struct DisasContext ctx;
+    struct DisasContext *dc = &ctx;
+    uint32_t next_page_start;
+    target_ulong npc;
+    int num_insns;
+    int max_insns;
+
+    qemu_log_try_set_file(stderr);
+
+    if (env->pregs[PR_VR] == 32) {
+        dc->decoder = crisv32_decoder;
+        dc->clear_locked_irq = 0;
+    } else {
+        dc->decoder = crisv10_decoder;
+        dc->clear_locked_irq = 1;
+    }
+
+    /* Odd PC indicates that branch is rexecuting due to exception in the
+     * delayslot, like in real hw.
+     */
+    pc_start = tb->pc & ~1;
+    dc->env = env;
+    dc->tb = tb;
+
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
+
+    dc->is_jmp = DISAS_NEXT;
+    dc->ppc = pc_start;
+    dc->pc = pc_start;
+    dc->singlestep_enabled = env->singlestep_enabled;
+    dc->flags_uptodate = 1;
+    dc->flagx_known = 1;
+    dc->flags_x = tb->flags & X_FLAG;
+    dc->cc_x_uptodate = 0;
+    dc->cc_mask = 0;
+    dc->update_cc = 0;
+    dc->clear_prefix = 0;
+
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->cc_size_uptodate = -1;
+
+    /* Decode TB flags.  */
+    dc->tb_flags = tb->flags & (S_FLAG | P_FLAG | U_FLAG \
+            | X_FLAG | PFIX_FLAG);
+    dc->delayed_branch = !!(tb->flags & 7);
+    if (dc->delayed_branch) {
+        dc->jmp = JMP_INDIRECT;
+    } else {
+        dc->jmp = JMP_NOJMP;
+    }
+
+    dc->cpustate_changed = 0;
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log(
+                "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
+                "pid=%x usp=%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n",
+                search_pc, dc->pc, dc->ppc,
+                (uint64_t)tb->flags,
+                env->btarget, (unsigned)tb->flags & 7,
+                env->pregs[PR_CCS],
+                env->pregs[PR_PID], env->pregs[PR_USP],
+                env->regs[0], env->regs[1], env->regs[2], env->regs[3],
+                env->regs[4], env->regs[5], env->regs[6], env->regs[7],
+                env->regs[8], env->regs[9],
+                env->regs[10], env->regs[11],
+                env->regs[12], env->regs[13],
+                env->regs[14], env->regs[15]);
+        qemu_log("--------------\n");
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+    }
+
+    next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+    lj = -1;
+    num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+
+    gen_icount_start();
+    do {
+        check_breakpoint(env, dc);
+
+        if (search_pc) {
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+            if (lj < j) {
+                lj++;
+                while (lj < j) {
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
+                }
+            }
+            if (dc->delayed_branch == 1) {
+                tcg_ctx.gen_opc_pc[lj] = dc->ppc | 1;
+            } else {
+                tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            }
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        }
+
+        /* Pretty disas.  */
+        LOG_DIS("%8.8x:\t", dc->pc);
+
+        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+            gen_io_start();
+        }
+        dc->clear_x = 1;
+
+        insn_len = dc->decoder(env, dc);
+        dc->ppc = dc->pc;
+        dc->pc += insn_len;
+        if (dc->clear_x) {
+            cris_clear_x_flag(dc);
+        }
+
+        num_insns++;
+        /* Check for delayed branches here. If we do it before
+           actually generating any host code, the simulator will just
+           loop doing nothing for on this program location.  */
+        if (dc->delayed_branch) {
+            dc->delayed_branch--;
+            if (dc->delayed_branch == 0) {
+                if (tb->flags & 7) {
+                    t_gen_mov_env_TN(dslot, tcg_const_tl(0));
+                }
+                if (dc->cpustate_changed || !dc->flagx_known
+                    || (dc->flags_x != (tb->flags & X_FLAG))) {
+                    cris_store_direct_jmp(dc);
+                }
+
+                if (dc->clear_locked_irq) {
+                    dc->clear_locked_irq = 0;
+                    t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
+                }
+
+                if (dc->jmp == JMP_DIRECT_CC) {
+                    int l1;
+
+                    l1 = gen_new_label();
+                    cris_evaluate_flags(dc);
+
+                    /* Conditional jmp.  */
+                    tcg_gen_brcondi_tl(TCG_COND_EQ,
+                               env_btaken, 0, l1);
+                    gen_goto_tb(dc, 1, dc->jmp_pc);
+                    gen_set_label(l1);
+                    gen_goto_tb(dc, 0, dc->pc);
+                    dc->is_jmp = DISAS_TB_JUMP;
+                    dc->jmp = JMP_NOJMP;
+                } else if (dc->jmp == JMP_DIRECT) {
+                    cris_evaluate_flags(dc);
+                    gen_goto_tb(dc, 0, dc->jmp_pc);
+                    dc->is_jmp = DISAS_TB_JUMP;
+                    dc->jmp = JMP_NOJMP;
+                } else {
+                    t_gen_cc_jmp(env_btarget, tcg_const_tl(dc->pc));
+                    dc->is_jmp = DISAS_JUMP;
+                }
+                break;
+            }
+        }
+
+        /* If we are rexecuting a branch due to exceptions on
+           delay slots dont break.  */
+        if (!(tb->pc & 1) && env->singlestep_enabled) {
+            break;
+        }
+    } while (!dc->is_jmp && !dc->cpustate_changed
+            && tcg_ctx.gen_opc_ptr < gen_opc_end
+            && !singlestep
+            && (dc->pc < next_page_start)
+            && num_insns < max_insns);
+
+    if (dc->clear_locked_irq) {
+        t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
+    }
+
+    npc = dc->pc;
 
         if (tb->cflags & CF_LAST_IO)
             gen_io_end();
-	/* Force an update if the per-tb cpu state has changed.  */
-	if (dc->is_jmp == DISAS_NEXT
-	    && (dc->cpustate_changed || !dc->flagx_known 
-	    || (dc->flags_x != (tb->flags & X_FLAG)))) {
-		dc->is_jmp = DISAS_UPDATE;
-		tcg_gen_movi_tl(env_pc, npc);
-	}
-	/* Broken branch+delayslot sequence.  */
-	if (dc->delayed_branch == 1) {
-		/* Set env->dslot to the size of the branch insn.  */
-		t_gen_mov_env_TN(dslot, tcg_const_tl(dc->pc - dc->ppc));
-		cris_store_direct_jmp(dc);
-	}
-
-	cris_evaluate_flags (dc);
-
-	if (unlikely(env->singlestep_enabled)) {
-		if (dc->is_jmp == DISAS_NEXT)
-			tcg_gen_movi_tl(env_pc, npc);
-		t_gen_raise_exception(EXCP_DEBUG);
-	} else {
-		switch(dc->is_jmp) {
-			case DISAS_NEXT:
-				gen_goto_tb(dc, 1, npc);
-				break;
-			default:
-			case DISAS_JUMP:
-			case DISAS_UPDATE:
-				/* indicate that the hash table must be used
-				   to find the next TB */
-				tcg_gen_exit_tb(0);
-				break;
-			case DISAS_SWI:
-			case DISAS_TB_JUMP:
-				/* nothing more to generate */
-				break;
-		}
-	}
-        gen_icount_end(tb, num_insns);
-	*gen_opc_ptr = INDEX_op_end;
-	if (search_pc) {
-		j = gen_opc_ptr - gen_opc_buf;
-		lj++;
-		while (lj <= j)
-			gen_opc_instr_start[lj++] = 0;
-	} else {
-		tb->size = dc->pc - pc_start;
-                tb->icount = num_insns;
-	}
+    /* Force an update if the per-tb cpu state has changed.  */
+    if (dc->is_jmp == DISAS_NEXT
+        && (dc->cpustate_changed || !dc->flagx_known
+        || (dc->flags_x != (tb->flags & X_FLAG)))) {
+        dc->is_jmp = DISAS_UPDATE;
+        tcg_gen_movi_tl(env_pc, npc);
+    }
+    /* Broken branch+delayslot sequence.  */
+    if (dc->delayed_branch == 1) {
+        /* Set env->dslot to the size of the branch insn.  */
+        t_gen_mov_env_TN(dslot, tcg_const_tl(dc->pc - dc->ppc));
+        cris_store_direct_jmp(dc);
+    }
+
+    cris_evaluate_flags(dc);
+
+    if (unlikely(env->singlestep_enabled)) {
+        if (dc->is_jmp == DISAS_NEXT) {
+            tcg_gen_movi_tl(env_pc, npc);
+        }
+        t_gen_raise_exception(EXCP_DEBUG);
+    } else {
+        switch (dc->is_jmp) {
+        case DISAS_NEXT:
+            gen_goto_tb(dc, 1, npc);
+            break;
+        default:
+        case DISAS_JUMP:
+        case DISAS_UPDATE:
+            /* indicate that the hash table must be used
+                   to find the next TB */
+            tcg_gen_exit_tb(0);
+            break;
+        case DISAS_SWI:
+        case DISAS_TB_JUMP:
+            /* nothing more to generate */
+            break;
+        }
+    }
+    gen_icount_end(tb, num_insns);
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
+    if (search_pc) {
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+        lj++;
+        while (lj <= j) {
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
+        }
+    } else {
+        tb->size = dc->pc - pc_start;
+        tb->icount = num_insns;
+    }
 
 #ifdef DEBUG_DISAS
 #if !DISAS_CRIS
-	if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
-		log_target_disas(pc_start, dc->pc - pc_start,
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        log_target_disas(env, pc_start, dc->pc - pc_start,
                                  dc->env->pregs[PR_VR]);
-		qemu_log("\nisize=%d osize=%td\n",
-			dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
-	}
+        qemu_log("\nisize=%d osize=%td\n",
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf);
+    }
 #endif
 #endif
 }
@@ -3432,41 +3471,45 @@ void gen_intermediate_code_pc (CPUCRISState *env, struct TranslationBlock *tb)
 void cpu_dump_state (CPUCRISState *env, FILE *f, fprintf_function cpu_fprintf,
                      int flags)
 {
-	int i;
-	uint32_t srs;
-
-	if (!env || !f)
-		return;
-
-	cpu_fprintf(f, "PC=%x CCS=%x btaken=%d btarget=%x\n"
-		    "cc_op=%d cc_src=%d cc_dest=%d cc_result=%x cc_mask=%x\n",
-		    env->pc, env->pregs[PR_CCS], env->btaken, env->btarget,
-		    env->cc_op,
-		    env->cc_src, env->cc_dest, env->cc_result, env->cc_mask);
-
-
-	for (i = 0; i < 16; i++) {
-		cpu_fprintf(f, "%s=%8.8x ",regnames[i], env->regs[i]);
-		if ((i + 1) % 4 == 0)
-			cpu_fprintf(f, "\n");
-	}
-	cpu_fprintf(f, "\nspecial regs:\n");
-	for (i = 0; i < 16; i++) {
-		cpu_fprintf(f, "%s=%8.8x ", pregnames[i], env->pregs[i]);
-		if ((i + 1) % 4 == 0)
-			cpu_fprintf(f, "\n");
-	}
-	srs = env->pregs[PR_SRS];
-	cpu_fprintf(f, "\nsupport function regs bank %x:\n", srs);
-	if (srs < 256) {
-		for (i = 0; i < 16; i++) {
-			cpu_fprintf(f, "s%2.2d=%8.8x ",
-				    i, env->sregs[srs][i]);
-			if ((i + 1) % 4 == 0)
-				cpu_fprintf(f, "\n");
-		}
-	}
-	cpu_fprintf(f, "\n\n");
+    int i;
+    uint32_t srs;
+
+    if (!env || !f) {
+        return;
+    }
+
+    cpu_fprintf(f, "PC=%x CCS=%x btaken=%d btarget=%x\n"
+            "cc_op=%d cc_src=%d cc_dest=%d cc_result=%x cc_mask=%x\n",
+            env->pc, env->pregs[PR_CCS], env->btaken, env->btarget,
+            env->cc_op,
+            env->cc_src, env->cc_dest, env->cc_result, env->cc_mask);
+
+
+    for (i = 0; i < 16; i++) {
+        cpu_fprintf(f, "%s=%8.8x ", regnames[i], env->regs[i]);
+        if ((i + 1) % 4 == 0) {
+            cpu_fprintf(f, "\n");
+        }
+    }
+    cpu_fprintf(f, "\nspecial regs:\n");
+    for (i = 0; i < 16; i++) {
+        cpu_fprintf(f, "%s=%8.8x ", pregnames[i], env->pregs[i]);
+        if ((i + 1) % 4 == 0) {
+            cpu_fprintf(f, "\n");
+        }
+    }
+    srs = env->pregs[PR_SRS];
+    cpu_fprintf(f, "\nsupport function regs bank %x:\n", srs);
+    if (srs < ARRAY_SIZE(env->sregs)) {
+        for (i = 0; i < 16; i++) {
+            cpu_fprintf(f, "s%2.2d=%8.8x ",
+                    i, env->sregs[srs][i]);
+            if ((i + 1) % 4 == 0) {
+                cpu_fprintf(f, "\n");
+            }
+        }
+    }
+    cpu_fprintf(f, "\n\n");
 
 }
 
@@ -3475,11 +3518,11 @@ struct
     uint32_t vr;
     const char *name;
 } cris_cores[] = {
-	{8, "crisv8"},
-	{9, "crisv9"},
-	{10, "crisv10"},
-	{11, "crisv11"},
-	{32, "crisv32"},
+    {8, "crisv8"},
+    {9, "crisv9"},
+    {10, "crisv10"},
+    {11, "crisv11"},
+    {32, "crisv32"},
 };
 
 void cris_cpu_list(FILE *f, fprintf_function cpu_fprintf)
@@ -3578,5 +3621,5 @@ CRISCPU *cpu_cris_init(const char *cpu_model)
 
 void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
 {
-	env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-cris/translate_v10.c b/target-cris/translate_v10.c
index 3629629d9d0..d2cca892e08 100644
--- a/target-cris/translate_v10.c
+++ b/target-cris/translate_v10.c
@@ -164,8 +164,8 @@ static unsigned int crisv10_post_memaddr(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static int dec10_prep_move_m(DisasContext *dc, int s_ext, int memsize,
-                           TCGv dst)
+static int dec10_prep_move_m(CPUCRISState *env, DisasContext *dc,
+                             int s_ext, int memsize, TCGv dst)
 {
     unsigned int rs;
     uint32_t imm;
@@ -182,17 +182,17 @@ static int dec10_prep_move_m(DisasContext *dc, int s_ext, int memsize,
         if (memsize != 4) {
             if (s_ext) {
                 if (memsize == 1)
-                    imm = ldsb_code(dc->pc + 2);
+                    imm = cpu_ldsb_code(env, dc->pc + 2);
                 else
-                    imm = ldsw_code(dc->pc + 2);
+                    imm = cpu_ldsw_code(env, dc->pc + 2);
             } else {
                 if (memsize == 1)
-                    imm = ldub_code(dc->pc + 2);
+                    imm = cpu_ldub_code(env, dc->pc + 2);
                 else
-                    imm = lduw_code(dc->pc + 2);
+                    imm = cpu_lduw_code(env, dc->pc + 2);
             }
         } else
-            imm = ldl_code(dc->pc + 2);
+            imm = cpu_ldl_code(env, dc->pc + 2);
 
         tcg_gen_movi_tl(dst, imm);
 
@@ -289,7 +289,7 @@ static unsigned int dec10_quick_imm(DisasContext *dc)
             } else {
                 /* BTST */
                 cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-                gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->dst],
+                gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->dst],
                            tcg_const_tl(imm), cpu_PR[PR_CCS]);
             }
             break;
@@ -723,7 +723,7 @@ static unsigned int dec10_reg(DisasContext *dc)
                 LOG_DIS("btst $r%d, $r%d sz=%d\n", dc->src, dc->dst, size);
                 cris_cc_mask(dc, CC_MASK_NZVC);
                 cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-                gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->dst],
+                gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->dst],
                            cpu_R[dc->src], cpu_PR[PR_CCS]);
                 break;
             case CRISV10_REG_DSTEP:
@@ -752,7 +752,8 @@ static unsigned int dec10_reg(DisasContext *dc)
     return insn_len;
 }
 
-static unsigned int dec10_ind_move_m_r(DisasContext *dc, unsigned int size)
+static unsigned int dec10_ind_move_m_r(CPUCRISState *env, DisasContext *dc,
+                                       unsigned int size)
 {
     unsigned int insn_len = 2;
     TCGv t;
@@ -762,7 +763,7 @@ static unsigned int dec10_ind_move_m_r(DisasContext *dc, unsigned int size)
 
     cris_cc_mask(dc, CC_MASK_NZVC);
     t = tcg_temp_new();
-    insn_len += dec10_prep_move_m(dc, 0, size, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t);
     cris_alu(dc, CC_OP_MOVE, cpu_R[dc->dst], cpu_R[dc->dst], t, size);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -789,7 +790,7 @@ static unsigned int dec10_ind_move_r_m(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static unsigned int dec10_ind_move_m_pr(DisasContext *dc)
+static unsigned int dec10_ind_move_m_pr(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2, rd = dc->dst;
     TCGv t, addr;
@@ -799,7 +800,7 @@ static unsigned int dec10_ind_move_m_pr(DisasContext *dc)
 
     addr = tcg_temp_new();
     t = tcg_temp_new();
-    insn_len += dec10_prep_move_m(dc, 0, 4, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, 4, t);
     if (rd == 15) {
         tcg_gen_mov_tl(env_btarget, t);
         cris_prepare_jmp(dc, JMP_INDIRECT);
@@ -899,14 +900,15 @@ static void dec10_movem_m_r(DisasContext *dc)
     tcg_temp_free(t0);
 }
 
-static int dec10_ind_alu(DisasContext *dc, int op, unsigned int size)
+static int dec10_ind_alu(CPUCRISState *env, DisasContext *dc,
+                         int op, unsigned int size)
 {
     int insn_len = 0;
     int rd = dc->dst;
     TCGv t[2];
 
     cris_alu_m_alloc_temps(t);
-    insn_len += dec10_prep_move_m(dc, 0, size, t[0]);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t[0]);
     cris_alu(dc, op, cpu_R[dc->dst], cpu_R[rd], t[0], size);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -920,14 +922,15 @@ static int dec10_ind_alu(DisasContext *dc, int op, unsigned int size)
     return insn_len;
 }
 
-static int dec10_ind_bound(DisasContext *dc, unsigned int size)
+static int dec10_ind_bound(CPUCRISState *env, DisasContext *dc,
+                           unsigned int size)
 {
     int insn_len = 0;
     int rd = dc->dst;
     TCGv t;
 
     t = tcg_temp_local_new();
-    insn_len += dec10_prep_move_m(dc, 0, size, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t);
     cris_alu(dc, CC_OP_BOUND, cpu_R[dc->dst], cpu_R[rd], t, 4);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -940,7 +943,7 @@ static int dec10_ind_bound(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static int dec10_alux_m(DisasContext *dc, int op)
+static int dec10_alux_m(CPUCRISState *env, DisasContext *dc, int op)
 {
     unsigned int size = (dc->size & 1) ? 2 : 1;
     unsigned int sx = !!(dc->size & 2);
@@ -953,7 +956,7 @@ static int dec10_alux_m(DisasContext *dc, int op)
     t = tcg_temp_new();
 
     cris_cc_mask(dc, CC_MASK_NZVC);
-    insn_len += dec10_prep_move_m(dc, sx, size, t);
+    insn_len += dec10_prep_move_m(env, dc, sx, size, t);
     cris_alu(dc, op, cpu_R[dc->dst], cpu_R[rd], t, 4);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -966,7 +969,7 @@ static int dec10_alux_m(DisasContext *dc, int op)
     return insn_len;
 }
 
-static int dec10_dip(DisasContext *dc)
+static int dec10_dip(CPUCRISState *env, DisasContext *dc)
 {
     int insn_len = 2;
     uint32_t imm;
@@ -974,7 +977,7 @@ static int dec10_dip(DisasContext *dc)
     LOG_DIS("dip pc=%x opcode=%d r%d r%d\n",
               dc->pc, dc->opcode, dc->src, dc->dst);
     if (dc->src == 15) {
-        imm = ldl_code(dc->pc + 2);
+        imm = cpu_ldl_code(env, dc->pc + 2);
         tcg_gen_movi_tl(cpu_PR[PR_PREFIX], imm);
         if (dc->postinc)
             insn_len += 4;
@@ -989,7 +992,7 @@ static int dec10_dip(DisasContext *dc)
     return insn_len;
 }
 
-static int dec10_bdap_m(DisasContext *dc, int size)
+static int dec10_bdap_m(CPUCRISState *env, DisasContext *dc, int size)
 {
     int insn_len = 2;
     int rd = dc->dst;
@@ -1014,13 +1017,13 @@ static int dec10_bdap_m(DisasContext *dc, int size)
     }
 #endif
     /* Now the rest of the modes are truly indirect.  */
-    insn_len += dec10_prep_move_m(dc, 1, size, cpu_PR[PR_PREFIX]);
+    insn_len += dec10_prep_move_m(env, dc, 1, size, cpu_PR[PR_PREFIX]);
     tcg_gen_add_tl(cpu_PR[PR_PREFIX], cpu_PR[PR_PREFIX], cpu_R[rd]);
     cris_set_prefix(dc);
     return insn_len;
 }
 
-static unsigned int dec10_ind(DisasContext *dc)
+static unsigned int dec10_ind(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
     unsigned int size = dec10_size(dc->size);
@@ -1031,7 +1034,7 @@ static unsigned int dec10_ind(DisasContext *dc)
     if (dc->size != 3) {
         switch (dc->opcode) {
             case CRISV10_IND_MOVE_M_R:
-                return dec10_ind_move_m_r(dc, size);
+                return dec10_ind_move_m_r(env, dc, size);
                 break;
             case CRISV10_IND_MOVE_R_M:
                 return dec10_ind_move_r_m(dc, size);
@@ -1039,7 +1042,7 @@ static unsigned int dec10_ind(DisasContext *dc)
             case CRISV10_IND_CMP:
                 LOG_DIS("cmp size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_CMP, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_CMP, size);
                 break;
             case CRISV10_IND_TEST:
                 LOG_DIS("test size=%d op=%d %d\n",  size, dc->src, dc->dst);
@@ -1047,7 +1050,7 @@ static unsigned int dec10_ind(DisasContext *dc)
                 cris_evaluate_flags(dc);
                 cris_cc_mask(dc, CC_MASK_NZVC);
                 cris_alu_m_alloc_temps(t);
-                insn_len += dec10_prep_move_m(dc, 0, size, t[0]);
+                insn_len += dec10_prep_move_m(env, dc, 0, size, t[0]);
                 tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
                 cris_alu(dc, CC_OP_CMP, cpu_R[dc->dst],
                          t[0], tcg_const_tl(0), size);
@@ -1056,39 +1059,39 @@ static unsigned int dec10_ind(DisasContext *dc)
             case CRISV10_IND_ADD:
                 LOG_DIS("add size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_ADD, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_ADD, size);
                 break;
             case CRISV10_IND_SUB:
                 LOG_DIS("sub size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_SUB, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_SUB, size);
                 break;
             case CRISV10_IND_BOUND:
                 LOG_DIS("bound size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_bound(dc, size);
+                insn_len += dec10_ind_bound(env, dc, size);
                 break;
             case CRISV10_IND_AND:
                 LOG_DIS("and size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_AND, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_AND, size);
                 break;
             case CRISV10_IND_OR:
                 LOG_DIS("or size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_OR, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_OR, size);
                 break;
             case CRISV10_IND_MOVX:
-                insn_len = dec10_alux_m(dc, CC_OP_MOVE);
+                insn_len = dec10_alux_m(env, dc, CC_OP_MOVE);
                 break;
             case CRISV10_IND_ADDX:
-                insn_len = dec10_alux_m(dc, CC_OP_ADD);
+                insn_len = dec10_alux_m(env, dc, CC_OP_ADD);
                 break;
             case CRISV10_IND_SUBX:
-                insn_len = dec10_alux_m(dc, CC_OP_SUB);
+                insn_len = dec10_alux_m(env, dc, CC_OP_SUB);
                 break;
             case CRISV10_IND_CMPX:
-                insn_len = dec10_alux_m(dc, CC_OP_CMP);
+                insn_len = dec10_alux_m(env, dc, CC_OP_CMP);
                 break;
             case CRISV10_IND_MUL:
                 /* This is a reg insn coded in the mem indir space.  */
@@ -1097,7 +1100,7 @@ static unsigned int dec10_ind(DisasContext *dc)
                 dec10_reg_mul(dc, size, dc->ir & (1 << 10));
                 break;
             case CRISV10_IND_BDAP_M:
-                insn_len = dec10_bdap_m(dc, size);
+                insn_len = dec10_bdap_m(env, dc, size);
                 break;
             default:
                 LOG_DIS("pc=%x var-ind.%d %d r%d r%d\n",
@@ -1110,7 +1113,7 @@ static unsigned int dec10_ind(DisasContext *dc)
 
     switch (dc->opcode) {
         case CRISV10_IND_MOVE_M_SPR:
-            insn_len = dec10_ind_move_m_pr(dc);
+            insn_len = dec10_ind_move_m_pr(env, dc);
             break;
         case CRISV10_IND_MOVE_SPR_M:
             insn_len = dec10_ind_move_pr_m(dc);
@@ -1119,7 +1122,7 @@ static unsigned int dec10_ind(DisasContext *dc)
             if (dc->src == 15) {
                 LOG_DIS("jump.%d %d r%d r%d direct\n", size,
                          dc->opcode, dc->src, dc->dst);
-                imm = ldl_code(dc->pc + 2);
+                imm = cpu_ldl_code(env, dc->pc + 2);
                 if (dc->mode == CRISV10_MODE_AUTOINC)
                     insn_len += size;
 
@@ -1168,24 +1171,24 @@ static unsigned int dec10_ind(DisasContext *dc)
             dc->delayed_branch--; /* v10 has no dslot here.  */
             break;
         case CRISV10_IND_MOVX:
-            insn_len = dec10_alux_m(dc, CC_OP_MOVE);
+            insn_len = dec10_alux_m(env, dc, CC_OP_MOVE);
             break;
         case CRISV10_IND_ADDX:
-            insn_len = dec10_alux_m(dc, CC_OP_ADD);
+            insn_len = dec10_alux_m(env, dc, CC_OP_ADD);
             break;
         case CRISV10_IND_SUBX:
-            insn_len = dec10_alux_m(dc, CC_OP_SUB);
+            insn_len = dec10_alux_m(env, dc, CC_OP_SUB);
             break;
         case CRISV10_IND_CMPX:
-            insn_len = dec10_alux_m(dc, CC_OP_CMP);
+            insn_len = dec10_alux_m(env, dc, CC_OP_CMP);
             break;
         case CRISV10_IND_DIP:
-            insn_len = dec10_dip(dc);
+            insn_len = dec10_dip(env, dc);
             break;
         case CRISV10_IND_BCC_M:
 
             cris_cc_mask(dc, 0);
-            imm = ldsw_code(dc->pc + 2);
+            imm = cpu_ldsw_code(env, dc->pc + 2);
             simm = (int16_t)imm;
             simm += 4;
 
@@ -1202,7 +1205,7 @@ static unsigned int dec10_ind(DisasContext *dc)
     return insn_len;
 }
 
-static unsigned int crisv10_decoder(DisasContext *dc)
+static unsigned int crisv10_decoder(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
 
@@ -1210,7 +1213,7 @@ static unsigned int crisv10_decoder(DisasContext *dc)
         tcg_gen_debug_insn_start(dc->pc);
 
     /* Load a halfword onto the instruction register.  */
-    dc->ir = lduw_code(dc->pc);
+    dc->ir = cpu_lduw_code(env, dc->pc);
 
     /* Now decode it.  */
     dc->opcode   = EXTRACT_FIELD(dc->ir, 6, 9);
@@ -1235,7 +1238,7 @@ static unsigned int crisv10_decoder(DisasContext *dc)
             break;
         case CRISV10_MODE_AUTOINC:
         case CRISV10_MODE_INDIRECT:
-            insn_len = dec10_ind(dc);
+            insn_len = dec10_ind(env, dc);
             break;
     }
 
diff --git a/target-i386/arch_dump.c b/target-i386/arch_dump.c
index 4240278eddb..2cd2f7f09e0 100644
--- a/target-i386/arch_dump.c
+++ b/target-i386/arch_dump.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "dump.h"
+#include "exec/cpu-all.h"
+#include "sysemu/dump.h"
 #include "elf.h"
 
 #ifdef TARGET_X86_64
@@ -403,7 +403,7 @@ int cpu_get_dump_info(ArchDumpInfo *info)
     } else {
         info->d_class = ELFCLASS32;
 
-        QLIST_FOREACH(block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
             if (block->offset + block->length > UINT_MAX) {
                 /* The memory size is greater than 4G */
                 info->d_class = ELFCLASS64;
diff --git a/target-i386/arch_memory_mapping.c b/target-i386/arch_memory_mapping.c
index 8e5a56a3a83..844893f44d9 100644
--- a/target-i386/arch_memory_mapping.c
+++ b/target-i386/arch_memory_mapping.c
@@ -12,14 +12,14 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 /* PAE Paging or IA-32e Paging */
-static void walk_pte(MemoryMappingList *list, target_phys_addr_t pte_start_addr,
+static void walk_pte(MemoryMappingList *list, hwaddr pte_start_addr,
                      int32_t a20_mask, target_ulong start_line_addr)
 {
-    target_phys_addr_t pte_addr, start_paddr;
+    hwaddr pte_addr, start_paddr;
     uint64_t pte;
     target_ulong start_vaddr;
     int i;
@@ -46,10 +46,10 @@ static void walk_pte(MemoryMappingList *list, target_phys_addr_t pte_start_addr,
 
 /* 32-bit Paging */
 static void walk_pte2(MemoryMappingList *list,
-                      target_phys_addr_t pte_start_addr, int32_t a20_mask,
+                      hwaddr pte_start_addr, int32_t a20_mask,
                       target_ulong start_line_addr)
 {
-    target_phys_addr_t pte_addr, start_paddr;
+    hwaddr pte_addr, start_paddr;
     uint32_t pte;
     target_ulong start_vaddr;
     int i;
@@ -75,10 +75,10 @@ static void walk_pte2(MemoryMappingList *list,
 }
 
 /* PAE Paging or IA-32e Paging */
-static void walk_pde(MemoryMappingList *list, target_phys_addr_t pde_start_addr,
+static void walk_pde(MemoryMappingList *list, hwaddr pde_start_addr,
                      int32_t a20_mask, target_ulong start_line_addr)
 {
-    target_phys_addr_t pde_addr, pte_start_addr, start_paddr;
+    hwaddr pde_addr, pte_start_addr, start_paddr;
     uint64_t pde;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -112,10 +112,10 @@ static void walk_pde(MemoryMappingList *list, target_phys_addr_t pde_start_addr,
 
 /* 32-bit Paging */
 static void walk_pde2(MemoryMappingList *list,
-                      target_phys_addr_t pde_start_addr, int32_t a20_mask,
+                      hwaddr pde_start_addr, int32_t a20_mask,
                       bool pse)
 {
-    target_phys_addr_t pde_addr, pte_start_addr, start_paddr;
+    hwaddr pde_addr, pte_start_addr, start_paddr, high_paddr;
     uint32_t pde;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -130,8 +130,13 @@ static void walk_pde2(MemoryMappingList *list,
 
         line_addr = (((unsigned int)i & 0x3ff) << 22);
         if ((pde & PG_PSE_MASK) && pse) {
-            /* 4 MB page */
-            start_paddr = (pde & ~0x3fffff) | ((pde & 0x1fe000) << 19);
+            /*
+             * 4 MB page:
+             * bits 39:32 are bits 20:13 of the PDE
+             * bit3 31:22 are bits 31:22 of the PDE
+             */
+            high_paddr = ((hwaddr)(pde & 0x1fe000) << 19);
+            start_paddr = (pde & ~0x3fffff) | high_paddr;
             if (cpu_physical_memory_is_io(start_paddr)) {
                 /* I/O region */
                 continue;
@@ -149,9 +154,9 @@ static void walk_pde2(MemoryMappingList *list,
 
 /* PAE Paging */
 static void walk_pdpe2(MemoryMappingList *list,
-                       target_phys_addr_t pdpe_start_addr, int32_t a20_mask)
+                       hwaddr pdpe_start_addr, int32_t a20_mask)
 {
-    target_phys_addr_t pdpe_addr, pde_start_addr;
+    hwaddr pdpe_addr, pde_start_addr;
     uint64_t pdpe;
     target_ulong line_addr;
     int i;
@@ -173,10 +178,10 @@ static void walk_pdpe2(MemoryMappingList *list,
 #ifdef TARGET_X86_64
 /* IA-32e Paging */
 static void walk_pdpe(MemoryMappingList *list,
-                      target_phys_addr_t pdpe_start_addr, int32_t a20_mask,
+                      hwaddr pdpe_start_addr, int32_t a20_mask,
                       target_ulong start_line_addr)
 {
-    target_phys_addr_t pdpe_addr, pde_start_addr, start_paddr;
+    hwaddr pdpe_addr, pde_start_addr, start_paddr;
     uint64_t pdpe;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -210,9 +215,9 @@ static void walk_pdpe(MemoryMappingList *list,
 
 /* IA-32e Paging */
 static void walk_pml4e(MemoryMappingList *list,
-                       target_phys_addr_t pml4e_start_addr, int32_t a20_mask)
+                       hwaddr pml4e_start_addr, int32_t a20_mask)
 {
-    target_phys_addr_t pml4e_addr, pdpe_start_addr;
+    hwaddr pml4e_addr, pdpe_start_addr;
     uint64_t pml4e;
     target_ulong line_addr;
     int i;
@@ -242,20 +247,20 @@ int cpu_get_memory_mapping(MemoryMappingList *list, CPUArchState *env)
     if (env->cr[4] & CR4_PAE_MASK) {
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
-            target_phys_addr_t pml4e_addr;
+            hwaddr pml4e_addr;
 
             pml4e_addr = (env->cr[3] & ~0xfff) & env->a20_mask;
             walk_pml4e(list, pml4e_addr, env->a20_mask);
         } else
 #endif
         {
-            target_phys_addr_t pdpe_addr;
+            hwaddr pdpe_addr;
 
             pdpe_addr = (env->cr[3] & ~0x1f) & env->a20_mask;
             walk_pdpe2(list, pdpe_addr, env->a20_mask);
         }
     } else {
-        target_phys_addr_t pde_addr;
+        hwaddr pde_addr;
         bool pse;
 
         pde_addr = (env->cr[3] & ~0xfff) & env->a20_mask;
diff --git a/target-i386/cc_helper.c b/target-i386/cc_helper.c
index 07892f9049f..9422003f246 100644
--- a/target-i386/cc_helper.c
+++ b/target-i386/cc_helper.c
@@ -353,6 +353,16 @@ void helper_sti(CPUX86State *env)
     env->eflags |= IF_MASK;
 }
 
+void helper_clac(CPUX86State *env)
+{
+    env->eflags &= ~AC_MASK;
+}
+
+void helper_stac(CPUX86State *env)
+{
+    env->eflags |= AC_MASK;
+}
+
 #if 0
 /* vm86plus instructions */
 void helper_cli_vm(CPUX86State *env)
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index 59011404804..332916a1857 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -20,9 +20,9 @@
 #ifndef QEMU_I386_CPU_QOM_H
 #define QEMU_I386_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
-#include "error.h"
+#include "qapi/error.h"
 
 #ifdef TARGET_X86_64
 #define TYPE_X86_CPU "x86_64-cpu"
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 423e00905d0..5c108e17abc 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -22,13 +22,16 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
+#include "sysemu/cpus.h"
+#include "topology.h"
 
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qapi/qmp/qerror.h"
 
-#include "qapi/qapi-visit-core.h"
-#include "arch_init.h"
+#include "qapi/visitor.h"
+#include "sysemu/arch_init.h"
 
 #include "hyperv.h"
 
@@ -37,6 +40,25 @@
 #include <linux/kvm_para.h>
 #endif
 
+#include "sysemu/sysemu.h"
+#ifndef CONFIG_USER_ONLY
+#include "hw/xen.h"
+#include "hw/sysbus.h"
+#include "hw/apic_internal.h"
+#endif
+
+static void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1,
+                                     uint32_t vendor2, uint32_t vendor3)
+{
+    int i;
+    for (i = 0; i < 4; i++) {
+        dst[i] = vendor1 >> (8 * i);
+        dst[i + 4] = vendor2 >> (8 * i);
+        dst[i + 8] = vendor3 >> (8 * i);
+    }
+    dst[CPUID_VENDOR_SZ] = '\0';
+}
+
 /* feature flags taken from "Intel Processor Identification and the CPUID
  * Instruction" and AMD's "CPUID Specification".  In cases of disagreement
  * between feature naming conventions, aliases may be added.
@@ -59,34 +81,54 @@ static const char *ext_feature_name[] = {
     NULL, "pcid", "dca", "sse4.1|sse4_1",
     "sse4.2|sse4_2", "x2apic", "movbe", "popcnt",
     "tsc-deadline", "aes", "xsave", "osxsave",
-    "avx", NULL, NULL, "hypervisor",
+    "avx", "f16c", "rdrand", "hypervisor",
 };
+/* Feature names that are already defined on feature_name[] but are set on
+ * CPUID[8000_0001].EDX on AMD CPUs don't have their names on
+ * ext2_feature_name[]. They are copied automatically to cpuid_ext2_features
+ * if and only if CPU vendor is AMD.
+ */
 static const char *ext2_feature_name[] = {
-    "fpu", "vme", "de", "pse",
-    "tsc", "msr", "pae", "mce",
-    "cx8" /* AMD CMPXCHG8B */, "apic", NULL, "syscall",
-    "mtrr", "pge", "mca", "cmov",
-    "pat", "pse36", NULL, NULL /* Linux mp */,
-    "nx|xd", NULL, "mmxext", "mmx",
-    "fxsr", "fxsr_opt|ffxsr", "pdpe1gb" /* AMD Page1GB */, "rdtscp",
+    NULL /* fpu */, NULL /* vme */, NULL /* de */, NULL /* pse */,
+    NULL /* tsc */, NULL /* msr */, NULL /* pae */, NULL /* mce */,
+    NULL /* cx8 */ /* AMD CMPXCHG8B */, NULL /* apic */, NULL, "syscall",
+    NULL /* mtrr */, NULL /* pge */, NULL /* mca */, NULL /* cmov */,
+    NULL /* pat */, NULL /* pse36 */, NULL, NULL /* Linux mp */,
+    "nx|xd", NULL, "mmxext", NULL /* mmx */,
+    NULL /* fxsr */, "fxsr_opt|ffxsr", "pdpe1gb" /* AMD Page1GB */, "rdtscp",
     NULL, "lm|i64", "3dnowext", "3dnow",
 };
 static const char *ext3_feature_name[] = {
     "lahf_lm" /* AMD LahfSahf */, "cmp_legacy", "svm", "extapic" /* AMD ExtApicSpace */,
     "cr8legacy" /* AMD AltMovCr8 */, "abm", "sse4a", "misalignsse",
     "3dnowprefetch", "osvw", "ibs", "xop",
-    "skinit", "wdt", NULL, NULL,
-    "fma4", NULL, "cvt16", "nodeid_msr",
+    "skinit", "wdt", NULL, "lwp",
+    "fma4", "tce", NULL, "nodeid_msr",
+    NULL, "tbm", "topoext", "perfctr_core",
+    "perfctr_nb", NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
+static const char *ext4_feature_name[] = {
+    NULL, NULL, "xstore", "xstore-en",
+    NULL, NULL, "xcrypt", "xcrypt-en",
+    "ace2", "ace2-en", "phe", "phe-en",
+    "pmm", "pmm-en", NULL, NULL,
+    NULL, NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
 };
 
 static const char *kvm_feature_name[] = {
-    "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock", "kvm_asyncpf", NULL, "kvm_pv_eoi", NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock",
+    "kvm_asyncpf", "kvm_steal_time", "kvm_pv_eoi", NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
 };
 
 static const char *svm_feature_name[] = {
@@ -100,19 +142,97 @@ static const char *svm_feature_name[] = {
     NULL, NULL, NULL, NULL,
 };
 
+static const char *cpuid_7_0_ebx_feature_name[] = {
+    "fsgsbase", NULL, NULL, "bmi1", "hle", "avx2", NULL, "smep",
+    "bmi2", "erms", "invpcid", "rtm", NULL, NULL, NULL, NULL,
+    NULL, NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+};
+
+typedef struct FeatureWordInfo {
+    const char **feat_names;
+    uint32_t cpuid_eax; /* Input EAX for CPUID */
+    int cpuid_reg;      /* R_* register constant */
+} FeatureWordInfo;
+
+static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
+    [FEAT_1_EDX] = {
+        .feat_names = feature_name,
+        .cpuid_eax = 1, .cpuid_reg = R_EDX,
+    },
+    [FEAT_1_ECX] = {
+        .feat_names = ext_feature_name,
+        .cpuid_eax = 1, .cpuid_reg = R_ECX,
+    },
+    [FEAT_8000_0001_EDX] = {
+        .feat_names = ext2_feature_name,
+        .cpuid_eax = 0x80000001, .cpuid_reg = R_EDX,
+    },
+    [FEAT_8000_0001_ECX] = {
+        .feat_names = ext3_feature_name,
+        .cpuid_eax = 0x80000001, .cpuid_reg = R_ECX,
+    },
+    [FEAT_C000_0001_EDX] = {
+        .feat_names = ext4_feature_name,
+        .cpuid_eax = 0xC0000001, .cpuid_reg = R_EDX,
+    },
+    [FEAT_KVM] = {
+        .feat_names = kvm_feature_name,
+        .cpuid_eax = KVM_CPUID_FEATURES, .cpuid_reg = R_EAX,
+    },
+    [FEAT_SVM] = {
+        .feat_names = svm_feature_name,
+        .cpuid_eax = 0x8000000A, .cpuid_reg = R_EDX,
+    },
+    [FEAT_7_0_EBX] = {
+        .feat_names = cpuid_7_0_ebx_feature_name,
+        .cpuid_eax = 7, .cpuid_reg = R_EBX,
+    },
+};
+
+const char *get_register_name_32(unsigned int reg)
+{
+    static const char *reg_names[CPU_NB_REGS32] = {
+        [R_EAX] = "EAX",
+        [R_ECX] = "ECX",
+        [R_EDX] = "EDX",
+        [R_EBX] = "EBX",
+        [R_ESP] = "ESP",
+        [R_EBP] = "EBP",
+        [R_ESI] = "ESI",
+        [R_EDI] = "EDI",
+    };
+
+    if (reg > CPU_NB_REGS32) {
+        return NULL;
+    }
+    return reg_names[reg];
+}
+
 /* collects per-function cpuid data
  */
 typedef struct model_features_t {
     uint32_t *guest_feat;
     uint32_t *host_feat;
-    uint32_t check_feat;
-    const char **flag_names;
-    uint32_t cpuid;
-    } model_features_t;
+    FeatureWord feat_word;
+} model_features_t;
 
 int check_cpuid = 0;
 int enforce_cpuid = 0;
 
+static uint32_t kvm_default_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
+        (1 << KVM_FEATURE_NOP_IO_DELAY) |
+        (1 << KVM_FEATURE_CLOCKSOURCE2) |
+        (1 << KVM_FEATURE_ASYNC_PF) |
+        (1 << KVM_FEATURE_STEAL_TIME) |
+        (1 << KVM_FEATURE_PV_EOI) |
+        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
+
+void disable_kvm_pv_eoi(void)
+{
+    kvm_default_features &= ~(1UL << KVM_FEATURE_PV_EOI);
+}
+
 void host_cpuid(uint32_t function, uint32_t count,
                 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)
 {
@@ -210,37 +330,34 @@ static bool lookup_feature(uint32_t *pval, const char *s, const char *e,
     return found;
 }
 
-static void add_flagname_to_bitmaps(const char *flagname, uint32_t *features,
-                                    uint32_t *ext_features,
-                                    uint32_t *ext2_features,
-                                    uint32_t *ext3_features,
-                                    uint32_t *kvm_features,
-                                    uint32_t *svm_features)
+static void add_flagname_to_bitmaps(const char *flagname,
+                                    FeatureWordArray words)
 {
-    if (!lookup_feature(features, flagname, NULL, feature_name) &&
-        !lookup_feature(ext_features, flagname, NULL, ext_feature_name) &&
-        !lookup_feature(ext2_features, flagname, NULL, ext2_feature_name) &&
-        !lookup_feature(ext3_features, flagname, NULL, ext3_feature_name) &&
-        !lookup_feature(kvm_features, flagname, NULL, kvm_feature_name) &&
-        !lookup_feature(svm_features, flagname, NULL, svm_feature_name))
-            fprintf(stderr, "CPU feature %s not found\n", flagname);
+    FeatureWord w;
+    for (w = 0; w < FEATURE_WORDS; w++) {
+        FeatureWordInfo *wi = &feature_word_info[w];
+        if (wi->feat_names &&
+            lookup_feature(&words[w], flagname, NULL, wi->feat_names)) {
+            break;
+        }
+    }
+    if (w == FEATURE_WORDS) {
+        fprintf(stderr, "CPU feature %s not found\n", flagname);
+    }
 }
 
 typedef struct x86_def_t {
-    struct x86_def_t *next;
     const char *name;
     uint32_t level;
-    uint32_t vendor1, vendor2, vendor3;
+    /* vendor is zero-terminated, 12 character ASCII string */
+    char vendor[CPUID_VENDOR_SZ + 1];
     int family;
     int model;
     int stepping;
-    int tsc_khz;
     uint32_t features, ext_features, ext2_features, ext3_features;
     uint32_t kvm_features, svm_features;
     uint32_t xlevel;
     char model_id[48];
-    int vendor_override;
-    uint32_t flags;
     /* Store the results of Centaur's CPUID instructions */
     uint32_t ext4_features;
     uint32_t xlevel2;
@@ -259,7 +376,6 @@ typedef struct x86_def_t {
           CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_PGE | CPUID_CMOV | \
           CPUID_PAT | CPUID_FXSR | CPUID_MMX | CPUID_SSE | CPUID_SSE2 | \
           CPUID_PAE | CPUID_SEP | CPUID_APIC)
-#define EXT2_FEATURE_MASK 0x0183F3FF
 
 #define TCG_FEATURES (CPUID_FP87 | CPUID_PSE | CPUID_TSC | CPUID_MSR | \
           CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | CPUID_SEP | \
@@ -272,12 +388,12 @@ typedef struct x86_def_t {
           /* missing:
           CPUID_VME, CPUID_DTS, CPUID_SS, CPUID_HT, CPUID_TM, CPUID_PBE */
 #define TCG_EXT_FEATURES (CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | \
-          CPUID_EXT_CX16 | CPUID_EXT_POPCNT | \
+          CPUID_EXT_SSSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT | \
           CPUID_EXT_HYPERVISOR)
           /* missing:
           CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_EST,
           CPUID_EXT_TM2, CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_XSAVE */
-#define TCG_EXT2_FEATURES ((TCG_FEATURES & EXT2_FEATURE_MASK) | \
+#define TCG_EXT2_FEATURES ((TCG_FEATURES & CPUID_EXT2_AMD_ALIASES) | \
           CPUID_EXT2_NX | CPUID_EXT2_MMXEXT | CPUID_EXT2_RDTSCP | \
           CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT)
           /* missing:
@@ -285,20 +401,15 @@ typedef struct x86_def_t {
 #define TCG_EXT3_FEATURES (CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM | \
           CPUID_EXT3_CR8LEG | CPUID_EXT3_ABM | CPUID_EXT3_SSE4A)
 #define TCG_SVM_FEATURES 0
+#define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP)
 
-/* maintains list of cpu model definitions
- */
-static x86_def_t *x86_defs = {NULL};
-
-/* built-in cpu model definitions (deprecated)
+/* built-in CPU model definitions
  */
 static x86_def_t builtin_x86_defs[] = {
     {
         .name = "qemu64",
         .level = 4,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -306,7 +417,7 @@ static x86_def_t builtin_x86_defs[] = {
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .ext3_features = CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
             CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
@@ -315,9 +426,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "phenom",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 16,
         .model = 2,
         .stepping = 3,
@@ -326,7 +435,7 @@ static x86_def_t builtin_x86_defs[] = {
             CPUID_PSE36 | CPUID_VME | CPUID_HT,
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_CX16 |
             CPUID_EXT_POPCNT,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX |
             CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_MMXEXT |
             CPUID_EXT2_FFXSR | CPUID_EXT2_PDPE1GB | CPUID_EXT2_RDTSCP,
@@ -343,6 +452,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "core2duo",
         .level = 10,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 15,
         .stepping = 11,
@@ -361,9 +471,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "kvm64",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -374,7 +482,7 @@ static x86_def_t builtin_x86_defs[] = {
         /* Missing: CPUID_EXT_POPCNT, CPUID_EXT_MONITOR */
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         /* Missing: CPUID_EXT3_LAHF_LM, CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
                     CPUID_EXT3_CR8LEG, CPUID_EXT3_ABM, CPUID_EXT3_SSE4A,
@@ -387,6 +495,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "qemu32",
         .level = 4,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 3,
         .stepping = 3,
@@ -397,13 +506,14 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "kvm32",
         .level = 5,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 15,
         .model = 6,
         .stepping = 1,
         .features = PPRO_FEATURES |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_PSE36,
         .ext_features = CPUID_EXT_SSE3,
-        .ext2_features = PPRO_FEATURES & EXT2_FEATURE_MASK,
+        .ext2_features = PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES,
         .ext3_features = 0,
         .xlevel = 0x80000008,
         .model_id = "Common 32-bit KVM processor"
@@ -411,6 +521,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "coreduo",
         .level = 10,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 14,
         .stepping = 8,
@@ -426,6 +537,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "486",
         .level = 1,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 4,
         .model = 0,
         .stepping = 0,
@@ -435,6 +547,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium",
         .level = 1,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 5,
         .model = 4,
         .stepping = 3,
@@ -444,6 +557,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium2",
         .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 5,
         .stepping = 2,
@@ -453,6 +567,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium3",
         .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 7,
         .stepping = 3,
@@ -462,20 +577,21 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "athlon",
         .level = 2,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 6,
         .model = 2,
         .stepping = 3,
-        .features = PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR | CPUID_MCA,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) | CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
+        .features = PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR |
+            CPUID_MCA,
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
+            CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
         .xlevel = 0x80000008,
     },
     {
         .name = "n270",
         /* original is on level 10 */
         .level = 5,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 28,
         .stepping = 2,
@@ -485,13 +601,274 @@ static x86_def_t builtin_x86_defs[] = {
             /* Some CPUs got no CPUID_SEP */
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 |
             CPUID_EXT_DSCPL | CPUID_EXT_EST | CPUID_EXT_TM2 | CPUID_EXT_XTPR,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) | CPUID_EXT2_NX,
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
+            CPUID_EXT2_NX,
         .ext3_features = CPUID_EXT3_LAHF_LM,
         .xlevel = 0x8000000A,
         .model_id = "Intel(R) Atom(TM) CPU N270   @ 1.60GHz",
     },
+    {
+        .name = "Conroe",
+        .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSSE3 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Celeron_4x0 (Conroe/Merom Class Core 2)",
+    },
+    {
+        .name = "Penryn",
+        .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core 2 Duo P9xxx (Penryn Class Core 2)",
+    },
+    {
+        .name = "Nehalem",
+        .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core i7 9xx (Nehalem Class Core i7)",
+    },
+    {
+        .name = "Westmere",
+        .level = 11,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 44,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Westmere E56xx/L56xx/X56xx (Nehalem-C)",
+    },
+    {
+        .name = "SandyBridge",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 42,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_POPCNT |
+             CPUID_EXT_X2APIC | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+             CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Xeon E312xx (Sandy Bridge)",
+    },
+    {
+        .name = "Haswell",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 60,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_POPCNT | CPUID_EXT_X2APIC | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
+             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
+             CPUID_EXT_PCID,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+             CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .cpuid_7_0_ebx_features = CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
+            CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
+            CPUID_7_0_EBX_RTM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core Processor (Haswell)",
+    },
+    {
+        .name = "Opteron_G1",
+        .level = 5,
+        .vendor = CPUID_VENDOR_AMD,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 240 (Gen 1 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G2",
+        .level = 5,
+        .vendor = CPUID_VENDOR_AMD,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_CX16 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
+             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
+             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
+             CPUID_EXT2_APIC | CPUID_EXT2_CX8 | CPUID_EXT2_MCE |
+             CPUID_EXT2_PAE | CPUID_EXT2_MSR | CPUID_EXT2_TSC | CPUID_EXT2_PSE |
+             CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_SVM | CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 22xx (Gen 2 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G3",
+        .level = 5,
+        .vendor = CPUID_VENDOR_AMD,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_POPCNT | CPUID_EXT_CX16 | CPUID_EXT_MONITOR |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
+             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
+             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
+             CPUID_EXT2_APIC | CPUID_EXT2_CX8 | CPUID_EXT2_MCE |
+             CPUID_EXT2_PAE | CPUID_EXT2_MSR | CPUID_EXT2_TSC | CPUID_EXT2_PSE |
+             CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_MISALIGNSSE | CPUID_EXT3_SSE4A |
+             CPUID_EXT3_ABM | CPUID_EXT3_SVM | CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 23xx (Gen 3 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G4",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_AMD,
+        .family = 21,
+        .model = 1,
+        .stepping = 2,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_FMA4 | CPUID_EXT3_XOP |
+             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
+             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
+             CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000001A,
+        .model_id = "AMD Opteron 62xx class CPU",
+    },
+    {
+        .name = "Opteron_G5",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_AMD,
+        .family = 21,
+        .model = 2,
+        .stepping = 0,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_F16C | CPUID_EXT_AVX | CPUID_EXT_XSAVE |
+             CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_FMA |
+             CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_TBM | CPUID_EXT3_FMA4 | CPUID_EXT3_XOP |
+             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
+             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
+             CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000001A,
+        .model_id = "AMD Opteron 63xx class CPU",
+    },
 };
 
+#ifdef CONFIG_KVM
 static int cpu_x86_fill_model_id(char *str)
 {
     uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
@@ -506,106 +883,134 @@ static int cpu_x86_fill_model_id(char *str)
     }
     return 0;
 }
+#endif
 
-static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
+/* Fill a x86_def_t struct with information about the host CPU, and
+ * the CPU features supported by the host hardware + host kernel
+ *
+ * This function may be called only if KVM is enabled.
+ */
+static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
 {
+#ifdef CONFIG_KVM
+    KVMState *s = kvm_state;
     uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
 
+    assert(kvm_enabled());
+
     x86_cpu_def->name = "host";
     host_cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->level = eax;
-    x86_cpu_def->vendor1 = ebx;
-    x86_cpu_def->vendor2 = edx;
-    x86_cpu_def->vendor3 = ecx;
+    x86_cpu_vendor_words2str(x86_cpu_def->vendor, ebx, edx, ecx);
 
     host_cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
     x86_cpu_def->family = ((eax >> 8) & 0x0F) + ((eax >> 20) & 0xFF);
     x86_cpu_def->model = ((eax >> 4) & 0x0F) | ((eax & 0xF0000) >> 12);
     x86_cpu_def->stepping = eax & 0x0F;
-    x86_cpu_def->ext_features = ecx;
-    x86_cpu_def->features = edx;
 
-    if (kvm_enabled() && x86_cpu_def->level >= 7) {
-        x86_cpu_def->cpuid_7_0_ebx_features = kvm_arch_get_supported_cpuid(kvm_state, 0x7, 0, R_EBX);
+    x86_cpu_def->level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
+    x86_cpu_def->features = kvm_arch_get_supported_cpuid(s, 0x1, 0, R_EDX);
+    x86_cpu_def->ext_features = kvm_arch_get_supported_cpuid(s, 0x1, 0, R_ECX);
+
+    if (x86_cpu_def->level >= 7) {
+        x86_cpu_def->cpuid_7_0_ebx_features =
+                    kvm_arch_get_supported_cpuid(s, 0x7, 0, R_EBX);
     } else {
         x86_cpu_def->cpuid_7_0_ebx_features = 0;
     }
 
-    host_cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->xlevel = eax;
+    x86_cpu_def->xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0, R_EAX);
+    x86_cpu_def->ext2_features =
+                kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX);
+    x86_cpu_def->ext3_features =
+                kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_ECX);
 
-    host_cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->ext2_features = edx;
-    x86_cpu_def->ext3_features = ecx;
     cpu_x86_fill_model_id(x86_cpu_def->model_id);
-    x86_cpu_def->vendor_override = 0;
 
     /* Call Centaur's CPUID instruction. */
-    if (x86_cpu_def->vendor1 == CPUID_VENDOR_VIA_1 &&
-        x86_cpu_def->vendor2 == CPUID_VENDOR_VIA_2 &&
-        x86_cpu_def->vendor3 == CPUID_VENDOR_VIA_3) {
+    if (!strcmp(x86_cpu_def->vendor, CPUID_VENDOR_VIA)) {
         host_cpuid(0xC0000000, 0, &eax, &ebx, &ecx, &edx);
+        eax = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
         if (eax >= 0xC0000001) {
             /* Support VIA max extended level */
             x86_cpu_def->xlevel2 = eax;
             host_cpuid(0xC0000001, 0, &eax, &ebx, &ecx, &edx);
-            x86_cpu_def->ext4_features = edx;
+            x86_cpu_def->ext4_features =
+                    kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
         }
     }
 
-    /*
-     * Every SVM feature requires emulation support in KVM - so we can't just
-     * read the host features here. KVM might even support SVM features not
-     * available on the host hardware. Just set all bits and mask out the
-     * unsupported ones later.
-     */
-    x86_cpu_def->svm_features = -1;
+    /* Other KVM-specific feature fields: */
+    x86_cpu_def->svm_features =
+        kvm_arch_get_supported_cpuid(s, 0x8000000A, 0, R_EDX);
+    x86_cpu_def->kvm_features =
+        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
 
-    return 0;
+#endif /* CONFIG_KVM */
 }
 
-static int unavailable_host_feature(struct model_features_t *f, uint32_t mask)
+static int unavailable_host_feature(FeatureWordInfo *f, uint32_t mask)
 {
     int i;
 
     for (i = 0; i < 32; ++i)
         if (1 << i & mask) {
-            fprintf(stderr, "warning: host cpuid %04x_%04x lacks requested"
-                " flag '%s' [0x%08x]\n",
-                f->cpuid >> 16, f->cpuid & 0xffff,
-                f->flag_names[i] ? f->flag_names[i] : "[reserved]", mask);
+            const char *reg = get_register_name_32(f->cpuid_reg);
+            assert(reg);
+            fprintf(stderr, "warning: host doesn't support requested feature: "
+                "CPUID.%02XH:%s%s%s [bit %d]\n",
+                f->cpuid_eax, reg,
+                f->feat_names[i] ? "." : "",
+                f->feat_names[i] ? f->feat_names[i] : "", i);
             break;
         }
     return 0;
 }
 
-/* best effort attempt to inform user requested cpu flags aren't making
- * their way to the guest.  Note: ft[].check_feat ideally should be
- * specified via a guest_def field to suppress report of extraneous flags.
+/* Check if all requested cpu flags are making their way to the guest
+ *
+ * Returns 0 if all flags are supported by the host, non-zero otherwise.
+ *
+ * This function may be called only if KVM is enabled.
  */
-static int check_features_against_host(x86_def_t *guest_def)
+static int kvm_check_features_against_host(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     x86_def_t host_def;
     uint32_t mask;
     int rv, i;
     struct model_features_t ft[] = {
-        {&guest_def->features, &host_def.features,
-            ~0, feature_name, 0x00000000},
-        {&guest_def->ext_features, &host_def.ext_features,
-            ~CPUID_EXT_HYPERVISOR, ext_feature_name, 0x00000001},
-        {&guest_def->ext2_features, &host_def.ext2_features,
-            ~PPRO_FEATURES, ext2_feature_name, 0x80000000},
-        {&guest_def->ext3_features, &host_def.ext3_features,
-            ~CPUID_EXT3_SVM, ext3_feature_name, 0x80000001}};
-
-    cpu_x86_fill_host(&host_def);
-    for (rv = 0, i = 0; i < ARRAY_SIZE(ft); ++i)
-        for (mask = 1; mask; mask <<= 1)
-            if (ft[i].check_feat & mask && *ft[i].guest_feat & mask &&
+        {&env->cpuid_features, &host_def.features,
+            FEAT_1_EDX },
+        {&env->cpuid_ext_features, &host_def.ext_features,
+            FEAT_1_ECX },
+        {&env->cpuid_ext2_features, &host_def.ext2_features,
+            FEAT_8000_0001_EDX },
+        {&env->cpuid_ext3_features, &host_def.ext3_features,
+            FEAT_8000_0001_ECX },
+        {&env->cpuid_ext4_features, &host_def.ext4_features,
+            FEAT_C000_0001_EDX },
+        {&env->cpuid_7_0_ebx_features, &host_def.cpuid_7_0_ebx_features,
+            FEAT_7_0_EBX },
+        {&env->cpuid_svm_features, &host_def.svm_features,
+            FEAT_SVM },
+        {&env->cpuid_kvm_features, &host_def.kvm_features,
+            FEAT_KVM },
+    };
+
+    assert(kvm_enabled());
+
+    kvm_cpu_fill_host(&host_def);
+    for (rv = 0, i = 0; i < ARRAY_SIZE(ft); ++i) {
+        FeatureWord w = ft[i].feat_word;
+        FeatureWordInfo *wi = &feature_word_info[w];
+        for (mask = 1; mask; mask <<= 1) {
+            if (*ft[i].guest_feat & mask &&
                 !(*ft[i].host_feat & mask)) {
-                    unavailable_host_feature(&ft[i], mask);
-                    rv = 1;
-                }
+                unavailable_host_feature(wi, mask);
+                rv = 1;
+            }
+        }
+    }
     return rv;
 }
 
@@ -758,15 +1163,10 @@ static char *x86_cpuid_get_vendor(Object *obj, Error **errp)
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     char *value;
-    int i;
 
-    value = (char *)g_malloc(12 + 1);
-    for (i = 0; i < 4; i++) {
-        value[i    ] = env->cpuid_vendor1 >> (8 * i);
-        value[i + 4] = env->cpuid_vendor2 >> (8 * i);
-        value[i + 8] = env->cpuid_vendor3 >> (8 * i);
-    }
-    value[12] = '\0';
+    value = (char *)g_malloc(CPUID_VENDOR_SZ + 1);
+    x86_cpu_vendor_words2str(value, env->cpuid_vendor1, env->cpuid_vendor2,
+                             env->cpuid_vendor3);
     return value;
 }
 
@@ -777,7 +1177,7 @@ static void x86_cpuid_set_vendor(Object *obj, const char *value,
     CPUX86State *env = &cpu->env;
     int i;
 
-    if (strlen(value) != 12) {
+    if (strlen(value) != CPUID_VENDOR_SZ) {
         error_set(errp, QERR_PROPERTY_VALUE_BAD, "",
                   "vendor", value);
         return;
@@ -791,7 +1191,6 @@ static void x86_cpuid_set_vendor(Object *obj, const char *value,
         env->cpuid_vendor2 |= ((uint8_t)value[i + 4]) << (8 * i);
         env->cpuid_vendor3 |= ((uint8_t)value[i + 8]) << (8 * i);
     }
-    env->cpuid_vendor_override = 1;
 }
 
 static char *x86_cpuid_get_model_id(Object *obj, Error **errp)
@@ -846,7 +1245,7 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
 {
     X86CPU *cpu = X86_CPU(obj);
     const int64_t min = 0;
-    const int64_t max = INT_MAX;
+    const int64_t max = INT64_MAX;
     int64_t value;
 
     visit_type_int(v, &value, name, errp);
@@ -862,148 +1261,116 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
     cpu->env.tsc_khz = value / 1000;
 }
 
-static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
+static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *name)
 {
-    unsigned int i;
     x86_def_t *def;
+    int i;
 
-    char *s = g_strdup(cpu_model);
-    char *featurestr, *name = strtok(s, ",");
-    /* Features to be added*/
-    uint32_t plus_features = 0, plus_ext_features = 0;
-    uint32_t plus_ext2_features = 0, plus_ext3_features = 0;
-    uint32_t plus_kvm_features = 0, plus_svm_features = 0;
-    /* Features to be removed */
-    uint32_t minus_features = 0, minus_ext_features = 0;
-    uint32_t minus_ext2_features = 0, minus_ext3_features = 0;
-    uint32_t minus_kvm_features = 0, minus_svm_features = 0;
-    uint32_t numvalue;
+    if (name == NULL) {
+        return -1;
+    }
+    if (kvm_enabled() && strcmp(name, "host") == 0) {
+        kvm_cpu_fill_host(x86_cpu_def);
+        return 0;
+    }
 
-    for (def = x86_defs; def; def = def->next)
-        if (name && !strcmp(name, def->name))
-            break;
-    if (kvm_enabled() && name && strcmp(name, "host") == 0) {
-        cpu_x86_fill_host(x86_cpu_def);
-    } else if (!def) {
-        goto error;
-    } else {
-        memcpy(x86_cpu_def, def, sizeof(*def));
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
+        def = &builtin_x86_defs[i];
+        if (strcmp(name, def->name) == 0) {
+            memcpy(x86_cpu_def, def, sizeof(*def));
+            /* sysenter isn't supported in compatibility mode on AMD,
+             * syscall isn't supported in compatibility mode on Intel.
+             * Normally we advertise the actual CPU vendor, but you can
+             * override this using the 'vendor' property if you want to use
+             * KVM's sysenter/syscall emulation in compatibility mode and
+             * when doing cross vendor migration
+             */
+            if (kvm_enabled()) {
+                uint32_t  ebx = 0, ecx = 0, edx = 0;
+                host_cpuid(0, 0, NULL, &ebx, &ecx, &edx);
+                x86_cpu_vendor_words2str(x86_cpu_def->vendor, ebx, edx, ecx);
+            }
+            return 0;
+        }
     }
 
-#if defined(CONFIG_KVM)
-    plus_kvm_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
-        (1 << KVM_FEATURE_NOP_IO_DELAY) | 
-        (1 << KVM_FEATURE_MMU_OP) |
-        (1 << KVM_FEATURE_CLOCKSOURCE2) |
-        (1 << KVM_FEATURE_ASYNC_PF) | 
-        (1 << KVM_FEATURE_STEAL_TIME) |
-        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
-#else
-    plus_kvm_features = 0;
-#endif
+    return -1;
+}
 
-    add_flagname_to_bitmaps("hypervisor", &plus_features,
-        &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
-        &plus_kvm_features, &plus_svm_features);
+/* Parse "+feature,-feature,feature=foo" CPU feature string
+ */
+static void cpu_x86_parse_featurestr(X86CPU *cpu, char *features, Error **errp)
+{
+    char *featurestr; /* Single 'key=value" string being parsed */
+    /* Features to be added */
+    FeatureWordArray plus_features = { 0 };
+    /* Features to be removed */
+    FeatureWordArray minus_features = { 0 };
+    uint32_t numvalue;
+    CPUX86State *env = &cpu->env;
 
-    featurestr = strtok(NULL, ",");
+    featurestr = features ? strtok(features, ",") : NULL;
 
     while (featurestr) {
         char *val;
         if (featurestr[0] == '+') {
-            add_flagname_to_bitmaps(featurestr + 1, &plus_features,
-                            &plus_ext_features, &plus_ext2_features,
-                            &plus_ext3_features, &plus_kvm_features,
-                            &plus_svm_features);
+            add_flagname_to_bitmaps(featurestr + 1, plus_features);
         } else if (featurestr[0] == '-') {
-            add_flagname_to_bitmaps(featurestr + 1, &minus_features,
-                            &minus_ext_features, &minus_ext2_features,
-                            &minus_ext3_features, &minus_kvm_features,
-                            &minus_svm_features);
+            add_flagname_to_bitmaps(featurestr + 1, minus_features);
         } else if ((val = strchr(featurestr, '='))) {
             *val = 0; val++;
             if (!strcmp(featurestr, "family")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xff + 0xf) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->family = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "model")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xff) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->model = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "stepping")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xf) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->stepping = numvalue ;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "level")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->level = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "xlevel")) {
                 char *err;
+                char num[32];
+
                 numvalue = strtoul(val, &err, 0);
                 if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s\n", val);
+                    goto out;
                 }
                 if (numvalue < 0x80000000) {
+                    fprintf(stderr, "xlevel value shall always be >= 0x80000000"
+                            ", fixup will be removed in future versions\n");
                     numvalue += 0x80000000;
                 }
-                x86_cpu_def->xlevel = numvalue;
+                snprintf(num, sizeof(num), "%" PRIu32, numvalue);
+                object_property_parse(OBJECT(cpu), num, featurestr, errp);
             } else if (!strcmp(featurestr, "vendor")) {
-                if (strlen(val) != 12) {
-                    fprintf(stderr, "vendor string must be 12 chars long\n");
-                    goto error;
-                }
-                x86_cpu_def->vendor1 = 0;
-                x86_cpu_def->vendor2 = 0;
-                x86_cpu_def->vendor3 = 0;
-                for(i = 0; i < 4; i++) {
-                    x86_cpu_def->vendor1 |= ((uint8_t)val[i    ]) << (8 * i);
-                    x86_cpu_def->vendor2 |= ((uint8_t)val[i + 4]) << (8 * i);
-                    x86_cpu_def->vendor3 |= ((uint8_t)val[i + 8]) << (8 * i);
-                }
-                x86_cpu_def->vendor_override = 1;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "model_id")) {
-                pstrcpy(x86_cpu_def->model_id, sizeof(x86_cpu_def->model_id),
-                        val);
+                object_property_parse(OBJECT(cpu), val, "model-id", errp);
             } else if (!strcmp(featurestr, "tsc_freq")) {
                 int64_t tsc_freq;
                 char *err;
+                char num[32];
 
                 tsc_freq = strtosz_suffix_unit(val, &err,
                                                STRTOSZ_DEFSUFFIX_B, 1000);
                 if (tsc_freq < 0 || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s\n", val);
+                    goto out;
                 }
-                x86_cpu_def->tsc_khz = tsc_freq / 1000;
+                snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
+                object_property_parse(OBJECT(cpu), num, "tsc-frequency", errp);
             } else if (!strcmp(featurestr, "hv_spinlocks")) {
                 char *err;
                 numvalue = strtoul(val, &err, 0);
                 if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s\n", val);
+                    goto out;
                 }
                 hyperv_set_spinlock_retries(numvalue);
             } else {
-                fprintf(stderr, "unrecognized feature %s\n", featurestr);
-                goto error;
+                error_setg(errp, "unrecognized feature %s\n", featurestr);
+                goto out;
             }
         } else if (!strcmp(featurestr, "check")) {
             check_cpuid = 1;
@@ -1014,33 +1381,34 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
         } else if (!strcmp(featurestr, "hv_vapic")) {
             hyperv_enable_vapic_recommended(true);
         } else {
-            fprintf(stderr, "feature string `%s' not in format (+feature|-feature|feature=xyz)\n", featurestr);
-            goto error;
+            error_setg(errp, "feature string `%s' not in format (+feature|"
+                       "-feature|feature=xyz)\n", featurestr);
+            goto out;
+        }
+        if (error_is_set(errp)) {
+            goto out;
         }
         featurestr = strtok(NULL, ",");
     }
-    x86_cpu_def->features |= plus_features;
-    x86_cpu_def->ext_features |= plus_ext_features;
-    x86_cpu_def->ext2_features |= plus_ext2_features;
-    x86_cpu_def->ext3_features |= plus_ext3_features;
-    x86_cpu_def->kvm_features |= plus_kvm_features;
-    x86_cpu_def->svm_features |= plus_svm_features;
-    x86_cpu_def->features &= ~minus_features;
-    x86_cpu_def->ext_features &= ~minus_ext_features;
-    x86_cpu_def->ext2_features &= ~minus_ext2_features;
-    x86_cpu_def->ext3_features &= ~minus_ext3_features;
-    x86_cpu_def->kvm_features &= ~minus_kvm_features;
-    x86_cpu_def->svm_features &= ~minus_svm_features;
-    if (check_cpuid) {
-        if (check_features_against_host(x86_cpu_def) && enforce_cpuid)
-            goto error;
-    }
-    g_free(s);
-    return 0;
-
-error:
-    g_free(s);
-    return -1;
+    env->cpuid_features |= plus_features[FEAT_1_EDX];
+    env->cpuid_ext_features |= plus_features[FEAT_1_ECX];
+    env->cpuid_ext2_features |= plus_features[FEAT_8000_0001_EDX];
+    env->cpuid_ext3_features |= plus_features[FEAT_8000_0001_ECX];
+    env->cpuid_ext4_features |= plus_features[FEAT_C000_0001_EDX];
+    env->cpuid_kvm_features |= plus_features[FEAT_KVM];
+    env->cpuid_svm_features |= plus_features[FEAT_SVM];
+    env->cpuid_7_0_ebx_features |= plus_features[FEAT_7_0_EBX];
+    env->cpuid_features &= ~minus_features[FEAT_1_EDX];
+    env->cpuid_ext_features &= ~minus_features[FEAT_1_ECX];
+    env->cpuid_ext2_features &= ~minus_features[FEAT_8000_0001_EDX];
+    env->cpuid_ext3_features &= ~minus_features[FEAT_8000_0001_ECX];
+    env->cpuid_ext4_features &= ~minus_features[FEAT_C000_0001_EDX];
+    env->cpuid_kvm_features &= ~minus_features[FEAT_KVM];
+    env->cpuid_svm_features &= ~minus_features[FEAT_SVM];
+    env->cpuid_7_0_ebx_features &= ~minus_features[FEAT_7_0_EBX];
+
+out:
+    return;
 }
 
 /* generate a composite string into buf of all cpuid names in featureset
@@ -1073,81 +1441,43 @@ static void listflags(char *buf, int bufsize, uint32_t fbits,
         }
 }
 
-/* generate CPU information:
- * -?        list model names
- * -?model   list model names/IDs
- * -?dump    output all model (x86_def_t) data
- * -?cpuid   list all recognized cpuid flag names
- */
-void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
+/* generate CPU information. */
+void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    unsigned char model = !strcmp("?model", optarg);
-    unsigned char dump = !strcmp("?dump", optarg);
-    unsigned char cpuid = !strcmp("?cpuid", optarg);
     x86_def_t *def;
     char buf[256];
+    int i;
 
-    if (cpuid) {
-        (*cpu_fprintf)(f, "Recognized CPUID flags:\n");
-        listflags(buf, sizeof (buf), (uint32_t)~0, feature_name, 1);
-        (*cpu_fprintf)(f, "  f_edx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext_feature_name, 1);
-        (*cpu_fprintf)(f, "  f_ecx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext2_feature_name, 1);
-        (*cpu_fprintf)(f, "  extf_edx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext3_feature_name, 1);
-        (*cpu_fprintf)(f, "  extf_ecx: %s\n", buf);
-        return;
-    }
-    for (def = x86_defs; def; def = def->next) {
-        snprintf(buf, sizeof (buf), def->flags ? "[%s]": "%s", def->name);
-        if (model || dump) {
-            (*cpu_fprintf)(f, "x86 %16s  %-48s\n", buf, def->model_id);
-        } else {
-            (*cpu_fprintf)(f, "x86 %16s\n", buf);
-        }
-        if (dump) {
-            memcpy(buf, &def->vendor1, sizeof (def->vendor1));
-            memcpy(buf + 4, &def->vendor2, sizeof (def->vendor2));
-            memcpy(buf + 8, &def->vendor3, sizeof (def->vendor3));
-            buf[12] = '\0';
-            (*cpu_fprintf)(f,
-                "  family %d model %d stepping %d level %d xlevel 0x%x"
-                " vendor \"%s\"\n",
-                def->family, def->model, def->stepping, def->level,
-                def->xlevel, buf);
-            listflags(buf, sizeof (buf), def->features, feature_name, 0);
-            (*cpu_fprintf)(f, "  feature_edx %08x (%s)\n", def->features,
-                buf);
-            listflags(buf, sizeof (buf), def->ext_features, ext_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  feature_ecx %08x (%s)\n", def->ext_features,
-                buf);
-            listflags(buf, sizeof (buf), def->ext2_features, ext2_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  extfeature_edx %08x (%s)\n",
-                def->ext2_features, buf);
-            listflags(buf, sizeof (buf), def->ext3_features, ext3_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  extfeature_ecx %08x (%s)\n",
-                def->ext3_features, buf);
-            (*cpu_fprintf)(f, "\n");
-        }
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
+        def = &builtin_x86_defs[i];
+        snprintf(buf, sizeof(buf), "%s", def->name);
+        (*cpu_fprintf)(f, "x86 %16s  %-48s\n", buf, def->model_id);
     }
     if (kvm_enabled()) {
         (*cpu_fprintf)(f, "x86 %16s\n", "[host]");
     }
+    (*cpu_fprintf)(f, "\nRecognized CPUID flags:\n");
+    listflags(buf, sizeof(buf), (uint32_t)~0, feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext2_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext3_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
 }
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
     CpuDefinitionInfoList *cpu_list = NULL;
     x86_def_t *def;
+    int i;
 
-    for (def = x86_defs; def; def = def->next) {
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
         CpuDefinitionInfoList *entry;
         CpuDefinitionInfo *info;
 
+        def = &builtin_x86_defs[i];
         info = g_malloc0(sizeof(*info));
         info->name = g_strdup(def->name);
 
@@ -1160,26 +1490,61 @@ CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
     return cpu_list;
 }
 
+#ifdef CONFIG_KVM
+static void filter_features_for_kvm(X86CPU *cpu)
+{
+    CPUX86State *env = &cpu->env;
+    KVMState *s = kvm_state;
+
+    env->cpuid_features &=
+        kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
+    env->cpuid_ext_features &=
+        kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
+    env->cpuid_ext2_features &=
+        kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX);
+    env->cpuid_ext3_features &=
+        kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_ECX);
+    env->cpuid_svm_features  &=
+        kvm_arch_get_supported_cpuid(s, 0x8000000A, 0, R_EDX);
+    env->cpuid_7_0_ebx_features &=
+        kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX);
+    env->cpuid_kvm_features &=
+        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
+    env->cpuid_ext4_features &=
+        kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
+
+}
+#endif
+
 int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
 {
     CPUX86State *env = &cpu->env;
     x86_def_t def1, *def = &def1;
     Error *error = NULL;
+    char *name, *features;
+    gchar **model_pieces;
 
     memset(def, 0, sizeof(*def));
 
-    if (cpu_x86_find_by_name(def, cpu_model) < 0)
-        return -1;
-    if (def->vendor1) {
-        env->cpuid_vendor1 = def->vendor1;
-        env->cpuid_vendor2 = def->vendor2;
-        env->cpuid_vendor3 = def->vendor3;
-    } else {
-        env->cpuid_vendor1 = CPUID_VENDOR_INTEL_1;
-        env->cpuid_vendor2 = CPUID_VENDOR_INTEL_2;
-        env->cpuid_vendor3 = CPUID_VENDOR_INTEL_3;
+    model_pieces = g_strsplit(cpu_model, ",", 2);
+    if (!model_pieces[0]) {
+        error_setg(&error, "Invalid/empty CPU model name");
+        goto out;
+    }
+    name = model_pieces[0];
+    features = model_pieces[1];
+
+    if (cpu_x86_find_by_name(def, name) < 0) {
+        error_setg(&error, "Unable to find CPU definition: %s", name);
+        goto out;
     }
-    env->cpuid_vendor_override = def->vendor_override;
+
+    if (kvm_enabled()) {
+        def->kvm_features |= kvm_default_features;
+    }
+    def->ext_features |= CPUID_EXT_HYPERVISOR;
+
+    object_property_set_str(OBJECT(cpu), def->vendor, "vendor", &error);
     object_property_set_int(OBJECT(cpu), def->level, "level", &error);
     object_property_set_int(OBJECT(cpu), def->family, "family", &error);
     object_property_set_int(OBJECT(cpu), def->model, "model", &error);
@@ -1192,23 +1557,19 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
     env->cpuid_kvm_features = def->kvm_features;
     env->cpuid_svm_features = def->svm_features;
     env->cpuid_ext4_features = def->ext4_features;
-    env->cpuid_7_0_ebx = def->cpuid_7_0_ebx_features;
+    env->cpuid_7_0_ebx_features = def->cpuid_7_0_ebx_features;
     env->cpuid_xlevel2 = def->xlevel2;
-    object_property_set_int(OBJECT(cpu), (int64_t)def->tsc_khz * 1000,
-                            "tsc-frequency", &error);
-    if (!kvm_enabled()) {
-        env->cpuid_features &= TCG_FEATURES;
-        env->cpuid_ext_features &= TCG_EXT_FEATURES;
-        env->cpuid_ext2_features &= (TCG_EXT2_FEATURES
-#ifdef TARGET_X86_64
-            | CPUID_EXT2_SYSCALL | CPUID_EXT2_LM
-#endif
-            );
-        env->cpuid_ext3_features &= TCG_EXT3_FEATURES;
-        env->cpuid_svm_features &= TCG_SVM_FEATURES;
-    }
+
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", &error);
-    if (error_is_set(&error)) {
+    if (error) {
+        goto out;
+    }
+
+    cpu_x86_parse_featurestr(cpu, features, &error);
+out:
+    g_strfreev(model_pieces);
+    if (error) {
+        fprintf(stderr, "%s\n", error_get_pretty(error));
         error_free(error);
         return -1;
     }
@@ -1216,109 +1577,6 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-/* copy vendor id string to 32 bit register, nul pad as needed
- */
-static void cpyid(const char *s, uint32_t *id)
-{
-    char *d = (char *)id;
-    char i;
-
-    for (i = sizeof (*id); i--; )
-        *d++ = *s ? *s++ : '\0';
-}
-
-/* interpret radix and convert from string to arbitrary scalar,
- * otherwise flag failure
- */
-#define setscalar(pval, str, perr)                      \
-{                                                       \
-    char *pend;                                         \
-    unsigned long ul;                                   \
-                                                        \
-    ul = strtoul(str, &pend, 0);                        \
-    *str && !*pend ? (*pval = ul) : (*perr = 1);        \
-}
-
-/* map cpuid options to feature bits, otherwise return failure
- * (option tags in *str are delimited by whitespace)
- */
-static void setfeatures(uint32_t *pval, const char *str,
-    const char **featureset, int *perr)
-{
-    const char *p, *q;
-
-    for (q = p = str; *p || *q; q = p) {
-        while (iswhite(*p))
-            q = ++p;
-        while (*p && !iswhite(*p))
-            ++p;
-        if (!*q && !*p)
-            return;
-        if (!lookup_feature(pval, q, p, featureset)) {
-            fprintf(stderr, "error: feature \"%.*s\" not available in set\n",
-                (int)(p - q), q);
-            *perr = 1;
-            return;
-        }
-    }
-}
-
-/* map config file options to x86_def_t form
- */
-static int cpudef_setfield(const char *name, const char *str, void *opaque)
-{
-    x86_def_t *def = opaque;
-    int err = 0;
-
-    if (!strcmp(name, "name")) {
-        g_free((void *)def->name);
-        def->name = g_strdup(str);
-    } else if (!strcmp(name, "model_id")) {
-        strncpy(def->model_id, str, sizeof (def->model_id));
-    } else if (!strcmp(name, "level")) {
-        setscalar(&def->level, str, &err)
-    } else if (!strcmp(name, "vendor")) {
-        cpyid(&str[0], &def->vendor1);
-        cpyid(&str[4], &def->vendor2);
-        cpyid(&str[8], &def->vendor3);
-    } else if (!strcmp(name, "family")) {
-        setscalar(&def->family, str, &err)
-    } else if (!strcmp(name, "model")) {
-        setscalar(&def->model, str, &err)
-    } else if (!strcmp(name, "stepping")) {
-        setscalar(&def->stepping, str, &err)
-    } else if (!strcmp(name, "feature_edx")) {
-        setfeatures(&def->features, str, feature_name, &err);
-    } else if (!strcmp(name, "feature_ecx")) {
-        setfeatures(&def->ext_features, str, ext_feature_name, &err);
-    } else if (!strcmp(name, "extfeature_edx")) {
-        setfeatures(&def->ext2_features, str, ext2_feature_name, &err);
-    } else if (!strcmp(name, "extfeature_ecx")) {
-        setfeatures(&def->ext3_features, str, ext3_feature_name, &err);
-    } else if (!strcmp(name, "xlevel")) {
-        setscalar(&def->xlevel, str, &err)
-    } else {
-        fprintf(stderr, "error: unknown option [%s = %s]\n", name, str);
-        return (1);
-    }
-    if (err) {
-        fprintf(stderr, "error: bad option value [%s = %s]\n", name, str);
-        return (1);
-    }
-    return (0);
-}
-
-/* register config file entry as x86_def_t
- */
-static int cpudef_register(QemuOpts *opts, void *opaque)
-{
-    x86_def_t *def = g_malloc0(sizeof (x86_def_t));
-
-    qemu_opt_foreach(opts, cpudef_setfield, def, 1);
-    def->next = x86_defs;
-    x86_defs = def;
-    return (0);
-}
 
 void cpu_clear_apic_feature(CPUX86State *env)
 {
@@ -1327,8 +1585,7 @@ void cpu_clear_apic_feature(CPUX86State *env)
 
 #endif /* !CONFIG_USER_ONLY */
 
-/* register "cpudef" models defined in configuration file.  Here we first
- * preload any built-in definitions
+/* Initialize list of CPU models, filling some non-static fields if necessary
  */
 void x86_cpudef_setup(void)
 {
@@ -1336,24 +1593,20 @@ void x86_cpudef_setup(void)
     static const char *model_with_versions[] = { "qemu32", "qemu64", "athlon" };
 
     for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); ++i) {
-        builtin_x86_defs[i].next = x86_defs;
-        builtin_x86_defs[i].flags = 1;
+        x86_def_t *def = &builtin_x86_defs[i];
 
         /* Look for specific "cpudef" models that */
         /* have the QEMU version in .model_id */
         for (j = 0; j < ARRAY_SIZE(model_with_versions); j++) {
-            if (strcmp(model_with_versions[j], builtin_x86_defs[i].name) == 0) {
-                pstrcpy(builtin_x86_defs[i].model_id, sizeof(builtin_x86_defs[i].model_id), "QEMU Virtual CPU version ");
-                pstrcat(builtin_x86_defs[i].model_id, sizeof(builtin_x86_defs[i].model_id), qemu_get_version());
+            if (strcmp(model_with_versions[j], def->name) == 0) {
+                pstrcpy(def->model_id, sizeof(def->model_id),
+                        "QEMU Virtual CPU version ");
+                pstrcat(def->model_id, sizeof(def->model_id),
+                        qemu_get_version());
                 break;
             }
         }
-
-        x86_defs = &builtin_x86_defs[i];
     }
-#if !defined(CONFIG_USER_ONLY)
-    qemu_opts_foreach(qemu_find_opts("cpudef"), cpudef_register, NULL, 0);
-#endif
 }
 
 static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
@@ -1362,22 +1615,15 @@ static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
     *ebx = env->cpuid_vendor1;
     *edx = env->cpuid_vendor2;
     *ecx = env->cpuid_vendor3;
-
-    /* sysenter isn't supported on compatibility mode on AMD, syscall
-     * isn't supported in compatibility mode on Intel.
-     * Normally we advertise the actual cpu vendor, but you can override
-     * this if you want to use KVM's sysenter/syscall emulation
-     * in compatibility mode and when doing cross vendor migration
-     */
-    if (kvm_enabled() && ! env->cpuid_vendor_override) {
-        host_cpuid(0, 0, NULL, ebx, ecx, edx);
-    }
 }
 
 void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                    uint32_t *eax, uint32_t *ebx,
                    uint32_t *ecx, uint32_t *edx)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
     /* test if maximum index reached */
     if (index & 0x80000000) {
         if (index > env->cpuid_xlevel) {
@@ -1389,7 +1635,11 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                     index = env->cpuid_xlevel;
                 }
             } else {
-                index =  env->cpuid_xlevel;
+                /* Intel documentation states that invalid EAX input will
+                 * return the same information as EAX=cpuid_level
+                 * (Intel SDM Vol. 2A - Instruction Set Reference - CPUID)
+                 */
+                index =  env->cpuid_level;
             }
         }
     } else {
@@ -1407,8 +1657,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
         *ecx = env->cpuid_ext_features;
         *edx = env->cpuid_features;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ebx |= (env->nr_cores * env->nr_threads) << 16;
+        if (cs->nr_cores * cs->nr_threads > 1) {
+            *ebx |= (cs->nr_cores * cs->nr_threads) << 16;
             *edx |= 1 << 28;    /* HTT bit */
         }
         break;
@@ -1421,8 +1671,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 4:
         /* cache info: needed for Core compatibility */
-        if (env->nr_cores > 1) {
-            *eax = (env->nr_cores - 1) << 26;
+        if (cs->nr_cores > 1) {
+            *eax = (cs->nr_cores - 1) << 26;
         } else {
             *eax = 0;
         }
@@ -1441,8 +1691,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                 break;
             case 2: /* L2 cache info */
                 *eax |= 0x0000143;
-                if (env->nr_threads > 1) {
-                    *eax |= (env->nr_threads - 1) << 14;
+                if (cs->nr_threads > 1) {
+                    *eax |= (cs->nr_threads - 1) << 14;
                 }
                 *ebx = 0x3c0003f;
                 *ecx = 0x0000fff;
@@ -1474,7 +1724,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         /* Structured Extended Feature Flags Enumeration Leaf */
         if (count == 0) {
             *eax = 0; /* Maximum ECX value for sub-leaves */
-            *ebx = env->cpuid_7_0_ebx; /* Feature flags */
+            *ebx = env->cpuid_7_0_ebx_features; /* Feature flags */
             *ecx = 0; /* Reserved */
             *edx = 0; /* Reserved */
         } else {
@@ -1494,7 +1744,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
     case 0xA:
         /* Architectural Performance Monitoring Leaf */
         if (kvm_enabled()) {
-            KVMState *s = env->kvm_state;
+            KVMState *s = cs->kvm_state;
 
             *eax = kvm_arch_get_supported_cpuid(s, 0xA, count, R_EAX);
             *ebx = kvm_arch_get_supported_cpuid(s, 0xA, count, R_EBX);
@@ -1517,7 +1767,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
             break;
         }
         if (kvm_enabled()) {
-            KVMState *s = env->kvm_state;
+            KVMState *s = cs->kvm_state;
 
             *eax = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EAX);
             *ebx = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EBX);
@@ -1546,7 +1796,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
          * discards multiple thread information if it is set.
          * So dont set it here for Intel to make Linux guests happy.
          */
-        if (env->nr_cores * env->nr_threads > 1) {
+        if (cs->nr_cores * cs->nr_threads > 1) {
             uint32_t tebx, tecx, tedx;
             get_cpuid_vendor(env, &tebx, &tecx, &tedx);
             if (tebx != CPUID_VENDOR_INTEL_1 ||
@@ -1594,22 +1844,22 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = 0;
         *ecx = 0;
         *edx = 0;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ecx |= (env->nr_cores * env->nr_threads) - 1;
+        if (cs->nr_cores * cs->nr_threads > 1) {
+            *ecx |= (cs->nr_cores * cs->nr_threads) - 1;
         }
         break;
     case 0x8000000A:
-	if (env->cpuid_ext3_features & CPUID_EXT3_SVM) {
-		*eax = 0x00000001; /* SVM Revision */
-		*ebx = 0x00000010; /* nr of ASIDs */
-		*ecx = 0;
-		*edx = env->cpuid_svm_features; /* optional features */
-	} else {
-		*eax = 0;
-		*ebx = 0;
-		*ecx = 0;
-		*edx = 0;
-	}
+        if (env->cpuid_ext3_features & CPUID_EXT3_SVM) {
+            *eax = 0x00000001; /* SVM Revision */
+            *ebx = 0x00000010; /* nr of ASIDs */
+            *ecx = 0;
+            *edx = env->cpuid_svm_features; /* optional features */
+        } else {
+            *eax = 0;
+            *ebx = 0;
+            *ecx = 0;
+            *edx = 0;
+        }
         break;
     case 0xC0000000:
         *eax = env->cpuid_xlevel2;
@@ -1652,8 +1902,8 @@ static void x86_cpu_reset(CPUState *s)
     int i;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
-        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
+        log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
     }
 
     xcc->parent_reset(s);
@@ -1726,7 +1976,7 @@ static void x86_cpu_reset(CPUState *s)
 
 #if !defined(CONFIG_USER_ONLY)
     /* We hard-wire the BSP to the first CPU. */
-    if (env->cpu_index == 0) {
+    if (s->cpu_index == 0) {
         apic_designate_bsp(env->apic_state);
     }
 
@@ -1764,12 +2014,102 @@ static void mce_init(X86CPU *cpu)
     }
 }
 
+#define MSI_ADDR_BASE 0xfee00000
+
+#ifndef CONFIG_USER_ONLY
+static void x86_cpu_apic_init(X86CPU *cpu, Error **errp)
+{
+    static int apic_mapped;
+    CPUX86State *env = &cpu->env;
+    APICCommonState *apic;
+    const char *apic_type = "apic";
+
+    if (kvm_irqchip_in_kernel()) {
+        apic_type = "kvm-apic";
+    } else if (xen_enabled()) {
+        apic_type = "xen-apic";
+    }
+
+    env->apic_state = qdev_try_create(NULL, apic_type);
+    if (env->apic_state == NULL) {
+        error_setg(errp, "APIC device '%s' could not be created", apic_type);
+        return;
+    }
+
+    object_property_add_child(OBJECT(cpu), "apic",
+                              OBJECT(env->apic_state), NULL);
+    qdev_prop_set_uint8(env->apic_state, "id", env->cpuid_apic_id);
+    /* TODO: convert to link<> */
+    apic = APIC_COMMON(env->apic_state);
+    apic->cpu = cpu;
+
+    if (qdev_init(env->apic_state)) {
+        error_setg(errp, "APIC device '%s' could not be initialized",
+                   object_get_typename(OBJECT(env->apic_state)));
+        return;
+    }
+
+    /* XXX: mapping more APICs at the same memory location */
+    if (apic_mapped == 0) {
+        /* NOTE: the APIC is directly connected to the CPU - it is not
+           on the global memory bus. */
+        /* XXX: what if the base changes? */
+        sysbus_mmio_map(SYS_BUS_DEVICE(env->apic_state), 0, MSI_ADDR_BASE);
+        apic_mapped = 1;
+    }
+}
+#endif
+
 void x86_cpu_realize(Object *obj, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
+    CPUX86State *env = &cpu->env;
+
+    if (env->cpuid_7_0_ebx_features && env->cpuid_level < 7) {
+        env->cpuid_level = 7;
+    }
+
+    /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
+     * CPUID[1].EDX.
+     */
+    if (env->cpuid_vendor1 == CPUID_VENDOR_AMD_1 &&
+        env->cpuid_vendor2 == CPUID_VENDOR_AMD_2 &&
+        env->cpuid_vendor3 == CPUID_VENDOR_AMD_3) {
+        env->cpuid_ext2_features &= ~CPUID_EXT2_AMD_ALIASES;
+        env->cpuid_ext2_features |= (env->cpuid_features
+           & CPUID_EXT2_AMD_ALIASES);
+    }
+
+    if (!kvm_enabled()) {
+        env->cpuid_features &= TCG_FEATURES;
+        env->cpuid_ext_features &= TCG_EXT_FEATURES;
+        env->cpuid_ext2_features &= (TCG_EXT2_FEATURES
+#ifdef TARGET_X86_64
+            | CPUID_EXT2_SYSCALL | CPUID_EXT2_LM
+#endif
+            );
+        env->cpuid_ext3_features &= TCG_EXT3_FEATURES;
+        env->cpuid_svm_features &= TCG_SVM_FEATURES;
+    } else {
+#ifdef CONFIG_KVM
+        filter_features_for_kvm(cpu);
+#endif
+        if (check_cpuid && kvm_check_features_against_host(cpu)
+            && enforce_cpuid) {
+            error_setg(errp, "Host's CPU doesn't support requested features");
+            return;
+        }
+    }
 
 #ifndef CONFIG_USER_ONLY
     qemu_register_reset(x86_cpu_machine_reset_cb, cpu);
+
+    if (cpu->env.cpuid_features & CPUID_APIC || smp_cpus > 1) {
+        x86_cpu_apic_init(cpu, errp);
+        if (error_is_set(errp)) {
+            return;
+        }
+    }
 #endif
 
     mce_init(cpu);
@@ -1777,8 +2117,42 @@ void x86_cpu_realize(Object *obj, Error **errp)
     cpu_reset(CPU(cpu));
 }
 
+/* Enables contiguous-apic-ID mode, for compatibility */
+static bool compat_apic_id_mode;
+
+void enable_compat_apic_id_mode(void)
+{
+    compat_apic_id_mode = true;
+}
+
+/* Calculates initial APIC ID for a specific CPU index
+ *
+ * Currently we need to be able to calculate the APIC ID from the CPU index
+ * alone (without requiring a CPU object), as the QEMU<->Seabios interfaces have
+ * no concept of "CPU index", and the NUMA tables on fw_cfg need the APIC ID of
+ * all CPUs up to max_cpus.
+ */
+uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index)
+{
+    uint32_t correct_id;
+    static bool warned;
+
+    correct_id = x86_apicid_from_cpu_idx(smp_cores, smp_threads, cpu_index);
+    if (compat_apic_id_mode) {
+        if (cpu_index != correct_id && !warned) {
+            error_report("APIC IDs set in compatibility mode, "
+                         "CPU topology won't match the configuration");
+            warned = true;
+        }
+        return cpu_index;
+    } else {
+        return correct_id;
+    }
+}
+
 static void x86_cpu_initfn(Object *obj)
 {
+    CPUState *cs = CPU(obj);
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     static int inited;
@@ -1810,7 +2184,7 @@ static void x86_cpu_initfn(Object *obj)
                         x86_cpuid_get_tsc_freq,
                         x86_cpuid_set_tsc_freq, NULL, NULL, NULL);
 
-    env->cpuid_apic_id = env->cpu_index;
+    env->cpuid_apic_id = x86_cpu_apic_id_from_index(cs->cpu_index);
 
     /* init various static tables used in TCG mode */
     if (tcg_enabled() && !inited) {
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 0677502dcc3..62508dc688d 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -44,9 +44,9 @@
 
 #define CPUArchState struct CPUX86State
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define R_EAX 0
 #define R_ECX 1
@@ -123,8 +123,8 @@
 
 /* hidden flags - used internally by qemu to represent additional cpu
    states. Only the CPL, INHIBIT_IRQ, SMM and SVMI are not
-   redundant. We avoid using the IOPL_MASK, TF_MASK and VM_MASK bit
-   position to ease oring with eflags. */
+   redundant. We avoid using the IOPL_MASK, TF_MASK, VM_MASK and AC_MASK
+   bit positions to ease oring with eflags. */
 /* current cpl */
 #define HF_CPL_SHIFT         0
 /* true if soft mmu is being used */
@@ -147,10 +147,12 @@
 #define HF_CS64_SHIFT       15 /* only used on x86_64: 64 bit code segment  */
 #define HF_RF_SHIFT         16 /* must be same as eflags */
 #define HF_VM_SHIFT         17 /* must be same as eflags */
+#define HF_AC_SHIFT         18 /* must be same as eflags */
 #define HF_SMM_SHIFT        19 /* CPU in SMM mode */
 #define HF_SVME_SHIFT       20 /* SVME enabled (copy of EFER.SVME) */
 #define HF_SVMI_SHIFT       21 /* SVM intercepts are active */
 #define HF_OSFXSR_SHIFT     22 /* CR4.OSFXSR */
+#define HF_SMAP_SHIFT       23 /* CR4.SMAP */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
@@ -168,10 +170,12 @@
 #define HF_CS64_MASK         (1 << HF_CS64_SHIFT)
 #define HF_RF_MASK           (1 << HF_RF_SHIFT)
 #define HF_VM_MASK           (1 << HF_VM_SHIFT)
+#define HF_AC_MASK           (1 << HF_AC_SHIFT)
 #define HF_SMM_MASK          (1 << HF_SMM_SHIFT)
 #define HF_SVME_MASK         (1 << HF_SVME_SHIFT)
 #define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
+#define HF_SMAP_MASK         (1 << HF_SMAP_SHIFT)
 
 /* hflags2 */
 
@@ -210,6 +214,13 @@
 #define CR4_OSFXSR_SHIFT 9
 #define CR4_OSFXSR_MASK (1 << CR4_OSFXSR_SHIFT)
 #define CR4_OSXMMEXCPT_MASK  (1 << 10)
+#define CR4_VMXE_MASK   (1 << 13)
+#define CR4_SMXE_MASK   (1 << 14)
+#define CR4_FSGSBASE_MASK (1 << 16)
+#define CR4_PCIDE_MASK  (1 << 17)
+#define CR4_OSXSAVE_MASK (1 << 18)
+#define CR4_SMEP_MASK   (1 << 20)
+#define CR4_SMAP_MASK   (1 << 21)
 
 #define DR6_BD          (1 << 13)
 #define DR6_BS          (1 << 14)
@@ -220,6 +231,12 @@
 #define DR7_TYPE_SHIFT  16
 #define DR7_LEN_SHIFT   18
 #define DR7_FIXED_1     0x00000400
+#define DR7_LOCAL_BP_MASK    0x55
+#define DR7_MAX_BP           4
+#define DR7_TYPE_BP_INST     0x0
+#define DR7_TYPE_DATA_WR     0x1
+#define DR7_TYPE_IO_RW       0x2
+#define DR7_TYPE_DATA_RW     0x3
 
 #define PG_PRESENT_BIT	0
 #define PG_RW_BIT	1
@@ -284,6 +301,7 @@
 #define MSR_IA32_APICBASE_BSP           (1<<8)
 #define MSR_IA32_APICBASE_ENABLE        (1<<11)
 #define MSR_IA32_APICBASE_BASE          (0xfffff<<12)
+#define MSR_TSC_ADJUST                  0x0000003b
 #define MSR_IA32_TSCDEADLINE            0x6e0
 
 #define MSR_MTRRcap			0xfe
@@ -349,6 +367,21 @@
 
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
+/* CPUID feature words */
+typedef enum FeatureWord {
+    FEAT_1_EDX,         /* CPUID[1].EDX */
+    FEAT_1_ECX,         /* CPUID[1].ECX */
+    FEAT_7_0_EBX,       /* CPUID[EAX=7,ECX=0].EBX */
+    FEAT_8000_0001_EDX, /* CPUID[8000_0001].EDX */
+    FEAT_8000_0001_ECX, /* CPUID[8000_0001].ECX */
+    FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
+    FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
+    FEAT_SVM,           /* CPUID[8000_000A].EDX */
+    FEATURE_WORDS,
+} FeatureWord;
+
+typedef uint32_t FeatureWordArray[FEATURE_WORDS];
+
 /* cpuid_features bits */
 #define CPUID_FP87 (1 << 0)
 #define CPUID_VME  (1 << 1)
@@ -382,6 +415,7 @@
 #define CPUID_PBE (1 << 31)
 
 #define CPUID_EXT_SSE3     (1 << 0)
+#define CPUID_EXT_PCLMULQDQ (1 << 1)
 #define CPUID_EXT_DTES64   (1 << 2)
 #define CPUID_EXT_MONITOR  (1 << 3)
 #define CPUID_EXT_DSCPL    (1 << 4)
@@ -391,9 +425,11 @@
 #define CPUID_EXT_TM2      (1 << 8)
 #define CPUID_EXT_SSSE3    (1 << 9)
 #define CPUID_EXT_CID      (1 << 10)
+#define CPUID_EXT_FMA      (1 << 12)
 #define CPUID_EXT_CX16     (1 << 13)
 #define CPUID_EXT_XTPR     (1 << 14)
 #define CPUID_EXT_PDCM     (1 << 15)
+#define CPUID_EXT_PCID     (1 << 17)
 #define CPUID_EXT_DCA      (1 << 18)
 #define CPUID_EXT_SSE41    (1 << 19)
 #define CPUID_EXT_SSE42    (1 << 20)
@@ -401,14 +437,36 @@
 #define CPUID_EXT_MOVBE    (1 << 22)
 #define CPUID_EXT_POPCNT   (1 << 23)
 #define CPUID_EXT_TSC_DEADLINE_TIMER (1 << 24)
+#define CPUID_EXT_AES      (1 << 25)
 #define CPUID_EXT_XSAVE    (1 << 26)
 #define CPUID_EXT_OSXSAVE  (1 << 27)
+#define CPUID_EXT_AVX      (1 << 28)
+#define CPUID_EXT_F16C     (1 << 29)
+#define CPUID_EXT_RDRAND   (1 << 30)
 #define CPUID_EXT_HYPERVISOR  (1 << 31)
 
+#define CPUID_EXT2_FPU     (1 << 0)
+#define CPUID_EXT2_VME     (1 << 1)
+#define CPUID_EXT2_DE      (1 << 2)
+#define CPUID_EXT2_PSE     (1 << 3)
+#define CPUID_EXT2_TSC     (1 << 4)
+#define CPUID_EXT2_MSR     (1 << 5)
+#define CPUID_EXT2_PAE     (1 << 6)
+#define CPUID_EXT2_MCE     (1 << 7)
+#define CPUID_EXT2_CX8     (1 << 8)
+#define CPUID_EXT2_APIC    (1 << 9)
 #define CPUID_EXT2_SYSCALL (1 << 11)
+#define CPUID_EXT2_MTRR    (1 << 12)
+#define CPUID_EXT2_PGE     (1 << 13)
+#define CPUID_EXT2_MCA     (1 << 14)
+#define CPUID_EXT2_CMOV    (1 << 15)
+#define CPUID_EXT2_PAT     (1 << 16)
+#define CPUID_EXT2_PSE36   (1 << 17)
 #define CPUID_EXT2_MP      (1 << 19)
 #define CPUID_EXT2_NX      (1 << 20)
 #define CPUID_EXT2_MMXEXT  (1 << 22)
+#define CPUID_EXT2_MMX     (1 << 23)
+#define CPUID_EXT2_FXSR    (1 << 24)
 #define CPUID_EXT2_FFXSR   (1 << 25)
 #define CPUID_EXT2_PDPE1GB (1 << 26)
 #define CPUID_EXT2_RDTSCP  (1 << 27)
@@ -416,6 +474,17 @@
 #define CPUID_EXT2_3DNOWEXT (1 << 30)
 #define CPUID_EXT2_3DNOW   (1 << 31)
 
+/* CPUID[8000_0001].EDX bits that are aliase of CPUID[1].EDX bits on AMD CPUs */
+#define CPUID_EXT2_AMD_ALIASES (CPUID_EXT2_FPU | CPUID_EXT2_VME | \
+                                CPUID_EXT2_DE | CPUID_EXT2_PSE | \
+                                CPUID_EXT2_TSC | CPUID_EXT2_MSR | \
+                                CPUID_EXT2_PAE | CPUID_EXT2_MCE | \
+                                CPUID_EXT2_CX8 | CPUID_EXT2_APIC | \
+                                CPUID_EXT2_MTRR | CPUID_EXT2_PGE | \
+                                CPUID_EXT2_MCA | CPUID_EXT2_CMOV | \
+                                CPUID_EXT2_PAT | CPUID_EXT2_PSE36 | \
+                                CPUID_EXT2_MMX | CPUID_EXT2_FXSR)
+
 #define CPUID_EXT3_LAHF_LM (1 << 0)
 #define CPUID_EXT3_CMP_LEG (1 << 1)
 #define CPUID_EXT3_SVM     (1 << 2)
@@ -427,7 +496,17 @@
 #define CPUID_EXT3_3DNOWPREFETCH (1 << 8)
 #define CPUID_EXT3_OSVW    (1 << 9)
 #define CPUID_EXT3_IBS     (1 << 10)
+#define CPUID_EXT3_XOP     (1 << 11)
 #define CPUID_EXT3_SKINIT  (1 << 12)
+#define CPUID_EXT3_WDT     (1 << 13)
+#define CPUID_EXT3_LWP     (1 << 15)
+#define CPUID_EXT3_FMA4    (1 << 16)
+#define CPUID_EXT3_TCE     (1 << 17)
+#define CPUID_EXT3_NODEID  (1 << 19)
+#define CPUID_EXT3_TBM     (1 << 21)
+#define CPUID_EXT3_TOPOEXT (1 << 22)
+#define CPUID_EXT3_PERFCORE (1 << 23)
+#define CPUID_EXT3_PERFNB  (1 << 24)
 
 #define CPUID_SVM_NPT          (1 << 0)
 #define CPUID_SVM_LBRV         (1 << 1)
@@ -440,17 +519,32 @@
 #define CPUID_SVM_PAUSEFILTER  (1 << 10)
 #define CPUID_SVM_PFTHRESHOLD  (1 << 12)
 
+#define CPUID_7_0_EBX_FSGSBASE (1 << 0)
+#define CPUID_7_0_EBX_BMI1     (1 << 3)
+#define CPUID_7_0_EBX_HLE      (1 << 4)
+#define CPUID_7_0_EBX_AVX2     (1 << 5)
+#define CPUID_7_0_EBX_SMEP     (1 << 7)
+#define CPUID_7_0_EBX_BMI2     (1 << 8)
+#define CPUID_7_0_EBX_ERMS     (1 << 9)
+#define CPUID_7_0_EBX_INVPCID  (1 << 10)
+#define CPUID_7_0_EBX_RTM      (1 << 11)
+#define CPUID_7_0_EBX_RDSEED   (1 << 18)
+#define CPUID_7_0_EBX_ADX      (1 << 19)
+#define CPUID_7_0_EBX_SMAP     (1 << 20)
+
+#define CPUID_VENDOR_SZ      12
+
 #define CPUID_VENDOR_INTEL_1 0x756e6547 /* "Genu" */
 #define CPUID_VENDOR_INTEL_2 0x49656e69 /* "ineI" */
 #define CPUID_VENDOR_INTEL_3 0x6c65746e /* "ntel" */
+#define CPUID_VENDOR_INTEL "GenuineIntel"
 
 #define CPUID_VENDOR_AMD_1   0x68747541 /* "Auth" */
 #define CPUID_VENDOR_AMD_2   0x69746e65 /* "enti" */
 #define CPUID_VENDOR_AMD_3   0x444d4163 /* "cAMD" */
+#define CPUID_VENDOR_AMD   "AuthenticAMD"
 
-#define CPUID_VENDOR_VIA_1   0x746e6543 /* "Cent" */
-#define CPUID_VENDOR_VIA_2   0x48727561 /* "aurH" */
-#define CPUID_VENDOR_VIA_3   0x736c7561 /* "auls" */
+#define CPUID_VENDOR_VIA   "CentaurHauls"
 
 #define CPUID_MWAIT_IBE     (1 << 1) /* Interrupts can exit capability */
 #define CPUID_MWAIT_EMX     (1 << 0) /* enumeration supported */
@@ -615,7 +709,7 @@ typedef struct {
 #define CPU_NB_REGS CPU_NB_REGS32
 #endif
 
-#define NB_MMU_MODES 2
+#define NB_MMU_MODES 3
 
 typedef enum TPRAccess {
     TPR_ACCESS_READ,
@@ -702,6 +796,7 @@ typedef struct CPUX86State {
     uint64_t pv_eoi_en_msr;
 
     uint64_t tsc;
+    uint64_t tsc_adjust;
     uint64_t tsc_deadline;
 
     uint64_t mcg_status;
@@ -740,12 +835,11 @@ typedef struct CPUX86State {
     uint32_t cpuid_ext2_features;
     uint32_t cpuid_ext3_features;
     uint32_t cpuid_apic_id;
-    int cpuid_vendor_override;
     /* Store the results of Centaur's CPUID instructions */
     uint32_t cpuid_xlevel2;
     uint32_t cpuid_ext4_features;
     /* Flags from CPUID[EAX=7,ECX=0].EBX */
-    uint32_t cpuid_7_0_ebx;
+    uint32_t cpuid_7_0_ebx_features;
 
     /* MTRRs */
     uint64_t mtrr_fixed[11];
@@ -792,7 +886,7 @@ typedef struct CPUX86State {
 
 X86CPU *cpu_x86_init(const char *cpu_model);
 int cpu_x86_exec(CPUX86State *s);
-void x86_cpu_list (FILE *f, fprintf_function cpu_fprintf, const char *optarg);
+void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 void x86_cpudef_setup(void);
 int cpu_x86_support_mca_broadcast(CPUX86State *env);
 
@@ -859,9 +953,11 @@ static inline void cpu_x86_load_seg_cache(CPUX86State *env,
     }
 }
 
-static inline void cpu_x86_load_seg_cache_sipi(CPUX86State *env,
+static inline void cpu_x86_load_seg_cache_sipi(X86CPU *cpu,
                                                int sipi_vector)
 {
+    CPUX86State *env = &cpu->env;
+
     env->eip = 0;
     cpu_x86_load_seg_cache(env, R_CS, sipi_vector << 8,
                            sipi_vector << 12,
@@ -917,9 +1013,20 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
 #define cpu_handle_mmu_fault cpu_x86_handle_mmu_fault
 void cpu_x86_set_a20(CPUX86State *env, int a20_state);
 
-static inline int hw_breakpoint_enabled(unsigned long dr7, int index)
+static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
 {
-    return (dr7 >> (index * 2)) & 3;
+    return (dr7 >> (index * 2)) & 1;
+}
+
+static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 2;
+
+}
+static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return hw_global_breakpoint_enabled(dr7, index) ||
+           hw_local_breakpoint_enabled(dr7, index);
 }
 
 static inline int hw_breakpoint_type(unsigned long dr7, int index)
@@ -935,7 +1042,7 @@ static inline int hw_breakpoint_len(unsigned long dr7, int index)
 
 void hw_breakpoint_insert(CPUX86State *env, int index);
 void hw_breakpoint_remove(CPUX86State *env, int index);
-int check_hw_breakpoints(CPUX86State *env, int force_dr6_update);
+bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
 void breakpoint_handler(CPUX86State *env);
 
 /* will be suppressed */
@@ -947,10 +1054,6 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
 void cpu_smm_update(CPUX86State *env);
 uint64_t cpu_get_tsc(CPUX86State *env);
 
-/* used to debug */
-#define X86_DUMP_FPU  0x0001 /* dump FPU state too */
-#define X86_DUMP_CCOP 0x0002 /* dump qemu flag cache */
-
 #define TARGET_PAGE_BITS 12
 
 #ifdef TARGET_X86_64
@@ -976,7 +1079,7 @@ static inline CPUX86State *cpu_init(const char *cpu_model)
 #define cpu_exec cpu_x86_exec
 #define cpu_gen_code cpu_x86_gen_code
 #define cpu_signal_handler cpu_x86_signal_handler
-#define cpu_list_id x86_cpu_list
+#define cpu_list x86_cpu_list
 #define cpudef_setup	x86_cpudef_setup
 
 #define CPU_SAVE_VERSION 12
@@ -984,10 +1087,15 @@ static inline CPUX86State *cpu_init(const char *cpu_model)
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
-#define MMU_USER_IDX 1
+#define MMU_MODE2_SUFFIX _ksmap /* Kernel with SMAP override */
+#define MMU_KERNEL_IDX  0
+#define MMU_USER_IDX    1
+#define MMU_KSMAP_IDX   2
 static inline int cpu_mmu_index (CPUX86State *env)
 {
-    return (env->hflags & HF_CPL_MASK) == 3 ? 1 : 0;
+    return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
+        ((env->hflags & HF_SMAP_MASK) && (env->eflags & AC_MASK))
+        ? MMU_KSMAP_IDX : MMU_KERNEL_IDX;
 }
 
 #undef EAX
@@ -1042,15 +1150,17 @@ static inline void cpu_clone_regs(CPUX86State *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "svm.h"
 
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/apic.h"
 #endif
 
-static inline bool cpu_has_work(CPUX86State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
+
     return ((env->interrupt_request & (CPU_INTERRUPT_HARD |
                                        CPU_INTERRUPT_POLL)) &&
             (env->eflags & IF_MASK)) ||
@@ -1060,7 +1170,7 @@ static inline bool cpu_has_work(CPUX86State *env)
                                       CPU_INTERRUPT_MCE));
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUX86State *env, TranslationBlock *tb)
 {
@@ -1073,7 +1183,7 @@ static inline void cpu_get_tb_cpu_state(CPUX86State *env, target_ulong *pc,
     *cs_base = env->segs[R_CS].base;
     *pc = *cs_base + env->eip;
     *flags = env->hflags |
-        (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK));
+        (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK | AC_MASK));
 }
 
 void do_cpu_init(X86CPU *cpu);
@@ -1082,7 +1192,7 @@ void do_cpu_sipi(X86CPU *cpu);
 #define MCE_INJECT_BROADCAST    1
 #define MCE_INJECT_UNCOND_AO    2
 
-void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
+void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
                         uint64_t status, uint64_t mcg_status, uint64_t addr,
                         uint64_t misc, int flags);
 
@@ -1139,4 +1249,12 @@ void do_smm_enter(CPUX86State *env1);
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 
+void disable_kvm_pv_eoi(void);
+
+/* Return name of 32-bit register, from a R_* constant */
+const char *get_register_name_32(unsigned int reg);
+
+uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index);
+void enable_compat_apic_id_mode(void);
+
 #endif /* CPU_I386_H */
diff --git a/target-i386/excp_helper.c b/target-i386/excp_helper.c
index aaa5ca20907..179ea82f0fc 100644
--- a/target-i386/excp_helper.c
+++ b/target-i386/excp_helper.c
@@ -18,8 +18,8 @@
  */
 
 #include "cpu.h"
-#include "qemu-log.h"
-#include "sysemu.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 #include "helper.h"
 
 #if 0
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index dfc34a6253f..44f3d279445 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -22,7 +22,7 @@
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #define FPU_RC_MASK         0xc00
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 8a5da3d7c09..547c25ee9dc 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -18,10 +18,10 @@
  */
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #ifndef CONFIG_USER_ONLY
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 #endif
 
 //#define DEBUG_MMU
@@ -284,7 +284,7 @@ void cpu_dump_state(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
         cpu_fprintf(f, "\nDR6=" TARGET_FMT_lx " DR7=" TARGET_FMT_lx "\n",
                     env->dr[6], env->dr[7]);
     }
-    if (flags & X86_DUMP_CCOP) {
+    if (flags & CPU_DUMP_CCOP) {
         if ((unsigned)env->cc_op < CC_OP_NB)
             snprintf(cc_op_name, sizeof(cc_op_name), "%s", cc_op_str[env->cc_op]);
         else
@@ -303,7 +303,7 @@ void cpu_dump_state(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
         }
     }
     cpu_fprintf(f, "EFER=%016" PRIx64 "\n", env->efer);
-    if (flags & X86_DUMP_FPU) {
+    if (flags & CPU_DUMP_FPU) {
         int fptag;
         fptag = 0;
         for(i = 0; i < 8; i++) {
@@ -443,17 +443,27 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 #if defined(DEBUG_MMU)
     printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
 #endif
-    if ((new_cr4 & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK)) !=
-        (env->cr[4] & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK))) {
+    if ((new_cr4 ^ env->cr[4]) &
+        (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK |
+         CR4_SMEP_MASK | CR4_SMAP_MASK)) {
         tlb_flush(env, 1);
     }
     /* SSE handling */
-    if (!(env->cpuid_features & CPUID_SSE))
+    if (!(env->cpuid_features & CPUID_SSE)) {
         new_cr4 &= ~CR4_OSFXSR_MASK;
-    if (new_cr4 & CR4_OSFXSR_MASK)
+    }
+    env->hflags &= ~HF_OSFXSR_MASK;
+    if (new_cr4 & CR4_OSFXSR_MASK) {
         env->hflags |= HF_OSFXSR_MASK;
-    else
-        env->hflags &= ~HF_OSFXSR_MASK;
+    }
+
+    if (!(env->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)) {
+        new_cr4 &= ~CR4_SMAP_MASK;
+    }
+    env->hflags &= ~HF_SMAP_MASK;
+    if (new_cr4 & CR4_SMAP_MASK) {
+        env->hflags |= HF_SMAP_MASK;
+    }
 
     env->cr[4] = new_cr4;
 }
@@ -493,7 +503,7 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     uint64_t ptep, pte;
     target_ulong pde_addr, pte_addr;
     int error_code, is_dirty, prot, page_size, is_write, is_user;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t page_offset;
     target_ulong vaddr, virt_addr;
 
@@ -591,17 +601,38 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             /* 2 MB page */
             page_size = 2048 * 1024;
             ptep ^= PG_NX_MASK;
-            if ((ptep & PG_NX_MASK) && is_write1 == 2)
+            if ((ptep & PG_NX_MASK) && is_write1 == 2) {
                 goto do_fault_protect;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            }
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pde & PG_DIRTY_MASK);
             if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
@@ -635,15 +666,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             ptep ^= PG_NX_MASK;
             if ((ptep & PG_NX_MASK) && is_write1 == 2)
                 goto do_fault_protect;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pte & PG_DIRTY_MASK);
             if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
@@ -670,15 +721,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
         /* if PSE bit is set, then we use a 4MB page */
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
             page_size = 4096 * 1024;
-            if (is_user) {
-                if (!(pde & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(pde & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(pde & PG_RW_MASK))
+                }
+                if (is_write && !(pde & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (pde & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (pde & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(pde & PG_RW_MASK))
+                    is_write && !(pde & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pde & PG_DIRTY_MASK);
             if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
@@ -707,15 +778,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             }
             /* combine pde and pte user and rw protections */
             ptep = pte & pde;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pte & PG_DIRTY_MASK);
             if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
@@ -762,8 +853,9 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     if (is_user)
         error_code |= PG_ERROR_U_MASK;
     if (is_write1 == 2 &&
-        (env->efer & MSR_EFER_NXE) &&
-        (env->cr[4] & CR4_PAE_MASK))
+        (((env->efer & MSR_EFER_NXE) &&
+          (env->cr[4] & CR4_PAE_MASK)) ||
+         (env->cr[4] & CR4_SMEP_MASK)))
         error_code |= PG_ERROR_I_D_MASK;
     if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
         /* cr2 is not modified in case of exceptions */
@@ -777,11 +869,11 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     return 1;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
 {
     target_ulong pde_addr, pte_addr;
     uint64_t pte;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t page_offset;
     int page_size;
 
@@ -874,30 +966,35 @@ target_phys_addr_t cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
 
 void hw_breakpoint_insert(CPUX86State *env, int index)
 {
-    int type, err = 0;
+    int type = 0, err = 0;
 
     switch (hw_breakpoint_type(env->dr[7], index)) {
-    case 0:
-        if (hw_breakpoint_enabled(env->dr[7], index))
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(env->dr[7], index)) {
             err = cpu_breakpoint_insert(env, env->dr[index], BP_CPU,
                                         &env->cpu_breakpoint[index]);
+        }
         break;
-    case 1:
+    case DR7_TYPE_DATA_WR:
         type = BP_CPU | BP_MEM_WRITE;
-        goto insert_wp;
-    case 2:
-         /* No support for I/O watchpoints yet */
         break;
-    case 3:
+    case DR7_TYPE_IO_RW:
+        /* No support for I/O watchpoints yet */
+        break;
+    case DR7_TYPE_DATA_RW:
         type = BP_CPU | BP_MEM_ACCESS;
-    insert_wp:
+        break;
+    }
+
+    if (type != 0) {
         err = cpu_watchpoint_insert(env, env->dr[index],
                                     hw_breakpoint_len(env->dr[7], index),
                                     type, &env->cpu_watchpoint[index]);
-        break;
     }
-    if (err)
+
+    if (err) {
         env->cpu_breakpoint[index] = NULL;
+    }
 }
 
 void hw_breakpoint_remove(CPUX86State *env, int index)
@@ -905,39 +1002,60 @@ void hw_breakpoint_remove(CPUX86State *env, int index)
     if (!env->cpu_breakpoint[index])
         return;
     switch (hw_breakpoint_type(env->dr[7], index)) {
-    case 0:
-        if (hw_breakpoint_enabled(env->dr[7], index))
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(env->dr[7], index)) {
             cpu_breakpoint_remove_by_ref(env, env->cpu_breakpoint[index]);
+        }
         break;
-    case 1:
-    case 3:
+    case DR7_TYPE_DATA_WR:
+    case DR7_TYPE_DATA_RW:
         cpu_watchpoint_remove_by_ref(env, env->cpu_watchpoint[index]);
         break;
-    case 2:
+    case DR7_TYPE_IO_RW:
         /* No support for I/O watchpoints yet */
         break;
     }
 }
 
-int check_hw_breakpoints(CPUX86State *env, int force_dr6_update)
+bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
 {
     target_ulong dr6;
-    int reg, type;
-    int hit_enabled = 0;
+    int reg;
+    bool hit_enabled = false;
 
     dr6 = env->dr[6] & ~0xf;
-    for (reg = 0; reg < 4; reg++) {
-        type = hw_breakpoint_type(env->dr[7], reg);
-        if ((type == 0 && env->dr[reg] == env->eip) ||
-            ((type & 1) && env->cpu_watchpoint[reg] &&
-             (env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT))) {
+    for (reg = 0; reg < DR7_MAX_BP; reg++) {
+        bool bp_match = false;
+        bool wp_match = false;
+
+        switch (hw_breakpoint_type(env->dr[7], reg)) {
+        case DR7_TYPE_BP_INST:
+            if (env->dr[reg] == env->eip) {
+                bp_match = true;
+            }
+            break;
+        case DR7_TYPE_DATA_WR:
+        case DR7_TYPE_DATA_RW:
+            if (env->cpu_watchpoint[reg] &&
+                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
+                wp_match = true;
+            }
+            break;
+        case DR7_TYPE_IO_RW:
+            break;
+        }
+        if (bp_match || wp_match) {
             dr6 |= 1 << reg;
-            if (hw_breakpoint_enabled(env->dr[7], reg))
-                hit_enabled = 1;
+            if (hw_breakpoint_enabled(env->dr[7], reg)) {
+                hit_enabled = true;
+            }
         }
     }
-    if (hit_enabled || force_dr6_update)
+
+    if (hit_enabled || force_dr6_update) {
         env->dr[6] = dr6;
+    }
+
     return hit_enabled;
 }
 
@@ -948,16 +1066,17 @@ void breakpoint_handler(CPUX86State *env)
     if (env->watchpoint_hit) {
         if (env->watchpoint_hit->flags & BP_CPU) {
             env->watchpoint_hit = NULL;
-            if (check_hw_breakpoints(env, 0))
+            if (check_hw_breakpoints(env, false)) {
                 raise_exception(env, EXCP01_DB);
-            else
+            } else {
                 cpu_resume_from_signal(env, NULL);
+            }
         }
     } else {
         QTAILQ_FOREACH(bp, &env->breakpoints, entry)
             if (bp->pc == env->eip) {
                 if (bp->flags & BP_CPU) {
-                    check_hw_breakpoints(env, 1);
+                    check_hw_breakpoints(env, true);
                     raise_exception(env, EXCP01_DB);
                 }
                 break;
@@ -967,7 +1086,7 @@ void breakpoint_handler(CPUX86State *env)
 
 typedef struct MCEInjectionParams {
     Monitor *mon;
-    CPUX86State *env;
+    X86CPU *cpu;
     int bank;
     uint64_t status;
     uint64_t mcg_status;
@@ -979,7 +1098,8 @@ typedef struct MCEInjectionParams {
 static void do_inject_x86_mce(void *data)
 {
     MCEInjectionParams *params = data;
-    CPUX86State *cenv = params->env;
+    CPUX86State *cenv = &params->cpu->env;
+    CPUState *cpu = CPU(params->cpu);
     uint64_t *banks = cenv->mce_banks + 4 * params->bank;
 
     cpu_synchronize_state(cenv);
@@ -1002,7 +1122,7 @@ static void do_inject_x86_mce(void *data)
         if ((cenv->mcg_cap & MCG_CTL_P) && cenv->mcg_ctl != ~(uint64_t)0) {
             monitor_printf(params->mon,
                            "CPU %d: Uncorrected error reporting disabled\n",
-                           cenv->cpu_index);
+                           cpu->cpu_index);
             return;
         }
 
@@ -1014,7 +1134,7 @@ static void do_inject_x86_mce(void *data)
             monitor_printf(params->mon,
                            "CPU %d: Uncorrected error reporting disabled for"
                            " bank %d\n",
-                           cenv->cpu_index, params->bank);
+                           cpu->cpu_index, params->bank);
             return;
         }
 
@@ -1023,7 +1143,7 @@ static void do_inject_x86_mce(void *data)
             monitor_printf(params->mon,
                            "CPU %d: Previous MCE still in progress, raising"
                            " triple fault\n",
-                           cenv->cpu_index);
+                           cpu->cpu_index);
             qemu_log_mask(CPU_LOG_RESET, "Triple fault\n");
             qemu_system_reset_request();
             return;
@@ -1049,13 +1169,14 @@ static void do_inject_x86_mce(void *data)
     }
 }
 
-void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
+void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
                         uint64_t status, uint64_t mcg_status, uint64_t addr,
                         uint64_t misc, int flags)
 {
+    CPUX86State *cenv = &cpu->env;
     MCEInjectionParams params = {
         .mon = mon,
-        .env = cenv,
+        .cpu = cpu,
         .bank = bank,
         .status = status,
         .mcg_status = mcg_status,
@@ -1084,7 +1205,7 @@ void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
         return;
     }
 
-    run_on_cpu(cenv, do_inject_x86_mce, &params);
+    run_on_cpu(CPU(cpu), do_inject_x86_mce, &params);
     if (flags & MCE_INJECT_BROADCAST) {
         params.bank = 1;
         params.status = MCI_STATUS_VAL | MCI_STATUS_UC;
@@ -1095,23 +1216,20 @@ void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
             if (cenv == env) {
                 continue;
             }
-            params.env = env;
-            run_on_cpu(cenv, do_inject_x86_mce, &params);
+            params.cpu = x86_env_get_cpu(env);
+            run_on_cpu(CPU(cpu), do_inject_x86_mce, &params);
         }
     }
 }
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access)
 {
-    TranslationBlock *tb;
-
     if (kvm_enabled()) {
         env->tpr_access_type = access;
 
         cpu_interrupt(env, CPU_INTERRUPT_TPR);
     } else {
-        tb = tb_find_pc(env->mem_io_pc);
-        cpu_restore_state(tb, env, env->mem_io_pc);
+        cpu_restore_state(env, env->mem_io_pc);
 
         apic_handle_tpr_access_report(env->apic_state, env->eip, access);
     }
@@ -1151,6 +1269,7 @@ X86CPU *cpu_x86_init(const char *cpu_model)
 {
     X86CPU *cpu;
     CPUX86State *env;
+    Error *error = NULL;
 
     cpu = X86_CPU(object_new(TYPE_X86_CPU));
     env = &cpu->env;
@@ -1161,8 +1280,12 @@ X86CPU *cpu_x86_init(const char *cpu_model)
         return NULL;
     }
 
-    x86_cpu_realize(OBJECT(cpu), NULL);
-
+    x86_cpu_realize(OBJECT(cpu), &error);
+    if (error) {
+        error_free(error);
+        object_delete(OBJECT(cpu));
+        return NULL;
+    }
     return cpu;
 }
 
diff --git a/target-i386/helper.h b/target-i386/helper.h
index ab6af638e69..9ed720d0ed8 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -1,7 +1,7 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_FLAGS_2(cc_compute_all, TCG_CALL_PURE, i32, env, int)
-DEF_HELPER_FLAGS_2(cc_compute_c, TCG_CALL_PURE, i32, env, int)
+DEF_HELPER_FLAGS_2(cc_compute_all, TCG_CALL_NO_SE, i32, env, int)
+DEF_HELPER_FLAGS_2(cc_compute_c, TCG_CALL_NO_SE, i32, env, int)
 
 DEF_HELPER_0(lock, void)
 DEF_HELPER_0(unlock, void)
@@ -67,6 +67,8 @@ DEF_HELPER_3(raise_interrupt, void, env, int, int)
 DEF_HELPER_2(raise_exception, void, env, int)
 DEF_HELPER_1(cli, void, env)
 DEF_HELPER_1(sti, void, env)
+DEF_HELPER_1(clac, void, env)
+DEF_HELPER_1(stac, void, env)
 DEF_HELPER_1(set_inhibit_irq, void, env)
 DEF_HELPER_1(reset_inhibit_irq, void, env)
 DEF_HELPER_3(boundw, void, env, tl, int)
@@ -218,4 +220,4 @@ DEF_HELPER_3(rclq, tl, env, tl, tl)
 DEF_HELPER_3(rcrq, tl, env, tl, tl)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-i386/int_helper.c b/target-i386/int_helper.c
index f39747e806b..84b812dccac 100644
--- a/target-i386/int_helper.c
+++ b/target-i386/int_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 //#define DEBUG_MULDIV
diff --git a/target-i386/ioport-user.c b/target-i386/ioport-user.c
index 03fac22d22b..f7636e0a870 100644
--- a/target-i386/ioport-user.c
+++ b/target-i386/ioport-user.c
@@ -21,7 +21,7 @@
 
 #include "qemu.h"
 #include "qemu-common.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 
 void cpu_outb(pio_addr_t addr, uint8_t val)
 {
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index ffc294ec392..9ebf1816d99 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -21,16 +21,18 @@
 #include <linux/kvm_para.h>
 
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #include "cpu.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
+#include "qemu/config-file.h"
 #include "hw/pc.h"
 #include "hw/apic.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "hyperv.h"
+#include "hw/pci/pci.h"
 
 //#define DEBUG_KVM
 
@@ -61,6 +63,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 
 static bool has_msr_star;
 static bool has_msr_hsave_pa;
+static bool has_msr_tsc_adjust;
 static bool has_msr_tsc_deadline;
 static bool has_msr_async_pf_en;
 static bool has_msr_pv_eoi_en;
@@ -97,6 +100,19 @@ static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
     return cpuid;
 }
 
+/* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough
+ * for all entries.
+ */
+static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s)
+{
+    struct kvm_cpuid2 *cpuid;
+    int max = 1;
+    while ((cpuid = try_get_cpuid(s, max)) == NULL) {
+        max *= 2;
+    }
+    return cpuid;
+}
+
 struct kvm_para_features {
     int cap;
     int feature;
@@ -122,60 +138,98 @@ static int get_para_features(KVMState *s)
 }
 
 
+/* Returns the value for a specific register on the cpuid entry
+ */
+static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg)
+{
+    uint32_t ret = 0;
+    switch (reg) {
+    case R_EAX:
+        ret = entry->eax;
+        break;
+    case R_EBX:
+        ret = entry->ebx;
+        break;
+    case R_ECX:
+        ret = entry->ecx;
+        break;
+    case R_EDX:
+        ret = entry->edx;
+        break;
+    }
+    return ret;
+}
+
+/* Find matching entry for function/index on kvm_cpuid2 struct
+ */
+static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid,
+                                                 uint32_t function,
+                                                 uint32_t index)
+{
+    int i;
+    for (i = 0; i < cpuid->nent; ++i) {
+        if (cpuid->entries[i].function == function &&
+            cpuid->entries[i].index == index) {
+            return &cpuid->entries[i];
+        }
+    }
+    /* not found: */
+    return NULL;
+}
+
 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
                                       uint32_t index, int reg)
 {
     struct kvm_cpuid2 *cpuid;
-    int i, max;
     uint32_t ret = 0;
     uint32_t cpuid_1_edx;
-    int has_kvm_features = 0;
+    bool found = false;
 
-    max = 1;
-    while ((cpuid = try_get_cpuid(s, max)) == NULL) {
-        max *= 2;
+    cpuid = get_supported_cpuid(s);
+
+    struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index);
+    if (entry) {
+        found = true;
+        ret = cpuid_entry_get_reg(entry, reg);
     }
 
-    for (i = 0; i < cpuid->nent; ++i) {
-        if (cpuid->entries[i].function == function &&
-            cpuid->entries[i].index == index) {
-            if (cpuid->entries[i].function == KVM_CPUID_FEATURES) {
-                has_kvm_features = 1;
-            }
-            switch (reg) {
-            case R_EAX:
-                ret = cpuid->entries[i].eax;
-                break;
-            case R_EBX:
-                ret = cpuid->entries[i].ebx;
-                break;
-            case R_ECX:
-                ret = cpuid->entries[i].ecx;
-                break;
-            case R_EDX:
-                ret = cpuid->entries[i].edx;
-                switch (function) {
-                case 1:
-                    /* KVM before 2.6.30 misreports the following features */
-                    ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
-                    break;
-                case 0x80000001:
-                    /* On Intel, kvm returns cpuid according to the Intel spec,
-                     * so add missing bits according to the AMD spec:
-                     */
-                    cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
-                    ret |= cpuid_1_edx & 0x183f7ff;
-                    break;
-                }
-                break;
-            }
+    /* Fixups for the data returned by KVM, below */
+
+    if (function == 1 && reg == R_EDX) {
+        /* KVM before 2.6.30 misreports the following features */
+        ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
+    } else if (function == 1 && reg == R_ECX) {
+        /* We can set the hypervisor flag, even if KVM does not return it on
+         * GET_SUPPORTED_CPUID
+         */
+        ret |= CPUID_EXT_HYPERVISOR;
+        /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it
+         * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER,
+         * and the irqchip is in the kernel.
+         */
+        if (kvm_irqchip_in_kernel() &&
+                kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
+            ret |= CPUID_EXT_TSC_DEADLINE_TIMER;
         }
+
+        /* x2apic is reported by GET_SUPPORTED_CPUID, but it can't be enabled
+         * without the in-kernel irqchip
+         */
+        if (!kvm_irqchip_in_kernel()) {
+            ret &= ~CPUID_EXT_X2APIC;
+        }
+    } else if (function == 0x80000001 && reg == R_EDX) {
+        /* On Intel, kvm returns cpuid according to the Intel spec,
+         * so add missing bits according to the AMD spec:
+         */
+        cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
+        ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
     }
 
     g_free(cpuid);
 
     /* fallback for older kernels */
-    if (!has_kvm_features && (function == KVM_CPUID_FEATURES)) {
+    if ((function == KVM_CPUID_FEATURES) && !found) {
         ret = get_para_features(s);
     }
 
@@ -228,8 +282,9 @@ static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
     return -ENOSYS;
 }
 
-static void kvm_mce_inject(CPUX86State *env, target_phys_addr_t paddr, int code)
+static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
 {
+    CPUX86State *env = &cpu->env;
     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
     uint64_t mcg_status = MCG_STATUS_MCIP;
@@ -241,7 +296,7 @@ static void kvm_mce_inject(CPUX86State *env, target_phys_addr_t paddr, int code)
         status |= 0xc0;
         mcg_status |= MCG_STATUS_RIPV;
     }
-    cpu_x86_inject_mce(NULL, env, 9, status, mcg_status, paddr,
+    cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
                        (MCM_ADDR_PHYS << 6) | 0xc,
                        cpu_x86_support_mca_broadcast(env) ?
                        MCE_INJECT_BROADCAST : 0);
@@ -253,15 +308,17 @@ static void hardware_memory_error(void)
     exit(1);
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
 {
+    X86CPU *cpu = X86_CPU(c);
+    CPUX86State *env = &cpu->env;
     ram_addr_t ram_addr;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
 
     if ((env->mcg_cap & MCG_SER_P) && addr
         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
-            !kvm_physical_memory_addr_from_host(env->kvm_state, addr, &paddr)) {
+            !kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
                     "QEMU itself instead of guest system!\n");
             /* Hope we are lucky for AO MCE */
@@ -272,7 +329,7 @@ int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
             }
         }
         kvm_hwpoison_page_add(ram_addr);
-        kvm_mce_inject(env, paddr, code);
+        kvm_mce_inject(cpu, paddr, code);
     } else {
         if (code == BUS_MCEERR_AO) {
             return 0;
@@ -289,18 +346,18 @@ int kvm_arch_on_sigbus(int code, void *addr)
 {
     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
         ram_addr_t ram_addr;
-        target_phys_addr_t paddr;
+        hwaddr paddr;
 
         /* Hope we are lucky for AO MCE */
         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
-            !kvm_physical_memory_addr_from_host(first_cpu->kvm_state, addr,
-                                                &paddr)) {
+            !kvm_physical_memory_addr_from_host(CPU(first_cpu)->kvm_state,
+                                                addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
                     "QEMU itself instead of guest system!: %p\n", addr);
             return 0;
         }
         kvm_hwpoison_page_add(ram_addr);
-        kvm_mce_inject(first_cpu, paddr, code);
+        kvm_mce_inject(x86_env_get_cpu(first_cpu), paddr, code);
     } else {
         if (code == BUS_MCEERR_AO) {
             return 0;
@@ -313,8 +370,10 @@ int kvm_arch_on_sigbus(int code, void *addr)
     return 0;
 }
 
-static int kvm_inject_mce_oldstyle(CPUX86State *env)
+static int kvm_inject_mce_oldstyle(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
+
     if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
         unsigned int bank, bank_num = env->mcg_cap & 0xff;
         struct kvm_x86_mce mce;
@@ -338,7 +397,7 @@ static int kvm_inject_mce_oldstyle(CPUX86State *env)
         mce.addr = env->mce_banks[bank * 4 + 2];
         mce.misc = env->mce_banks[bank * 4 + 3];
 
-        return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
+        return kvm_vcpu_ioctl(CPU(cpu), KVM_X86_SET_MCE, &mce);
     }
     return 0;
 }
@@ -352,37 +411,28 @@ static void cpu_update_state(void *opaque, int running, RunState state)
     }
 }
 
-int kvm_arch_init_vcpu(CPUX86State *env)
+unsigned long kvm_arch_vcpu_id(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    return cpu->env.cpuid_apic_id;
+}
+
+#define KVM_MAX_CPUID_ENTRIES  100
+
+int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct {
         struct kvm_cpuid2 cpuid;
-        struct kvm_cpuid_entry2 entries[100];
+        struct kvm_cpuid_entry2 entries[KVM_MAX_CPUID_ENTRIES];
     } QEMU_PACKED cpuid_data;
-    KVMState *s = env->kvm_state;
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
     uint32_t limit, i, j, cpuid_i;
     uint32_t unused;
     struct kvm_cpuid_entry2 *c;
     uint32_t signature[3];
     int r;
 
-    env->cpuid_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
-
-    i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
-    j = env->cpuid_ext_features & CPUID_EXT_TSC_DEADLINE_TIMER;
-    env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
-    env->cpuid_ext_features |= i;
-    if (j && kvm_irqchip_in_kernel() &&
-        kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
-        env->cpuid_ext_features |= CPUID_EXT_TSC_DEADLINE_TIMER;
-    }
-
-    env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
-                                                             0, R_EDX);
-    env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
-                                                             0, R_ECX);
-    env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(s, 0x8000000A,
-                                                             0, R_EDX);
-
     cpuid_i = 0;
 
     /* Paravirtualization CPUIDs */
@@ -403,8 +453,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     c = &cpuid_data.entries[cpuid_i++];
     memset(c, 0, sizeof(*c));
     c->function = KVM_CPUID_FEATURES;
-    c->eax = env->cpuid_kvm_features &
-        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
+    c->eax = env->cpuid_kvm_features;
 
     if (hyperv_enabled()) {
         memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
@@ -461,6 +510,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0; i <= limit; i++) {
+        if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+            fprintf(stderr, "unsupported level value: 0x%x\n", limit);
+            abort();
+        }
         c = &cpuid_data.entries[cpuid_i++];
 
         switch (i) {
@@ -475,6 +528,11 @@ int kvm_arch_init_vcpu(CPUX86State *env)
             times = c->eax & 0xff;
 
             for (j = 1; j < times; ++j) {
+                if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                    fprintf(stderr, "cpuid_data is full, no space for "
+                            "cpuid(eax:2):eax & 0xf = 0x%x\n", times);
+                    abort();
+                }
                 c = &cpuid_data.entries[cpuid_i++];
                 c->function = i;
                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
@@ -503,6 +561,11 @@ int kvm_arch_init_vcpu(CPUX86State *env)
                 if (i == 0xd && c->eax == 0) {
                     continue;
                 }
+                if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                    fprintf(stderr, "cpuid_data is full, no space for "
+                            "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
+                    abort();
+                }
                 c = &cpuid_data.entries[cpuid_i++];
             }
             break;
@@ -516,6 +579,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0x80000000; i <= limit; i++) {
+        if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+            fprintf(stderr, "unsupported xlevel value: 0x%x\n", limit);
+            abort();
+        }
         c = &cpuid_data.entries[cpuid_i++];
 
         c->function = i;
@@ -525,11 +592,13 @@ int kvm_arch_init_vcpu(CPUX86State *env)
 
     /* Call Centaur's CPUID instructions they are supported. */
     if (env->cpuid_xlevel2 > 0) {
-        env->cpuid_ext4_features &=
-            kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
 
         for (i = 0xC0000000; i <= limit; i++) {
+            if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                fprintf(stderr, "unsupported xlevel2 value: 0x%x\n", limit);
+                abort();
+            }
             c = &cpuid_data.entries[cpuid_i++];
 
             c->function = i;
@@ -542,12 +611,12 @@ int kvm_arch_init_vcpu(CPUX86State *env)
 
     if (((env->cpuid_version >> 8)&0xF) >= 6
         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
-        && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
+        && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
         uint64_t mcg_cap;
         int banks;
         int ret;
 
-        ret = kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks);
+        ret = kvm_get_mce_cap_supported(cs->kvm_state, &mcg_cap, &banks);
         if (ret < 0) {
             fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
             return ret;
@@ -558,7 +627,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
         }
         mcg_cap &= MCE_CAP_DEF;
         mcg_cap |= banks;
-        ret = kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, &mcg_cap);
+        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &mcg_cap);
         if (ret < 0) {
             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
             return ret;
@@ -570,14 +639,14 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     qemu_add_vm_change_state_handler(cpu_update_state, env);
 
     cpuid_data.cpuid.padding = 0;
-    r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
+    r = kvm_vcpu_ioctl(cs, KVM_SET_CPUID2, &cpuid_data);
     if (r) {
         return r;
     }
 
-    r = kvm_check_extension(env->kvm_state, KVM_CAP_TSC_CONTROL);
+    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL);
     if (r && env->tsc_khz) {
-        r = kvm_vcpu_ioctl(env, KVM_SET_TSC_KHZ, env->tsc_khz);
+        r = kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz);
         if (r < 0) {
             fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
             return r;
@@ -591,9 +660,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     return 0;
 }
 
-void kvm_arch_reset_vcpu(CPUX86State *env)
+void kvm_arch_reset_vcpu(CPUState *cs)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
 
     env->exception_injected = -1;
     env->interrupt_injected = -1;
@@ -644,6 +714,10 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_hsave_pa = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_TSC_ADJUST) {
+                    has_msr_tsc_adjust = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_IA32_TSCDEADLINE) {
                     has_msr_tsc_deadline = true;
                     continue;
@@ -784,13 +858,14 @@ static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
     }
 }
 
-static int kvm_getput_regs(CPUX86State *env, int set)
+static int kvm_getput_regs(X86CPU *cpu, int set)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_regs regs;
     int ret = 0;
 
     if (!set) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_REGS, &regs);
         if (ret < 0) {
             return ret;
         }
@@ -819,14 +894,15 @@ static int kvm_getput_regs(CPUX86State *env, int set)
     kvm_getput_reg(&regs.rip, &env->eip, set);
 
     if (set) {
-        ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_REGS, &regs);
     }
 
     return ret;
 }
 
-static int kvm_put_fpu(CPUX86State *env)
+static int kvm_put_fpu(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_fpu fpu;
     int i;
 
@@ -844,7 +920,7 @@ static int kvm_put_fpu(CPUX86State *env)
     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
     fpu.mxcsr = env->mxcsr;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu);
 }
 
 #define XSAVE_FCW_FSW     0
@@ -857,14 +933,15 @@ static int kvm_put_fpu(CPUX86State *env)
 #define XSAVE_XSTATE_BV   128
 #define XSAVE_YMMH_SPACE  144
 
-static int kvm_put_xsave(CPUX86State *env)
+static int kvm_put_xsave(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xsave* xsave = env->kvm_xsave_buf;
     uint16_t cwd, swd, twd;
     int i, r;
 
     if (!kvm_has_xsave()) {
-        return kvm_put_fpu(env);
+        return kvm_put_fpu(cpu);
     }
 
     memset(xsave, 0, sizeof(struct kvm_xsave));
@@ -887,12 +964,13 @@ static int kvm_put_xsave(CPUX86State *env)
     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
             sizeof env->ymmh_regs);
-    r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
+    r = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
     return r;
 }
 
-static int kvm_put_xcrs(CPUX86State *env)
+static int kvm_put_xcrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xcrs xcrs;
 
     if (!kvm_has_xcrs()) {
@@ -903,11 +981,12 @@ static int kvm_put_xcrs(CPUX86State *env)
     xcrs.flags = 0;
     xcrs.xcrs[0].xcr = 0;
     xcrs.xcrs[0].value = env->xcr0;
-    return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XCRS, &xcrs);
 }
 
-static int kvm_put_sregs(CPUX86State *env)
+static int kvm_put_sregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_sregs sregs;
 
     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
@@ -952,7 +1031,7 @@ static int kvm_put_sregs(CPUX86State *env)
 
     sregs.efer = env->efer;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
 }
 
 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
@@ -962,8 +1041,9 @@ static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
     entry->data = value;
 }
 
-static int kvm_put_msrs(CPUX86State *env, int level)
+static int kvm_put_msrs(X86CPU *cpu, int level)
 {
+    CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
         struct kvm_msr_entry entries[100];
@@ -981,6 +1061,9 @@ static int kvm_put_msrs(CPUX86State *env, int level)
     if (has_msr_hsave_pa) {
         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
     }
+    if (has_msr_tsc_adjust) {
+        kvm_msr_entry_set(&msrs[n++], MSR_TSC_ADJUST, env->tsc_adjust);
+    }
     if (has_msr_tsc_deadline) {
         kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
     }
@@ -1044,17 +1127,18 @@ static int kvm_put_msrs(CPUX86State *env, int level)
 
     msr_data.info.nmsrs = n;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
 
 }
 
 
-static int kvm_get_fpu(CPUX86State *env)
+static int kvm_get_fpu(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_fpu fpu;
     int i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_FPU, &fpu);
     if (ret < 0) {
         return ret;
     }
@@ -1075,17 +1159,18 @@ static int kvm_get_fpu(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_xsave(CPUX86State *env)
+static int kvm_get_xsave(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xsave* xsave = env->kvm_xsave_buf;
     int ret, i;
     uint16_t cwd, swd, twd;
 
     if (!kvm_has_xsave()) {
-        return kvm_get_fpu(env);
+        return kvm_get_fpu(cpu);
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XSAVE, xsave);
     if (ret < 0) {
         return ret;
     }
@@ -1113,8 +1198,9 @@ static int kvm_get_xsave(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_xcrs(CPUX86State *env)
+static int kvm_get_xcrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     int i, ret;
     struct kvm_xcrs xcrs;
 
@@ -1122,7 +1208,7 @@ static int kvm_get_xcrs(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XCRS, &xcrs);
     if (ret < 0) {
         return ret;
     }
@@ -1137,13 +1223,14 @@ static int kvm_get_xcrs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_sregs(CPUX86State *env)
+static int kvm_get_sregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_sregs sregs;
     uint32_t hflags;
     int bit, i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
     if (ret < 0) {
         return ret;
     }
@@ -1221,8 +1308,9 @@ static int kvm_get_sregs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_msrs(CPUX86State *env)
+static int kvm_get_msrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
         struct kvm_msr_entry entries[100];
@@ -1241,6 +1329,9 @@ static int kvm_get_msrs(CPUX86State *env)
     if (has_msr_hsave_pa) {
         msrs[n++].index = MSR_VM_HSAVE_PA;
     }
+    if (has_msr_tsc_adjust) {
+        msrs[n++].index = MSR_TSC_ADJUST;
+    }
     if (has_msr_tsc_deadline) {
         msrs[n++].index = MSR_IA32_TSCDEADLINE;
     }
@@ -1279,7 +1370,7 @@ static int kvm_get_msrs(CPUX86State *env)
     }
 
     msr_data.info.nmsrs = n;
-    ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
     if (ret < 0) {
         return ret;
     }
@@ -1318,6 +1409,9 @@ static int kvm_get_msrs(CPUX86State *env)
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
             break;
+        case MSR_TSC_ADJUST:
+            env->tsc_adjust = msrs[i].data;
+            break;
         case MSR_IA32_TSCDEADLINE:
             env->tsc_deadline = msrs[i].data;
             break;
@@ -1357,19 +1451,20 @@ static int kvm_get_msrs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_put_mp_state(CPUX86State *env)
+static int kvm_put_mp_state(X86CPU *cpu)
 {
-    struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
+    struct kvm_mp_state mp_state = { .mp_state = cpu->env.mp_state };
 
-    return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
 }
 
-static int kvm_get_mp_state(CPUX86State *env)
+static int kvm_get_mp_state(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_mp_state mp_state;
     int ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
     if (ret < 0) {
         return ret;
     }
@@ -1380,14 +1475,15 @@ static int kvm_get_mp_state(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_apic(CPUX86State *env)
+static int kvm_get_apic(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     DeviceState *apic = env->apic_state;
     struct kvm_lapic_state kapic;
     int ret;
 
     if (apic && kvm_irqchip_in_kernel()) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, &kapic);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_LAPIC, &kapic);
         if (ret < 0) {
             return ret;
         }
@@ -1397,21 +1493,23 @@ static int kvm_get_apic(CPUX86State *env)
     return 0;
 }
 
-static int kvm_put_apic(CPUX86State *env)
+static int kvm_put_apic(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     DeviceState *apic = env->apic_state;
     struct kvm_lapic_state kapic;
 
     if (apic && kvm_irqchip_in_kernel()) {
         kvm_put_apic_state(apic, &kapic);
 
-        return kvm_vcpu_ioctl(env, KVM_SET_LAPIC, &kapic);
+        return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_LAPIC, &kapic);
     }
     return 0;
 }
 
-static int kvm_put_vcpu_events(CPUX86State *env, int level)
+static int kvm_put_vcpu_events(X86CPU *cpu, int level)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_vcpu_events events;
 
     if (!kvm_has_vcpu_events()) {
@@ -1441,11 +1539,12 @@ static int kvm_put_vcpu_events(CPUX86State *env, int level)
             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
     }
 
-    return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
 }
 
-static int kvm_get_vcpu_events(CPUX86State *env)
+static int kvm_get_vcpu_events(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_vcpu_events events;
     int ret;
 
@@ -1453,7 +1552,7 @@ static int kvm_get_vcpu_events(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
     if (ret < 0) {
        return ret;
     }
@@ -1479,8 +1578,9 @@ static int kvm_get_vcpu_events(CPUX86State *env)
     return 0;
 }
 
-static int kvm_guest_debug_workarounds(CPUX86State *env)
+static int kvm_guest_debug_workarounds(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     int ret = 0;
     unsigned long reinject_trap = 0;
 
@@ -1508,8 +1608,9 @@ static int kvm_guest_debug_workarounds(CPUX86State *env)
     return ret;
 }
 
-static int kvm_put_debugregs(CPUX86State *env)
+static int kvm_put_debugregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_debugregs dbgregs;
     int i;
 
@@ -1524,11 +1625,12 @@ static int kvm_put_debugregs(CPUX86State *env)
     dbgregs.dr7 = env->dr[7];
     dbgregs.flags = 0;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEBUGREGS, &dbgregs);
 }
 
-static int kvm_get_debugregs(CPUX86State *env)
+static int kvm_get_debugregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_debugregs dbgregs;
     int i, ret;
 
@@ -1536,7 +1638,7 @@ static int kvm_get_debugregs(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_DEBUGREGS, &dbgregs);
     if (ret < 0) {
         return ret;
     }
@@ -1549,117 +1651,121 @@ static int kvm_get_debugregs(CPUX86State *env)
     return 0;
 }
 
-int kvm_arch_put_registers(CPUX86State *env, int level)
+int kvm_arch_put_registers(CPUState *cpu, int level)
 {
+    X86CPU *x86_cpu = X86_CPU(cpu);
     int ret;
 
-    assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
+    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
-    ret = kvm_getput_regs(env, 1);
+    ret = kvm_getput_regs(x86_cpu, 1);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_xsave(env);
+    ret = kvm_put_xsave(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_xcrs(env);
+    ret = kvm_put_xcrs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_sregs(env);
+    ret = kvm_put_sregs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     /* must be before kvm_put_msrs */
-    ret = kvm_inject_mce_oldstyle(env);
+    ret = kvm_inject_mce_oldstyle(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_msrs(env, level);
+    ret = kvm_put_msrs(x86_cpu, level);
     if (ret < 0) {
         return ret;
     }
     if (level >= KVM_PUT_RESET_STATE) {
-        ret = kvm_put_mp_state(env);
+        ret = kvm_put_mp_state(x86_cpu);
         if (ret < 0) {
             return ret;
         }
-        ret = kvm_put_apic(env);
+        ret = kvm_put_apic(x86_cpu);
         if (ret < 0) {
             return ret;
         }
     }
-    ret = kvm_put_vcpu_events(env, level);
+    ret = kvm_put_vcpu_events(x86_cpu, level);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_debugregs(env);
+    ret = kvm_put_debugregs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     /* must be last */
-    ret = kvm_guest_debug_workarounds(env);
+    ret = kvm_guest_debug_workarounds(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     return 0;
 }
 
-int kvm_arch_get_registers(CPUX86State *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
+    X86CPU *cpu = X86_CPU(cs);
     int ret;
 
-    assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
+    assert(cpu_is_stopped(cs) || qemu_cpu_is_self(cs));
 
-    ret = kvm_getput_regs(env, 0);
+    ret = kvm_getput_regs(cpu, 0);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_xsave(env);
+    ret = kvm_get_xsave(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_xcrs(env);
+    ret = kvm_get_xcrs(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_sregs(env);
+    ret = kvm_get_sregs(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_msrs(env);
+    ret = kvm_get_msrs(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_mp_state(env);
+    ret = kvm_get_mp_state(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_apic(env);
+    ret = kvm_get_apic(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_vcpu_events(env);
+    ret = kvm_get_vcpu_events(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_debugregs(env);
+    ret = kvm_get_debugregs(cpu);
     if (ret < 0) {
         return ret;
     }
     return 0;
 }
 
-void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
 {
+    X86CPU *x86_cpu = X86_CPU(cpu);
+    CPUX86State *env = &x86_cpu->env;
     int ret;
 
     /* Inject NMI */
     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
         DPRINTF("injected NMI\n");
-        ret = kvm_vcpu_ioctl(env, KVM_NMI);
+        ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
         if (ret < 0) {
             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
                     strerror(-ret));
@@ -1687,7 +1793,7 @@ void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
 
                 intr.irq = irq;
                 DPRINTF("injected interrupt %d\n", irq);
-                ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
+                ret = kvm_vcpu_ioctl(cpu, KVM_INTERRUPT, &intr);
                 if (ret < 0) {
                     fprintf(stderr,
                             "KVM: injection failed, interrupt lost (%s)\n",
@@ -1711,8 +1817,11 @@ void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
     }
 }
 
-void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
+    X86CPU *x86_cpu = X86_CPU(cpu);
+    CPUX86State *env = &x86_cpu->env;
+
     if (run->if_flag) {
         env->eflags |= IF_MASK;
     } else {
@@ -1722,9 +1831,10 @@ void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
     cpu_set_apic_base(env->apic_state, run->apic_base);
 }
 
-int kvm_arch_process_async_events(CPUX86State *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
 
     if (env->interrupt_request & CPU_INTERRUPT_MCE) {
         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
@@ -1780,8 +1890,10 @@ int kvm_arch_process_async_events(CPUX86State *env)
     return env->halted;
 }
 
-static int kvm_handle_halt(CPUX86State *env)
+static int kvm_handle_halt(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
+
     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
           (env->eflags & IF_MASK)) &&
         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
@@ -1792,9 +1904,11 @@ static int kvm_handle_halt(CPUX86State *env)
     return 0;
 }
 
-static int kvm_handle_tpr_access(CPUX86State *env)
+static int kvm_handle_tpr_access(X86CPU *cpu)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUX86State *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
 
     apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
                                   run->tpr_access.is_write ? TPR_ACCESS_WRITE
@@ -1802,8 +1916,9 @@ static int kvm_handle_tpr_access(CPUX86State *env)
     return 1;
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_insert_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
     static const uint8_t int3 = 0xcc;
 
     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
@@ -1813,8 +1928,9 @@ int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp
     return 0;
 }
 
-int kvm_arch_remove_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_remove_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
     uint8_t int3;
 
     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
@@ -1908,14 +2024,16 @@ void kvm_arch_remove_all_hw_breakpoints(void)
 
 static CPUWatchpoint hw_watchpoint;
 
-static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
+static int kvm_handle_debug(X86CPU *cpu,
+                            struct kvm_debug_exit_arch *arch_info)
 {
+    CPUX86State *env = &cpu->env;
     int ret = 0;
     int n;
 
     if (arch_info->exception == 1) {
         if (arch_info->dr6 & (1 << 14)) {
-            if (cpu_single_env->singlestep_enabled) {
+            if (env->singlestep_enabled) {
                 ret = EXCP_DEBUG;
             }
         } else {
@@ -1927,13 +2045,13 @@ static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
                         break;
                     case 0x1:
                         ret = EXCP_DEBUG;
-                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
+                        env->watchpoint_hit = &hw_watchpoint;
                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
                         hw_watchpoint.flags = BP_MEM_WRITE;
                         break;
                     case 0x3:
                         ret = EXCP_DEBUG;
-                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
+                        env->watchpoint_hit = &hw_watchpoint;
                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
                         hw_watchpoint.flags = BP_MEM_ACCESS;
                         break;
@@ -1941,22 +2059,22 @@ static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
                 }
             }
         }
-    } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
+    } else if (kvm_find_sw_breakpoint(CPU(cpu), arch_info->pc)) {
         ret = EXCP_DEBUG;
     }
     if (ret == 0) {
-        cpu_synchronize_state(cpu_single_env);
-        assert(cpu_single_env->exception_injected == -1);
+        cpu_synchronize_state(env);
+        assert(env->exception_injected == -1);
 
         /* pass to guest */
-        cpu_single_env->exception_injected = arch_info->exception;
-        cpu_single_env->has_error_code = 0;
+        env->exception_injected = arch_info->exception;
+        env->has_error_code = 0;
     }
 
     return ret;
 }
 
-void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
+void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
 {
     const uint8_t type_code[] = {
         [GDB_BREAKPOINT_HW] = 0x0,
@@ -1968,7 +2086,7 @@ void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
     };
     int n;
 
-    if (kvm_sw_breakpoints_active(env)) {
+    if (kvm_sw_breakpoints_active(cpu)) {
         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
     }
     if (nb_hw_breakpoint > 0) {
@@ -1993,21 +2111,22 @@ static bool host_supports_vmx(void)
 
 #define VMX_INVALID_GUEST_STATE 0x80000021
 
-int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
+    X86CPU *cpu = X86_CPU(cs);
     uint64_t code;
     int ret;
 
     switch (run->exit_reason) {
     case KVM_EXIT_HLT:
         DPRINTF("handle_hlt\n");
-        ret = kvm_handle_halt(env);
+        ret = kvm_handle_halt(cpu);
         break;
     case KVM_EXIT_SET_TPR:
         ret = 0;
         break;
     case KVM_EXIT_TPR_ACCESS:
-        ret = kvm_handle_tpr_access(env);
+        ret = kvm_handle_tpr_access(cpu);
         break;
     case KVM_EXIT_FAIL_ENTRY:
         code = run->fail_entry.hardware_entry_failure_reason;
@@ -2033,7 +2152,7 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
         break;
     case KVM_EXIT_DEBUG:
         DPRINTF("kvm_exit_debug\n");
-        ret = kvm_handle_debug(&run->debug.arch);
+        ret = kvm_handle_debug(cpu, &run->debug.arch);
         break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
@@ -2044,8 +2163,11 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
     return ret;
 }
 
-bool kvm_arch_stop_on_emulation_error(CPUX86State *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cs)
 {
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
     kvm_cpu_synchronize_state(env);
     return !(env->cr[0] & CR0_PE_MASK) ||
            ((env->segs[R_CS].selector  & 3) != 3);
@@ -2068,3 +2190,143 @@ void kvm_arch_init_irq_routing(KVMState *s)
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_routing_allowed = true;
 }
+
+/* Classic KVM device assignment interface. Will remain x86 only. */
+int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
+                          uint32_t flags, uint32_t *dev_id)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .segnr = dev_addr->domain,
+        .busnr = dev_addr->bus,
+        .devfn = PCI_DEVFN(dev_addr->slot, dev_addr->function),
+        .flags = flags,
+    };
+    int ret;
+
+    dev_data.assigned_dev_id =
+        (dev_addr->domain << 16) | (dev_addr->bus << 8) | dev_data.devfn;
+
+    ret = kvm_vm_ioctl(s, KVM_ASSIGN_PCI_DEVICE, &dev_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    *dev_id = dev_data.assigned_dev_id;
+
+    return 0;
+}
+
+int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .assigned_dev_id = dev_id,
+    };
+
+    return kvm_vm_ioctl(s, KVM_DEASSIGN_PCI_DEVICE, &dev_data);
+}
+
+static int kvm_assign_irq_internal(KVMState *s, uint32_t dev_id,
+                                   uint32_t irq_type, uint32_t guest_irq)
+{
+    struct kvm_assigned_irq assigned_irq = {
+        .assigned_dev_id = dev_id,
+        .guest_irq = guest_irq,
+        .flags = irq_type,
+    };
+
+    if (kvm_check_extension(s, KVM_CAP_ASSIGN_DEV_IRQ)) {
+        return kvm_vm_ioctl(s, KVM_ASSIGN_DEV_IRQ, &assigned_irq);
+    } else {
+        return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, &assigned_irq);
+    }
+}
+
+int kvm_device_intx_assign(KVMState *s, uint32_t dev_id, bool use_host_msi,
+                           uint32_t guest_irq)
+{
+    uint32_t irq_type = KVM_DEV_IRQ_GUEST_INTX |
+        (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX);
+
+    return kvm_assign_irq_internal(s, dev_id, irq_type, guest_irq);
+}
+
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .assigned_dev_id = dev_id,
+        .flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &dev_data);
+}
+
+static int kvm_deassign_irq_internal(KVMState *s, uint32_t dev_id,
+                                     uint32_t type)
+{
+    struct kvm_assigned_irq assigned_irq = {
+        .assigned_dev_id = dev_id,
+        .flags = type,
+    };
+
+    return kvm_vm_ioctl(s, KVM_DEASSIGN_DEV_IRQ, &assigned_irq);
+}
+
+int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_INTX |
+        (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX));
+}
+
+int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq)
+{
+    return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSI |
+                                              KVM_DEV_IRQ_GUEST_MSI, virq);
+}
+
+int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSI |
+                                                KVM_DEV_IRQ_HOST_MSI);
+}
+
+bool kvm_device_msix_supported(KVMState *s)
+{
+    /* The kernel lacks a corresponding KVM_CAP, so we probe by calling
+     * KVM_ASSIGN_SET_MSIX_NR with an invalid parameter. */
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, NULL) == -EFAULT;
+}
+
+int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
+                                 uint32_t nr_vectors)
+{
+    struct kvm_assigned_msix_nr msix_nr = {
+        .assigned_dev_id = dev_id,
+        .entry_nr = nr_vectors,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, &msix_nr);
+}
+
+int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
+                               int virq)
+{
+    struct kvm_assigned_msix_entry msix_entry = {
+        .assigned_dev_id = dev_id,
+        .gsi = virq,
+        .entry = vector,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_ENTRY, &msix_entry);
+}
+
+int kvm_device_msix_assign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSIX |
+                                              KVM_DEV_IRQ_GUEST_MSIX, 0);
+}
+
+int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
+                                                KVM_DEV_IRQ_HOST_MSIX);
+}
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index b82bbf401e0..4392ab43598 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -11,6 +11,28 @@
 #ifndef QEMU_KVM_I386_H
 #define QEMU_KVM_I386_H
 
+#include "sysemu/kvm.h"
+
 bool kvm_allows_irq0_override(void);
 
+int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
+                          uint32_t flags, uint32_t *dev_id);
+int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id);
+
+int kvm_device_intx_assign(KVMState *s, uint32_t dev_id,
+                           bool use_host_msi, uint32_t guest_irq);
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi);
+
+int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq);
+int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id);
+
+bool kvm_device_msix_supported(KVMState *s);
+int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
+                                 uint32_t nr_vectors);
+int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
+                               int virq);
+int kvm_device_msix_assign(KVMState *s, uint32_t dev_id);
+int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id);
+
 #endif
diff --git a/target-i386/machine.c b/target-i386/machine.c
index 477150887bb..8df6a6b645e 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -4,7 +4,7 @@
 #include "hw/isa.h"
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static const VMStateDescription vmstate_segment = {
     .name = "segment",
@@ -265,10 +265,11 @@ static int cpu_post_load(void *opaque, int version_id)
 
     cpu_breakpoint_remove_all(env, BP_CPU);
     cpu_watchpoint_remove_all(env, BP_CPU);
-    for (i = 0; i < 4; i++)
+    for (i = 0; i < DR7_MAX_BP; i++) {
         hw_breakpoint_insert(env, i);
-
+    }
     tlb_flush(env, 1);
+
     return 0;
 }
 
@@ -328,6 +329,24 @@ static const VMStateDescription vmstate_fpop_ip_dp = {
     }
 };
 
+static bool tsc_adjust_needed(void *opaque)
+{
+    CPUX86State *env = opaque;
+
+    return env->tsc_adjust != 0;
+}
+
+static const VMStateDescription vmstate_msr_tsc_adjust = {
+    .name = "cpu/msr_tsc_adjust",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT64(tsc_adjust, CPUX86State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool tscdeadline_needed(void *opaque)
 {
     CPUX86State *env = opaque;
@@ -478,6 +497,9 @@ static const VMStateDescription vmstate_cpu = {
             .vmsd = &vmstate_fpop_ip_dp,
             .needed = fpop_ip_dp_needed,
         }, {
+            .vmsd = &vmstate_msr_tsc_adjust,
+            .needed = tsc_adjust_needed,
+        }, {
             .vmsd = &vmstate_msr_tscdeadline,
             .needed = tscdeadline_needed,
         }, {
diff --git a/target-i386/mem_helper.c b/target-i386/mem_helper.c
index 7f99c7cfe3d..6cf9ba076e8 100644
--- a/target-i386/mem_helper.c
+++ b/target-i386/mem_helper.c
@@ -21,7 +21,7 @@
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* broken thread support */
@@ -114,16 +114,16 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #endif
 
@@ -135,19 +135,13 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 void tlb_fill(CPUX86State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_x86_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         raise_exception_err(env, env->exception_index, env->error_code);
     }
diff --git a/target-i386/misc_helper.c b/target-i386/misc_helper.c
index a02037963f3..b6d574019a3 100644
--- a/target-i386/misc_helper.c
+++ b/target-i386/misc_helper.c
@@ -18,11 +18,11 @@
  */
 
 #include "cpu.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* check if Port I/O is allowed in TSS */
@@ -110,7 +110,7 @@ void helper_into(CPUX86State *env, int next_eip_addend)
 void helper_single_step(CPUX86State *env)
 {
 #ifndef CONFIG_USER_ONLY
-    check_hw_breakpoints(env, 1);
+    check_hw_breakpoints(env, true);
     env->dr[6] |= DR6_BS;
 #endif
     raise_exception(env, EXCP01_DB);
@@ -197,11 +197,11 @@ void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
         env->dr[reg] = t0;
         hw_breakpoint_insert(env, reg);
     } else if (reg == 7) {
-        for (i = 0; i < 4; i++) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
             hw_breakpoint_remove(env, i);
         }
         env->dr[7] = t0;
-        for (i = 0; i < 4; i++) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
             hw_breakpoint_insert(env, i);
         }
     } else {
@@ -580,14 +580,17 @@ void helper_monitor(CPUX86State *env, target_ulong ptr)
 
 void helper_mwait(CPUX86State *env, int next_eip_addend)
 {
+    CPUState *cpu;
+
     if ((uint32_t)ECX != 0) {
         raise_exception(env, EXCP0D_GPF);
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_MWAIT, 0);
     EIP += next_eip_addend;
 
+    cpu = CPU(x86_env_get_cpu(env));
     /* XXX: not complete but not completely erroneous */
-    if (env->cpu_index != 0 || env->next_cpu != NULL) {
+    if (cpu->cpu_index != 0 || env->next_cpu != NULL) {
         /* more than one CPU: do not sleep because another CPU may
            wake this one */
     } else {
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 5fff8d59c6d..3247deeb60c 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -19,19 +19,19 @@
  */
 
 #include "cpu.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "helper.h"
 
 //#define DEBUG_PCALL
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #ifdef DEBUG_PCALL
 # define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
 # define LOG_PCALL_STATE(env)                                  \
-    log_cpu_state_mask(CPU_LOG_PCALL, (env), X86_DUMP_CCOP)
+    log_cpu_state_mask(CPU_LOG_PCALL, (env), CPU_DUMP_CCOP)
 #else
 # define LOG_PCALL(...) do { } while (0)
 # define LOG_PCALL_STATE(env) do { } while (0)
@@ -465,13 +465,14 @@ static void switch_tss(CPUX86State *env, int tss_selector,
 
 #ifndef CONFIG_USER_ONLY
     /* reset local breakpoints */
-    if (env->dr[7] & 0x55) {
-        for (i = 0; i < 4; i++) {
-            if (hw_breakpoint_enabled(env->dr[7], i) == 0x1) {
+    if (env->dr[7] & DR7_LOCAL_BP_MASK) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if (hw_local_breakpoint_enabled(env->dr[7], i) &&
+                !hw_global_breakpoint_enabled(env->dr[7], i)) {
                 hw_breakpoint_remove(env, i);
             }
         }
-        env->dr[7] &= ~0x55;
+        env->dr[7] &= ~DR7_LOCAL_BP_MASK;
     }
 #endif
 }
@@ -1177,7 +1178,7 @@ static void do_interrupt_all(CPUX86State *env, int intno, int is_int,
                 qemu_log(" EAX=" TARGET_FMT_lx, EAX);
             }
             qemu_log("\n");
-            log_cpu_state(env, X86_DUMP_CCOP);
+            log_cpu_state(env, CPU_DUMP_CCOP);
 #if 0
             {
                 int i;
diff --git a/target-i386/smm_helper.c b/target-i386/smm_helper.c
index 8b04eb2e001..eea2fe97826 100644
--- a/target-i386/smm_helper.c
+++ b/target-i386/smm_helper.c
@@ -47,7 +47,7 @@ void do_smm_enter(CPUX86State *env)
     int i, offset;
 
     qemu_log_mask(CPU_LOG_INT, "SMM: enter\n");
-    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
+    log_cpu_state_mask(CPU_LOG_INT, env, CPU_DUMP_CCOP);
 
     env->hflags |= HF_SMM_MASK;
     cpu_smm_update(env);
@@ -295,7 +295,7 @@ void helper_rsm(CPUX86State *env)
     cpu_smm_update(env);
 
     qemu_log_mask(CPU_LOG_INT, "SMM: after RSM\n");
-    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
+    log_cpu_state_mask(CPU_LOG_INT, env, CPU_DUMP_CCOP);
 }
 
 #endif /* !CONFIG_USER_ONLY */
diff --git a/target-i386/svm_helper.c b/target-i386/svm_helper.c
index 4943c37fee4..3f246e90739 100644
--- a/target-i386/svm_helper.c
+++ b/target-i386/svm_helper.c
@@ -18,11 +18,11 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* Secure Virtual Machine helpers */
@@ -85,7 +85,7 @@ void helper_svm_check_io(CPUX86State *env, uint32_t port, uint32_t param,
 }
 #else
 
-static inline void svm_save_seg(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_save_seg(CPUX86State *env, hwaddr addr,
                                 const SegmentCache *sc)
 {
     stw_phys(addr + offsetof(struct vmcb_seg, selector),
@@ -98,7 +98,7 @@ static inline void svm_save_seg(CPUX86State *env, target_phys_addr_t addr,
              ((sc->flags >> 8) & 0xff) | ((sc->flags >> 12) & 0x0f00));
 }
 
-static inline void svm_load_seg(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_load_seg(CPUX86State *env, hwaddr addr,
                                 SegmentCache *sc)
 {
     unsigned int flags;
@@ -110,7 +110,7 @@ static inline void svm_load_seg(CPUX86State *env, target_phys_addr_t addr,
     sc->flags = ((flags & 0xff) << 8) | ((flags & 0x0f00) << 12);
 }
 
-static inline void svm_load_seg_cache(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_load_seg_cache(CPUX86State *env, hwaddr addr,
                                       int seg_reg)
 {
     SegmentCache sc1, *sc = &sc1;
diff --git a/target-i386/topology.h b/target-i386/topology.h
new file mode 100644
index 00000000000..24ed525453e
--- /dev/null
+++ b/target-i386/topology.h
@@ -0,0 +1,136 @@
+/*
+ *  x86 CPU topology data structures and functions
+ *
+ *  Copyright (c) 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef TARGET_I386_TOPOLOGY_H
+#define TARGET_I386_TOPOLOGY_H
+
+/* This file implements the APIC-ID-based CPU topology enumeration logic,
+ * documented at the following document:
+ *   Intel® 64 Architecture Processor Topology Enumeration
+ *   http://software.intel.com/en-us/articles/intel-64-architecture-processor-topology-enumeration/
+ *
+ * This code should be compatible with AMD's "Extended Method" described at:
+ *   AMD CPUID Specification (Publication #25481)
+ *   Section 3: Multiple Core Calcuation
+ * as long as:
+ *  nr_threads is set to 1;
+ *  OFFSET_IDX is assumed to be 0;
+ *  CPUID Fn8000_0008_ECX[ApicIdCoreIdSize[3:0]] is set to apicid_core_width().
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include "qemu/bitops.h"
+
+/* APIC IDs can be 32-bit, but beware: APIC IDs > 255 require x2APIC support
+ */
+typedef uint32_t apic_id_t;
+
+/* Return the bit width needed for 'count' IDs
+ */
+static unsigned apicid_bitwidth_for_count(unsigned count)
+{
+    g_assert(count >= 1);
+    if (count == 1) {
+        return 0;
+    }
+    return bitops_flsl(count - 1) + 1;
+}
+
+/* Bit width of the SMT_ID (thread ID) field on the APIC ID
+ */
+static inline unsigned apicid_smt_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_threads);
+}
+
+/* Bit width of the Core_ID field
+ */
+static inline unsigned apicid_core_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_cores);
+}
+
+/* Bit offset of the Core_ID field
+ */
+static inline unsigned apicid_core_offset(unsigned nr_cores,
+                                          unsigned nr_threads)
+{
+    return apicid_smt_width(nr_cores, nr_threads);
+}
+
+/* Bit offset of the Pkg_ID (socket ID) field
+ */
+static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_core_offset(nr_cores, nr_threads) +
+           apicid_core_width(nr_cores, nr_threads);
+}
+
+/* Make APIC ID for the CPU based on Pkg_ID, Core_ID, SMT_ID
+ *
+ * The caller must make sure core_id < nr_cores and smt_id < nr_threads.
+ */
+static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
+                                             unsigned nr_threads,
+                                             unsigned pkg_id,
+                                             unsigned core_id,
+                                             unsigned smt_id)
+{
+    return (pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
+           (core_id << apicid_core_offset(nr_cores, nr_threads)) |
+           smt_id;
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on (contiguous) CPU index
+ */
+static inline void x86_topo_ids_from_idx(unsigned nr_cores,
+                                         unsigned nr_threads,
+                                         unsigned cpu_index,
+                                         unsigned *pkg_id,
+                                         unsigned *core_id,
+                                         unsigned *smt_id)
+{
+    unsigned core_index = cpu_index / nr_threads;
+    *smt_id = cpu_index % nr_threads;
+    *core_id = core_index % nr_cores;
+    *pkg_id = core_index / nr_cores;
+}
+
+/* Make APIC ID for the CPU 'cpu_index'
+ *
+ * 'cpu_index' is a sequential, contiguous ID for the CPU.
+ */
+static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
+                                                unsigned nr_threads,
+                                                unsigned cpu_index)
+{
+    unsigned pkg_id, core_id, smt_id;
+    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index,
+                          &pkg_id, &core_id, &smt_id);
+    return apicid_from_topo_ids(nr_cores, nr_threads, pkg_id, core_id, smt_id);
+}
+
+#endif /* TARGET_I386_TOPOLOGY_H */
diff --git a/target-i386/translate.c b/target-i386/translate.c
index eb0cabcf191..32d21f52d02 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -24,7 +24,7 @@
 #include <signal.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -65,7 +65,7 @@ static TCGv cpu_tmp5;
 
 static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 #ifdef TARGET_X86_64
 static int x86_64_hregs;
@@ -107,6 +107,7 @@ typedef struct DisasContext {
     int cpuid_ext_features;
     int cpuid_ext2_features;
     int cpuid_ext3_features;
+    int cpuid_7_0_ebx_features;
 } DisasContext;
 
 static void gen_eob(DisasContext *s);
@@ -2017,7 +2018,8 @@ static void gen_shifti(DisasContext *s1, int op, int ot, int d, int c)
     }
 }
 
-static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_ptr)
+static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm,
+                          int *reg_ptr, int *offset_ptr)
 {
     target_long disp;
     int havesib;
@@ -2043,7 +2045,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
 
         if (base == 4) {
             havesib = 1;
-            code = cpu_ldub_code(cpu_single_env, s->pc++);
+            code = cpu_ldub_code(env, s->pc++);
             scale = (code >> 6) & 3;
             index = ((code >> 3) & 7) | REX_X(s);
             base = (code & 7);
@@ -2054,7 +2056,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
         case 0:
             if ((base & 7) == 5) {
                 base = -1;
-                disp = (int32_t)cpu_ldl_code(cpu_single_env, s->pc);
+                disp = (int32_t)cpu_ldl_code(env, s->pc);
                 s->pc += 4;
                 if (CODE64(s) && !havesib) {
                     disp += s->pc + s->rip_offset;
@@ -2064,11 +2066,11 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             }
             break;
         case 1:
-            disp = (int8_t)cpu_ldub_code(cpu_single_env, s->pc++);
+            disp = (int8_t)cpu_ldub_code(env, s->pc++);
             break;
         default:
         case 2:
-            disp = (int32_t)cpu_ldl_code(cpu_single_env, s->pc);
+            disp = (int32_t)cpu_ldl_code(env, s->pc);
             s->pc += 4;
             break;
         }
@@ -2131,7 +2133,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
         switch (mod) {
         case 0:
             if (rm == 6) {
-                disp = cpu_lduw_code(cpu_single_env, s->pc);
+                disp = cpu_lduw_code(env, s->pc);
                 s->pc += 2;
                 gen_op_movl_A0_im(disp);
                 rm = 0; /* avoid SS override */
@@ -2141,11 +2143,11 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             }
             break;
         case 1:
-            disp = (int8_t)cpu_ldub_code(cpu_single_env, s->pc++);
+            disp = (int8_t)cpu_ldub_code(env, s->pc++);
             break;
         default:
         case 2:
-            disp = cpu_lduw_code(cpu_single_env, s->pc);
+            disp = cpu_lduw_code(env, s->pc);
             s->pc += 2;
             break;
         }
@@ -2201,7 +2203,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
     *offset_ptr = disp;
 }
 
-static void gen_nop_modrm(DisasContext *s, int modrm)
+static void gen_nop_modrm(CPUX86State *env, DisasContext *s, int modrm)
 {
     int mod, rm, base, code;
 
@@ -2215,7 +2217,7 @@ static void gen_nop_modrm(DisasContext *s, int modrm)
         base = rm;
 
         if (base == 4) {
-            code = cpu_ldub_code(cpu_single_env, s->pc++);
+            code = cpu_ldub_code(env, s->pc++);
             base = (code & 7);
         }
 
@@ -2275,7 +2277,8 @@ static void gen_add_A0_ds_seg(DisasContext *s)
 
 /* generate modrm memory load or store of 'reg'. TMP0 is used if reg ==
    OR_TMP0 */
-static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_store)
+static void gen_ldst_modrm(CPUX86State *env, DisasContext *s, int modrm,
+                           int ot, int reg, int is_store)
 {
     int mod, rm, opreg, disp;
 
@@ -2292,7 +2295,7 @@ static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_s
                 gen_op_mov_reg_T0(ot, reg);
         }
     } else {
-        gen_lea_modrm(s, modrm, &opreg, &disp);
+        gen_lea_modrm(env, s, modrm, &opreg, &disp);
         if (is_store) {
             if (reg != OR_TMP0)
                 gen_op_mov_TN_reg(ot, 0, reg);
@@ -2305,22 +2308,22 @@ static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_s
     }
 }
 
-static inline uint32_t insn_get(DisasContext *s, int ot)
+static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, int ot)
 {
     uint32_t ret;
 
     switch(ot) {
     case OT_BYTE:
-        ret = cpu_ldub_code(cpu_single_env, s->pc);
+        ret = cpu_ldub_code(env, s->pc);
         s->pc++;
         break;
     case OT_WORD:
-        ret = cpu_lduw_code(cpu_single_env, s->pc);
+        ret = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         break;
     default:
     case OT_LONG:
-        ret = cpu_ldl_code(cpu_single_env, s->pc);
+        ret = cpu_ldl_code(env, s->pc);
         s->pc += 4;
         break;
     }
@@ -3166,7 +3169,8 @@ static const struct SSEOpHelper_eppi sse_op_table7[256] = {
     [0x63] = SSE42_OP(pcmpistri),
 };
 
-static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
+static void gen_sse(CPUX86State *env, DisasContext *s, int b,
+                    target_ulong pc_start, int rex_r)
 {
     int b1, op1_offset, op2_offset, is_xmm, val, ot;
     int modrm, mod, rm, reg, reg_addr, offset_addr;
@@ -3229,7 +3233,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         gen_helper_enter_mmx(cpu_env);
     }
 
-    modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+    modrm = cpu_ldub_code(env, s->pc++);
     reg = ((modrm >> 3) & 7);
     if (is_xmm)
         reg |= rex_r;
@@ -3240,7 +3244,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x0e7: /* movntq */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             break;
         case 0x1e7: /* movntdq */
@@ -3248,20 +3252,20 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12b: /* movntps */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             break;
         case 0x3f0: /* lddqu */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             break;
         case 0x22b: /* movntss */
         case 0x32b: /* movntsd */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (b1 & 1) {
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,
                     xmm_regs[reg]));
@@ -3274,12 +3278,12 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x6e: /* movd mm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == 2) {
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 0);
                 tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
             } else
 #endif
             {
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx));
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -3289,14 +3293,14 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x16e: /* movd xmm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == 2) {
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
                 gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
             } else
 #endif
             {
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -3305,7 +3309,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x6f: /* movq mm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             } else {
                 rm = (modrm & 7);
@@ -3322,7 +3326,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x16f: /* movdqa xmm, ea */
         case 0x26f: /* movdqu xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3332,7 +3336,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x210: /* movss xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
                 gen_op_movl_T0_0();
@@ -3347,7 +3351,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x310: /* movsd xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
                 gen_op_movl_T0_0();
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
@@ -3361,7 +3365,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x012: /* movlps */
         case 0x112: /* movlpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 /* movhlps */
@@ -3372,7 +3376,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x212: /* movsldup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3388,7 +3392,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x312: /* movddup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3401,7 +3405,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x016: /* movhps */
         case 0x116: /* movhpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
             } else {
                 /* movlhps */
@@ -3412,7 +3416,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x216: /* movshdup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3433,8 +3437,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
                 if (b1 == 1 && reg != 0)
                     goto illegal_op;
-                field_length = cpu_ldub_code(cpu_single_env, s->pc++) & 0x3F;
-                bit_index = cpu_ldub_code(cpu_single_env, s->pc++) & 0x3F;
+                field_length = cpu_ldub_code(env, s->pc++) & 0x3F;
+                bit_index = cpu_ldub_code(env, s->pc++) & 0x3F;
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                     offsetof(CPUX86State,xmm_regs[reg]));
                 if (b1 == 1)
@@ -3452,13 +3456,13 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (s->dflag == 2) {
                 tcg_gen_ld_i64(cpu_T[0], cpu_env, 
                                offsetof(CPUX86State,fpregs[reg].mmx));
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 1);
             } else
 #endif
             {
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 1);
             }
             break;
         case 0x17e: /* movd ea, xmm */
@@ -3466,18 +3470,18 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (s->dflag == 2) {
                 tcg_gen_ld_i64(cpu_T[0], cpu_env, 
                                offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 1);
             } else
 #endif
             {
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 1);
             }
             break;
         case 0x27e: /* movq xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3488,7 +3492,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x7f: /* movq ea, mm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             } else {
                 rm = (modrm & 7);
@@ -3503,7 +3507,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x17f: /* movdqa ea, xmm */
         case 0x27f: /* movdqu ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3513,7 +3517,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x211: /* movss ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
                 gen_op_st_T0_A0(OT_LONG + s->mem_index);
             } else {
@@ -3524,7 +3528,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x311: /* movsd ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3535,7 +3539,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x013: /* movlps */
         case 0x113: /* movlpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 goto illegal_op;
@@ -3544,7 +3548,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x017: /* movhps */
         case 0x117: /* movhpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
             } else {
                 goto illegal_op;
@@ -3559,7 +3563,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (b1 >= 2) {
 	        goto illegal_op;
             }
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (is_xmm) {
                 gen_op_movl_T0_im(val);
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
@@ -3609,7 +3613,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12a: /* cvtpi2pd */
             gen_helper_enter_mmx(cpu_env);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,mmx_t0);
                 gen_ldq_env_A0(s->mem_index, op2_offset);
             } else {
@@ -3632,7 +3636,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x22a: /* cvtsi2ss */
         case 0x32a: /* cvtsi2sd */
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
             op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             if (ot == OT_LONG) {
@@ -3654,7 +3658,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12d: /* cvtpd2pi */
             gen_helper_enter_mmx(cpu_env);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,xmm_t0);
                 gen_ldo_env_A0(s->mem_index, op2_offset);
             } else {
@@ -3685,7 +3689,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x32d: /* cvtsd2si */
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if ((b >> 8) & 1) {
                     gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_t0.XMM_Q(0)));
                 } else {
@@ -3717,8 +3721,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0xc4: /* pinsrw */
         case 0x1c4:
             s->rip_offset = 1;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
+            val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
                 tcg_gen_st16_tl(cpu_T[0], cpu_env,
@@ -3734,7 +3738,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (mod != 3)
                 goto illegal_op;
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
                 rm = (modrm & 7) | REX_B(s);
@@ -3751,7 +3755,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x1d6: /* movq ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3795,7 +3799,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                 goto crc32;
         case 0x038:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             rm = modrm & 7;
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
@@ -3816,7 +3820,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
                 } else {
                     op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     switch (b) {
                     case 0x20: case 0x30: /* pmovsxbw, pmovzxbw */
                     case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
@@ -3851,7 +3855,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
                 } else {
                     op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldq_env_A0(s->mem_index, op2_offset);
                 }
             }
@@ -3869,7 +3873,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x338: /* crc32 */
         crc32:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
 
             if (b != 0xf0 && b != 0xf1)
@@ -3889,7 +3893,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
             gen_op_mov_TN_reg(OT_LONG, 0, reg);
             tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
             gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
                              cpu_T[0], tcg_const_i32(8 << ot));
 
@@ -3899,7 +3903,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x03a:
         case 0x13a:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             rm = modrm & 7;
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
@@ -3918,9 +3922,9 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                 ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3)
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 reg = ((modrm >> 3) & 7) | rex_r;
-                val = cpu_ldub_code(cpu_single_env, s->pc++);
+                val = cpu_ldub_code(env, s->pc++);
                 switch (b) {
                 case 0x14: /* pextrb */
                     tcg_gen_ld8u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
@@ -4050,7 +4054,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
                 } else {
                     op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldo_env_A0(s->mem_index, op2_offset);
                 }
             } else {
@@ -4059,11 +4063,11 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
                 } else {
                     op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldq_env_A0(s->mem_index, op2_offset);
                 }
             }
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
 
             if ((b & 0xfc) == 0x60) { /* pcmpXstrX */
                 s->cc_op = CC_OP_EFLAGS;
@@ -4094,7 +4098,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         if (is_xmm) {
             op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,xmm_t0);
                 if (b1 >= 2 && ((b >= 0x50 && b <= 0x5f && b != 0x5b) ||
                                 b == 0xc2)) {
@@ -4117,7 +4121,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         } else {
             op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,mmx_t0);
                 gen_ldq_env_A0(s->mem_index, op2_offset);
             } else {
@@ -4129,7 +4133,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x0f: /* 3DNow! data insns */
             if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
                 goto illegal_op;
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             sse_fn_epp = sse_op_table5[val];
             if (!sse_fn_epp) {
                 goto illegal_op;
@@ -4140,7 +4144,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x70: /* pshufx insn */
         case 0xc6: /* pshufx insn */
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
             /* XXX: introduce a new table? */
@@ -4149,7 +4153,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0xc2:
             /* compare insns */
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (val >= 8)
                 goto illegal_op;
             sse_fn_epp = sse_op_table4[val][b1];
@@ -4194,7 +4198,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
 /* convert one instruction. s->is_jmp is set if the translation must
    be stopped. Return the next pc value */
-static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
+static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
+                               target_ulong pc_start)
 {
     int b, prefixes, aflag, dflag;
     int shift, ot;
@@ -4202,8 +4207,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     target_ulong next_eip, tval;
     int rex_w, rex_r;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(pc_start);
+    }
     s->pc = pc_start;
     prefixes = 0;
     aflag = s->code32;
@@ -4218,7 +4224,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #endif
     s->rip_offset = 0; /* for relative ip address */
  next_byte:
-    b = cpu_ldub_code(cpu_single_env, s->pc);
+    b = cpu_ldub_code(env, s->pc);
     s->pc++;
     /* check prefixes */
 #ifdef TARGET_X86_64
@@ -4333,7 +4339,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x0f:
         /**************************/
         /* extended op code */
-        b = cpu_ldub_code(cpu_single_env, s->pc++) | 0x100;
+        b = cpu_ldub_code(env, s->pc++) | 0x100;
         goto reswitch;
 
         /**************************/
@@ -4358,12 +4364,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
             switch(f) {
             case 0: /* OP Ev, Gv */
-                modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+                modrm = cpu_ldub_code(env, s->pc++);
                 reg = ((modrm >> 3) & 7) | rex_r;
                 mod = (modrm >> 6) & 3;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     opreg = OR_TMP0;
                 } else if (op == OP_XORL && rm == reg) {
                 xor_zero:
@@ -4380,12 +4386,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_op(s, op, ot, opreg);
                 break;
             case 1: /* OP Gv, Ev */
-                modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+                modrm = cpu_ldub_code(env, s->pc++);
                 mod = (modrm >> 6) & 3;
                 reg = ((modrm >> 3) & 7) | rex_r;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_op_ld_T1_A0(ot + s->mem_index);
                 } else if (op == OP_XORL && rm == reg) {
                     goto xor_zero;
@@ -4395,7 +4401,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_op(s, op, ot, reg);
                 break;
             case 2: /* OP A, Iv */
-                val = insn_get(s, ot);
+                val = insn_get(env, s, ot);
                 gen_op_movl_T1_im(val);
                 gen_op(s, op, ot, OR_EAX);
                 break;
@@ -4417,7 +4423,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             else
                 ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
             op = (modrm >> 3) & 7;
@@ -4427,7 +4433,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     s->rip_offset = 1;
                 else
                     s->rip_offset = insn_const_size(ot);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
                 opreg = rm;
@@ -4438,10 +4444,10 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             case 0x80:
             case 0x81:
             case 0x82:
-                val = insn_get(s, ot);
+                val = insn_get(env, s, ot);
                 break;
             case 0x83:
-                val = (int8_t)insn_get(s, OT_BYTE);
+                val = (int8_t)insn_get(env, s, OT_BYTE);
                 break;
             }
             gen_op_movl_T1_im(val);
@@ -4466,14 +4472,14 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (mod != 3) {
             if (op == 0)
                 s->rip_offset = insn_const_size(ot);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
             gen_op_mov_TN_reg(ot, 0, rm);
@@ -4481,7 +4487,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
         switch(op) {
         case 0: /* test */
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             gen_op_movl_T1_im(val);
             gen_op_testl_T0_T1_cc();
             s->cc_op = CC_OP_LOGICB + ot;
@@ -4698,7 +4704,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
@@ -4717,7 +4723,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             }
         }
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (op >= 2 && op != 3 && op != 5)
                 gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
@@ -4810,10 +4816,10 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_TN_reg(ot, 1, reg);
         gen_op_testl_T0_T1_cc();
         s->cc_op = CC_OP_LOGICB + ot;
@@ -4825,7 +4831,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        val = insn_get(s, ot);
+        val = insn_get(env, s, ot);
 
         gen_op_mov_TN_reg(ot, 0, OR_EAX);
         gen_op_movl_T1_im(val);
@@ -4875,18 +4881,18 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x69: /* imul Gv, Ev, I */
     case 0x6b:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         if (b == 0x69)
             s->rip_offset = insn_const_size(ot);
         else if (b == 0x6b)
             s->rip_offset = 1;
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         if (b == 0x69) {
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             gen_op_movl_T1_im(val);
         } else if (b == 0x6b) {
-            val = (int8_t)insn_get(s, OT_BYTE);
+            val = (int8_t)insn_get(env, s, OT_BYTE);
             gen_op_movl_T1_im(val);
         } else {
             gen_op_mov_TN_reg(ot, 1, reg);
@@ -4939,7 +4945,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
@@ -4950,7 +4956,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_op_mov_reg_T1(ot, reg);
             gen_op_mov_reg_T0(ot, rm);
         } else {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_mov_TN_reg(ot, 0, reg);
             gen_op_ld_T1_A0(ot + s->mem_index);
             gen_op_addl_T0_T1();
@@ -4970,7 +4976,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 ot = OT_BYTE;
             else
                 ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             t0 = tcg_temp_local_new();
@@ -4982,7 +4988,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 rm = (modrm & 7) | REX_B(s);
                 gen_op_mov_v_reg(ot, t0, rm);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_mov_tl(a0, cpu_A0);
                 gen_op_ld_v(ot + s->mem_index, t0, a0);
                 rm = 0; /* avoid warning */
@@ -5018,7 +5024,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x1c7: /* cmpxchg8b */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if ((mod == 3) || ((modrm & 0x38) != 0x8))
             goto illegal_op;
@@ -5029,7 +5035,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_jmp_im(pc_start - s->cs_base);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_helper_cmpxchg16b(cpu_env, cpu_A0);
         } else
 #endif        
@@ -5039,7 +5045,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_jmp_im(pc_start - s->cs_base);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_helper_cmpxchg8b(cpu_env, cpu_A0);
         }
         s->cc_op = CC_OP_EFLAGS;
@@ -5080,9 +5086,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = dflag + OT_WORD;
         }
         if (b == 0x68)
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
         else
-            val = (int8_t)insn_get(s, OT_BYTE);
+            val = (int8_t)insn_get(env, s, OT_BYTE);
         gen_op_movl_T0_im(val);
         gen_push_T0(s);
         break;
@@ -5092,7 +5098,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         } else {
             ot = dflag + OT_WORD;
         }
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         gen_pop_T0(s);
         if (mod == 3) {
@@ -5103,7 +5109,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         } else {
             /* NOTE: order is important too for MMU exceptions */
             s->popl_esp_hack = 1 << ot;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             s->popl_esp_hack = 0;
             gen_pop_update(s);
         }
@@ -5111,9 +5117,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xc8: /* enter */
         {
             int level;
-            val = cpu_lduw_code(cpu_single_env, s->pc);
+            val = cpu_lduw_code(env, s->pc);
             s->pc += 2;
-            level = cpu_ldub_code(cpu_single_env, s->pc++);
+            level = cpu_ldub_code(env, s->pc++);
             gen_enter(s, val, level);
         }
         break;
@@ -5193,11 +5199,11 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
         /* generate a generic store */
-        gen_ldst_modrm(s, modrm, ot, reg, 1);
+        gen_ldst_modrm(env, s, modrm, ot, reg, 1);
         break;
     case 0xc6:
     case 0xc7: /* mov Ev, Iv */
@@ -5205,13 +5211,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod != 3) {
             s->rip_offset = insn_const_size(ot);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         }
-        val = insn_get(s, ot);
+        val = insn_get(env, s, ot);
         gen_op_movl_T0_im(val);
         if (mod != 3)
             gen_op_st_T0_A0(ot + s->mem_index);
@@ -5224,18 +5230,18 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = OT_WORD + dflag;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_reg_T0(ot, reg);
         break;
     case 0x8e: /* mov seg, Gv */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
-        gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
         gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
         if (reg == R_SS) {
             /* if reg == SS, inhibit interrupts/trace */
@@ -5251,7 +5257,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x8c: /* mov Gv, seg */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         if (reg >= 6)
@@ -5261,7 +5267,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD + dflag;
         else
             ot = OT_WORD;
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
         break;
 
     case 0x1b6: /* movzbS Gv, Eb */
@@ -5274,7 +5280,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             d_ot = dflag + OT_WORD;
             /* ot is the size of source */
             ot = (b & 1) + OT_BYTE;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
@@ -5298,7 +5304,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 }
                 gen_op_mov_reg_T0(d_ot, reg);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if (b & 8) {
                     gen_op_lds_T0_A0(ot + s->mem_index);
                 } else {
@@ -5311,7 +5317,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
     case 0x8d: /* lea */
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
@@ -5320,7 +5326,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         s->override = -1;
         val = s->addseg;
         s->addseg = 0;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         s->addseg = val;
         gen_op_mov_reg_A0(ot - OT_WORD, reg);
         break;
@@ -5338,16 +5344,16 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 ot = dflag + OT_WORD;
 #ifdef TARGET_X86_64
             if (s->aflag == 2) {
-                offset_addr = cpu_ldq_code(cpu_single_env, s->pc);
+                offset_addr = cpu_ldq_code(env, s->pc);
                 s->pc += 8;
                 gen_op_movq_A0_im(offset_addr);
             } else
 #endif
             {
                 if (s->aflag) {
-                    offset_addr = insn_get(s, OT_LONG);
+                    offset_addr = insn_get(env, s, OT_LONG);
                 } else {
-                    offset_addr = insn_get(s, OT_WORD);
+                    offset_addr = insn_get(env, s, OT_WORD);
                 }
                 gen_op_movl_A0_im(offset_addr);
             }
@@ -5385,7 +5391,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_op_mov_reg_T0(OT_BYTE, R_EAX);
         break;
     case 0xb0 ... 0xb7: /* mov R, Ib */
-        val = insn_get(s, OT_BYTE);
+        val = insn_get(env, s, OT_BYTE);
         gen_op_movl_T0_im(val);
         gen_op_mov_reg_T0(OT_BYTE, (b & 7) | REX_B(s));
         break;
@@ -5394,7 +5400,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (dflag == 2) {
             uint64_t tmp;
             /* 64 bit case */
-            tmp = cpu_ldq_code(cpu_single_env, s->pc);
+            tmp = cpu_ldq_code(env, s->pc);
             s->pc += 8;
             reg = (b & 7) | REX_B(s);
             gen_movtl_T0_im(tmp);
@@ -5403,7 +5409,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #endif
         {
             ot = dflag ? OT_LONG : OT_WORD;
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             reg = (b & 7) | REX_B(s);
             gen_op_movl_T0_im(val);
             gen_op_mov_reg_T0(ot, reg);
@@ -5422,7 +5428,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
@@ -5433,7 +5439,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_op_mov_reg_T0(ot, rm);
             gen_op_mov_reg_T1(ot, reg);
         } else {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_mov_TN_reg(ot, 0, reg);
             /* for xchg, lock is implicit */
             if (!(prefixes & PREFIX_LOCK))
@@ -5465,12 +5471,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         op = R_GS;
     do_lxx:
         ot = dflag ? OT_LONG : OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         gen_op_ld_T1_A0(ot + s->mem_index);
         gen_add_A0_im(s, 1 << (ot - OT_WORD + 1));
         /* load the segment first to handle exceptions properly */
@@ -5497,7 +5503,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             else
                 ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             mod = (modrm >> 6) & 3;
             op = (modrm >> 3) & 7;
 
@@ -5505,7 +5511,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 if (shift == 2) {
                     s->rip_offset = 1;
                 }
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
                 opreg = (modrm & 7) | REX_B(s);
@@ -5516,7 +5522,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_shift(s, op, ot, opreg, OR_ECX);
             } else {
                 if (shift == 2) {
-                    shift = cpu_ldub_code(cpu_single_env, s->pc++);
+                    shift = cpu_ldub_code(env, s->pc++);
                 }
                 gen_shifti(s, op, ot, opreg, shift);
             }
@@ -5550,12 +5556,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         shift = 0;
     do_shiftd:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         reg = ((modrm >> 3) & 7) | rex_r;
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             opreg = OR_TMP0;
         } else {
             opreg = rm;
@@ -5563,7 +5569,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_op_mov_TN_reg(ot, 1, reg);
 
         if (shift) {
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             tcg_gen_movi_tl(cpu_T3, val);
         } else {
             tcg_gen_mov_tl(cpu_T3, cpu_regs[R_ECX]);
@@ -5580,13 +5586,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
             break;
         }
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = modrm & 7;
         op = ((b & 7) << 3) | ((modrm >> 3) & 7);
         if (mod != 3) {
             /* memory op */
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             switch(op) {
             case 0x00 ... 0x07: /* fxxxs */
             case 0x10 ... 0x17: /* fixxxl */
@@ -6211,7 +6217,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag ? OT_LONG : OT_WORD;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T0_im(val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
@@ -6231,7 +6237,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag ? OT_LONG : OT_WORD;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T0_im(val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
@@ -6293,7 +6299,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /************************/
         /* control */
     case 0xc2: /* ret im */
-        val = cpu_ldsw_code(cpu_single_env, s->pc);
+        val = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
         gen_pop_T0(s);
         if (CODE64(s) && s->dflag)
@@ -6313,7 +6319,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_eob(s);
         break;
     case 0xca: /* lret im */
-        val = cpu_ldsw_code(cpu_single_env, s->pc);
+        val = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
     do_lret:
         if (s->pe && !s->vm86) {
@@ -6369,9 +6375,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xe8: /* call im */
         {
             if (dflag)
-                tval = (int32_t)insn_get(s, OT_LONG);
+                tval = (int32_t)insn_get(env, s, OT_LONG);
             else
-                tval = (int16_t)insn_get(s, OT_WORD);
+                tval = (int16_t)insn_get(env, s, OT_WORD);
             next_eip = s->pc - s->cs_base;
             tval += next_eip;
             if (s->dflag == 0)
@@ -6390,8 +6396,8 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (CODE64(s))
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            offset = insn_get(s, ot);
-            selector = insn_get(s, OT_WORD);
+            offset = insn_get(env, s, ot);
+            selector = insn_get(env, s, OT_WORD);
 
             gen_op_movl_T0_im(selector);
             gen_op_movl_T1_imu(offset);
@@ -6399,9 +6405,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         goto do_lcall;
     case 0xe9: /* jmp im */
         if (dflag)
-            tval = (int32_t)insn_get(s, OT_LONG);
+            tval = (int32_t)insn_get(env, s, OT_LONG);
         else
-            tval = (int16_t)insn_get(s, OT_WORD);
+            tval = (int16_t)insn_get(env, s, OT_WORD);
         tval += s->pc - s->cs_base;
         if (s->dflag == 0)
             tval &= 0xffff;
@@ -6416,28 +6422,28 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (CODE64(s))
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            offset = insn_get(s, ot);
-            selector = insn_get(s, OT_WORD);
+            offset = insn_get(env, s, ot);
+            selector = insn_get(env, s, OT_WORD);
 
             gen_op_movl_T0_im(selector);
             gen_op_movl_T1_imu(offset);
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
-        tval = (int8_t)insn_get(s, OT_BYTE);
+        tval = (int8_t)insn_get(env, s, OT_BYTE);
         tval += s->pc - s->cs_base;
         if (s->dflag == 0)
             tval &= 0xffff;
         gen_jmp(s, tval);
         break;
     case 0x70 ... 0x7f: /* jcc Jb */
-        tval = (int8_t)insn_get(s, OT_BYTE);
+        tval = (int8_t)insn_get(env, s, OT_BYTE);
         goto do_jcc;
     case 0x180 ... 0x18f: /* jcc Jv */
         if (dflag) {
-            tval = (int32_t)insn_get(s, OT_LONG);
+            tval = (int32_t)insn_get(env, s, OT_LONG);
         } else {
-            tval = (int16_t)insn_get(s, OT_WORD);
+            tval = (int16_t)insn_get(env, s, OT_WORD);
         }
     do_jcc:
         next_eip = s->pc - s->cs_base;
@@ -6448,9 +6454,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
 
     case 0x190 ... 0x19f: /* setcc Gv */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         gen_setcc(s, b);
-        gen_ldst_modrm(s, modrm, OT_BYTE, OR_TMP0, 1);
+        gen_ldst_modrm(env, s, modrm, OT_BYTE, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
         {
@@ -6458,12 +6464,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             TCGv t0;
 
             ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             t0 = tcg_temp_local_new();
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -6555,7 +6561,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             }
             gen_pop_update(s);
             s->cc_op = CC_OP_EFLAGS;
-            /* abort translation because TF flag may change */
+            /* abort translation because TF/AC flag may change */
             gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
@@ -6616,19 +6622,19 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /* bit operations */
     case 0x1ba: /* bt/bts/btr/btc Gv, im */
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         op = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         if (mod != 3) {
             s->rip_offset = 1;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
             gen_op_mov_TN_reg(ot, 0, rm);
         }
         /* load shift */
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T1_im(val);
         if (op < 4)
             goto illegal_op;
@@ -6647,13 +6653,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         op = 3;
     do_btx:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         gen_op_mov_TN_reg(OT_LONG, 1, reg);
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             /* specific case: we need to add a displacement */
             gen_exts(ot, cpu_T[1]);
             tcg_gen_sari_tl(cpu_tmp0, cpu_T[1], 3 + ot);
@@ -6708,9 +6714,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             TCGv t0;
 
             ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_ldst_modrm(s,modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s,modrm, ot, OR_TMP0, 0);
             gen_extu(ot, cpu_T[0]);
             t0 = tcg_temp_local_new();
             tcg_gen_mov_tl(t0, cpu_T[0]);
@@ -6780,7 +6786,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xd4: /* aam */
         if (CODE64(s))
             goto illegal_op;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         if (val == 0) {
             gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base);
         } else {
@@ -6791,7 +6797,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xd5: /* aad */
         if (CODE64(s))
             goto illegal_op;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_helper_aad(cpu_env, tcg_const_i32(val));
         s->cc_op = CC_OP_LOGICB;
         break;
@@ -6825,7 +6831,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base);
         break;
     case 0xcd: /* int N */
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         if (s->vm86 && s->iopl != 3) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -6847,7 +6853,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_debug(s, pc_start - s->cs_base);
 #else
         /* start debug */
-        tb_flush(cpu_single_env);
+        tb_flush(env);
         cpu_set_log(CPU_LOG_INT | CPU_LOG_TB_IN_ASM);
 #endif
         break;
@@ -6895,13 +6901,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (CODE64(s))
             goto illegal_op;
         ot = dflag ? OT_LONG : OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         gen_op_mov_TN_reg(ot, 0, reg);
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         if (ot == OT_WORD) {
@@ -6942,7 +6948,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         {
             int l1, l2, l3;
 
-            tval = (int8_t)insn_get(s, OT_BYTE);
+            tval = (int8_t)insn_get(env, s, OT_BYTE);
             next_eip = s->pc - s->cs_base;
             tval += next_eip;
             if (s->dflag == 0)
@@ -7022,7 +7028,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
     case 0x134: /* sysenter */
         /* For Intel SYSENTER is valid on 64-bit */
-        if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
+        if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -7035,7 +7041,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
     case 0x135: /* sysexit */
         /* For Intel SYSEXIT is valid on 64-bit */
-        if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
+        if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -7086,7 +7092,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x100:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7098,7 +7104,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD;
             if (mod == 3)
                 ot += s->dflag;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 2: /* lldt */
             if (!s->pe || s->vm86)
@@ -7107,7 +7113,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_lldt(cpu_env, cpu_tmp2_i32);
@@ -7121,7 +7127,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD;
             if (mod == 3)
                 ot += s->dflag;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 3: /* ltr */
             if (!s->pe || s->vm86)
@@ -7130,7 +7136,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_ltr(cpu_env, cpu_tmp2_i32);
@@ -7140,7 +7146,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 5: /* verw */
             if (!s->pe || s->vm86)
                 goto illegal_op;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             if (op == 4) {
@@ -7155,7 +7161,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x101:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         rm = modrm & 7;
@@ -7164,7 +7170,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (mod == 3)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
             gen_op_st_T0_A0(OT_WORD + s->mem_index);
             gen_add_A0_im(s, 2);
@@ -7205,12 +7211,30 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
                     gen_eob(s);
                     break;
+                case 2: /* clac */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
+                        s->cpl != 0) {
+                        goto illegal_op;
+                    }
+                    gen_helper_clac(cpu_env);
+                    gen_jmp_im(s->pc - s->cs_base);
+                    gen_eob(s);
+                    break;
+                case 3: /* stac */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
+                        s->cpl != 0) {
+                        goto illegal_op;
+                    }
+                    gen_helper_stac(cpu_env);
+                    gen_jmp_im(s->pc - s->cs_base);
+                    gen_eob(s);
+                    break;
                 default:
                     goto illegal_op;
                 }
             } else { /* sidt */
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
                 gen_op_st_T0_A0(OT_WORD + s->mem_index);
                 gen_add_A0_im(s, 2);
@@ -7312,7 +7336,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             } else {
                 gen_svm_check_intercept(s, pc_start,
                                         op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T1_A0(OT_WORD + s->mem_index);
                 gen_add_A0_im(s, 2);
                 gen_op_ld_T0_A0(CODE64(s) + OT_LONG + s->mem_index);
@@ -7334,14 +7358,14 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #else
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
 #endif
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 1);
             break;
         case 6: /* lmsw */
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_helper_lmsw(cpu_env, cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
@@ -7355,7 +7379,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     if (s->cc_op != CC_OP_DYNAMIC)
                         gen_op_set_cc_op(s->cc_op);
                     gen_jmp_im(pc_start - s->cs_base);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_helper_invlpg(cpu_env, cpu_A0);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
@@ -7422,7 +7446,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             /* d_ot is the size of destination */
             d_ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
@@ -7434,7 +7458,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
                 gen_op_mov_reg_T0(d_ot, reg);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if (d_ot == OT_QUAD) {
                     gen_op_lds_T0_A0(OT_LONG + s->mem_index);
                 } else {
@@ -7454,12 +7478,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             t1 = tcg_temp_local_new();
             t2 = tcg_temp_local_new();
             ot = OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = (modrm >> 3) & 7;
             mod = (modrm >> 6) & 3;
             rm = modrm & 7;
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
                 a0 = tcg_temp_local_new();
                 tcg_gen_mov_tl(a0, cpu_A0);
@@ -7502,9 +7526,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (!s->pe || s->vm86)
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
             t0 = tcg_temp_local_new();
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
@@ -7523,7 +7547,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x118:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7533,24 +7557,24 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 3: /* prefetchnt0 */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             /* nothing more to do */
             break;
         default: /* nop (multi byte) */
-            gen_nop_modrm(s, modrm);
+            gen_nop_modrm(env, s, modrm);
             break;
         }
         break;
     case 0x119 ... 0x11f: /* nop (multi byte) */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
-        gen_nop_modrm(s, modrm);
+        modrm = cpu_ldub_code(env, s->pc++);
+        gen_nop_modrm(env, s, modrm);
         break;
     case 0x120: /* mov reg, crN */
     case 0x122: /* mov crN, reg */
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
              * AMD documentation (24594.pdf) and testing of
              * intel 386 and 486 processors all show that the mod bits
@@ -7596,7 +7620,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
              * AMD documentation (24594.pdf) and testing of
              * intel 386 and 486 processors all show that the mod bits
@@ -7640,16 +7664,16 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (!(s->cpuid_features & CPUID_SSE2))
             goto illegal_op;
         ot = s->dflag == 2 ? OT_QUAD : OT_LONG;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         reg = ((modrm >> 3) & 7) | rex_r;
         /* generate a generic store */
-        gen_ldst_modrm(s, modrm, ot, reg, 1);
+        gen_ldst_modrm(env, s, modrm, ot, reg, 1);
         break;
     case 0x1ae:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7661,7 +7685,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
@@ -7675,7 +7699,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
@@ -7691,7 +7715,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK) ||
                 mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (op == 2) {
                 gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -7716,7 +7740,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 /* clflush */
                 if (!(s->cpuid_features & CPUID_CLFLUSH))
                     goto illegal_op;
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             }
             break;
         default:
@@ -7724,11 +7748,11 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x10d: /* 3DNow! prefetch(w) */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         /* ignore for now */
         break;
     case 0x1aa: /* rsm */
@@ -7747,8 +7771,8 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (!(s->cpuid_ext_features & CPUID_EXT_POPCNT))
             goto illegal_op;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
-        reg = ((modrm >> 3) & 7);
+        modrm = cpu_ldub_code(env, s->pc++);
+        reg = ((modrm >> 3) & 7) | rex_r;
 
         if (s->prefix & PREFIX_DATA)
             ot = OT_WORD;
@@ -7757,7 +7781,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = OT_QUAD;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_helper_popcnt(cpu_T[0], cpu_env, cpu_T[0], tcg_const_i32(ot));
         gen_op_mov_reg_T0(ot, reg);
 
@@ -7774,7 +7798,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x1c2:
     case 0x1c4 ... 0x1c6:
     case 0x1d0 ... 0x1fe:
-        gen_sse(s, b, pc_start, rex_r);
+        gen_sse(env, s, b, pc_start, rex_r);
         break;
     default:
         goto illegal_op;
@@ -7900,15 +7924,13 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     /* select memory access functions */
     dc->mem_index = 0;
     if (flags & HF_SOFTMMU_MASK) {
-        if (dc->cpl == 3)
-            dc->mem_index = 2 * 4;
-        else
-            dc->mem_index = 1 * 4;
+        dc->mem_index = (cpu_mmu_index(env) + 1) << 2;
     }
     dc->cpuid_features = env->cpuid_features;
     dc->cpuid_ext_features = env->cpuid_ext_features;
     dc->cpuid_ext2_features = env->cpuid_ext2_features;
     dc->cpuid_ext3_features = env->cpuid_ext3_features;
+    dc->cpuid_7_0_ebx_features = env->cpuid_7_0_ebx_features;
 #ifdef TARGET_X86_64
     dc->lma = (flags >> HF_LMA_SHIFT) & 1;
     dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
@@ -7940,7 +7962,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     cpu_ptr0 = tcg_temp_new_ptr();
     cpu_ptr1 = tcg_temp_new_ptr();
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
@@ -7962,21 +7984,21 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = pc_ptr;
+            tcg_ctx.gen_opc_pc[lj] = pc_ptr;
             gen_opc_cc_op[lj] = dc->cc_op;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
-        pc_ptr = disas_insn(dc, pc_ptr);
+        pc_ptr = disas_insn(env, dc, pc_ptr);
         num_insns++;
         /* stop translation if indicated */
         if (dc->is_jmp)
@@ -7993,7 +8015,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
             break;
         }
         /* if too long translation, stop generation too */
-        if (gen_opc_ptr >= gen_opc_end ||
+        if (tcg_ctx.gen_opc_ptr >= gen_opc_end ||
             (pc_ptr - pc_start) >= (TARGET_PAGE_SIZE - 32) ||
             num_insns >= max_insns) {
             gen_jmp_im(pc_ptr - dc->cs_base);
@@ -8009,13 +8031,13 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     if (tb->cflags & CF_LAST_IO)
         gen_io_end();
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     /* we don't forget to fill the last values */
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     }
 
 #ifdef DEBUG_DISAS
@@ -8029,7 +8051,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
         else
 #endif
             disas_flags = !dc->code32;
-        log_target_disas(pc_start, pc_ptr - pc_start, disas_flags);
+        log_target_disas(env, pc_start, pc_ptr - pc_start, disas_flags);
         qemu_log("\n");
     }
 #endif
@@ -8058,16 +8080,17 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos)
         int i;
         qemu_log("RESTORE:\n");
         for(i = 0;i <= pc_pos; i++) {
-            if (gen_opc_instr_start[i]) {
-                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i, gen_opc_pc[i]);
+            if (tcg_ctx.gen_opc_instr_start[i]) {
+                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i,
+                        tcg_ctx.gen_opc_pc[i]);
             }
         }
         qemu_log("pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
-                pc_pos, gen_opc_pc[pc_pos] - tb->cs_base,
+                pc_pos, tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base,
                 (uint32_t)tb->cs_base);
     }
 #endif
-    env->eip = gen_opc_pc[pc_pos] - tb->cs_base;
+    env->eip = tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base;
     cc_op = gen_opc_cc_op[pc_pos];
     if (cc_op != CC_OP_DYNAMIC)
         env->cc_op = cc_op;
diff --git a/target-lm32/Makefile.objs b/target-lm32/Makefile.objs
index 2e0e093e1f9..ca20f21443b 100644
--- a/target-lm32/Makefile.objs
+++ b/target-lm32/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-lm32/cpu-qom.h b/target-lm32/cpu-qom.h
index 4ae2eddafbc..400cdbd554a 100644
--- a/target-lm32/cpu-qom.h
+++ b/target-lm32/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_LM32_CPU_QOM_H
 #define QEMU_LM32_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_LM32_CPU "lm32-cpu"
diff --git a/target-lm32/cpu.c b/target-lm32/cpu.c
index caa48340754..eca2dca427b 100644
--- a/target-lm32/cpu.c
+++ b/target-lm32/cpu.c
@@ -30,7 +30,7 @@ static void lm32_cpu_reset(CPUState *s)
     CPULM32State *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index da80469f513..4e202db32c2 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -26,7 +26,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 struct CPULM32State;
 typedef struct CPULM32State CPULM32State;
 
@@ -238,7 +238,7 @@ static inline int cpu_interrupts_enabled(CPULM32State *env)
     return env->ie & IE_IE;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline target_ulong cpu_get_pc(CPULM32State *env)
 {
@@ -253,12 +253,14 @@ static inline void cpu_get_tb_cpu_state(CPULM32State *env, target_ulong *pc,
     *flags = 0;
 }
 
-static inline bool cpu_has_work(CPULM32State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPULM32State *env = &LM32_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPULM32State *env, TranslationBlock *tb)
 {
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 1ea477fea31..d76ea3fe096 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 int cpu_lm32_handle_mmu_fault(CPULM32State *env, target_ulong address, int rw,
                               int mmu_idx)
@@ -37,7 +37,7 @@ int cpu_lm32_handle_mmu_fault(CPULM32State *env, target_ulong address, int rw,
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPULM32State *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPULM32State *env, target_ulong addr)
 {
     return addr & TARGET_PAGE_MASK;
 }
diff --git a/target-lm32/helper.h b/target-lm32/helper.h
index 9d335efc45a..3ea15a6e801 100644
--- a/target-lm32/helper.h
+++ b/target-lm32/helper.h
@@ -1,14 +1,14 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_0(hlt, void)
-DEF_HELPER_1(wcsr_im, void, i32)
-DEF_HELPER_1(wcsr_ip, void, i32)
-DEF_HELPER_1(wcsr_jtx, void, i32)
-DEF_HELPER_1(wcsr_jrx, void, i32)
-DEF_HELPER_0(rcsr_im, i32)
-DEF_HELPER_0(rcsr_ip, i32)
-DEF_HELPER_0(rcsr_jtx, i32)
-DEF_HELPER_0(rcsr_jrx, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_1(hlt, void, env)
+DEF_HELPER_2(wcsr_im, void, env, i32)
+DEF_HELPER_2(wcsr_ip, void, env, i32)
+DEF_HELPER_2(wcsr_jtx, void, env, i32)
+DEF_HELPER_2(wcsr_jrx, void, env, i32)
+DEF_HELPER_1(rcsr_im, i32, env)
+DEF_HELPER_1(rcsr_ip, i32, env)
+DEF_HELPER_1(rcsr_jtx, i32, env)
+DEF_HELPER_1(rcsr_jrx, i32, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-lm32/op_helper.c b/target-lm32/op_helper.c
index 51edc1a0e16..53410b176eb 100644
--- a/target-lm32/op_helper.c
+++ b/target-lm32/op_helper.c
@@ -1,8 +1,7 @@
 #include <assert.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "hw/lm32_pic.h"
 #include "hw/lm32_juart.h"
@@ -10,63 +9,63 @@
 #if !defined(CONFIG_USER_ONLY)
 #define MMUSUFFIX _mmu
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPULM32State *env, uint32_t index)
 {
     env->exception_index = index;
     cpu_loop_exit(env);
 }
 
-void helper_hlt(void)
+void helper_hlt(CPULM32State *env)
 {
     env->halted = 1;
     env->exception_index = EXCP_HLT;
     cpu_loop_exit(env);
 }
 
-void helper_wcsr_im(uint32_t im)
+void helper_wcsr_im(CPULM32State *env, uint32_t im)
 {
     lm32_pic_set_im(env->pic_state, im);
 }
 
-void helper_wcsr_ip(uint32_t im)
+void helper_wcsr_ip(CPULM32State *env, uint32_t im)
 {
     lm32_pic_set_ip(env->pic_state, im);
 }
 
-void helper_wcsr_jtx(uint32_t jtx)
+void helper_wcsr_jtx(CPULM32State *env, uint32_t jtx)
 {
     lm32_juart_set_jtx(env->juart_state, jtx);
 }
 
-void helper_wcsr_jrx(uint32_t jrx)
+void helper_wcsr_jrx(CPULM32State *env, uint32_t jrx)
 {
     lm32_juart_set_jrx(env->juart_state, jrx);
 }
 
-uint32_t helper_rcsr_im(void)
+uint32_t helper_rcsr_im(CPULM32State *env)
 {
     return lm32_pic_get_im(env->pic_state);
 }
 
-uint32_t helper_rcsr_ip(void)
+uint32_t helper_rcsr_ip(CPULM32State *env)
 {
     return lm32_pic_get_ip(env->pic_state);
 }
 
-uint32_t helper_rcsr_jtx(void)
+uint32_t helper_rcsr_jtx(CPULM32State *env)
 {
     return lm32_juart_get_jtx(env->juart_state);
 }
 
-uint32_t helper_rcsr_jrx(void)
+uint32_t helper_rcsr_jrx(CPULM32State *env)
 {
     return lm32_juart_get_jrx(env->juart_state);
 }
@@ -74,31 +73,19 @@ uint32_t helper_rcsr_jrx(void)
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPULM32State *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPULM32State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPULM32State *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     ret = cpu_lm32_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
 
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index 872a2ba656b..6b873401747 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "helper.h"
 #include "tcg-op.h"
 
@@ -53,7 +53,7 @@ static TCGv cpu_deba;
 static TCGv cpu_bp[4];
 static TCGv cpu_wp[4];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 enum {
     OP_FMT_RI,
@@ -116,7 +116,7 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index)
 {
     TCGv_i32 tmp = tcg_const_i32(index);
 
-    gen_helper_raise_exception(tmp);
+    gen_helper_raise_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
 }
 
@@ -179,7 +179,7 @@ static void dec_and(DisasContext *dc)
     } else  {
         if (dc->r0 == 0 && dc->r1 == 0 && dc->r2 == 0) {
             tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
-            gen_helper_hlt();
+            gen_helper_hlt(cpu_env);
         } else {
             tcg_gen_and_tl(cpu_R[dc->r2], cpu_R[dc->r0], cpu_R[dc->r1]);
         }
@@ -601,10 +601,10 @@ static void dec_rcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_ie);
         break;
     case CSR_IM:
-        gen_helper_rcsr_im(cpu_R[dc->r2]);
+        gen_helper_rcsr_im(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_IP:
-        gen_helper_rcsr_ip(cpu_R[dc->r2]);
+        gen_helper_rcsr_ip(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_CC:
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_cc);
@@ -622,10 +622,10 @@ static void dec_rcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_deba);
         break;
     case CSR_JTX:
-        gen_helper_rcsr_jtx(cpu_R[dc->r2]);
+        gen_helper_rcsr_jtx(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_JRX:
-        gen_helper_rcsr_jrx(cpu_R[dc->r2]);
+        gen_helper_rcsr_jrx(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_ICC:
     case CSR_DCC:
@@ -812,7 +812,7 @@ static void dec_wcsr(DisasContext *dc)
         if (use_icount) {
             gen_io_start();
         }
-        gen_helper_wcsr_im(cpu_R[dc->r1]);
+        gen_helper_wcsr_im(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
         if (use_icount) {
             gen_io_end();
@@ -824,7 +824,7 @@ static void dec_wcsr(DisasContext *dc)
         if (use_icount) {
             gen_io_start();
         }
-        gen_helper_wcsr_ip(cpu_R[dc->r1]);
+        gen_helper_wcsr_ip(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
         if (use_icount) {
             gen_io_end();
@@ -844,10 +844,10 @@ static void dec_wcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_deba, cpu_R[dc->r1]);
         break;
     case CSR_JTX:
-        gen_helper_wcsr_jtx(cpu_R[dc->r1]);
+        gen_helper_wcsr_jtx(cpu_env, cpu_R[dc->r1]);
         break;
     case CSR_JRX:
-        gen_helper_wcsr_jrx(cpu_R[dc->r1]);
+        gen_helper_wcsr_jrx(cpu_env, cpu_R[dc->r1]);
         break;
     case CSR_DC:
         tcg_gen_mov_tl(cpu_dc, cpu_R[dc->r1]);
@@ -940,15 +940,13 @@ static const DecoderInfo decinfo[] = {
     dec_cmpne
 };
 
-static inline void decode(DisasContext *dc)
+static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    uint32_t ir;
-
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
     }
 
-    dc->ir = ir = ldl_code(dc->pc);
+    dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
     /* try guessing 'empty' instruction memory, although it may be a valid
@@ -1020,7 +1018,7 @@ static void gen_intermediate_code_internal(CPULM32State *env,
     dc->env = env;
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -1049,16 +1047,16 @@ static void gen_intermediate_code_internal(CPULM32State *env,
         check_breakpoint(env, dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         /* Pretty disas.  */
@@ -1068,12 +1066,12 @@ static void gen_intermediate_code_internal(CPULM32State *env,
             gen_io_start();
         }
 
-        decode(dc);
+        decode(dc, cpu_ldl_code(env, dc->pc));
         dc->pc += 4;
         num_insns++;
 
     } while (!dc->is_jmp
-         && gen_opc_ptr < gen_opc_end
+         && tcg_ctx.gen_opc_ptr < gen_opc_end
          && !env->singlestep_enabled
          && !singlestep
          && (dc->pc < next_page_start)
@@ -1107,12 +1105,12 @@ static void gen_intermediate_code_internal(CPULM32State *env,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -1122,9 +1120,10 @@ static void gen_intermediate_code_internal(CPULM32State *env,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 }
@@ -1173,7 +1172,7 @@ void cpu_dump_state(CPULM32State *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
 
 void lm32_translate_init(void)
diff --git a/target-m68k/Makefile.objs b/target-m68k/Makefile.objs
index cda60157f53..7eccfab0e43 100644
--- a/target-m68k/Makefile.objs
+++ b/target-m68k/Makefile.objs
@@ -1,5 +1,3 @@
 obj-y += m68k-semi.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-m68k/cpu-qom.h b/target-m68k/cpu-qom.h
index 805786b04d4..170daa7c962 100644
--- a/target-m68k/cpu-qom.h
+++ b/target-m68k/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_M68K_CPU_QOM_H
 #define QEMU_M68K_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_M68K_CPU "m68k-cpu"
 
diff --git a/target-m68k/cpu.c b/target-m68k/cpu.c
index 3e70bb0ead4..5c7803181df 100644
--- a/target-m68k/cpu.c
+++ b/target-m68k/cpu.c
@@ -35,7 +35,7 @@ static void m68k_cpu_reset(CPUState *s)
     CPUM68KState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -55,6 +55,22 @@ static void m68k_cpu_reset(CPUState *s)
 
 /* CPU models */
 
+static ObjectClass *m68k_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && (object_class_dynamic_cast(oc, TYPE_M68K_CPU) == NULL ||
+                       object_class_is_abstract(oc))) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void m5206_cpu_initfn(Object *obj)
 {
     M68kCPU *cpu = M68K_CPU(obj);
@@ -134,6 +150,8 @@ static void m68k_cpu_class_init(ObjectClass *c, void *data)
 
     mcc->parent_reset = cc->reset;
     cc->reset = m68k_cpu_reset;
+
+    cc->class_by_name = m68k_cpu_class_by_name;
 }
 
 static void register_cpu_type(const M68kCPUInfo *info)
@@ -144,7 +162,7 @@ static void register_cpu_type(const M68kCPUInfo *info)
         .instance_init = info->instance_init,
     };
 
-    type_register_static(&type_info);
+    type_register(&type_info);
 }
 
 static const TypeInfo m68k_cpu_type_info = {
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 5e6ee509694..adaf56c4716 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -26,9 +26,9 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define MAX_QREGS 32
 
@@ -103,9 +103,6 @@ typedef struct CPUM68KState {
     uint32_t rambar0;
     uint32_t cacr;
 
-    /* ??? remove this.  */
-    uint32_t t1;
-
     int pending_vector;
     int pending_level;
 
@@ -245,7 +242,7 @@ static inline void cpu_clone_regs(CPUM68KState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
@@ -257,12 +254,14 @@ static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
             | ((env->macsr >> 4) & 0xf);        /* Bits 0-3 */
 }
 
-static inline bool cpu_has_work(CPUM68KState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUM68KState *env = &M68K_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUM68KState *env, TranslationBlock *tb)
 {
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index eac0053c5de..f66e12b6ba3 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -19,17 +19,12 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 #include "helpers.h"
 
 #define SIGNBIT (1u << 31)
 
-typedef struct M68kCPUListState {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} M68kCPUListState;
-
 /* Sort alphabetically, except for "any". */
 static gint m68k_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -51,7 +46,7 @@ static gint m68k_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void m68k_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *c = data;
-    M68kCPUListState *s = user_data;
+    CPUListState *s = user_data;
 
     (*s->cpu_fprintf)(s->file, "%s\n",
                       object_class_get_name(c));
@@ -59,7 +54,7 @@ static void m68k_cpu_list_entry(gpointer data, gpointer user_data)
 
 void m68k_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    M68kCPUListState s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
@@ -102,12 +97,14 @@ CPUM68KState *cpu_m68k_init(const char *cpu_model)
 {
     M68kCPU *cpu;
     CPUM68KState *env;
+    ObjectClass *oc;
     static int inited;
 
-    if (object_class_by_name(cpu_model) == NULL) {
+    oc = cpu_class_by_name(TYPE_M68K_CPU, cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = M68K_CPU(object_new(cpu_model));
+    cpu = M68K_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
 
     if (!inited) {
@@ -289,7 +286,7 @@ int cpu_m68k_handle_mmu_fault (CPUM68KState *env, target_ulong address, int rw,
 /* MMU */
 
 /* TODO: This will need fixing once the MMU is implemented.  */
-target_phys_addr_t cpu_get_phys_page_debug(CPUM68KState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUM68KState *env, target_ulong addr)
 {
     return addr;
 }
diff --git a/target-m68k/helpers.h b/target-m68k/helpers.h
index cb8a0c7d4a7..2b024502ba8 100644
--- a/target-m68k/helpers.h
+++ b/target-m68k/helpers.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_1(bitrev, i32, i32)
 DEF_HELPER_1(ff1, i32, i32)
@@ -49,6 +49,6 @@ DEF_HELPER_3(set_mac_exts, void, env, i32, i32)
 DEF_HELPER_3(set_mac_extu, void, env, i32, i32)
 
 DEF_HELPER_2(flush_flags, void, env, i32)
-DEF_HELPER_1(raise_exception, void, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-m68k/m68k-semi.c b/target-m68k/m68k-semi.c
index 3bb30cd1f76..239fadbad59 100644
--- a/target-m68k/m68k-semi.c
+++ b/target-m68k/m68k-semi.c
@@ -33,10 +33,10 @@
 #define SEMIHOSTING_HEAP_SIZE (128 * 1024 * 1024)
 #else
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "softmmu-semi.h"
+#include "exec/gdbstub.h"
+#include "exec/softmmu-semi.h"
 #endif
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define HOSTED_EXIT  0
 #define HOSTED_INIT_SIM 1
@@ -133,37 +133,61 @@ static void translate_stat(CPUM68KState *env, target_ulong addr, struct stat *s)
     unlock_user(p, addr, sizeof(struct m68k_gdb_stat));
 }
 
+static void m68k_semi_return_u32(CPUM68KState *env, uint32_t ret, uint32_t err)
+{
+    target_ulong args = env->dregs[1];
+    if (put_user_u32(ret, args) ||
+        put_user_u32(err, args + 4)) {
+        /* The m68k semihosting ABI does not provide any way to report this
+         * error to the guest, so the best we can do is log it in qemu.
+         * It is always a guest error not to pass us a valid argument block.
+         */
+        qemu_log_mask(LOG_GUEST_ERROR, "m68k-semihosting: return value "
+                      "discarded because argument block not writable\n");
+    }
+}
+
+static void m68k_semi_return_u64(CPUM68KState *env, uint64_t ret, uint32_t err)
+{
+    target_ulong args = env->dregs[1];
+    if (put_user_u32(ret >> 32, args) ||
+        put_user_u32(ret, args + 4) ||
+        put_user_u32(err, args + 8)) {
+        /* No way to report this via m68k semihosting ABI; just log it */
+        qemu_log_mask(LOG_GUEST_ERROR, "m68k-semihosting: return value "
+                      "discarded because argument block not writable\n");
+    }
+}
+
 static int m68k_semi_is_fseek;
 
 static void m68k_semi_cb(CPUM68KState *env, target_ulong ret, target_ulong err)
 {
-    target_ulong args;
-
-    args = env->dregs[1];
     if (m68k_semi_is_fseek) {
         /* FIXME: We've already lost the high bits of the fseek
            return value.  */
-        /* FIXME - handle put_user() failure */
-        put_user_u32(0, args);
-        args += 4;
+        m68k_semi_return_u64(env, ret, err);
         m68k_semi_is_fseek = 0;
+    } else {
+        m68k_semi_return_u32(env, ret, err);
     }
-    /* FIXME - handle put_user() failure */
-    put_user_u32(ret, args);
-    put_user_u32(errno, args + 4);
 }
 
-#define ARG(n)					\
-({						\
-    target_ulong __arg;				\
-    /* FIXME - handle get_user() failure */	\
-    get_user_ual(__arg, args + (n) * 4);	\
-    __arg;					\
-})
-#define PARG(x) ((unsigned long)ARG(x))
+/* Read the input value from the argument block; fail the semihosting
+ * call if the memory read fails.
+ */
+#define GET_ARG(n) do {                                 \
+    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
+        result = -1;                                    \
+        errno = EFAULT;                                 \
+        goto failed;                                    \
+    }                                                   \
+} while (0)
+
 void do_m68k_semihosting(CPUM68KState *env, int nr)
 {
     uint32_t args;
+    target_ulong arg0, arg1, arg2, arg3;
     void *p;
     void *q;
     uint32_t len;
@@ -175,27 +199,33 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
         gdb_exit(env, env->dregs[0]);
         exit(env->dregs[0]);
     case HOSTED_OPEN:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(m68k_semi_cb, "open,%s,%x,%x", ARG(0), (int)ARG(1),
-                           ARG(2), ARG(3));
+            gdb_do_syscall(m68k_semi_cb, "open,%s,%x,%x", arg0, (int)arg1,
+                           arg2, arg3);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = open(p, translate_openflags(ARG(2)), ARG(3));
-                unlock_user(p, ARG(0), 0);
+                result = open(p, translate_openflags(arg2), arg3);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_CLOSE:
         {
             /* Ignore attempts to close stdin/out/err.  */
-            int fd = ARG(0);
+            GET_ARG(0);
+            int fd = arg0;
             if (fd > 2) {
                 if (use_gdb_syscalls()) {
-                    gdb_do_syscall(m68k_semi_cb, "close,%x", ARG(0));
+                    gdb_do_syscall(m68k_semi_cb, "close,%x", arg0);
                     return;
                 } else {
                     result = close(fd);
@@ -206,123 +236,147 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
             break;
         }
     case HOSTED_READ:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "read,%x,%x,%x",
-                           ARG(0), ARG(1), len);
+                           arg0, arg1, len);
             return;
         } else {
-            if (!(p = lock_user(VERIFY_WRITE, ARG(1), len, 0))) {
+            p = lock_user(VERIFY_WRITE, arg1, len, 0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = read(ARG(0), p, len);
-                unlock_user(p, ARG(1), len);
+                result = read(arg0, p, len);
+                unlock_user(p, arg1, len);
             }
         }
         break;
     case HOSTED_WRITE:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "write,%x,%x,%x",
-                           ARG(0), ARG(1), len);
+                           arg0, arg1, len);
             return;
         } else {
-            if (!(p = lock_user(VERIFY_READ, ARG(1), len, 1))) {
+            p = lock_user(VERIFY_READ, arg1, len, 1);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = write(ARG(0), p, len);
-                unlock_user(p, ARG(0), 0);
+                result = write(arg0, p, len);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_LSEEK:
         {
             uint64_t off;
-            off = (uint32_t)ARG(2) | ((uint64_t)ARG(1) << 32);
+            GET_ARG(0);
+            GET_ARG(1);
+            GET_ARG(2);
+            GET_ARG(3);
+            off = (uint32_t)arg2 | ((uint64_t)arg1 << 32);
             if (use_gdb_syscalls()) {
                 m68k_semi_is_fseek = 1;
                 gdb_do_syscall(m68k_semi_cb, "fseek,%x,%lx,%x",
-                               ARG(0), off, ARG(3));
+                               arg0, off, arg3);
             } else {
-                off = lseek(ARG(0), off, ARG(3));
-                /* FIXME - handle put_user() failure */
-                put_user_u32(off >> 32, args);
-                put_user_u32(off, args + 4);
-                put_user_u32(errno, args + 8);
+                off = lseek(arg0, off, arg3);
+                m68k_semi_return_u64(env, off, errno);
             }
             return;
         }
     case HOSTED_RENAME:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "rename,%s,%s",
-                           ARG(0), (int)ARG(1), ARG(2), (int)ARG(3));
+                           arg0, (int)arg1, arg2, (int)arg3);
             return;
         } else {
-            p = lock_user_string(ARG(0));
-            q = lock_user_string(ARG(2));
+            p = lock_user_string(arg0);
+            q = lock_user_string(arg2);
             if (!p || !q) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = rename(p, q);
             }
-            unlock_user(p, ARG(0), 0);
-            unlock_user(q, ARG(2), 0);
+            unlock_user(p, arg0, 0);
+            unlock_user(q, arg2, 0);
         }
         break;
     case HOSTED_UNLINK:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "unlink,%s",
-                           ARG(0), (int)ARG(1));
+                           arg0, (int)arg1);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = unlink(p);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_STAT:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "stat,%s,%x",
-                           ARG(0), (int)ARG(1), ARG(2));
+                           arg0, (int)arg1, arg2);
             return;
         } else {
             struct stat s;
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = stat(p, &s);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
             if (result == 0) {
-                translate_stat(env, ARG(2), &s);
+                translate_stat(env, arg2, &s);
             }
         }
         break;
     case HOSTED_FSTAT:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "fstat,%x,%x",
-                           ARG(0), ARG(1));
+                           arg0, arg1);
             return;
         } else {
             struct stat s;
-            result = fstat(ARG(0), &s);
+            result = fstat(arg0, &s);
             if (result == 0) {
-                translate_stat(env, ARG(1), &s);
+                translate_stat(env, arg1, &s);
             }
         }
         break;
     case HOSTED_GETTIMEOFDAY:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "gettimeofday,%x,%x",
-                           ARG(0), ARG(1));
+                           arg0, arg1);
             return;
         } else {
             qemu_timeval tv;
@@ -330,37 +384,41 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
             result = qemu_gettimeofday(&tv);
             if (result != 0) {
                 if (!(p = lock_user(VERIFY_WRITE,
-                                    ARG(0), sizeof(struct gdb_timeval), 0))) {
+                                    arg0, sizeof(struct gdb_timeval), 0))) {
                     /* FIXME - check error code? */
                     result = -1;
                 } else {
                     p->tv_sec = cpu_to_be32(tv.tv_sec);
                     p->tv_usec = cpu_to_be64(tv.tv_usec);
-                    unlock_user(p, ARG(0), sizeof(struct gdb_timeval));
+                    unlock_user(p, arg0, sizeof(struct gdb_timeval));
                 }
             }
         }
         break;
     case HOSTED_ISATTY:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(m68k_semi_cb, "isatty,%x", ARG(0));
+            gdb_do_syscall(m68k_semi_cb, "isatty,%x", arg0);
             return;
         } else {
-            result = isatty(ARG(0));
+            result = isatty(arg0);
         }
         break;
     case HOSTED_SYSTEM:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "system,%s",
-                           ARG(0), (int)ARG(1));
+                           arg0, (int)arg1);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = system(p);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
@@ -402,7 +460,6 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
         cpu_abort(env, "Unsupported semihosting syscall %d\n", nr);
         result = 0;
     }
-    /* FIXME - handle put_user() failure */
-    put_user_u32(result, args);
-    put_user_u32(errno, args + 4);
+failed:
+    m68k_semi_return_u32(env, result, errno);
 }
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 1971a578926..16df24c0caa 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -17,17 +17,16 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helpers.h"
 
 #if defined(CONFIG_USER_ONLY)
 
-void do_interrupt(CPUM68KState *env1)
+void do_interrupt(CPUM68KState *env)
 {
-    env1->exception_index = -1;
+    env->exception_index = -1;
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env1)
+void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
 }
 
@@ -35,66 +34,55 @@ void do_interrupt_m68k_hardirq(CPUM68KState *env1)
 
 extern int semihosting_enabled;
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUM68KState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUM68KState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUM68KState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_m68k_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
-static void do_rte(void)
+static void do_rte(CPUM68KState *env)
 {
     uint32_t sp;
     uint32_t fmt;
 
     sp = env->aregs[7];
-    fmt = ldl_kernel(sp);
-    env->pc = ldl_kernel(sp + 4);
+    fmt = cpu_ldl_kernel(env, sp);
+    env->pc = cpu_ldl_kernel(env, sp + 4);
     sp |= (fmt >> 28) & 3;
     env->sr = fmt & 0xffff;
     m68k_switch_sp(env);
     env->aregs[7] = sp + 8;
 }
 
-static void do_interrupt_all(int is_hw)
+static void do_interrupt_all(CPUM68KState *env, int is_hw)
 {
     uint32_t sp;
     uint32_t fmt;
@@ -108,14 +96,14 @@ static void do_interrupt_all(int is_hw)
         switch (env->exception_index) {
         case EXCP_RTE:
             /* Return from an exception.  */
-            do_rte();
+            do_rte(env);
             return;
         case EXCP_HALT_INSN:
             if (semihosting_enabled
                     && (env->sr & SR_S) != 0
                     && (env->pc & 3) == 0
-                    && lduw_code(env->pc - 4) == 0x4e71
-                    && ldl_code(env->pc) == 0x4e7bf000) {
+                    && cpu_lduw_code(env, env->pc - 4) == 0x4e71
+                    && cpu_ldl_code(env, env->pc) == 0x4e7bf000) {
                 env->pc += 4;
                 do_m68k_semihosting(env, env->dregs[0]);
                 return;
@@ -151,44 +139,34 @@ static void do_interrupt_all(int is_hw)
     /* ??? This could cause MMU faults.  */
     sp &= ~3;
     sp -= 4;
-    stl_kernel(sp, retaddr);
+    cpu_stl_kernel(env, sp, retaddr);
     sp -= 4;
-    stl_kernel(sp, fmt);
+    cpu_stl_kernel(env, sp, fmt);
     env->aregs[7] = sp;
     /* Jump to vector.  */
-    env->pc = ldl_kernel(env->vbr + vector);
+    env->pc = cpu_ldl_kernel(env, env->vbr + vector);
 }
 
-void do_interrupt(CPUM68KState *env1)
+void do_interrupt(CPUM68KState *env)
 {
-    CPUM68KState *saved_env;
-
-    saved_env = env;
-    env = env1;
-    do_interrupt_all(0);
-    env = saved_env;
+    do_interrupt_all(env, 0);
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env1)
+void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
-    CPUM68KState *saved_env;
-
-    saved_env = env;
-    env = env1;
-    do_interrupt_all(1);
-    env = saved_env;
+    do_interrupt_all(env, 1);
 }
 #endif
 
-static void raise_exception(int tt)
+static void raise_exception(CPUM68KState *env, int tt)
 {
     env->exception_index = tt;
     cpu_loop_exit(env);
 }
 
-void HELPER(raise_exception)(uint32_t tt)
+void HELPER(raise_exception)(CPUM68KState *env, uint32_t tt)
 {
-    raise_exception(tt);
+    raise_exception(env, tt);
 }
 
 void HELPER(divu)(CPUM68KState *env, uint32_t word)
@@ -202,14 +180,12 @@ void HELPER(divu)(CPUM68KState *env, uint32_t word)
     num = env->div1;
     den = env->div2;
     /* ??? This needs to make sure the throwing location is accurate.  */
-    if (den == 0)
-        raise_exception(EXCP_DIV0);
+    if (den == 0) {
+        raise_exception(env, EXCP_DIV0);
+    }
     quot = num / den;
     rem = num % den;
     flags = 0;
-    /* Avoid using a PARAM1 of zero.  This breaks dyngen because it uses
-       the address of a symbol, and gcc knows symbols can't have address
-       zero.  */
     if (word && quot > 0xffff)
         flags |= CCF_V;
     if (quot == 0)
@@ -231,8 +207,9 @@ void HELPER(divs)(CPUM68KState *env, uint32_t word)
 
     num = env->div1;
     den = env->div2;
-    if (den == 0)
-        raise_exception(EXCP_DIV0);
+    if (den == 0) {
+        raise_exception(env, EXCP_DIV0);
+    }
     quot = num / den;
     rem = num % den;
     flags = 0;
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 9fc1e3102d9..e763195f867 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -19,9 +19,9 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helpers.h"
 #define GEN_HELPER 1
@@ -61,7 +61,7 @@ static TCGv NULL_QREG;
 /* Used to distinguish stores from bad addressing modes.  */
 static TCGv store_dummy;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void m68k_tcg_init(void)
 {
@@ -150,18 +150,24 @@ static void *gen_throws_exception;
 #define OS_SINGLE 4
 #define OS_DOUBLE 5
 
-typedef void (*disas_proc)(DisasContext *, uint16_t);
+typedef void (*disas_proc)(CPUM68KState *env, DisasContext *s, uint16_t insn);
 
 #ifdef DEBUG_DISPATCH
-#define DISAS_INSN(name) \
-  static void real_disas_##name (DisasContext *s, uint16_t insn); \
-  static void disas_##name (DisasContext *s, uint16_t insn) { \
-    qemu_log("Dispatch " #name "\n"); \
-    real_disas_##name(s, insn); } \
-  static void real_disas_##name (DisasContext *s, uint16_t insn)
+#define DISAS_INSN(name)                                                \
+    static void real_disas_##name(CPUM68KState *env, DisasContext *s,   \
+                                  uint16_t insn);                       \
+    static void disas_##name(CPUM68KState *env, DisasContext *s,        \
+                             uint16_t insn)                             \
+    {                                                                   \
+        qemu_log("Dispatch " #name "\n");                               \
+        real_disas_##name(s, env, insn);                                \
+    }                                                                   \
+    static void real_disas_##name(CPUM68KState *env, DisasContext *s,   \
+                                  uint16_t insn)
 #else
-#define DISAS_INSN(name) \
-  static void disas_##name (DisasContext *s, uint16_t insn)
+#define DISAS_INSN(name)                                                \
+    static void disas_##name(CPUM68KState *env, DisasContext *s,        \
+                             uint16_t insn)
 #endif
 
 /* Generate a load from the specified address.  Narrow values are
@@ -257,12 +263,12 @@ static TCGv gen_ldst(DisasContext *s, int opsize, TCGv addr, TCGv val,
 }
 
 /* Read a 32-bit immediate constant.  */
-static inline uint32_t read_im32(DisasContext *s)
+static inline uint32_t read_im32(CPUM68KState *env, DisasContext *s)
 {
     uint32_t im;
-    im = ((uint32_t)lduw_code(s->pc)) << 16;
+    im = ((uint32_t)cpu_lduw_code(env, s->pc)) << 16;
     s->pc += 2;
-    im |= lduw_code(s->pc);
+    im |= cpu_lduw_code(env, s->pc);
     s->pc += 2;
     return im;
 }
@@ -288,7 +294,8 @@ static TCGv gen_addr_index(uint16_t ext, TCGv tmp)
 
 /* Handle a base + index + displacement effective addresss.
    A NULL_QREG base means pc-relative.  */
-static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
+static TCGv gen_lea_indexed(CPUM68KState *env, DisasContext *s, int opsize,
+                            TCGv base)
 {
     uint32_t offset;
     uint16_t ext;
@@ -297,7 +304,7 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
     uint32_t bd, od;
 
     offset = s->pc;
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     if ((ext & 0x800) == 0 && !m68k_feature(s->env, M68K_FEATURE_WORD_INDEX))
@@ -311,10 +318,10 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
         if ((ext & 0x30) > 0x10) {
             /* base displacement */
             if ((ext & 0x30) == 0x20) {
-                bd = (int16_t)lduw_code(s->pc);
+                bd = (int16_t)cpu_lduw_code(env, s->pc);
                 s->pc += 2;
             } else {
-                bd = read_im32(s);
+                bd = read_im32(env, s);
             }
         } else {
             bd = 0;
@@ -360,10 +367,10 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
             if ((ext & 3) > 1) {
                 /* outer displacement */
                 if ((ext & 3) == 2) {
-                    od = (int16_t)lduw_code(s->pc);
+                    od = (int16_t)cpu_lduw_code(env, s->pc);
                     s->pc += 2;
                 } else {
-                    od = read_im32(s);
+                    od = read_im32(env, s);
                 }
             } else {
                 od = 0;
@@ -492,7 +499,8 @@ static inline TCGv gen_extend(TCGv val, int opsize, int sign)
 
 /* Generate code for an "effective address".  Does not adjust the base
    register for autoincrement addressing modes.  */
-static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
+static TCGv gen_lea(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                    int opsize)
 {
     TCGv reg;
     TCGv tmp;
@@ -514,29 +522,29 @@ static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
     case 5: /* Indirect displacement.  */
         reg = AREG(insn, 0);
         tmp = tcg_temp_new();
-        ext = lduw_code(s->pc);
+        ext = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         tcg_gen_addi_i32(tmp, reg, (int16_t)ext);
         return tmp;
     case 6: /* Indirect index + displacement.  */
         reg = AREG(insn, 0);
-        return gen_lea_indexed(s, opsize, reg);
+        return gen_lea_indexed(env, s, opsize, reg);
     case 7: /* Other */
         switch (insn & 7) {
         case 0: /* Absolute short.  */
-            offset = ldsw_code(s->pc);
+            offset = cpu_ldsw_code(env, s->pc);
             s->pc += 2;
             return tcg_const_i32(offset);
         case 1: /* Absolute long.  */
-            offset = read_im32(s);
+            offset = read_im32(env, s);
             return tcg_const_i32(offset);
         case 2: /* pc displacement  */
             offset = s->pc;
-            offset += ldsw_code(s->pc);
+            offset += cpu_ldsw_code(env, s->pc);
             s->pc += 2;
             return tcg_const_i32(offset);
         case 3: /* pc index+displacement.  */
-            return gen_lea_indexed(s, opsize, NULL_QREG);
+            return gen_lea_indexed(env, s, opsize, NULL_QREG);
         case 4: /* Immediate.  */
         default:
             return NULL_QREG;
@@ -548,15 +556,16 @@ static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
 
 /* Helper function for gen_ea. Reuse the computed address between the
    for read/write operands.  */
-static inline TCGv gen_ea_once(DisasContext *s, uint16_t insn, int opsize,
-                              TCGv val, TCGv *addrp, ea_what what)
+static inline TCGv gen_ea_once(CPUM68KState *env, DisasContext *s,
+                               uint16_t insn, int opsize, TCGv val,
+                               TCGv *addrp, ea_what what)
 {
     TCGv tmp;
 
     if (addrp && what == EA_STORE) {
         tmp = *addrp;
     } else {
-        tmp = gen_lea(s, insn, opsize);
+        tmp = gen_lea(env, s, insn, opsize);
         if (IS_NULL_QREG(tmp))
             return tmp;
         if (addrp)
@@ -568,8 +577,8 @@ static inline TCGv gen_ea_once(DisasContext *s, uint16_t insn, int opsize,
 /* Generate code to load/store a value ito/from an EA.  If VAL > 0 this is
    a write otherwise it is a read (0 == sign extend, -1 == zero extend).
    ADDRP is non-null for readwrite operands.  */
-static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
-                   TCGv *addrp, ea_what what)
+static TCGv gen_ea(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                   int opsize, TCGv val, TCGv *addrp, ea_what what)
 {
     TCGv reg;
     TCGv result;
@@ -609,7 +618,7 @@ static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
             if (addrp && what == EA_STORE) {
                 tmp = *addrp;
             } else {
-                tmp = gen_lea(s, insn, opsize);
+                tmp = gen_lea(env, s, insn, opsize);
                 if (IS_NULL_QREG(tmp))
                     return tmp;
                 if (addrp)
@@ -626,33 +635,35 @@ static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
         return result;
     case 5: /* Indirect displacement.  */
     case 6: /* Indirect index + displacement.  */
-        return gen_ea_once(s, insn, opsize, val, addrp, what);
+        return gen_ea_once(env, s, insn, opsize, val, addrp, what);
     case 7: /* Other */
         switch (insn & 7) {
         case 0: /* Absolute short.  */
         case 1: /* Absolute long.  */
         case 2: /* pc displacement  */
         case 3: /* pc index+displacement.  */
-            return gen_ea_once(s, insn, opsize, val, addrp, what);
+            return gen_ea_once(env, s, insn, opsize, val, addrp, what);
         case 4: /* Immediate.  */
             /* Sign extend values for consistency.  */
             switch (opsize) {
             case OS_BYTE:
-                if (what == EA_LOADS)
-                    offset = ldsb_code(s->pc + 1);
-                else
-                    offset = ldub_code(s->pc + 1);
+                if (what == EA_LOADS) {
+                    offset = cpu_ldsb_code(env, s->pc + 1);
+                } else {
+                    offset = cpu_ldub_code(env, s->pc + 1);
+                }
                 s->pc += 2;
                 break;
             case OS_WORD:
-                if (what == EA_LOADS)
-                    offset = ldsw_code(s->pc);
-                else
-                    offset = lduw_code(s->pc);
+                if (what == EA_LOADS) {
+                    offset = cpu_ldsw_code(env, s->pc);
+                } else {
+                    offset = cpu_lduw_code(env, s->pc);
+                }
                 s->pc += 2;
                 break;
             case OS_LONG:
-                offset = read_im32(s);
+                offset = read_im32(env, s);
                 break;
             default:
                 qemu_assert(0, "Bad immediate operand");
@@ -815,7 +826,7 @@ static void gen_exception(DisasContext *s, uint32_t where, int nr)
 {
     gen_flush_cc_op(s);
     gen_jmp_im(s, where);
-    gen_helper_raise_exception(tcg_const_i32(nr));
+    gen_helper_raise_exception(cpu_env, tcg_const_i32(nr));
 }
 
 static inline void gen_addr_fault(DisasContext *s)
@@ -823,20 +834,21 @@ static inline void gen_addr_fault(DisasContext *s)
     gen_exception(s, s->insn_pc, EXCP_ADDRESS);
 }
 
-#define SRC_EA(result, opsize, op_sign, addrp) do { \
-    result = gen_ea(s, insn, opsize, NULL_QREG, addrp, op_sign ? EA_LOADS : EA_LOADU); \
-    if (IS_NULL_QREG(result)) { \
-        gen_addr_fault(s); \
-        return; \
-    } \
+#define SRC_EA(env, result, opsize, op_sign, addrp) do {                \
+        result = gen_ea(env, s, insn, opsize, NULL_QREG, addrp,         \
+                        op_sign ? EA_LOADS : EA_LOADU);                 \
+        if (IS_NULL_QREG(result)) {                                     \
+            gen_addr_fault(s);                                          \
+            return;                                                     \
+        }                                                               \
     } while (0)
 
-#define DEST_EA(insn, opsize, val, addrp) do { \
-    TCGv ea_result = gen_ea(s, insn, opsize, val, addrp, EA_STORE); \
-    if (IS_NULL_QREG(ea_result)) { \
-        gen_addr_fault(s); \
-        return; \
-    } \
+#define DEST_EA(env, insn, opsize, val, addrp) do {                     \
+        TCGv ea_result = gen_ea(env, s, insn, opsize, val, addrp, EA_STORE); \
+        if (IS_NULL_QREG(ea_result)) {                                  \
+            gen_addr_fault(s);                                          \
+            return;                                                     \
+        }                                                               \
     } while (0)
 
 /* Generate a jump to an immediate address.  */
@@ -872,8 +884,7 @@ DISAS_INSN(undef_fpu)
 DISAS_INSN(undef)
 {
     gen_exception(s, s->pc - 2, EXCP_UNSUPPORTED);
-    cpu_abort(cpu_single_env, "Illegal instruction: %04x @ %08x",
-              insn, s->pc - 2);
+    cpu_abort(env, "Illegal instruction: %04x @ %08x", insn, s->pc - 2);
 }
 
 DISAS_INSN(mulw)
@@ -890,7 +901,7 @@ DISAS_INSN(mulw)
         tcg_gen_ext16s_i32(tmp, reg);
     else
         tcg_gen_ext16u_i32(tmp, reg);
-    SRC_EA(src, OS_WORD, sign, NULL);
+    SRC_EA(env, src, OS_WORD, sign, NULL);
     tcg_gen_mul_i32(tmp, tmp, src);
     tcg_gen_mov_i32(reg, tmp);
     /* Unlike m68k, coldfire always clears the overflow bit.  */
@@ -911,7 +922,7 @@ DISAS_INSN(divw)
     } else {
         tcg_gen_ext16u_i32(QREG_DIV1, reg);
     }
-    SRC_EA(src, OS_WORD, sign, NULL);
+    SRC_EA(env, src, OS_WORD, sign, NULL);
     tcg_gen_mov_i32(QREG_DIV2, src);
     if (sign) {
         gen_helper_divs(cpu_env, tcg_const_i32(1));
@@ -934,7 +945,7 @@ DISAS_INSN(divl)
     TCGv reg;
     uint16_t ext;
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext & 0x87f8) {
         gen_exception(s, s->pc - 4, EXCP_UNSUPPORTED);
@@ -943,7 +954,7 @@ DISAS_INSN(divl)
     num = DREG(ext, 12);
     reg = DREG(ext, 0);
     tcg_gen_mov_i32(QREG_DIV1, num);
-    SRC_EA(den, OS_LONG, 0, NULL);
+    SRC_EA(env, den, OS_LONG, 0, NULL);
     tcg_gen_mov_i32(QREG_DIV2, den);
     if (ext & 0x0800) {
         gen_helper_divs(cpu_env, tcg_const_i32(0));
@@ -973,11 +984,11 @@ DISAS_INSN(addsub)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(tmp, OS_LONG, 0, &addr);
+        SRC_EA(env, tmp, OS_LONG, 0, &addr);
         src = reg;
     } else {
         tmp = reg;
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
     }
     if (add) {
         tcg_gen_add_i32(dest, tmp, src);
@@ -990,7 +1001,7 @@ DISAS_INSN(addsub)
     }
     gen_update_cc_add(dest, src);
     if (insn & 0x100) {
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1020,7 +1031,7 @@ DISAS_INSN(bitop_reg)
     else
         opsize = OS_LONG;
     op = (insn >> 6) & 3;
-    SRC_EA(src1, opsize, 0, op ? &addr: NULL);
+    SRC_EA(env, src1, opsize, 0, op ? &addr: NULL);
     src2 = DREG(insn, 9);
     dest = tcg_temp_new();
 
@@ -1055,7 +1066,7 @@ DISAS_INSN(bitop_reg)
         break;
     }
     if (op)
-        DEST_EA(insn, opsize, dest, &addr);
+        DEST_EA(env, insn, opsize, dest, &addr);
 }
 
 DISAS_INSN(sats)
@@ -1086,9 +1097,9 @@ DISAS_INSN(movem)
     TCGv tmp;
     int is_load;
 
-    mask = lduw_code(s->pc);
+    mask = cpu_lduw_code(env, s->pc);
     s->pc += 2;
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1130,14 +1141,14 @@ DISAS_INSN(bitop_im)
         opsize = OS_LONG;
     op = (insn >> 6) & 3;
 
-    bitnum = lduw_code(s->pc);
+    bitnum = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (bitnum & 0xff00) {
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
         return;
     }
 
-    SRC_EA(src1, opsize, 0, op ? &addr: NULL);
+    SRC_EA(env, src1, opsize, 0, op ? &addr: NULL);
 
     gen_flush_flags(s);
     if (opsize == OS_BYTE)
@@ -1172,7 +1183,7 @@ DISAS_INSN(bitop_im)
         default: /* btst */
             break;
         }
-        DEST_EA(insn, opsize, tmp, &addr);
+        DEST_EA(env, insn, opsize, tmp, &addr);
     }
 }
 
@@ -1185,8 +1196,8 @@ DISAS_INSN(arith_im)
     TCGv addr;
 
     op = (insn >> 9) & 7;
-    SRC_EA(src1, OS_LONG, 0, (op == 6) ? NULL : &addr);
-    im = read_im32(s);
+    SRC_EA(env, src1, OS_LONG, 0, (op == 6) ? NULL : &addr);
+    im = read_im32(env, s);
     dest = tcg_temp_new();
     switch (op) {
     case 0: /* ori */
@@ -1225,7 +1236,7 @@ DISAS_INSN(arith_im)
         abort();
     }
     if (op != 6) {
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     }
 }
 
@@ -1257,7 +1268,7 @@ DISAS_INSN(move)
     default:
         abort();
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     op = (insn >> 6) & 7;
     if (op == 1) {
         /* movea */
@@ -1268,7 +1279,7 @@ DISAS_INSN(move)
         /* normal move */
         uint16_t dest_ea;
         dest_ea = ((insn >> 9) & 7) | (op << 3);
-        DEST_EA(dest_ea, opsize, src, NULL);
+        DEST_EA(env, dest_ea, opsize, src, NULL);
         /* This will be correct because loads sign extend.  */
         gen_logic_cc(s, src);
     }
@@ -1289,7 +1300,7 @@ DISAS_INSN(lea)
     TCGv tmp;
 
     reg = AREG(insn, 9);
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1314,7 +1325,7 @@ DISAS_INSN(clr)
     default:
         abort();
     }
-    DEST_EA(insn, opsize, tcg_const_i32(0), NULL);
+    DEST_EA(env, insn, opsize, tcg_const_i32(0), NULL);
     gen_logic_cc(s, tcg_const_i32(0));
 }
 
@@ -1363,7 +1374,8 @@ static void gen_set_sr_im(DisasContext *s, uint16_t val, int ccr_only)
     }
 }
 
-static void gen_set_sr(DisasContext *s, uint16_t insn, int ccr_only)
+static void gen_set_sr(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                       int ccr_only)
 {
     TCGv tmp;
     TCGv reg;
@@ -1383,17 +1395,17 @@ static void gen_set_sr(DisasContext *s, uint16_t insn, int ccr_only)
     else if ((insn & 0x3f) == 0x3c)
       {
         uint16_t val;
-        val = lduw_code(s->pc);
+        val = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         gen_set_sr_im(s, val, ccr_only);
       }
     else
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
 }
 
 DISAS_INSN(move_to_ccr)
 {
-    gen_set_sr(s, insn, 1);
+    gen_set_sr(env, s, insn, 1);
 }
 
 DISAS_INSN(not)
@@ -1424,7 +1436,7 @@ DISAS_INSN(pea)
 {
     TCGv tmp;
 
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1470,7 +1482,7 @@ DISAS_INSN(tst)
     default:
         abort();
     }
-    SRC_EA(tmp, opsize, 1, NULL);
+    SRC_EA(env, tmp, opsize, 1, NULL);
     gen_logic_cc(s, tmp);
 }
 
@@ -1492,10 +1504,10 @@ DISAS_INSN(tas)
     TCGv addr;
 
     dest = tcg_temp_new();
-    SRC_EA(src1, OS_BYTE, 1, &addr);
+    SRC_EA(env, src1, OS_BYTE, 1, &addr);
     gen_logic_cc(s, src1);
     tcg_gen_ori_i32(dest, src1, 0x80);
-    DEST_EA(insn, OS_BYTE, dest, &addr);
+    DEST_EA(env, insn, OS_BYTE, dest, &addr);
 }
 
 DISAS_INSN(mull)
@@ -1507,14 +1519,14 @@ DISAS_INSN(mull)
 
     /* The upper 32 bits of the product are discarded, so
        muls.l and mulu.l are functionally equivalent.  */
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext & 0x87ff) {
         gen_exception(s, s->pc - 4, EXCP_UNSUPPORTED);
         return;
     }
     reg = DREG(ext, 12);
-    SRC_EA(src1, OS_LONG, 0, NULL);
+    SRC_EA(env, src1, OS_LONG, 0, NULL);
     dest = tcg_temp_new();
     tcg_gen_mul_i32(dest, src1, reg);
     tcg_gen_mov_i32(reg, dest);
@@ -1528,7 +1540,7 @@ DISAS_INSN(link)
     TCGv reg;
     TCGv tmp;
 
-    offset = ldsw_code(s->pc);
+    offset = cpu_ldsw_code(env, s->pc);
     s->pc += 2;
     reg = AREG(insn, 0);
     tmp = tcg_temp_new();
@@ -1572,7 +1584,7 @@ DISAS_INSN(jump)
 
     /* Load the target address first to ensure correct exception
        behavior.  */
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1592,7 +1604,7 @@ DISAS_INSN(addsubq)
     int val;
     TCGv addr;
 
-    SRC_EA(src1, OS_LONG, 0, &addr);
+    SRC_EA(env, src1, OS_LONG, 0, &addr);
     val = (insn >> 9) & 7;
     if (val == 0)
         val = 8;
@@ -1619,7 +1631,7 @@ DISAS_INSN(addsubq)
         }
         gen_update_cc_add(dest, src2);
     }
-    DEST_EA(insn, OS_LONG, dest, &addr);
+    DEST_EA(env, insn, OS_LONG, dest, &addr);
 }
 
 DISAS_INSN(tpf)
@@ -1634,7 +1646,7 @@ DISAS_INSN(tpf)
     case 4: /* No extension words.  */
         break;
     default:
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
     }
 }
 
@@ -1649,10 +1661,10 @@ DISAS_INSN(branch)
     op = (insn >> 8) & 0xf;
     offset = (int8_t)insn;
     if (offset == 0) {
-        offset = ldsw_code(s->pc);
+        offset = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
     } else if (offset == -1) {
-        offset = read_im32(s);
+        offset = read_im32(env, s);
     }
     if (op == 1) {
         /* bsr */
@@ -1691,7 +1703,7 @@ DISAS_INSN(mvzs)
         opsize = OS_WORD;
     else
         opsize = OS_BYTE;
-    SRC_EA(src, opsize, (insn & 0x80) == 0, NULL);
+    SRC_EA(env, src, opsize, (insn & 0x80) == 0, NULL);
     reg = DREG(insn, 9);
     tcg_gen_mov_i32(reg, src);
     gen_logic_cc(s, src);
@@ -1707,11 +1719,11 @@ DISAS_INSN(or)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(src, OS_LONG, 0, &addr);
+        SRC_EA(env, src, OS_LONG, 0, &addr);
         tcg_gen_or_i32(dest, src, reg);
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
         tcg_gen_or_i32(dest, src, reg);
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1723,7 +1735,7 @@ DISAS_INSN(suba)
     TCGv src;
     TCGv reg;
 
-    SRC_EA(src, OS_LONG, 0, NULL);
+    SRC_EA(env, src, OS_LONG, 0, NULL);
     reg = AREG(insn, 9);
     tcg_gen_sub_i32(reg, reg, src);
 }
@@ -1749,7 +1761,7 @@ DISAS_INSN(mov3q)
         val = -1;
     src = tcg_const_i32(val);
     gen_logic_cc(s, src);
-    DEST_EA(insn, OS_LONG, src, NULL);
+    DEST_EA(env, insn, OS_LONG, src, NULL);
 }
 
 DISAS_INSN(cmp)
@@ -1777,7 +1789,7 @@ DISAS_INSN(cmp)
     default:
         abort();
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_sub_i32(dest, reg, src);
@@ -1796,7 +1808,7 @@ DISAS_INSN(cmpa)
     } else {
         opsize = OS_WORD;
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     reg = AREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_sub_i32(dest, reg, src);
@@ -1811,12 +1823,12 @@ DISAS_INSN(eor)
     TCGv dest;
     TCGv addr;
 
-    SRC_EA(src, OS_LONG, 0, &addr);
+    SRC_EA(env, src, OS_LONG, 0, &addr);
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_xor_i32(dest, src, reg);
     gen_logic_cc(s, dest);
-    DEST_EA(insn, OS_LONG, dest, &addr);
+    DEST_EA(env, insn, OS_LONG, dest, &addr);
 }
 
 DISAS_INSN(and)
@@ -1829,11 +1841,11 @@ DISAS_INSN(and)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(src, OS_LONG, 0, &addr);
+        SRC_EA(env, src, OS_LONG, 0, &addr);
         tcg_gen_and_i32(dest, src, reg);
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
         tcg_gen_and_i32(dest, src, reg);
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1845,7 +1857,7 @@ DISAS_INSN(adda)
     TCGv src;
     TCGv reg;
 
-    SRC_EA(src, OS_LONG, 0, NULL);
+    SRC_EA(env, src, OS_LONG, 0, NULL);
     reg = AREG(insn, 9);
     tcg_gen_add_i32(reg, reg, src);
 }
@@ -1934,13 +1946,13 @@ DISAS_INSN(strldsr)
     uint32_t addr;
 
     addr = s->pc - 2;
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext != 0x46FC) {
         gen_exception(s, addr, EXCP_UNSUPPORTED);
         return;
     }
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (IS_USER(s) || (ext & SR_S) == 0) {
         gen_exception(s, addr, EXCP_PRIVILEGE);
@@ -1970,7 +1982,7 @@ DISAS_INSN(move_to_sr)
         gen_exception(s, s->pc - 2, EXCP_PRIVILEGE);
         return;
     }
-    gen_set_sr(s, insn, 0);
+    gen_set_sr(env, s, insn, 0);
     gen_lookup_tb(s);
 }
 
@@ -2008,7 +2020,7 @@ DISAS_INSN(stop)
         return;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     gen_set_sr_im(s, ext, 0);
@@ -2035,7 +2047,7 @@ DISAS_INSN(movec)
         return;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     if (ext & 0x8000) {
@@ -2100,7 +2112,7 @@ DISAS_INSN(fpu)
     int set_dest;
     int opsize;
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     opmode = ext & 0x7f;
     switch ((ext >> 13) & 7) {
@@ -2136,7 +2148,7 @@ DISAS_INSN(fpu)
                 tcg_gen_addi_i32(tmp32, tmp32, -8);
                 break;
             case 5:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
                 break;
@@ -2162,7 +2174,7 @@ DISAS_INSN(fpu)
         default:
             goto undef;
         }
-        DEST_EA(insn, opsize, tmp32, NULL);
+        DEST_EA(env, insn, opsize, tmp32, NULL);
         tcg_temp_free_i32(tmp32);
         return;
     case 4: /* fmove to control register.  */
@@ -2190,7 +2202,7 @@ DISAS_INSN(fpu)
                       (ext >> 10) & 7);
             goto undef;
         }
-        DEST_EA(insn, OS_LONG, tmp32, NULL);
+        DEST_EA(env, insn, OS_LONG, tmp32, NULL);
         break;
     case 6: /* fmovem */
     case 7:
@@ -2200,7 +2212,7 @@ DISAS_INSN(fpu)
             int i;
             if ((ext & 0x1f00) != 0x1000 || (ext & 0xff) == 0)
                 goto undef;
-            tmp32 = gen_lea(s, insn, OS_LONG);
+            tmp32 = gen_lea(env, s, insn, OS_LONG);
             if (IS_NULL_QREG(tmp32)) {
                 gen_addr_fault(s);
                 return;
@@ -2250,12 +2262,12 @@ DISAS_INSN(fpu)
                 tcg_gen_addi_i32(tmp32, tmp32, -8);
                 break;
             case 5:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
                 break;
             case 7:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 offset += s->pc - 2;
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
@@ -2275,7 +2287,7 @@ DISAS_INSN(fpu)
             }
             tcg_temp_free_i32(tmp32);
         } else {
-            SRC_EA(tmp32, opsize, 1, NULL);
+            SRC_EA(env, tmp32, opsize, 1, NULL);
             src = tcg_temp_new_i64();
             switch (opsize) {
             case OS_LONG:
@@ -2370,7 +2382,7 @@ DISAS_INSN(fpu)
 undef:
     /* FIXME: Is this right for offset addressing modes?  */
     s->pc -= 2;
-    disas_undef_fpu(s, insn);
+    disas_undef_fpu(env, s, insn);
 }
 
 DISAS_INSN(fbcc)
@@ -2381,10 +2393,10 @@ DISAS_INSN(fbcc)
     int l1;
 
     addr = s->pc;
-    offset = ldsw_code(s->pc);
+    offset = cpu_ldsw_code(env, s->pc);
     s->pc += 2;
     if (insn & (1 << 6)) {
-        offset = (offset << 16) | lduw_code(s->pc);
+        offset = (offset << 16) | cpu_lduw_code(env, s->pc);
         s->pc += 2;
     }
 
@@ -2506,18 +2518,18 @@ DISAS_INSN(mac)
         s->done_mac = 1;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     acc = ((insn >> 7) & 1) | ((ext >> 3) & 2);
     dual = ((insn & 0x30) != 0 && (ext & 3) != 0);
     if (dual && !m68k_feature(s->env, M68K_FEATURE_CF_EMAC_B)) {
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
         return;
     }
     if (insn & 0x30) {
         /* MAC with load.  */
-        tmp = gen_lea(s, insn, OS_LONG);
+        tmp = gen_lea(env, s, insn, OS_LONG);
         addr = tcg_temp_new();
         tcg_gen_and_i32(addr, tmp, QREG_MAC_MASK);
         /* Load the value now to ensure correct exception behavior.
@@ -2731,7 +2743,7 @@ DISAS_INSN(to_mac)
     int accnum;
     accnum = (insn >> 9) & 3;
     acc = MACREG(accnum);
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     if (s->env->macsr & MACSR_FI) {
         tcg_gen_ext_i32_i64(acc, val);
         tcg_gen_shli_i64(acc, acc, 8);
@@ -2748,7 +2760,7 @@ DISAS_INSN(to_mac)
 DISAS_INSN(to_macsr)
 {
     TCGv val;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     gen_helper_set_macsr(cpu_env, val);
     gen_lookup_tb(s);
 }
@@ -2756,7 +2768,7 @@ DISAS_INSN(to_macsr)
 DISAS_INSN(to_mask)
 {
     TCGv val;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     tcg_gen_ori_i32(QREG_MAC_MASK, val, 0xffff0000);
 }
 
@@ -2764,7 +2776,7 @@ DISAS_INSN(to_mext)
 {
     TCGv val;
     TCGv acc;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     acc = tcg_const_i32((insn & 0x400) ? 2 : 0);
     if (s->env->macsr & MACSR_FI)
         gen_helper_set_mac_extf(cpu_env, val, acc);
@@ -2941,10 +2953,14 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 {
     uint16_t insn;
 
-    insn = lduw_code(s->pc);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(s->pc);
+    }
+
+    insn = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
-    opcode_table[insn](s, insn);
+    opcode_table[insn](env, s, insn);
 }
 
 /* generate intermediate code for basic block 'tb'.  */
@@ -2966,7 +2982,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->env = env;
     dc->is_jmp = DISAS_NEXT;
@@ -2999,22 +3015,22 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
                 break;
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         dc->insn_pc = dc->pc;
 	disas_m68k_insn(env, dc);
         num_insns++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              (pc_offset) < (TARGET_PAGE_SIZE - 32) &&
@@ -3028,7 +3044,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
             gen_flush_cc_op(dc);
             tcg_gen_movi_i32(QREG_PC, dc->pc);
         }
-        gen_helper_raise_exception(tcg_const_i32(EXCP_DEBUG));
+        gen_helper_raise_exception(cpu_env, tcg_const_i32(EXCP_DEBUG));
     } else {
         switch(dc->is_jmp) {
         case DISAS_NEXT:
@@ -3048,21 +3064,21 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
         tb->icount = num_insns;
@@ -3105,5 +3121,5 @@ void cpu_dump_state(CPUM68KState *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-microblaze/Makefile.objs b/target-microblaze/Makefile.objs
index 4b09e8c6b59..afb87bcc801 100644
--- a/target-microblaze/Makefile.objs
+++ b/target-microblaze/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += mmu.o machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-microblaze/cpu-qom.h b/target-microblaze/cpu-qom.h
index 4b23303b6dd..f75549dc226 100644
--- a/target-microblaze/cpu-qom.h
+++ b/target-microblaze/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_MICROBLAZE_CPU_QOM_H
 #define QEMU_MICROBLAZE_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_MICROBLAZE_CPU "microblaze-cpu"
 
diff --git a/target-microblaze/cpu.c b/target-microblaze/cpu.c
index 34b3a9bfdc7..0f858fd8696 100644
--- a/target-microblaze/cpu.c
+++ b/target-microblaze/cpu.c
@@ -32,7 +32,7 @@ static void mb_cpu_reset(CPUState *s)
     CPUMBState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 4968c244e81..5621068d822 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -26,8 +26,8 @@
 
 #define CPUArchState struct CPUMBState
 
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 struct CPUMBState;
 typedef struct CPUMBState CPUMBState;
 #if !defined(CONFIG_USER_ONLY)
@@ -274,7 +274,6 @@ struct CPUMBState {
 
 MicroBlazeCPU *cpu_mb_init(const char *cpu_model);
 int cpu_mb_exec(CPUMBState *s);
-void cpu_mb_close(CPUMBState *s);
 void do_interrupt(CPUMBState *env);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -345,6 +344,7 @@ static inline void cpu_clone_regs(CPUMBState *env, target_ulong newsp)
 
 static inline void cpu_set_tls(CPUMBState *env, target_ulong newtls)
 {
+    env->regs[21] = newtls;
 }
 
 static inline int cpu_interrupts_enabled(CPUMBState *env)
@@ -352,7 +352,7 @@ static inline int cpu_interrupts_enabled(CPUMBState *env)
     return env->sregs[SR_MSR] & MSR_IE;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline target_ulong cpu_get_pc(CPUMBState *env)
 {
@@ -369,16 +369,18 @@ static inline void cpu_get_tb_cpu_state(CPUMBState *env, target_ulong *pc,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMBState *env1, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size);
 #endif
 
-static inline bool cpu_has_work(CPUMBState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUMBState *env = &MICROBLAZE_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUMBState *env, TranslationBlock *tb)
 {
diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
index 74fce268277..97aedc52bbb 100644
--- a/target-microblaze/helper.c
+++ b/target-microblaze/helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define D(x)
 #define DMMU(x)
@@ -198,7 +198,7 @@ void do_interrupt(CPUMBState *env)
             t = (env->sregs[SR_MSR] & (MSR_VM | MSR_UM)) << 1;
 
 #if 0
-#include "disas.h"
+#include "disas/disas.h"
 
 /* Useful instrumentation when debugging interrupt issues in either
    the models or in sw.  */
@@ -258,7 +258,7 @@ void do_interrupt(CPUMBState *env)
     }
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUMBState * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUMBState * env, target_ulong addr)
 {
     target_ulong vaddr, paddr = 0;
     struct microblaze_mmu_lookup lu;
diff --git a/target-microblaze/helper.h b/target-microblaze/helper.h
index 9dcfb0f1d5d..4e514294983 100644
--- a/target-microblaze/helper.h
+++ b/target-microblaze/helper.h
@@ -1,41 +1,41 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_0(debug, void)
-DEF_HELPER_FLAGS_3(carry, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32, i32, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_1(debug, void, env)
+DEF_HELPER_FLAGS_3(carry, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
 DEF_HELPER_2(cmp, i32, i32, i32)
 DEF_HELPER_2(cmpu, i32, i32, i32)
-DEF_HELPER_FLAGS_1(clz, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32)
-
-DEF_HELPER_2(divs, i32, i32, i32)
-DEF_HELPER_2(divu, i32, i32, i32)
-
-DEF_HELPER_2(fadd, i32, i32, i32)
-DEF_HELPER_2(frsub, i32, i32, i32)
-DEF_HELPER_2(fmul, i32, i32, i32)
-DEF_HELPER_2(fdiv, i32, i32, i32)
-DEF_HELPER_1(flt, i32, i32)
-DEF_HELPER_1(fint, i32, i32)
-DEF_HELPER_1(fsqrt, i32, i32)
-
-DEF_HELPER_2(fcmp_un, i32, i32, i32)
-DEF_HELPER_2(fcmp_lt, i32, i32, i32)
-DEF_HELPER_2(fcmp_eq, i32, i32, i32)
-DEF_HELPER_2(fcmp_le, i32, i32, i32)
-DEF_HELPER_2(fcmp_gt, i32, i32, i32)
-DEF_HELPER_2(fcmp_ne, i32, i32, i32)
-DEF_HELPER_2(fcmp_ge, i32, i32, i32)
-
-DEF_HELPER_FLAGS_2(pcmpbf, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32, i32)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
+
+DEF_HELPER_3(divs, i32, env, i32, i32)
+DEF_HELPER_3(divu, i32, env, i32, i32)
+
+DEF_HELPER_3(fadd, i32, env, i32, i32)
+DEF_HELPER_3(frsub, i32, env, i32, i32)
+DEF_HELPER_3(fmul, i32, env, i32, i32)
+DEF_HELPER_3(fdiv, i32, env, i32, i32)
+DEF_HELPER_2(flt, i32, env, i32)
+DEF_HELPER_2(fint, i32, env, i32)
+DEF_HELPER_2(fsqrt, i32, env, i32)
+
+DEF_HELPER_3(fcmp_un, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_lt, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_eq, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_le, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_gt, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_ne, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_ge, i32, env, i32, i32)
+
+DEF_HELPER_FLAGS_2(pcmpbf, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 #if !defined(CONFIG_USER_ONLY)
-DEF_HELPER_1(mmu_read, i32, i32)
-DEF_HELPER_2(mmu_write, void, i32, i32)
+DEF_HELPER_2(mmu_read, i32, env, i32)
+DEF_HELPER_3(mmu_write, void, env, i32, i32)
 #endif
 
-DEF_HELPER_4(memalign, void, i32, i32, i32, i32)
-DEF_HELPER_1(stackprot, void, i32)
+DEF_HELPER_5(memalign, void, env, i32, i32, i32, i32)
+DEF_HELPER_2(stackprot, void, env, i32)
 
 DEF_HELPER_2(get, i32, i32, i32)
 DEF_HELPER_3(put, void, i32, i32, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index 3b1f07243eb..1c62f3c68ff 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -20,53 +20,40 @@
 
 #include <assert.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define D(x)
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUMBState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUMBState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUMBState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     ret = cpu_mb_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
 
@@ -105,13 +92,13 @@ uint32_t helper_get(uint32_t id, uint32_t ctrl)
     return 0xdead0000 | id;
 }
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPUMBState *env, uint32_t index)
 {
     env->exception_index = index;
     cpu_loop_exit(env);
 }
 
-void helper_debug(void)
+void helper_debug(CPUMBState *env)
 {
     int i;
 
@@ -176,7 +163,7 @@ uint32_t helper_carry(uint32_t a, uint32_t b, uint32_t cf)
     return ncf;
 }
 
-static inline int div_prepare(uint32_t a, uint32_t b)
+static inline int div_prepare(CPUMBState *env, uint32_t a, uint32_t b)
 {
     if (b == 0) {
         env->sregs[SR_MSR] |= MSR_DZ;
@@ -184,7 +171,7 @@ static inline int div_prepare(uint32_t a, uint32_t b)
         if ((env->sregs[SR_MSR] & MSR_EE)
             && !(env->pvr.regs[2] & PVR2_DIV_ZERO_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_DIVZERO;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
         return 0;
     }
@@ -192,28 +179,30 @@ static inline int div_prepare(uint32_t a, uint32_t b)
     return 1;
 }
 
-uint32_t helper_divs(uint32_t a, uint32_t b)
+uint32_t helper_divs(CPUMBState *env, uint32_t a, uint32_t b)
 {
-    if (!div_prepare(a, b))
+    if (!div_prepare(env, a, b)) {
         return 0;
+    }
     return (int32_t)a / (int32_t)b;
 }
 
-uint32_t helper_divu(uint32_t a, uint32_t b)
+uint32_t helper_divu(CPUMBState *env, uint32_t a, uint32_t b)
 {
-    if (!div_prepare(a, b))
+    if (!div_prepare(env, a, b)) {
         return 0;
+    }
     return a / b;
 }
 
 /* raise FPU exception.  */
-static void raise_fpu_exception(void)
+static void raise_fpu_exception(CPUMBState *env)
 {
     env->sregs[SR_ESR] = ESR_EC_FPU;
-    helper_raise_exception(EXCP_HW_EXCP);
+    helper_raise_exception(env, EXCP_HW_EXCP);
 }
 
-static void update_fpu_flags(int flags)
+static void update_fpu_flags(CPUMBState *env, int flags)
 {
     int raise = 0;
 
@@ -236,11 +225,11 @@ static void update_fpu_flags(int flags)
     if (raise
         && (env->pvr.regs[2] & PVR2_FPU_EXC_MASK)
         && (env->sregs[SR_MSR] & MSR_EE)) {
-        raise_fpu_exception();
+        raise_fpu_exception(env);
     }
 }
 
-uint32_t helper_fadd(uint32_t a, uint32_t b)
+uint32_t helper_fadd(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -251,11 +240,11 @@ uint32_t helper_fadd(uint32_t a, uint32_t b)
     fd.f = float32_add(fa.f, fb.f, &env->fp_status);
 
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
     return fd.l;
 }
 
-uint32_t helper_frsub(uint32_t a, uint32_t b)
+uint32_t helper_frsub(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -265,11 +254,11 @@ uint32_t helper_frsub(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_sub(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
     return fd.l;
 }
 
-uint32_t helper_fmul(uint32_t a, uint32_t b)
+uint32_t helper_fmul(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -279,12 +268,12 @@ uint32_t helper_fmul(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_mul(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
 
-uint32_t helper_fdiv(uint32_t a, uint32_t b)
+uint32_t helper_fdiv(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -294,12 +283,12 @@ uint32_t helper_fdiv(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_div(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
 
-uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_un(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     uint32_t r = 0;
@@ -308,7 +297,7 @@ uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
     fb.l = b;
 
     if (float32_is_signaling_nan(fa.f) || float32_is_signaling_nan(fb.f)) {
-        update_fpu_flags(float_flag_invalid);
+        update_fpu_flags(env, float_flag_invalid);
         r = 1;
     }
 
@@ -319,7 +308,7 @@ uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
     return r;
 }
 
-uint32_t helper_fcmp_lt(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_lt(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int r;
@@ -330,12 +319,12 @@ uint32_t helper_fcmp_lt(uint32_t a, uint32_t b)
     fb.l = b;
     r = float32_lt(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_eq(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_eq(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags;
@@ -346,12 +335,12 @@ uint32_t helper_fcmp_eq(uint32_t a, uint32_t b)
     fb.l = b;
     r = float32_eq_quiet(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_le(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_le(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags;
@@ -362,13 +351,13 @@ uint32_t helper_fcmp_le(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = float32_le(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
 
     return r;
 }
 
-uint32_t helper_fcmp_gt(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_gt(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -378,11 +367,11 @@ uint32_t helper_fcmp_gt(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = float32_lt(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
     return r;
 }
 
-uint32_t helper_fcmp_ne(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_ne(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -392,12 +381,12 @@ uint32_t helper_fcmp_ne(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = !float32_eq_quiet(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_ge(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_ge(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -407,12 +396,12 @@ uint32_t helper_fcmp_ge(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = !float32_lt(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_flt(uint32_t a)
+uint32_t helper_flt(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fd, fa;
 
@@ -421,7 +410,7 @@ uint32_t helper_flt(uint32_t a)
     return fd.l;
 }
 
-uint32_t helper_fint(uint32_t a)
+uint32_t helper_fint(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fa;
     uint32_t r;
@@ -431,12 +420,12 @@ uint32_t helper_fint(uint32_t a)
     fa.l = a;
     r = float32_to_int32(fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return r;
 }
 
-uint32_t helper_fsqrt(uint32_t a)
+uint32_t helper_fsqrt(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fd, fa;
     int flags;
@@ -445,7 +434,7 @@ uint32_t helper_fsqrt(uint32_t a)
     fa.l = a;
     fd.l = float32_sqrt(fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
@@ -463,7 +452,8 @@ uint32_t helper_pcmpbf(uint32_t a, uint32_t b)
     return 0;
 }
 
-void helper_memalign(uint32_t addr, uint32_t dr, uint32_t wr, uint32_t mask)
+void helper_memalign(CPUMBState *env, uint32_t addr, uint32_t dr, uint32_t wr,
+                     uint32_t mask)
 {
     if (addr & mask) {
             qemu_log_mask(CPU_LOG_INT,
@@ -478,45 +468,39 @@ void helper_memalign(uint32_t addr, uint32_t dr, uint32_t wr, uint32_t mask)
             if (!(env->sregs[SR_MSR] & MSR_EE)) {
                 return;
             }
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
     }
 }
 
-void helper_stackprot(uint32_t addr)
+void helper_stackprot(CPUMBState *env, uint32_t addr)
 {
     if (addr < env->slr || addr > env->shr) {
             qemu_log("Stack protector violation at %x %x %x\n",
                      addr, env->slr, env->shr);
             env->sregs[SR_EAR] = addr;
             env->sregs[SR_ESR] = ESR_EC_STACKPROT;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
     }
 }
 
 #if !defined(CONFIG_USER_ONLY)
 /* Writes/reads to the MMU's special regs end up here.  */
-uint32_t helper_mmu_read(uint32_t rn)
+uint32_t helper_mmu_read(CPUMBState *env, uint32_t rn)
 {
     return mmu_read(env, rn);
 }
 
-void helper_mmu_write(uint32_t rn, uint32_t v)
+void helper_mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
 {
     mmu_write(env, rn, v);
 }
 
-void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMBState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
-    CPUMBState *saved_env;
-
-    saved_env = env;
-    env = env1;
-
     qemu_log_mask(CPU_LOG_INT, "Unassigned " TARGET_FMT_plx " wr=%d exe=%d\n",
              addr, is_write, is_exec);
     if (!(env->sregs[SR_MSR] & MSR_EE)) {
-        env = saved_env;
         return;
     }
 
@@ -524,14 +508,13 @@ void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
     if (is_exec) {
         if ((env->pvr.regs[2] & PVR2_IOPB_BUS_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_INSN_BUS;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
     } else {
         if ((env->pvr.regs[2] & PVR2_DOPB_BUS_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_DATA_BUS;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
     }
-    env = saved_env;
 }
 #endif
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index 7470149db3c..58ce71267d0 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "helper.h"
 #include "microblaze-decode.h"
@@ -50,7 +50,7 @@ static TCGv env_btaken;
 static TCGv env_btarget;
 static TCGv env_iflags;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
@@ -126,7 +126,7 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index)
 
     t_sync_flags(dc);
     tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
-    gen_helper_raise_exception(tmp);
+    gen_helper_raise_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     dc->is_jmp = DISAS_UPDATE;
 }
@@ -503,9 +503,9 @@ static void dec_msr(DisasContext *dc)
         sr &= 7;
         LOG_DIS("m%ss sr%d r%d imm=%x\n", to ? "t" : "f", sr, dc->ra, dc->imm);
         if (to)
-            gen_helper_mmu_write(tcg_const_tl(sr), cpu_R[dc->ra]);
+            gen_helper_mmu_write(cpu_env, tcg_const_tl(sr), cpu_R[dc->ra]);
         else
-            gen_helper_mmu_read(cpu_R[dc->rd], tcg_const_tl(sr));
+            gen_helper_mmu_read(cpu_R[dc->rd], cpu_env, tcg_const_tl(sr));
         return;
     }
 #endif
@@ -704,9 +704,11 @@ static void dec_div(DisasContext *dc)
     }
 
     if (u)
-        gen_helper_divu(cpu_R[dc->rd], *(dec_alu_op_b(dc)), cpu_R[dc->ra]);
+        gen_helper_divu(cpu_R[dc->rd], cpu_env, *(dec_alu_op_b(dc)),
+                        cpu_R[dc->ra]);
     else
-        gen_helper_divs(cpu_R[dc->rd], *(dec_alu_op_b(dc)), cpu_R[dc->ra]);
+        gen_helper_divs(cpu_R[dc->rd], cpu_env, *(dec_alu_op_b(dc)),
+                        cpu_R[dc->ra]);
     if (!dc->rd)
         tcg_gen_movi_tl(cpu_R[dc->rd], 0);
 }
@@ -838,7 +840,7 @@ static void dec_bit(DisasContext *dc)
             LOG_DIS("swapb r%d r%d\n", dc->rd, dc->ra);
             tcg_gen_bswap32_i32(cpu_R[dc->rd], cpu_R[dc->ra]);
             break;
-        case 0x1e1:
+        case 0x1e2:
             /*swaph */
             LOG_DIS("swaph r%d r%d\n", dc->rd, dc->ra);
             tcg_gen_rotri_i32(cpu_R[dc->rd], cpu_R[dc->ra], 16);
@@ -912,7 +914,7 @@ static inline TCGv *compute_ldst_addr(DisasContext *dc, TCGv *t)
         tcg_gen_add_tl(*t, cpu_R[dc->ra], cpu_R[dc->rb]);
 
         if (stackprot) {
-            gen_helper_stackprot(*t);
+            gen_helper_stackprot(cpu_env, *t);
         }
         return t;
     }
@@ -930,7 +932,7 @@ static inline TCGv *compute_ldst_addr(DisasContext *dc, TCGv *t)
     }
 
     if (stackprot) {
-        gen_helper_stackprot(*t);
+        gen_helper_stackprot(cpu_env, *t);
     }
     return t;
 }
@@ -1056,7 +1058,7 @@ static void dec_load(DisasContext *dc)
         gen_load(dc, v, *addr, size);
 
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
-        gen_helper_memalign(*addr, tcg_const_tl(dc->rd),
+        gen_helper_memalign(cpu_env, *addr, tcg_const_tl(dc->rd),
                             tcg_const_tl(0), tcg_const_tl(size - 1));
         if (dc->rd) {
             if (rev) {
@@ -1218,7 +1220,7 @@ static void dec_store(DisasContext *dc)
          *        the alignment checks in between the probe and the mem
          *        access.
          */
-        gen_helper_memalign(*addr, tcg_const_tl(dc->rd),
+        gen_helper_memalign(cpu_env, *addr, tcg_const_tl(dc->rd),
                             tcg_const_tl(1), tcg_const_tl(size - 1));
     }
 
@@ -1493,49 +1495,53 @@ static void dec_fpu(DisasContext *dc)
 
     switch (fpu_insn) {
         case 0:
-            gen_helper_fadd(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fadd(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 1:
-            gen_helper_frsub(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_frsub(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                             cpu_R[dc->rb]);
             break;
 
         case 2:
-            gen_helper_fmul(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fmul(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 3:
-            gen_helper_fdiv(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fdiv(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 4:
             switch ((dc->ir >> 4) & 7) {
                 case 0:
-                    gen_helper_fcmp_un(cpu_R[dc->rd],
+                    gen_helper_fcmp_un(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 1:
-                    gen_helper_fcmp_lt(cpu_R[dc->rd],
+                    gen_helper_fcmp_lt(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 2:
-                    gen_helper_fcmp_eq(cpu_R[dc->rd],
+                    gen_helper_fcmp_eq(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 3:
-                    gen_helper_fcmp_le(cpu_R[dc->rd],
+                    gen_helper_fcmp_le(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 4:
-                    gen_helper_fcmp_gt(cpu_R[dc->rd],
+                    gen_helper_fcmp_gt(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 5:
-                    gen_helper_fcmp_ne(cpu_R[dc->rd],
+                    gen_helper_fcmp_ne(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 6:
-                    gen_helper_fcmp_ge(cpu_R[dc->rd],
+                    gen_helper_fcmp_ge(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 default:
@@ -1552,21 +1558,21 @@ static void dec_fpu(DisasContext *dc)
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_flt(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_flt(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         case 6:
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_fint(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_fint(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         case 7:
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_fsqrt(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_fsqrt(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         default:
@@ -1654,15 +1660,15 @@ static struct decoder_info {
     {{0, 0}, dec_null}
 };
 
-static inline void decode(DisasContext *dc)
+static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    uint32_t ir;
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
+    }
 
-    dc->ir = ir = ldl_code(dc->pc);
+    dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
     if (dc->ir)
@@ -1735,7 +1741,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
     dc->tb = tb;
     org_flags = dc->synced_flags = dc->tb_flags = tb->flags;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->jmp = 0;
@@ -1778,15 +1784,15 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         check_breakpoint(env, dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-                        gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+                        tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         /* Pretty disas.  */
@@ -1796,7 +1802,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
             gen_io_start();
 
         dc->clear_imm = 1;
-	decode(dc);
+        decode(dc, cpu_ldl_code(env, dc->pc));
         if (dc->clear_imm)
             dc->tb_flags &= ~IMM_FLAG;
         dc->pc += 4;
@@ -1840,7 +1846,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         if (env->singlestep_enabled)
             break;
     } while (!dc->is_jmp && !dc->cpustate_changed
-         && gen_opc_ptr < gen_opc_end
+         && tcg_ctx.gen_opc_ptr < gen_opc_end
                  && !singlestep
          && (dc->pc < next_page_start)
                  && num_insns < max_insns);
@@ -1871,7 +1877,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         if (dc->is_jmp != DISAS_JUMP) {
             tcg_gen_movi_tl(cpu_SR[SR_PC], npc);
         }
-        gen_helper_raise_exception(tmp);
+        gen_helper_raise_exception(cpu_env, tmp);
         tcg_temp_free_i32(tmp);
     } else {
         switch(dc->is_jmp) {
@@ -1891,12 +1897,12 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
                 tb->icount = num_insns;
@@ -1907,10 +1913,11 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
 #if DISAS_GNU
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
 #endif
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 #endif
@@ -2007,5 +2014,5 @@ MicroBlazeCPU *cpu_mb_init(const char *cpu_model)
 
 void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->sregs[SR_PC] = gen_opc_pc[pc_pos];
+    env->sregs[SR_PC] = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-mips/Makefile.objs b/target-mips/Makefile.objs
index 2e0e093e1f9..119c816518e 100644
--- a/target-mips/Makefile.objs
+++ b/target-mips/Makefile.objs
@@ -1,4 +1,2 @@
-obj-y += translate.o op_helper.o helper.o cpu.o
+obj-y += translate.o dsp_helper.o op_helper.o lmi_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-mips/TODO b/target-mips/TODO
index 2a3546f6247..1d782d8027e 100644
--- a/target-mips/TODO
+++ b/target-mips/TODO
@@ -6,8 +6,7 @@ General
 - Unimplemented ASEs:
   - MDMX
   - SmartMIPS
-  - DSP r1
-  - DSP r2
+  - microMIPS DSP r1 & r2 encodings
 - MT ASE only partially implemented and not functional
 - Shadow register support only partially implemented,
   lacks set switching on interrupt/exception.
diff --git a/target-mips/cpu-qom.h b/target-mips/cpu-qom.h
index 6e2237123aa..2a4b8124023 100644
--- a/target-mips/cpu-qom.h
+++ b/target-mips/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_MIPS_CPU_QOM_H
 #define QEMU_MIPS_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #ifdef TARGET_MIPS64
 #define TYPE_MIPS_CPU "mips64-cpu"
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index 004406232bd..10ff46d6a79 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -29,8 +29,16 @@ static void mips_cpu_reset(CPUState *s)
     MIPSCPUClass *mcc = MIPS_CPU_GET_CLASS(cpu);
     CPUMIPSState *env = &cpu->env;
 
+    if (qemu_loglevel_mask(CPU_LOG_RESET)) {
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
+        log_cpu_state(env, 0);
+    }
+
     mcc->parent_reset(s);
 
+    memset(env, 0, offsetof(CPUMIPSState, breakpoints));
+    tlb_flush(env, 1);
+
     cpu_state_reset(env);
 }
 
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index ce3467f1404..5963d629732 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -12,8 +12,8 @@
 #include "config.h"
 #include "qemu-common.h"
 #include "mips-defs.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 
 struct CPUMIPSState;
 
@@ -37,11 +37,11 @@ typedef struct CPUMIPSTLBContext CPUMIPSTLBContext;
 struct CPUMIPSTLBContext {
     uint32_t nb_tlb;
     uint32_t tlb_in_use;
-    int (*map_address) (struct CPUMIPSState *env, target_phys_addr_t *physical, int *prot, target_ulong address, int rw, int access_type);
-    void (*helper_tlbwi) (void);
-    void (*helper_tlbwr) (void);
-    void (*helper_tlbp) (void);
-    void (*helper_tlbr) (void);
+    int (*map_address) (struct CPUMIPSState *env, hwaddr *physical, int *prot, target_ulong address, int rw, int access_type);
+    void (*helper_tlbwi)(struct CPUMIPSState *env);
+    void (*helper_tlbwr)(struct CPUMIPSState *env);
+    void (*helper_tlbp)(struct CPUMIPSState *env);
+    void (*helper_tlbr)(struct CPUMIPSState *env);
     union {
         struct {
             r4k_tlb_t tlb[MIPS_TLB_MAX];
@@ -415,7 +415,7 @@ struct CPUMIPSState {
     int error_code;
     uint32_t hflags;    /* CPU State */
     /* TMASK defines different execution modes */
-#define MIPS_HFLAG_TMASK  0x007FF
+#define MIPS_HFLAG_TMASK  0xC07FF
 #define MIPS_HFLAG_MODE   0x00007 /* execution modes                    */
     /* The KSU flags must be the lowest bits in hflags. The flag order
        must be the same as defined for CP0 Status. This allows to use
@@ -453,6 +453,9 @@ struct CPUMIPSState {
 #define MIPS_HFLAG_BDS32  0x10000 /* branch requires 32-bit delay slot  */
 #define MIPS_HFLAG_BX     0x20000 /* branch exchanges execution mode    */
 #define MIPS_HFLAG_BMASK  (MIPS_HFLAG_BMASK_BASE | MIPS_HFLAG_BMASK_EXT)
+    /* MIPS DSP resources access. */
+#define MIPS_HFLAG_DSP   0x40000  /* Enable access to MIPS DSP resources. */
+#define MIPS_HFLAG_DSPR2 0x80000  /* Enable access to MIPS DSPR2 resources. */
     target_ulong btarget;        /* Jump / branch target               */
     target_ulong bcond;          /* Branch condition (if needed)       */
 
@@ -479,18 +482,18 @@ struct CPUMIPSState {
 #include "cpu-qom.h"
 
 #if !defined(CONFIG_USER_ONLY)
-int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int no_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                         target_ulong address, int rw, int access_type);
-int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int fixed_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                            target_ulong address, int rw, int access_type);
-int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                      target_ulong address, int rw, int access_type);
-void r4k_helper_tlbwi (void);
-void r4k_helper_tlbwr (void);
-void r4k_helper_tlbp (void);
-void r4k_helper_tlbr (void);
+void r4k_helper_tlbwi(CPUMIPSState *env);
+void r4k_helper_tlbwr(CPUMIPSState *env);
+void r4k_helper_tlbp(CPUMIPSState *env);
+void r4k_helper_tlbr(CPUMIPSState *env);
 
-void cpu_unassigned_access(CPUMIPSState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMIPSState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size);
 #endif
 
@@ -557,7 +560,7 @@ static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
     return r;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Memory access type :
  * may be needed for precise access rights control and precise exceptions.
@@ -610,8 +613,9 @@ enum {
     EXCP_MDMX,
     EXCP_C2E,
     EXCP_CACHE, /* 32 */
+    EXCP_DSPDIS,
 
-    EXCP_LAST = EXCP_CACHE,
+    EXCP_LAST = EXCP_DSPDIS,
 };
 /* Dummy exception for conditional stores.  */
 #define EXCP_SC 0x100
@@ -658,7 +662,7 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
 void do_interrupt (CPUMIPSState *env);
 #if !defined(CONFIG_USER_ONLY)
 void r4k_invalidate_tlb (CPUMIPSState *env, int idx, int use_extra);
-target_phys_addr_t cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
+hwaddr cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
 		                               int rw);
 #endif
 
@@ -706,16 +710,17 @@ static inline int mips_vpe_active(CPUMIPSState *env)
     return active;
 }
 
-static inline int cpu_has_work(CPUMIPSState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
-    int has_work = 0;
+    CPUMIPSState *env = &MIPS_CPU(cpu)->env;
+    bool has_work = false;
 
     /* It is implementation dependent if non-enabled interrupts
        wake-up the CPU, however most of the implementations only
        check for interrupts that can be taken. */
     if ((env->interrupt_request & CPU_INTERRUPT_HARD) &&
         cpu_mips_hw_interrupts_pending(env)) {
-        has_work = 1;
+        has_work = true;
     }
 
     /* MIPS-MT has the ability to halt the CPU.  */
@@ -723,17 +728,17 @@ static inline int cpu_has_work(CPUMIPSState *env)
         /* The QEMU model will issue an _WAKE request whenever the CPUs
            should be woken up.  */
         if (env->interrupt_request & CPU_INTERRUPT_WAKE) {
-            has_work = 1;
+            has_work = true;
         }
 
         if (!mips_vpe_active(env)) {
-            has_work = 0;
+            has_work = false;
         }
     }
     return has_work;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUMIPSState *env, TranslationBlock *tb)
 {
@@ -742,4 +747,68 @@ static inline void cpu_pc_from_tb(CPUMIPSState *env, TranslationBlock *tb)
     env->hflags |= tb->flags & MIPS_HFLAG_BMASK;
 }
 
+static inline void compute_hflags(CPUMIPSState *env)
+{
+    env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
+                     MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
+                     MIPS_HFLAG_UX | MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2);
+    if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
+        !(env->CP0_Status & (1 << CP0St_ERL)) &&
+        !(env->hflags & MIPS_HFLAG_DM)) {
+        env->hflags |= (env->CP0_Status >> CP0St_KSU) & MIPS_HFLAG_KSU;
+    }
+#if defined(TARGET_MIPS64)
+    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
+        (env->CP0_Status & (1 << CP0St_PX)) ||
+        (env->CP0_Status & (1 << CP0St_UX))) {
+        env->hflags |= MIPS_HFLAG_64;
+    }
+    if (env->CP0_Status & (1 << CP0St_UX)) {
+        env->hflags |= MIPS_HFLAG_UX;
+    }
+#endif
+    if ((env->CP0_Status & (1 << CP0St_CU0)) ||
+        !(env->hflags & MIPS_HFLAG_KSU)) {
+        env->hflags |= MIPS_HFLAG_CP0;
+    }
+    if (env->CP0_Status & (1 << CP0St_CU1)) {
+        env->hflags |= MIPS_HFLAG_FPU;
+    }
+    if (env->CP0_Status & (1 << CP0St_FR)) {
+        env->hflags |= MIPS_HFLAG_F64;
+    }
+    if (env->insn_flags & ASE_DSPR2) {
+        /* Enables access MIPS DSP resources, now our cpu is DSP ASER2,
+           so enable to access DSPR2 resources. */
+        if (env->CP0_Status & (1 << CP0St_MX)) {
+            env->hflags |= MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2;
+        }
+
+    } else if (env->insn_flags & ASE_DSP) {
+        /* Enables access MIPS DSP resources, now our cpu is DSP ASE,
+           so enable to access DSP resources. */
+        if (env->CP0_Status & (1 << CP0St_MX)) {
+            env->hflags |= MIPS_HFLAG_DSP;
+        }
+
+    }
+    if (env->insn_flags & ISA_MIPS32R2) {
+        if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    } else if (env->insn_flags & ISA_MIPS32) {
+        if (env->hflags & MIPS_HFLAG_64) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    } else if (env->insn_flags & ISA_MIPS4) {
+        /* All supported MIPS IV CPUs use the XX (CU3) to enable
+           and disable the MIPS IV extensions to the MIPS III ISA.
+           Some other MIPS IV CPUs ignore the bit, so the check here
+           would be too restrictive for them.  */
+        if (env->CP0_Status & (1 << CP0St_CU3)) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    }
+}
+
 #endif /* !defined (__MIPS_CPU_H__) */
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
new file mode 100644
index 00000000000..4870e3dbbc1
--- /dev/null
+++ b/target-mips/dsp_helper.c
@@ -0,0 +1,4017 @@
+/*
+ * MIPS ASE DSP Instruction emulation helpers for QEMU.
+ *
+ * Copyright (c) 2012  Jia Liu <proljc@gmail.com>
+ *                     Dongxue Zhang <elta.era@gmail.com>
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/*** MIPS DSP internal functions begin ***/
+#define MIPSDSP_ABS(x) (((x) >= 0) ? x : -x)
+#define MIPSDSP_OVERFLOW(a, b, c, d) (!(!((a ^ b ^ -1) & (a ^ c) & d)))
+
+static inline void set_DSPControl_overflow_flag(uint32_t flag, int position,
+                                                CPUMIPSState *env)
+{
+    env->active_tc.DSPControl |= (target_ulong)flag << position;
+}
+
+static inline void set_DSPControl_carryflag(uint32_t flag, CPUMIPSState *env)
+{
+    env->active_tc.DSPControl |= (target_ulong)flag << 13;
+}
+
+static inline uint32_t get_DSPControl_carryflag(CPUMIPSState *env)
+{
+    return (env->active_tc.DSPControl >> 13) & 0x01;
+}
+
+static inline void set_DSPControl_24(uint32_t flag, int len, CPUMIPSState *env)
+{
+  uint32_t filter;
+
+  filter = ((0x01 << len) - 1) << 24;
+  filter = ~filter;
+
+  env->active_tc.DSPControl &= filter;
+  env->active_tc.DSPControl |= (target_ulong)flag << 24;
+}
+
+static inline uint32_t get_DSPControl_24(int len, CPUMIPSState *env)
+{
+  uint32_t filter;
+
+  filter = (0x01 << len) - 1;
+
+  return (env->active_tc.DSPControl >> 24) & filter;
+}
+
+static inline void set_DSPControl_pos(uint32_t pos, CPUMIPSState *env)
+{
+    target_ulong dspc;
+
+    dspc = env->active_tc.DSPControl;
+#ifndef TARGET_MIPS64
+    dspc = dspc & 0xFFFFFFC0;
+    dspc |= pos;
+#else
+    dspc = dspc & 0xFFFFFF80;
+    dspc |= pos;
+#endif
+    env->active_tc.DSPControl = dspc;
+}
+
+static inline uint32_t get_DSPControl_pos(CPUMIPSState *env)
+{
+    target_ulong dspc;
+    uint32_t pos;
+
+    dspc = env->active_tc.DSPControl;
+
+#ifndef TARGET_MIPS64
+    pos = dspc & 0x3F;
+#else
+    pos = dspc & 0x7F;
+#endif
+
+    return pos;
+}
+
+static inline void set_DSPControl_efi(uint32_t flag, CPUMIPSState *env)
+{
+    env->active_tc.DSPControl &= 0xFFFFBFFF;
+    env->active_tc.DSPControl |= (target_ulong)flag << 14;
+}
+
+#define DO_MIPS_SAT_ABS(size)                                          \
+static inline int##size##_t mipsdsp_sat_abs##size(int##size##_t a,         \
+                                                  CPUMIPSState *env)   \
+{                                                                      \
+    if (a == INT##size##_MIN) {                                        \
+        set_DSPControl_overflow_flag(1, 20, env);                      \
+        return INT##size##_MAX;                                        \
+    } else {                                                           \
+        return MIPSDSP_ABS(a);                                         \
+    }                                                                  \
+}
+DO_MIPS_SAT_ABS(8)
+DO_MIPS_SAT_ABS(16)
+DO_MIPS_SAT_ABS(32)
+#undef DO_MIPS_SAT_ABS
+
+/* get sum value */
+static inline int16_t mipsdsp_add_i16(int16_t a, int16_t b, CPUMIPSState *env)
+{
+    int16_t tempI;
+
+    tempI = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempI, 0x8000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempI;
+}
+
+static inline int16_t mipsdsp_sat_add_i16(int16_t a, int16_t b,
+                                          CPUMIPSState *env)
+{
+    int16_t tempS;
+
+    tempS = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempS, 0x8000)) {
+        if (a > 0) {
+            tempS = 0x7FFF;
+        } else {
+            tempS = 0x8000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempS;
+}
+
+static inline int32_t mipsdsp_sat_add_i32(int32_t a, int32_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t tempI;
+
+    tempI = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempI, 0x80000000)) {
+        if (a > 0) {
+            tempI = 0x7FFFFFFF;
+        } else {
+            tempI = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempI;
+}
+
+static inline uint8_t mipsdsp_add_u8(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+
+    if (temp & 0x0100) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFF;
+}
+
+static inline uint16_t mipsdsp_add_u16(uint16_t a, uint16_t b,
+                                       CPUMIPSState *env)
+{
+    uint32_t temp;
+
+    temp = (uint32_t)a + (uint32_t)b;
+
+    if (temp & 0x00010000) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFFFF;
+}
+
+static inline uint8_t mipsdsp_sat_add_u8(uint8_t a, uint8_t b,
+                                         CPUMIPSState *env)
+{
+    uint8_t  result;
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+    result = temp & 0xFF;
+
+    if (0x0100 & temp) {
+        result = 0xFF;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return result;
+}
+
+static inline uint16_t mipsdsp_sat_add_u16(uint16_t a, uint16_t b,
+                                           CPUMIPSState *env)
+{
+    uint16_t result;
+    uint32_t temp;
+
+    temp = (uint32_t)a + (uint32_t)b;
+    result = temp & 0xFFFF;
+
+    if (0x00010000 & temp) {
+        result = 0xFFFF;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return result;
+}
+
+static inline int32_t mipsdsp_sat32_acc_q31(int32_t acc, int32_t a,
+                                            CPUMIPSState *env)
+{
+    int64_t temp;
+    int32_t temp32, temp31, result;
+    int64_t temp_sum;
+
+#ifndef TARGET_MIPS64
+    temp = ((uint64_t)env->active_tc.HI[acc] << 32) |
+           (uint64_t)env->active_tc.LO[acc];
+#else
+    temp = (uint64_t)env->active_tc.LO[acc];
+#endif
+
+    temp_sum = (int64_t)a + temp;
+
+    temp32 = (temp_sum >> 32) & 0x01;
+    temp31 = (temp_sum >> 31) & 0x01;
+    result = temp_sum & 0xFFFFFFFF;
+
+    /* FIXME
+       This sat function may wrong, because user manual wrote:
+       temp127..0 ← temp + ( (signA) || a31..0
+       if ( temp32 ≠ temp31 ) then
+           if ( temp32 = 0 ) then
+               temp31..0 ← 0x80000000
+           else
+                temp31..0 ← 0x7FFFFFFF
+           endif
+           DSPControlouflag:16+acc ← 1
+       endif
+     */
+    if (temp32 != temp31) {
+        if (temp32 == 0) {
+            result = 0x7FFFFFFF;
+        } else {
+            result = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 16 + acc, env);
+    }
+
+    return result;
+}
+
+/* a[0] is LO, a[1] is HI. */
+static inline void mipsdsp_sat64_acc_add_q63(int64_t *ret,
+                                             int32_t ac,
+                                             int64_t *a,
+                                             CPUMIPSState *env)
+{
+    bool temp64;
+
+    ret[0] = env->active_tc.LO[ac] + a[0];
+    ret[1] = env->active_tc.HI[ac] + a[1];
+
+    if (((uint64_t)ret[0] < (uint64_t)env->active_tc.LO[ac]) &&
+        ((uint64_t)ret[0] < (uint64_t)a[0])) {
+        ret[1] += 1;
+    }
+    temp64 = ret[1] & 1;
+    if (temp64 != ((ret[0] >> 63) & 0x01)) {
+        if (temp64) {
+            ret[0] = (0x01ull << 63);
+            ret[1] = ~0ull;
+        } else {
+            ret[0] = (0x01ull << 63) - 1;
+            ret[1] = 0x00;
+        }
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    }
+}
+
+static inline void mipsdsp_sat64_acc_sub_q63(int64_t *ret,
+                                             int32_t ac,
+                                             int64_t *a,
+                                             CPUMIPSState *env)
+{
+    bool temp64;
+
+    ret[0] = env->active_tc.LO[ac] - a[0];
+    ret[1] = env->active_tc.HI[ac] - a[1];
+
+    if ((uint64_t)ret[0] > (uint64_t)env->active_tc.LO[ac]) {
+        ret[1] -= 1;
+    }
+    temp64 = ret[1] & 1;
+    if (temp64 != ((ret[0] >> 63) & 0x01)) {
+        if (temp64) {
+            ret[0] = (0x01ull << 63);
+            ret[1] = ~0ull;
+        } else {
+            ret[0] = (0x01ull << 63) - 1;
+            ret[1] = 0x00;
+        }
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    }
+}
+
+static inline int32_t mipsdsp_mul_i16_i16(int16_t a, int16_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = (int32_t)a * (int32_t)b;
+
+    if ((temp > (int)0x7FFF) || (temp < (int)0xFFFF8000)) {
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+    temp &= 0x0000FFFF;
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_mul_u16_u16(int32_t a, int32_t b)
+{
+    return a * b;
+}
+
+static inline int32_t mipsdsp_mul_i32_i32(int32_t a, int32_t b)
+{
+    return a * b;
+}
+
+static inline int32_t mipsdsp_sat16_mul_i16_i16(int16_t a, int16_t b,
+                                                CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = (int32_t)a * (int32_t)b;
+
+    if (temp > (int)0x7FFF) {
+        temp = 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else if (temp < (int)0xffff8000) {
+        temp = 0xFFFF8000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+    temp &= 0x0000FFFF;
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_mul_q15_q15_overflowflag21(uint16_t a, uint16_t b,
+                                                         CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = ((int32_t)(int16_t)a * (int32_t)(int16_t)b) << 1;
+    }
+
+    return temp;
+}
+
+/* right shift */
+static inline uint8_t mipsdsp_rshift_u8(uint8_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline uint16_t mipsdsp_rshift_u16(uint16_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int8_t mipsdsp_rashift8(int8_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int16_t mipsdsp_rashift16(int16_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int32_t mipsdsp_rashift32(int32_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int16_t mipsdsp_rshift1_add_q16(int16_t a, int16_t b)
+{
+    int32_t temp;
+
+    temp = (int32_t)a + (int32_t)b;
+
+    return (temp >> 1) & 0xFFFF;
+}
+
+/* round right shift */
+static inline int16_t mipsdsp_rrshift1_add_q16(int16_t a, int16_t b)
+{
+    int32_t temp;
+
+    temp = (int32_t)a + (int32_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFF;
+}
+
+static inline int32_t mipsdsp_rshift1_add_q32(int32_t a, int32_t b)
+{
+    int64_t temp;
+
+    temp = (int64_t)a + (int64_t)b;
+
+    return (temp >> 1) & 0xFFFFFFFF;
+}
+
+static inline int32_t mipsdsp_rrshift1_add_q32(int32_t a, int32_t b)
+{
+    int64_t temp;
+
+    temp = (int64_t)a + (int64_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFF;
+}
+
+static inline uint8_t mipsdsp_rshift1_add_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rrshift1_add_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b + 1;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rshift1_sub_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rrshift1_sub_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b + 1;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+/*  128 bits long. p[0] is LO, p[1] is HI. */
+static inline void mipsdsp_rndrashift_short_acc(int64_t *p,
+                                                int32_t ac,
+                                                int32_t shift,
+                                                CPUMIPSState *env)
+{
+    int64_t acc;
+
+    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
+          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
+    if (shift == 0) {
+        p[0] = acc << 1;
+        p[1] = (acc >> 63) & 0x01;
+    } else {
+        p[0] = acc >> (shift - 1);
+        p[1] = 0;
+    }
+}
+
+/* 128 bits long. p[0] is LO, p[1] is HI */
+static inline void mipsdsp_rashift_acc(uint64_t *p,
+                                       uint32_t ac,
+                                       uint32_t shift,
+                                       CPUMIPSState *env)
+{
+    uint64_t tempB, tempA;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+    shift = shift & 0x1F;
+
+    if (shift == 0) {
+        p[1] = tempB;
+        p[0] = tempA;
+    } else {
+        p[0] = (tempB << (64 - shift)) | (tempA >> shift);
+        p[1] = (int64_t)tempB >> shift;
+    }
+}
+
+/* 128 bits long. p[0] is LO, p[1] is HI , p[2] is sign of HI.*/
+static inline void mipsdsp_rndrashift_acc(uint64_t *p,
+                                          uint32_t ac,
+                                          uint32_t shift,
+                                          CPUMIPSState *env)
+{
+    int64_t tempB, tempA;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+    shift = shift & 0x3F;
+
+    if (shift == 0) {
+        p[2] = tempB >> 63;
+        p[1] = (tempB << 1) | (tempA >> 63);
+        p[0] = tempA << 1;
+    } else {
+        p[0] = (tempB << (65 - shift)) | (tempA >> (shift - 1));
+        p[1] = (int64_t)tempB >> (shift - 1);
+        if (tempB >= 0) {
+            p[2] = 0x0;
+        } else {
+            p[2] = ~0ull;
+        }
+    }
+}
+
+static inline int32_t mipsdsp_mul_q15_q15(int32_t ac, uint16_t a, uint16_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    } else {
+        temp = ((uint32_t)a * (uint32_t)b) << 1;
+    }
+
+    return temp;
+}
+
+static inline int64_t mipsdsp_mul_q31_q31(int32_t ac, uint32_t a, uint32_t b,
+                                          CPUMIPSState *env)
+{
+    uint64_t temp;
+
+    if ((a == 0x80000000) && (b == 0x80000000)) {
+        temp = (0x01ull << 63) - 1;
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    } else {
+        temp = ((uint64_t)a * (uint64_t)b) << 1;
+    }
+
+    return temp;
+}
+
+static inline uint16_t mipsdsp_mul_u8_u8(uint8_t a, uint8_t b)
+{
+    return (uint16_t)a * (uint16_t)b;
+}
+
+static inline uint16_t mipsdsp_mul_u8_u16(uint8_t a, uint16_t b,
+                                          CPUMIPSState *env)
+{
+    uint32_t tempI;
+
+    tempI = (uint32_t)a * (uint32_t)b;
+    if (tempI > 0x0000FFFF) {
+        tempI = 0x0000FFFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+
+    return tempI & 0x0000FFFF;
+}
+
+static inline uint64_t mipsdsp_mul_u32_u32(uint32_t a, uint32_t b)
+{
+    return (uint64_t)a * (uint64_t)b;
+}
+
+static inline int16_t mipsdsp_rndq15_mul_q15_q15(uint16_t a, uint16_t b,
+                                                 CPUMIPSState *env)
+{
+    uint32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFF0000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = (a * b) << 1;
+        temp = temp + 0x00008000;
+    }
+
+    return (temp & 0xFFFF0000) >> 16;
+}
+
+static inline int32_t mipsdsp_sat16_mul_q15_q15(uint16_t a, uint16_t b,
+                                                CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFF0000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = ((uint32_t)a * (uint32_t)b);
+        temp = temp << 1;
+    }
+
+    return (temp >> 16) & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_trunc16_sat16_round(int32_t a,
+                                                   CPUMIPSState *env)
+{
+    int64_t temp;
+
+    temp = (int32_t)a + 0x00008000;
+
+    if (a > (int)0x7fff8000) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 22, env);
+    }
+
+    return (temp >> 16) & 0xFFFF;
+}
+
+static inline uint8_t mipsdsp_sat8_reduce_precision(uint16_t a,
+                                                    CPUMIPSState *env)
+{
+    uint16_t mag;
+    uint32_t sign;
+
+    sign = (a >> 15) & 0x01;
+    mag = a & 0x7FFF;
+
+    if (sign == 0) {
+        if (mag > 0x7F80) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return 0xFF;
+        } else {
+            return (mag >> 7) & 0xFFFF;
+        }
+    } else {
+        set_DSPControl_overflow_flag(1, 22, env);
+        return 0x00;
+    }
+}
+
+static inline uint8_t mipsdsp_lshift8(uint8_t a, uint8_t s, CPUMIPSState *env)
+{
+    uint8_t sign;
+    uint8_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 7) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (8 - s)) - 1) << s) |
+                      ((a >> (6 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (6 - (s - 1));
+        }
+
+        if (discard != 0x00) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+static inline uint16_t mipsdsp_lshift16(uint16_t a, uint8_t s,
+                                        CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint16_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 15) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (16 - s)) - 1) << s) |
+                      ((a >> (14 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (14 - (s - 1));
+        }
+
+        if ((discard != 0x0000) && (discard != 0xFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+
+static inline uint32_t mipsdsp_lshift32(uint32_t a, uint8_t s,
+                                        CPUMIPSState *env)
+{
+    uint32_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        discard = (int32_t)a >> (31 - (s - 1));
+
+        if ((discard != 0x00000000) && (discard != 0xFFFFFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+static inline uint16_t mipsdsp_sat16_lshift(uint16_t a, uint8_t s,
+                                            CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint16_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 15) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (16 - s)) - 1) << s) |
+                      ((a >> (14 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (14 - (s - 1));
+        }
+
+        if ((discard != 0x0000) && (discard != 0xFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return (sign == 0) ? 0x7FFF : 0x8000;
+        } else {
+            return a << s;
+        }
+    }
+}
+
+static inline uint32_t mipsdsp_sat32_lshift(uint32_t a, uint8_t s,
+                                            CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint32_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 31) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (32 - s)) - 1) << s) |
+                      ((a >> (30 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (30 - (s - 1));
+        }
+
+        if ((discard != 0x00000000) && (discard != 0xFFFFFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return (sign == 0) ? 0x7FFFFFFF : 0x80000000;
+        } else {
+            return a << s;
+        }
+    }
+}
+
+static inline uint8_t mipsdsp_rnd8_rashift(uint8_t a, uint8_t s)
+{
+    uint32_t temp;
+
+    if (s == 0) {
+        temp = (uint32_t)a << 1;
+    } else {
+        temp = (int32_t)(int8_t)a >> (s - 1);
+    }
+
+    return (temp + 1) >> 1;
+}
+
+static inline uint16_t mipsdsp_rnd16_rashift(uint16_t a, uint8_t s)
+{
+    uint32_t temp;
+
+    if (s == 0) {
+        temp = (uint32_t)a << 1;
+    } else {
+        temp = (int32_t)(int16_t)a >> (s - 1);
+    }
+
+    return (temp + 1) >> 1;
+}
+
+static inline uint32_t mipsdsp_rnd32_rashift(uint32_t a, uint8_t s)
+{
+    int64_t temp;
+
+    if (s == 0) {
+        temp = (uint64_t)a << 1;
+    } else {
+        temp = (int64_t)(int32_t)a >> (s - 1);
+    }
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_sub_i16(int16_t a, int16_t b, CPUMIPSState *env)
+{
+    int16_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x8000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline uint16_t mipsdsp_sat16_sub(int16_t a, int16_t b,
+                                         CPUMIPSState *env)
+{
+    int16_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x8000)) {
+        if (a > 0) {
+            temp = 0x7FFF;
+        } else {
+            temp = 0x8000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline uint32_t mipsdsp_sat32_sub(int32_t a, int32_t b,
+                                         CPUMIPSState *env)
+{
+    int32_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x80000000)) {
+        if (a > 0) {
+            temp = 0x7FFFFFFF;
+        } else {
+            temp = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_rshift1_sub_q16(int16_t a, int16_t b)
+{
+    int32_t  temp;
+
+    temp = (int32_t)a - (int32_t)b;
+
+    return (temp >> 1) & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_rrshift1_sub_q16(int16_t a, int16_t b)
+{
+    int32_t  temp;
+
+    temp = (int32_t)a - (int32_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0x0000FFFF;
+}
+
+static inline uint32_t mipsdsp_rshift1_sub_q32(int32_t a, int32_t b)
+{
+    int64_t  temp;
+
+    temp = (int64_t)a - (int64_t)b;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint32_t mipsdsp_rrshift1_sub_q32(int32_t a, int32_t b)
+{
+    int64_t  temp;
+
+    temp = (int64_t)a - (int64_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_sub_u16_u16(uint16_t a, uint16_t b,
+                                           CPUMIPSState *env)
+{
+    uint8_t  temp16;
+    uint32_t temp;
+
+    temp = (uint32_t)a - (uint32_t)b;
+    temp16 = (temp >> 16) & 0x01;
+    if (temp16 == 1) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+    return temp & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_satu16_sub_u16_u16(uint16_t a, uint16_t b,
+                                                  CPUMIPSState *env)
+{
+    uint8_t  temp16;
+    uint32_t temp;
+
+    temp   = (uint32_t)a - (uint32_t)b;
+    temp16 = (temp >> 16) & 0x01;
+
+    if (temp16 == 1) {
+        temp = 0x0000;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x0000FFFF;
+}
+
+static inline uint8_t mipsdsp_sub_u8(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint8_t  temp8;
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+    temp8 = (temp >> 8) & 0x01;
+    if (temp8 == 1) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_satu8_sub(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint8_t  temp8;
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+    temp8 = (temp >> 8) & 0x01;
+    if (temp8 == 1) {
+        temp = 0x00;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x00FF;
+}
+
+static inline uint32_t mipsdsp_sub32(int32_t a, int32_t b, CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x80000000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_add_i32(int32_t a, int32_t b, CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, temp, 0x80000000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_cmp_eq(int32_t a, int32_t b)
+{
+    return a == b;
+}
+
+static inline int32_t mipsdsp_cmp_le(int32_t a, int32_t b)
+{
+    return a <= b;
+}
+
+static inline int32_t mipsdsp_cmp_lt(int32_t a, int32_t b)
+{
+    return a < b;
+}
+
+static inline int32_t mipsdsp_cmpu_eq(uint32_t a, uint32_t b)
+{
+    return a == b;
+}
+
+static inline int32_t mipsdsp_cmpu_le(uint32_t a, uint32_t b)
+{
+    return a <= b;
+}
+
+static inline int32_t mipsdsp_cmpu_lt(uint32_t a, uint32_t b)
+{
+    return a < b;
+}
+/*** MIPS DSP internal functions end ***/
+
+#define MIPSDSP_LHI 0xFFFFFFFF00000000ull
+#define MIPSDSP_LLO 0x00000000FFFFFFFFull
+#define MIPSDSP_HI  0xFFFF0000
+#define MIPSDSP_LO  0x0000FFFF
+#define MIPSDSP_Q3  0xFF000000
+#define MIPSDSP_Q2  0x00FF0000
+#define MIPSDSP_Q1  0x0000FF00
+#define MIPSDSP_Q0  0x000000FF
+
+#define MIPSDSP_SPLIT32_8(num, a, b, c, d)  \
+    do {                                    \
+        a = (num >> 24) & MIPSDSP_Q0;       \
+        b = (num >> 16) & MIPSDSP_Q0;       \
+        c = (num >> 8) & MIPSDSP_Q0;        \
+        d = num & MIPSDSP_Q0;               \
+    } while (0)
+
+#define MIPSDSP_SPLIT32_16(num, a, b)       \
+    do {                                    \
+        a = (num >> 16) & MIPSDSP_LO;       \
+        b = num & MIPSDSP_LO;               \
+    } while (0)
+
+#define MIPSDSP_RETURN32(a)             ((target_long)(int32_t)a)
+#define MIPSDSP_RETURN32_8(a, b, c, d)  ((target_long)(int32_t) \
+                                         (((uint32_t)a << 24) | \
+                                         (((uint32_t)b << 16) | \
+                                         (((uint32_t)c << 8) |  \
+                                          ((uint32_t)d & 0xFF)))))
+#define MIPSDSP_RETURN32_16(a, b)       ((target_long)(int32_t) \
+                                         (((uint32_t)a << 16) | \
+                                          ((uint32_t)b & 0xFFFF)))
+
+#ifdef TARGET_MIPS64
+#define MIPSDSP_SPLIT64_16(num, a, b, c, d)  \
+    do {                                     \
+        a = (num >> 48) & MIPSDSP_LO;        \
+        b = (num >> 32) & MIPSDSP_LO;        \
+        c = (num >> 16) & MIPSDSP_LO;        \
+        d = num & MIPSDSP_LO;                \
+    } while (0)
+
+#define MIPSDSP_SPLIT64_32(num, a, b)       \
+    do {                                    \
+        a = (num >> 32) & MIPSDSP_LLO;      \
+        b = num & MIPSDSP_LLO;              \
+    } while (0)
+
+#define MIPSDSP_RETURN64_16(a, b, c, d) (((uint64_t)a << 48) | \
+                                         ((uint64_t)b << 32) | \
+                                         ((uint64_t)c << 16) | \
+                                         (uint64_t)d)
+#define MIPSDSP_RETURN64_32(a, b)       (((uint64_t)a << 32) | (uint64_t)b)
+#endif
+
+/** DSP Arithmetic Sub-class insns **/
+#define ARITH_PH(name, func)                                      \
+target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    temph = mipsdsp_##func(rsh, rth);                             \
+    templ = mipsdsp_##func(rsl, rtl);                             \
+                                                                  \
+    return MIPSDSP_RETURN32_16(temph, templ);                     \
+}
+
+#define ARITH_PH_ENV(name, func)                                  \
+target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    temph = mipsdsp_##func(rsh, rth, env);                        \
+    templ = mipsdsp_##func(rsl, rtl, env);                        \
+                                                                  \
+    return MIPSDSP_RETURN32_16(temph, templ);                     \
+}
+
+
+ARITH_PH_ENV(addq, add_i16);
+ARITH_PH_ENV(addq_s, sat_add_i16);
+ARITH_PH_ENV(addu, add_u16);
+ARITH_PH_ENV(addu_s, sat_add_u16);
+
+ARITH_PH(addqh, rshift1_add_q16);
+ARITH_PH(addqh_r, rrshift1_add_q16);
+
+ARITH_PH_ENV(subq, sub_i16);
+ARITH_PH_ENV(subq_s, sat16_sub);
+ARITH_PH_ENV(subu, sub_u16_u16);
+ARITH_PH_ENV(subu_s, satu16_sub_u16_u16);
+
+ARITH_PH(subqh, rshift1_sub_q16);
+ARITH_PH(subqh_r, rrshift1_sub_q16);
+
+#undef ARITH_PH
+#undef ARITH_PH_ENV
+
+#ifdef TARGET_MIPS64
+#define ARITH_QH_ENV(name, func) \
+target_ulong helper_##name##_qh(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)           \
+{                                                            \
+    uint16_t rs3, rs2, rs1, rs0;                             \
+    uint16_t rt3, rt2, rt1, rt0;                             \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);              \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);              \
+                                                             \
+    tempD = mipsdsp_##func(rs3, rt3, env);                   \
+    tempC = mipsdsp_##func(rs2, rt2, env);                   \
+    tempB = mipsdsp_##func(rs1, rt1, env);                   \
+    tempA = mipsdsp_##func(rs0, rt0, env);                   \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+ARITH_QH_ENV(addq, add_i16);
+ARITH_QH_ENV(addq_s, sat_add_i16);
+ARITH_QH_ENV(addu, add_u16);
+ARITH_QH_ENV(addu_s, sat_add_u16);
+
+ARITH_QH_ENV(subq, sub_i16);
+ARITH_QH_ENV(subq_s, sat16_sub);
+ARITH_QH_ENV(subu, sub_u16_u16);
+ARITH_QH_ENV(subu_s, satu16_sub_u16_u16);
+
+#undef ARITH_QH_ENV
+
+#endif
+
+#define ARITH_W(name, func) \
+target_ulong helper_##name##_w(target_ulong rs, target_ulong rt) \
+{                                                                \
+    uint32_t rd;                                                 \
+    rd = mipsdsp_##func(rs, rt);                                 \
+    return MIPSDSP_RETURN32(rd);                                 \
+}
+
+#define ARITH_W_ENV(name, func) \
+target_ulong helper_##name##_w(target_ulong rs, target_ulong rt, \
+                               CPUMIPSState *env)                \
+{                                                                \
+    uint32_t rd;                                                 \
+    rd = mipsdsp_##func(rs, rt, env);                            \
+    return MIPSDSP_RETURN32(rd);                                 \
+}
+
+ARITH_W_ENV(addq_s, sat_add_i32);
+
+ARITH_W(addqh, rshift1_add_q32);
+ARITH_W(addqh_r, rrshift1_add_q32);
+
+ARITH_W_ENV(subq_s, sat32_sub);
+
+ARITH_W(subqh, rshift1_sub_q32);
+ARITH_W(subqh_r, rrshift1_sub_q32);
+
+#undef ARITH_W
+#undef ARITH_W_ENV
+
+target_ulong helper_absq_s_w(target_ulong rt, CPUMIPSState *env)
+{
+    uint32_t rd;
+
+    rd = mipsdsp_sat_abs32(rt, env);
+
+    return (target_ulong)rd;
+}
+
+
+#if defined(TARGET_MIPS64)
+
+#define ARITH_PW_ENV(name, func) \
+target_ulong helper_##name##_pw(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint32_t rs1, rs0;                                            \
+    uint32_t rt1, rt0;                                            \
+    uint32_t tempB, tempA;                                        \
+                                                                  \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                             \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    tempB = mipsdsp_##func(rs1, rt1, env);                        \
+    tempA = mipsdsp_##func(rs0, rt0, env);                        \
+                                                                  \
+    return MIPSDSP_RETURN64_32(tempB, tempA);                     \
+}
+
+ARITH_PW_ENV(addq, add_i32);
+ARITH_PW_ENV(addq_s, sat_add_i32);
+ARITH_PW_ENV(subq, sub32);
+ARITH_PW_ENV(subq_s, sat32_sub);
+
+#undef ARITH_PW_ENV
+
+#endif
+
+#define ARITH_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint8_t  rs0, rs1, rs2, rs3;                                  \
+    uint8_t  rt0, rt1, rt2, rt3;                                  \
+    uint8_t  temp0, temp1, temp2, temp3;                          \
+                                                                  \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                    \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                    \
+                                                                  \
+    temp0 = mipsdsp_##func(rs0, rt0);                             \
+    temp1 = mipsdsp_##func(rs1, rt1);                             \
+    temp2 = mipsdsp_##func(rs2, rt2);                             \
+    temp3 = mipsdsp_##func(rs3, rt3);                             \
+                                                                  \
+    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);        \
+}
+
+#define ARITH_QB_ENV(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)          \
+{                                                           \
+    uint8_t  rs0, rs1, rs2, rs3;                            \
+    uint8_t  rt0, rt1, rt2, rt3;                            \
+    uint8_t  temp0, temp1, temp2, temp3;                    \
+                                                            \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);              \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);              \
+                                                            \
+    temp0 = mipsdsp_##func(rs0, rt0, env);                  \
+    temp1 = mipsdsp_##func(rs1, rt1, env);                  \
+    temp2 = mipsdsp_##func(rs2, rt2, env);                  \
+    temp3 = mipsdsp_##func(rs3, rt3, env);                  \
+                                                            \
+    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);  \
+}
+
+ARITH_QB(adduh, rshift1_add_u8);
+ARITH_QB(adduh_r, rrshift1_add_u8);
+
+ARITH_QB_ENV(addu, add_u8);
+ARITH_QB_ENV(addu_s, sat_add_u8);
+
+#undef ADDU_QB
+#undef ADDU_QB_ENV
+
+#if defined(TARGET_MIPS64)
+#define ARITH_OB(name, func) \
+target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    int i;                                                        \
+    uint8_t rs_t[8], rt_t[8];                                     \
+    uint8_t temp[8];                                              \
+    uint64_t result;                                              \
+                                                                  \
+    result = 0;                                                   \
+                                                                  \
+    for (i = 0; i < 8; i++) {                                     \
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
+        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i]);               \
+        result |= (uint64_t)temp[i] << (8 * i);                   \
+    }                                                             \
+                                                                  \
+    return result;                                                \
+}
+
+#define ARITH_OB_ENV(name, func) \
+target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    int i;                                                        \
+    uint8_t rs_t[8], rt_t[8];                                     \
+    uint8_t temp[8];                                              \
+    uint64_t result;                                              \
+                                                                  \
+    result = 0;                                                   \
+                                                                  \
+    for (i = 0; i < 8; i++) {                                     \
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
+        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i], env);          \
+        result |= (uint64_t)temp[i] << (8 * i);                   \
+    }                                                             \
+                                                                  \
+    return result;                                                \
+}
+
+ARITH_OB_ENV(addu, add_u8);
+ARITH_OB_ENV(addu_s, sat_add_u8);
+
+ARITH_OB(adduh, rshift1_add_u8);
+ARITH_OB(adduh_r, rrshift1_add_u8);
+
+ARITH_OB_ENV(subu, sub_u8);
+ARITH_OB_ENV(subu_s, satu8_sub);
+
+ARITH_OB(subuh, rshift1_sub_u8);
+ARITH_OB(subuh_r, rrshift1_sub_u8);
+
+#undef ARITH_OB
+#undef ARITH_OB_ENV
+
+#endif
+
+#define SUBU_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs,               \
+                                target_ulong rt,               \
+                                CPUMIPSState *env)             \
+{                                                              \
+    uint8_t rs3, rs2, rs1, rs0;                                \
+    uint8_t rt3, rt2, rt1, rt0;                                \
+    uint8_t tempD, tempC, tempB, tempA;                        \
+                                                               \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                 \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                 \
+                                                               \
+    tempD = mipsdsp_##func(rs3, rt3, env);                     \
+    tempC = mipsdsp_##func(rs2, rt2, env);                     \
+    tempB = mipsdsp_##func(rs1, rt1, env);                     \
+    tempA = mipsdsp_##func(rs0, rt0, env);                     \
+                                                               \
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);     \
+}
+
+SUBU_QB(subu, sub_u8);
+SUBU_QB(subu_s, satu8_sub);
+
+#undef SUBU_QB
+
+#define SUBUH_QB(name, var) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint8_t rs3, rs2, rs1, rs0;                                   \
+    uint8_t rt3, rt2, rt1, rt0;                                   \
+    uint8_t tempD, tempC, tempB, tempA;                           \
+                                                                  \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                    \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                    \
+                                                                  \
+    tempD = ((uint16_t)rs3 - (uint16_t)rt3 + var) >> 1;           \
+    tempC = ((uint16_t)rs2 - (uint16_t)rt2 + var) >> 1;           \
+    tempB = ((uint16_t)rs1 - (uint16_t)rt1 + var) >> 1;           \
+    tempA = ((uint16_t)rs0 - (uint16_t)rt0 + var) >> 1;           \
+                                                                  \
+    return ((uint32_t)tempD << 24) | ((uint32_t)tempC << 16) |    \
+        ((uint32_t)tempB << 8) | ((uint32_t)tempA);               \
+}
+
+SUBUH_QB(subuh, 0);
+SUBUH_QB(subuh_r, 1);
+
+#undef SUBUH_QB
+
+target_ulong helper_addsc(target_ulong rs, target_ulong rt, CPUMIPSState *env)
+{
+    uint64_t temp, tempRs, tempRt;
+    int32_t flag;
+
+    tempRs = (uint64_t)rs & MIPSDSP_LLO;
+    tempRt = (uint64_t)rt & MIPSDSP_LLO;
+
+    temp = tempRs + tempRt;
+    flag = (temp & 0x0100000000ull) >> 32;
+    set_DSPControl_carryflag(flag, env);
+
+    return (target_long)(int32_t)(temp & MIPSDSP_LLO);
+}
+
+target_ulong helper_addwc(target_ulong rs, target_ulong rt, CPUMIPSState *env)
+{
+    uint32_t rd;
+    int32_t temp32, temp31;
+    int64_t tempL;
+
+    tempL = (int64_t)(int32_t)rs + (int64_t)(int32_t)rt +
+        get_DSPControl_carryflag(env);
+    temp31 = (tempL >> 31) & 0x01;
+    temp32 = (tempL >> 32) & 0x01;
+
+    if (temp31 != temp32) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    rd = tempL & MIPSDSP_LLO;
+
+    return (target_long)(int32_t)rd;
+}
+
+target_ulong helper_modsub(target_ulong rs, target_ulong rt)
+{
+    int32_t decr;
+    uint16_t lastindex;
+    target_ulong rd;
+
+    decr = rt & MIPSDSP_Q0;
+    lastindex = (rt >> 8) & MIPSDSP_LO;
+
+    if ((rs & MIPSDSP_LLO) == 0x00000000) {
+        rd = (target_ulong)lastindex;
+    } else {
+        rd = rs - decr;
+    }
+
+    return rd;
+}
+
+target_ulong helper_raddu_w_qb(target_ulong rs)
+{
+    uint8_t  rs3, rs2, rs1, rs0;
+    uint16_t temp;
+
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);
+
+    temp = (uint16_t)rs3 + (uint16_t)rs2 + (uint16_t)rs1 + (uint16_t)rs0;
+
+    return (target_ulong)temp;
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_raddu_l_ob(target_ulong rs)
+{
+    int i;
+    uint16_t rs_t[8];
+    uint64_t temp;
+
+    temp = 0;
+
+    for (i = 0; i < 8; i++) {
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;
+        temp += (uint64_t)rs_t[i];
+    }
+
+    return temp;
+}
+#endif
+
+target_ulong helper_absq_s_qb(target_ulong rt, CPUMIPSState *env)
+{
+    uint8_t tempD, tempC, tempB, tempA;
+
+    MIPSDSP_SPLIT32_8(rt, tempD, tempC, tempB, tempA);
+
+    tempD = mipsdsp_sat_abs8(tempD, env);
+    tempC = mipsdsp_sat_abs8(tempC, env);
+    tempB = mipsdsp_sat_abs8(tempB, env);
+    tempA = mipsdsp_sat_abs8(tempA, env);
+
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_absq_s_ph(target_ulong rt, CPUMIPSState *env)
+{
+    uint16_t tempB, tempA;
+
+    MIPSDSP_SPLIT32_16(rt, tempB, tempA);
+
+    tempB = mipsdsp_sat_abs16 (tempB, env);
+    tempA = mipsdsp_sat_abs16 (tempA, env);
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_absq_s_ob(target_ulong rt, CPUMIPSState *env)
+{
+    int i;
+    int8_t temp[8];
+    uint64_t result;
+
+    for (i = 0; i < 8; i++) {
+        temp[i] = (rt >> (8 * i)) & MIPSDSP_Q0;
+        temp[i] = mipsdsp_sat_abs8(temp[i], env);
+    }
+
+    for (i = 0; i < 8; i++) {
+        result = (uint64_t)(uint8_t)temp[i] << (8 * i);
+    }
+
+    return result;
+}
+
+target_ulong helper_absq_s_qh(target_ulong rt, CPUMIPSState *env)
+{
+    int16_t tempD, tempC, tempB, tempA;
+
+    MIPSDSP_SPLIT64_16(rt, tempD, tempC, tempB, tempA);
+
+    tempD = mipsdsp_sat_abs16(tempD, env);
+    tempC = mipsdsp_sat_abs16(tempC, env);
+    tempB = mipsdsp_sat_abs16(tempB, env);
+    tempA = mipsdsp_sat_abs16(tempA, env);
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_absq_s_pw(target_ulong rt, CPUMIPSState *env)
+{
+    int32_t tempB, tempA;
+
+    MIPSDSP_SPLIT64_32(rt, tempB, tempA);
+
+    tempB = mipsdsp_sat_abs32(tempB, env);
+    tempA = mipsdsp_sat_abs32(tempA, env);
+
+    return MIPSDSP_RETURN64_32(tempB, tempA);
+}
+#endif
+
+#define PRECR_QB_PH(name, a, b)\
+target_ulong helper_##name##_qb_ph(target_ulong rs, target_ulong rt) \
+{                                                                    \
+    uint8_t tempD, tempC, tempB, tempA;                              \
+                                                                     \
+    tempD = (rs >> a) & MIPSDSP_Q0;                                  \
+    tempC = (rs >> b) & MIPSDSP_Q0;                                  \
+    tempB = (rt >> a) & MIPSDSP_Q0;                                  \
+    tempA = (rt >> b) & MIPSDSP_Q0;                                  \
+                                                                     \
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);           \
+}
+
+PRECR_QB_PH(precr, 16, 0);
+PRECR_QB_PH(precrq, 24, 8);
+
+#undef PRECR_QB_OH
+
+target_ulong helper_precr_sra_ph_w(uint32_t sa, target_ulong rs,
+                                   target_ulong rt)
+{
+    uint16_t tempB, tempA;
+
+    tempB = ((int32_t)rt >> sa) & MIPSDSP_LO;
+    tempA = ((int32_t)rs >> sa) & MIPSDSP_LO;
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+target_ulong helper_precr_sra_r_ph_w(uint32_t sa,
+                                     target_ulong rs, target_ulong rt)
+{
+    uint64_t tempB, tempA;
+
+    /* If sa = 0, then (sa - 1) = -1 will case shift error, so we need else. */
+    if (sa == 0) {
+        tempB = (rt & MIPSDSP_LO) << 1;
+        tempA = (rs & MIPSDSP_LO) << 1;
+    } else {
+        tempB = ((int32_t)rt >> (sa - 1)) + 1;
+        tempA = ((int32_t)rs >> (sa - 1)) + 1;
+    }
+    rt = (((tempB >> 1) & MIPSDSP_LO) << 16) | ((tempA >> 1) & MIPSDSP_LO);
+
+    return (target_long)(int32_t)rt;
+}
+
+target_ulong helper_precrq_ph_w(target_ulong rs, target_ulong rt)
+{
+    uint16_t tempB, tempA;
+
+    tempB = (rs & MIPSDSP_HI) >> 16;
+    tempA = (rt & MIPSDSP_HI) >> 16;
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+target_ulong helper_precrq_rs_ph_w(target_ulong rs, target_ulong rt,
+                                   CPUMIPSState *env)
+{
+    uint16_t tempB, tempA;
+
+    tempB = mipsdsp_trunc16_sat16_round(rs, env);
+    tempA = mipsdsp_trunc16_sat16_round(rt, env);
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_precr_ob_qh(target_ulong rs, target_ulong rt)
+{
+    uint8_t rs6, rs4, rs2, rs0;
+    uint8_t rt6, rt4, rt2, rt0;
+    uint64_t temp;
+
+    rs6 = (rs >> 48) & MIPSDSP_Q0;
+    rs4 = (rs >> 32) & MIPSDSP_Q0;
+    rs2 = (rs >> 16) & MIPSDSP_Q0;
+    rs0 = rs & MIPSDSP_Q0;
+    rt6 = (rt >> 48) & MIPSDSP_Q0;
+    rt4 = (rt >> 32) & MIPSDSP_Q0;
+    rt2 = (rt >> 16) & MIPSDSP_Q0;
+    rt0 = rt & MIPSDSP_Q0;
+
+    temp = ((uint64_t)rs6 << 56) | ((uint64_t)rs4 << 48) |
+           ((uint64_t)rs2 << 40) | ((uint64_t)rs0 << 32) |
+           ((uint64_t)rt6 << 24) | ((uint64_t)rt4 << 16) |
+           ((uint64_t)rt2 << 8) | (uint64_t)rt0;
+
+    return temp;
+}
+
+#define PRECR_QH_PW(name, var) \
+target_ulong helper_precr_##name##_qh_pw(target_ulong rs, target_ulong rt, \
+                                    uint32_t sa)                      \
+{                                                                     \
+    uint16_t rs3, rs2, rs1, rs0;                                      \
+    uint16_t rt3, rt2, rt1, rt0;                                      \
+    uint16_t tempD, tempC, tempB, tempA;                              \
+                                                                      \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);                       \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                       \
+                                                                      \
+    /* When sa = 0, we use rt2, rt0, rs2, rs0;                        \
+     * when sa != 0, we use rt3, rt1, rs3, rs1. */                    \
+    if (sa == 0) {                                                    \
+        tempD = rt2 << var;                                           \
+        tempC = rt0 << var;                                           \
+        tempB = rs2 << var;                                           \
+        tempA = rs0 << var;                                           \
+    } else {                                                          \
+        tempD = (((int16_t)rt3 >> sa) + var) >> var;                  \
+        tempC = (((int16_t)rt1 >> sa) + var) >> var;                  \
+        tempB = (((int16_t)rs3 >> sa) + var) >> var;                  \
+        tempA = (((int16_t)rs1 >> sa) + var) >> var;                  \
+    }                                                                 \
+                                                                      \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);           \
+}
+
+PRECR_QH_PW(sra, 0);
+PRECR_QH_PW(sra_r, 1);
+
+#undef PRECR_QH_PW
+
+target_ulong helper_precrq_ob_qh(target_ulong rs, target_ulong rt)
+{
+    uint8_t rs6, rs4, rs2, rs0;
+    uint8_t rt6, rt4, rt2, rt0;
+    uint64_t temp;
+
+    rs6 = (rs >> 56) & MIPSDSP_Q0;
+    rs4 = (rs >> 40) & MIPSDSP_Q0;
+    rs2 = (rs >> 24) & MIPSDSP_Q0;
+    rs0 = (rs >> 8) & MIPSDSP_Q0;
+    rt6 = (rt >> 56) & MIPSDSP_Q0;
+    rt4 = (rt >> 40) & MIPSDSP_Q0;
+    rt2 = (rt >> 24) & MIPSDSP_Q0;
+    rt0 = (rt >> 8) & MIPSDSP_Q0;
+
+    temp = ((uint64_t)rs6 << 56) | ((uint64_t)rs4 << 48) |
+           ((uint64_t)rs2 << 40) | ((uint64_t)rs0 << 32) |
+           ((uint64_t)rt6 << 24) | ((uint64_t)rt4 << 16) |
+           ((uint64_t)rt2 << 8) | (uint64_t)rt0;
+
+    return temp;
+}
+
+target_ulong helper_precrq_qh_pw(target_ulong rs, target_ulong rt)
+{
+    uint16_t tempD, tempC, tempB, tempA;
+
+    tempD = (rs >> 48) & MIPSDSP_LO;
+    tempC = (rs >> 16) & MIPSDSP_LO;
+    tempB = (rt >> 48) & MIPSDSP_LO;
+    tempA = (rt >> 16) & MIPSDSP_LO;
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_precrq_rs_qh_pw(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    uint32_t rs2, rs0;
+    uint32_t rt2, rt0;
+    uint16_t tempD, tempC, tempB, tempA;
+
+    rs2 = (rs >> 32) & MIPSDSP_LLO;
+    rs0 = rs & MIPSDSP_LLO;
+    rt2 = (rt >> 32) & MIPSDSP_LLO;
+    rt0 = rt & MIPSDSP_LLO;
+
+    tempD = mipsdsp_trunc16_sat16_round(rs2, env);
+    tempC = mipsdsp_trunc16_sat16_round(rs0, env);
+    tempB = mipsdsp_trunc16_sat16_round(rt2, env);
+    tempA = mipsdsp_trunc16_sat16_round(rt0, env);
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_precrq_pw_l(target_ulong rs, target_ulong rt)
+{
+    uint32_t tempB, tempA;
+
+    tempB = (rs >> 32) & MIPSDSP_LLO;
+    tempA = (rt >> 32) & MIPSDSP_LLO;
+
+    return MIPSDSP_RETURN64_32(tempB, tempA);
+}
+#endif
+
+target_ulong helper_precrqu_s_qb_ph(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    uint8_t  tempD, tempC, tempB, tempA;
+    uint16_t rsh, rsl, rth, rtl;
+
+    rsh = (rs & MIPSDSP_HI) >> 16;
+    rsl =  rs & MIPSDSP_LO;
+    rth = (rt & MIPSDSP_HI) >> 16;
+    rtl =  rt & MIPSDSP_LO;
+
+    tempD = mipsdsp_sat8_reduce_precision(rsh, env);
+    tempC = mipsdsp_sat8_reduce_precision(rsl, env);
+    tempB = mipsdsp_sat8_reduce_precision(rth, env);
+    tempA = mipsdsp_sat8_reduce_precision(rtl, env);
+
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_precrqu_s_ob_qh(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    int i;
+    uint16_t rs3, rs2, rs1, rs0;
+    uint16_t rt3, rt2, rt1, rt0;
+    uint8_t temp[8];
+    uint64_t result;
+
+    result = 0;
+
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);
+
+    temp[7] = mipsdsp_sat8_reduce_precision(rs3, env);
+    temp[6] = mipsdsp_sat8_reduce_precision(rs2, env);
+    temp[5] = mipsdsp_sat8_reduce_precision(rs1, env);
+    temp[4] = mipsdsp_sat8_reduce_precision(rs0, env);
+    temp[3] = mipsdsp_sat8_reduce_precision(rt3, env);
+    temp[2] = mipsdsp_sat8_reduce_precision(rt2, env);
+    temp[1] = mipsdsp_sat8_reduce_precision(rt1, env);
+    temp[0] = mipsdsp_sat8_reduce_precision(rt0, env);
+
+    for (i = 0; i < 8; i++) {
+        result |= (uint64_t)temp[i] << (8 * i);
+    }
+
+    return result;
+}
+
+#define PRECEQ_PW(name, a, b) \
+target_ulong helper_preceq_pw_##name(target_ulong rt) \
+{                                                       \
+    uint16_t tempB, tempA;                              \
+    uint32_t tempBI, tempAI;                            \
+                                                        \
+    tempB = (rt >> a) & MIPSDSP_LO;                     \
+    tempA = (rt >> b) & MIPSDSP_LO;                     \
+                                                        \
+    tempBI = (uint32_t)tempB << 16;                     \
+    tempAI = (uint32_t)tempA << 16;                     \
+                                                        \
+    return MIPSDSP_RETURN64_32(tempBI, tempAI);         \
+}
+
+PRECEQ_PW(qhl, 48, 32);
+PRECEQ_PW(qhr, 16, 0);
+PRECEQ_PW(qhla, 48, 16);
+PRECEQ_PW(qhra, 32, 0);
+
+#undef PRECEQ_PW
+
+#endif
+
+#define PRECEQU_PH(name, a, b) \
+target_ulong helper_precequ_ph_##name(target_ulong rt) \
+{                                                        \
+    uint16_t tempB, tempA;                               \
+                                                         \
+    tempB = (rt >> a) & MIPSDSP_Q0;                      \
+    tempA = (rt >> b) & MIPSDSP_Q0;                      \
+                                                         \
+    tempB = tempB << 7;                                  \
+    tempA = tempA << 7;                                  \
+                                                         \
+    return MIPSDSP_RETURN32_16(tempB, tempA);            \
+}
+
+PRECEQU_PH(qbl, 24, 16);
+PRECEQU_PH(qbr, 8, 0);
+PRECEQU_PH(qbla, 24, 8);
+PRECEQU_PH(qbra, 16, 0);
+
+#undef PRECEQU_PH
+
+#if defined(TARGET_MIPS64)
+#define PRECEQU_QH(name, a, b, c, d) \
+target_ulong helper_precequ_qh_##name(target_ulong rt)       \
+{                                                            \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    tempD = (rt >> a) & MIPSDSP_Q0;                          \
+    tempC = (rt >> b) & MIPSDSP_Q0;                          \
+    tempB = (rt >> c) & MIPSDSP_Q0;                          \
+    tempA = (rt >> d) & MIPSDSP_Q0;                          \
+                                                             \
+    tempD = tempD << 7;                                      \
+    tempC = tempC << 7;                                      \
+    tempB = tempB << 7;                                      \
+    tempA = tempA << 7;                                      \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+PRECEQU_QH(obl, 56, 48, 40, 32);
+PRECEQU_QH(obr, 24, 16, 8, 0);
+PRECEQU_QH(obla, 56, 40, 24, 8);
+PRECEQU_QH(obra, 48, 32, 16, 0);
+
+#undef PRECEQU_QH
+
+#endif
+
+#define PRECEU_PH(name, a, b) \
+target_ulong helper_preceu_ph_##name(target_ulong rt) \
+{                                                     \
+    uint16_t tempB, tempA;                            \
+                                                      \
+    tempB = (rt >> a) & MIPSDSP_Q0;                   \
+    tempA = (rt >> b) & MIPSDSP_Q0;                   \
+                                                      \
+    return MIPSDSP_RETURN32_16(tempB, tempA);         \
+}
+
+PRECEU_PH(qbl, 24, 16);
+PRECEU_PH(qbr, 8, 0);
+PRECEU_PH(qbla, 24, 8);
+PRECEU_PH(qbra, 16, 0);
+
+#undef PRECEU_PH
+
+#if defined(TARGET_MIPS64)
+#define PRECEU_QH(name, a, b, c, d) \
+target_ulong helper_preceu_qh_##name(target_ulong rt)        \
+{                                                            \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    tempD = (rt >> a) & MIPSDSP_Q0;                          \
+    tempC = (rt >> b) & MIPSDSP_Q0;                          \
+    tempB = (rt >> c) & MIPSDSP_Q0;                          \
+    tempA = (rt >> d) & MIPSDSP_Q0;                          \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+PRECEU_QH(obl, 56, 48, 40, 32);
+PRECEU_QH(obr, 24, 16, 8, 0);
+PRECEU_QH(obla, 56, 40, 24, 8);
+PRECEU_QH(obra, 48, 32, 16, 0);
+
+#undef PRECEU_QH
+
+#endif
+
+/** DSP GPR-Based Shift Sub-class insns **/
+#define SHIFT_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong sa, target_ulong rt) \
+{                                                                    \
+    uint8_t rt3, rt2, rt1, rt0;                                      \
+                                                                     \
+    sa = sa & 0x07;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                       \
+                                                                     \
+    rt3 = mipsdsp_##func(rt3, sa);                                   \
+    rt2 = mipsdsp_##func(rt2, sa);                                   \
+    rt1 = mipsdsp_##func(rt1, sa);                                   \
+    rt0 = mipsdsp_##func(rt0, sa);                                   \
+                                                                     \
+    return MIPSDSP_RETURN32_8(rt3, rt2, rt1, rt0);                   \
+}
+
+#define SHIFT_QB_ENV(name, func) \
+target_ulong helper_##name##_qb(target_ulong sa, target_ulong rt,\
+                                CPUMIPSState *env) \
+{                                                                    \
+    uint8_t rt3, rt2, rt1, rt0;                                      \
+                                                                     \
+    sa = sa & 0x07;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                       \
+                                                                     \
+    rt3 = mipsdsp_##func(rt3, sa, env);                              \
+    rt2 = mipsdsp_##func(rt2, sa, env);                              \
+    rt1 = mipsdsp_##func(rt1, sa, env);                              \
+    rt0 = mipsdsp_##func(rt0, sa, env);                              \
+                                                                     \
+    return MIPSDSP_RETURN32_8(rt3, rt2, rt1, rt0);                   \
+}
+
+SHIFT_QB_ENV(shll, lshift8);
+SHIFT_QB(shrl, rshift_u8);
+
+SHIFT_QB(shra, rashift8);
+SHIFT_QB(shra_r, rnd8_rashift);
+
+#undef SHIFT_QB
+#undef SHIFT_QB_ENV
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_OB(name, func) \
+target_ulong helper_##name##_ob(target_ulong rt, target_ulong sa) \
+{                                                                        \
+    int i;                                                               \
+    uint8_t rt_t[8];                                                     \
+    uint64_t temp;                                                       \
+                                                                         \
+    sa = sa & 0x07;                                                      \
+    temp = 0;                                                            \
+                                                                         \
+    for (i = 0; i < 8; i++) {                                            \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                          \
+        rt_t[i] = mipsdsp_##func(rt_t[i], sa);                           \
+        temp |= (uint64_t)rt_t[i] << (8 * i);                            \
+    }                                                                    \
+                                                                         \
+    return temp;                                                         \
+}
+
+#define SHIFT_OB_ENV(name, func) \
+target_ulong helper_##name##_ob(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                       \
+{                                                                        \
+    int i;                                                               \
+    uint8_t rt_t[8];                                                     \
+    uint64_t temp;                                                       \
+                                                                         \
+    sa = sa & 0x07;                                                      \
+    temp = 0;                                                            \
+                                                                         \
+    for (i = 0; i < 8; i++) {                                            \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                          \
+        rt_t[i] = mipsdsp_##func(rt_t[i], sa, env);                      \
+        temp |= (uint64_t)rt_t[i] << (8 * i);                            \
+    }                                                                    \
+                                                                         \
+    return temp;                                                         \
+}
+
+SHIFT_OB_ENV(shll, lshift8);
+SHIFT_OB(shrl, rshift_u8);
+
+SHIFT_OB(shra, rashift8);
+SHIFT_OB(shra_r, rnd8_rashift);
+
+#undef SHIFT_OB
+#undef SHIFT_OB_ENV
+
+#endif
+
+#define SHIFT_PH(name, func) \
+target_ulong helper_##name##_ph(target_ulong sa, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t rth, rtl;                                            \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    rth = mipsdsp_##func(rth, sa, env);                           \
+    rtl = mipsdsp_##func(rtl, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN32_16(rth, rtl);                         \
+}
+
+SHIFT_PH(shll, lshift16);
+SHIFT_PH(shll_s, sat16_lshift);
+
+#undef SHIFT_PH
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_QH(name, func) \
+target_ulong helper_##name##_qh(target_ulong rt, target_ulong sa) \
+{                                                                 \
+    uint16_t rt3, rt2, rt1, rt0;                                  \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                   \
+                                                                  \
+    rt3 = mipsdsp_##func(rt3, sa);                                \
+    rt2 = mipsdsp_##func(rt2, sa);                                \
+    rt1 = mipsdsp_##func(rt1, sa);                                \
+    rt0 = mipsdsp_##func(rt0, sa);                                \
+                                                                  \
+    return MIPSDSP_RETURN64_16(rt3, rt2, rt1, rt0);               \
+}
+
+#define SHIFT_QH_ENV(name, func) \
+target_ulong helper_##name##_qh(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t rt3, rt2, rt1, rt0;                                  \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                   \
+                                                                  \
+    rt3 = mipsdsp_##func(rt3, sa, env);                           \
+    rt2 = mipsdsp_##func(rt2, sa, env);                           \
+    rt1 = mipsdsp_##func(rt1, sa, env);                           \
+    rt0 = mipsdsp_##func(rt0, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN64_16(rt3, rt2, rt1, rt0);               \
+}
+
+SHIFT_QH_ENV(shll, lshift16);
+SHIFT_QH_ENV(shll_s, sat16_lshift);
+
+SHIFT_QH(shrl, rshift_u16);
+SHIFT_QH(shra, rashift16);
+SHIFT_QH(shra_r, rnd16_rashift);
+
+#undef SHIFT_QH
+#undef SHIFT_QH_ENV
+
+#endif
+
+#define SHIFT_W(name, func) \
+target_ulong helper_##name##_w(target_ulong sa, target_ulong rt) \
+{                                                                       \
+    uint32_t temp;                                                      \
+                                                                        \
+    sa = sa & 0x1F;                                                     \
+    temp = mipsdsp_##func(rt, sa);                                      \
+                                                                        \
+    return (target_long)(int32_t)temp;                                  \
+}
+
+#define SHIFT_W_ENV(name, func) \
+target_ulong helper_##name##_w(target_ulong sa, target_ulong rt, \
+                               CPUMIPSState *env) \
+{                                                                       \
+    uint32_t temp;                                                      \
+                                                                        \
+    sa = sa & 0x1F;                                                     \
+    temp = mipsdsp_##func(rt, sa, env);                                 \
+                                                                        \
+    return (target_long)(int32_t)temp;                                  \
+}
+
+SHIFT_W_ENV(shll_s, sat32_lshift);
+SHIFT_W(shra_r, rnd32_rashift);
+
+#undef SHIFT_W
+#undef SHIFT_W_ENV
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_PW(name, func) \
+target_ulong helper_##name##_pw(target_ulong rt, target_ulong sa) \
+{                                                                 \
+    uint32_t rt1, rt0;                                            \
+                                                                  \
+    sa = sa & 0x1F;                                               \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    rt1 = mipsdsp_##func(rt1, sa);                                \
+    rt0 = mipsdsp_##func(rt0, sa);                                \
+                                                                  \
+    return MIPSDSP_RETURN64_32(rt1, rt0);                         \
+}
+
+#define SHIFT_PW_ENV(name, func) \
+target_ulong helper_##name##_pw(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint32_t rt1, rt0;                                            \
+                                                                  \
+    sa = sa & 0x1F;                                               \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    rt1 = mipsdsp_##func(rt1, sa, env);                           \
+    rt0 = mipsdsp_##func(rt0, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN64_32(rt1, rt0);                         \
+}
+
+SHIFT_PW_ENV(shll, lshift32);
+SHIFT_PW_ENV(shll_s, sat32_lshift);
+
+SHIFT_PW(shra, rashift32);
+SHIFT_PW(shra_r, rnd32_rashift);
+
+#undef SHIFT_PW
+#undef SHIFT_PW_ENV
+
+#endif
+
+#define SHIFT_PH(name, func) \
+target_ulong helper_##name##_ph(target_ulong sa, target_ulong rt) \
+{                                                                    \
+    uint16_t rth, rtl;                                               \
+                                                                     \
+    sa = sa & 0x0F;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                \
+                                                                     \
+    rth = mipsdsp_##func(rth, sa);                                   \
+    rtl = mipsdsp_##func(rtl, sa);                                   \
+                                                                     \
+    return MIPSDSP_RETURN32_16(rth, rtl);                            \
+}
+
+SHIFT_PH(shrl, rshift_u16);
+SHIFT_PH(shra, rashift16);
+SHIFT_PH(shra_r, rnd16_rashift);
+
+#undef SHIFT_PH
+
+/** DSP Multiply Sub-class insns **/
+/* Return value made up by two 16bits value.
+ * FIXME give the macro a better name.
+ */
+#define MUL_RETURN32_16_PH(name, func, \
+                           rsmov1, rsmov2, rsfilter, \
+                           rtmov1, rtmov2, rtfilter) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                           CPUMIPSState *env)                \
+{                                                            \
+    uint16_t rsB, rsA, rtB, rtA;                             \
+                                                             \
+    rsB = (rs >> rsmov1) & rsfilter;                         \
+    rsA = (rs >> rsmov2) & rsfilter;                         \
+    rtB = (rt >> rtmov1) & rtfilter;                         \
+    rtA = (rt >> rtmov2) & rtfilter;                         \
+                                                             \
+    rsB = mipsdsp_##func(rsB, rtB, env);                     \
+    rsA = mipsdsp_##func(rsA, rtA, env);                     \
+                                                             \
+    return MIPSDSP_RETURN32_16(rsB, rsA);                    \
+}
+
+MUL_RETURN32_16_PH(muleu_s_ph_qbl, mul_u8_u16, \
+                      24, 16, MIPSDSP_Q0, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(muleu_s_ph_qbr, mul_u8_u16, \
+                      8, 0, MIPSDSP_Q0, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mulq_rs_ph, rndq15_mul_q15_q15, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mul_ph, mul_i16_i16, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mul_s_ph, sat16_mul_i16_i16, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mulq_s_ph, sat16_mul_q15_q15, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+
+#undef MUL_RETURN32_16_PH
+
+#define MUL_RETURN32_32_ph(name, func, movbits) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                                  CPUMIPSState *env)         \
+{                                                            \
+    int16_t rsh, rth;                                        \
+    int32_t temp;                                            \
+                                                             \
+    rsh = (rs >> movbits) & MIPSDSP_LO;                      \
+    rth = (rt >> movbits) & MIPSDSP_LO;                      \
+    temp = mipsdsp_##func(rsh, rth, env);                    \
+                                                             \
+    return (target_long)(int32_t)temp;                       \
+}
+
+MUL_RETURN32_32_ph(muleq_s_w_phl, mul_q15_q15_overflowflag21, 16);
+MUL_RETURN32_32_ph(muleq_s_w_phr, mul_q15_q15_overflowflag21, 0);
+
+#undef MUL_RETURN32_32_ph
+
+#define MUL_VOID_PH(name, use_ac_env) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                          CPUMIPSState *env)                             \
+{                                                                        \
+    int16_t rsh, rsl, rth, rtl;                                          \
+    int32_t tempB, tempA;                                                \
+    int64_t acc, dotp;                                                   \
+                                                                         \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                                    \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                    \
+                                                                         \
+    if (use_ac_env == 1) {                                               \
+        tempB = mipsdsp_mul_q15_q15(ac, rsh, rth, env);                  \
+        tempA = mipsdsp_mul_q15_q15(ac, rsl, rtl, env);                  \
+    } else {                                                             \
+        tempB = mipsdsp_mul_u16_u16(rsh, rth);                           \
+        tempA = mipsdsp_mul_u16_u16(rsl, rtl);                           \
+    }                                                                    \
+                                                                         \
+    dotp = (int64_t)tempB - (int64_t)tempA;                              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                      \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);               \
+    dotp = dotp + acc;                                                   \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+                            ((dotp & MIPSDSP_LHI) >> 32);                \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(dotp & MIPSDSP_LLO);  \
+}
+
+MUL_VOID_PH(mulsaq_s_w_ph, 1);
+MUL_VOID_PH(mulsa_w_ph, 0);
+
+#undef MUL_VOID_PH
+
+#if defined(TARGET_MIPS64)
+#define MUL_RETURN64_16_QH(name, func, \
+                           rsmov1, rsmov2, rsmov3, rsmov4, rsfilter, \
+                           rtmov1, rtmov2, rtmov3, rtmov4, rtfilter) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,         \
+                           CPUMIPSState *env)                        \
+{                                                                    \
+    uint16_t rs3, rs2, rs1, rs0;                                     \
+    uint16_t rt3, rt2, rt1, rt0;                                     \
+    uint16_t tempD, tempC, tempB, tempA;                             \
+                                                                     \
+    rs3 = (rs >> rsmov1) & rsfilter;                                 \
+    rs2 = (rs >> rsmov2) & rsfilter;                                 \
+    rs1 = (rs >> rsmov3) & rsfilter;                                 \
+    rs0 = (rs >> rsmov4) & rsfilter;                                 \
+    rt3 = (rt >> rtmov1) & rtfilter;                                 \
+    rt2 = (rt >> rtmov2) & rtfilter;                                 \
+    rt1 = (rt >> rtmov3) & rtfilter;                                 \
+    rt0 = (rt >> rtmov4) & rtfilter;                                 \
+                                                                     \
+    tempD = mipsdsp_##func(rs3, rt3, env);                           \
+    tempC = mipsdsp_##func(rs2, rt2, env);                           \
+    tempB = mipsdsp_##func(rs1, rt1, env);                           \
+    tempA = mipsdsp_##func(rs0, rt0, env);                           \
+                                                                     \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);          \
+}
+
+MUL_RETURN64_16_QH(muleu_s_qh_obl, mul_u8_u16, \
+                   56, 48, 40, 32, MIPSDSP_Q0, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+MUL_RETURN64_16_QH(muleu_s_qh_obr, mul_u8_u16, \
+                   24, 16, 8, 0, MIPSDSP_Q0, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+MUL_RETURN64_16_QH(mulq_rs_qh, rndq15_mul_q15_q15, \
+                   48, 32, 16, 0, MIPSDSP_LO, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+
+#undef MUL_RETURN64_16_QH
+
+#define MUL_RETURN64_32_QH(name, \
+                           rsmov1, rsmov2, \
+                           rtmov1, rtmov2) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                           CPUMIPSState *env)                \
+{                                                            \
+    uint16_t rsB, rsA;                                       \
+    uint16_t rtB, rtA;                                       \
+    uint32_t tempB, tempA;                                   \
+                                                             \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                       \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                       \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                       \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                       \
+                                                             \
+    tempB = mipsdsp_mul_q15_q15(5, rsB, rtB, env);           \
+    tempA = mipsdsp_mul_q15_q15(5, rsA, rtA, env);           \
+                                                             \
+    return ((uint64_t)tempB << 32) | (uint64_t)tempA;        \
+}
+
+MUL_RETURN64_32_QH(muleq_s_pw_qhl, 48, 32, 48, 32);
+MUL_RETURN64_32_QH(muleq_s_pw_qhr, 16, 0, 16, 0);
+
+#undef MUL_RETURN64_32_QH
+
+void helper_mulsaq_s_w_qh(target_ulong rs, target_ulong rt, uint32_t ac,
+                          CPUMIPSState *env)
+{
+    int16_t rs3, rs2, rs1, rs0;
+    int16_t rt3, rt2, rt1, rt0;
+    int32_t tempD, tempC, tempB, tempA;
+    int64_t acc[2];
+    int64_t temp[2];
+    int64_t temp_sum;
+
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);
+
+    tempD = mipsdsp_mul_q15_q15(ac, rs3, rt3, env);
+    tempC = mipsdsp_mul_q15_q15(ac, rs2, rt2, env);
+    tempB = mipsdsp_mul_q15_q15(ac, rs1, rt1, env);
+    tempA = mipsdsp_mul_q15_q15(ac, rs0, rt0, env);
+
+    temp[0] = ((int32_t)tempD - (int32_t)tempC) +
+              ((int32_t)tempB - (int32_t)tempA);
+    temp[0] = (int64_t)(temp[0] << 30) >> 30;
+    if (((temp[0] >> 33) & 0x01) == 0) {
+        temp[1] = 0x00;
+    } else {
+        temp[1] = ~0ull;
+    }
+
+    acc[0] = env->active_tc.LO[ac];
+    acc[1] = env->active_tc.HI[ac];
+
+    temp_sum = acc[0] + temp[0];
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&
+       ((uint64_t)temp_sum < (uint64_t)temp[0])) {
+        acc[1] += 1;
+    }
+    acc[0] = temp_sum;
+    acc[1] += temp[1];
+
+    env->active_tc.HI[ac] = acc[1];
+    env->active_tc.LO[ac] = acc[0];
+}
+#endif
+
+#define DP_QB(name, func, is_add, rsmov1, rsmov2, rtmov1, rtmov2) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                   CPUMIPSState *env)                                    \
+{                                                                        \
+    uint8_t rs3, rs2;                                                    \
+    uint8_t rt3, rt2;                                                    \
+    uint16_t tempB, tempA;                                               \
+    uint64_t tempC, dotp;                                                \
+                                                                         \
+    rs3 = (rs >> rsmov1) & MIPSDSP_Q0;                                   \
+    rs2 = (rs >> rsmov2) & MIPSDSP_Q0;                                   \
+    rt3 = (rt >> rtmov1) & MIPSDSP_Q0;                                   \
+    rt2 = (rt >> rtmov2) & MIPSDSP_Q0;                                   \
+    tempB = mipsdsp_##func(rs3, rt3);                                    \
+    tempA = mipsdsp_##func(rs2, rt2);                                    \
+    dotp = (int64_t)tempB + (int64_t)tempA;                              \
+    if (is_add) {                                                        \
+        tempC = (((uint64_t)env->active_tc.HI[ac] << 32) |               \
+                 ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO))        \
+            + dotp;                                                      \
+    } else {                                                             \
+        tempC = (((uint64_t)env->active_tc.HI[ac] << 32) |               \
+                 ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO))        \
+            - dotp;                                                      \
+    }                                                                    \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+                            ((tempC & MIPSDSP_LHI) >> 32);               \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(tempC & MIPSDSP_LLO); \
+}
+
+DP_QB(dpau_h_qbl, mul_u8_u8, 1, 24, 16, 24, 16);
+DP_QB(dpau_h_qbr, mul_u8_u8, 1, 8, 0, 8, 0);
+DP_QB(dpsu_h_qbl, mul_u8_u8, 0, 24, 16, 24, 16);
+DP_QB(dpsu_h_qbr, mul_u8_u8, 0, 8, 0, 8, 0);
+
+#undef DP_QB
+
+#if defined(TARGET_MIPS64)
+#define DP_OB(name, add_sub, \
+              rsmov1, rsmov2, rsmov3, rsmov4, \
+              rtmov1, rtmov2, rtmov3, rtmov4) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,       \
+                       CPUMIPSState *env)                               \
+{                                                                       \
+    uint8_t rsD, rsC, rsB, rsA;                                         \
+    uint8_t rtD, rtC, rtB, rtA;                                         \
+    uint16_t tempD, tempC, tempB, tempA;                                \
+    uint64_t temp[2];                                                   \
+    uint64_t acc[2];                                                    \
+    uint64_t temp_sum;                                                  \
+                                                                        \
+    temp[0] = 0;                                                        \
+    temp[1] = 0;                                                        \
+                                                                        \
+    rsD = (rs >> rsmov1) & MIPSDSP_Q0;                                  \
+    rsC = (rs >> rsmov2) & MIPSDSP_Q0;                                  \
+    rsB = (rs >> rsmov3) & MIPSDSP_Q0;                                  \
+    rsA = (rs >> rsmov4) & MIPSDSP_Q0;                                  \
+    rtD = (rt >> rtmov1) & MIPSDSP_Q0;                                  \
+    rtC = (rt >> rtmov2) & MIPSDSP_Q0;                                  \
+    rtB = (rt >> rtmov3) & MIPSDSP_Q0;                                  \
+    rtA = (rt >> rtmov4) & MIPSDSP_Q0;                                  \
+                                                                        \
+    tempD = mipsdsp_mul_u8_u8(rsD, rtD);                                \
+    tempC = mipsdsp_mul_u8_u8(rsC, rtC);                                \
+    tempB = mipsdsp_mul_u8_u8(rsB, rtB);                                \
+    tempA = mipsdsp_mul_u8_u8(rsA, rtA);                                \
+                                                                        \
+    temp[0] = (uint64_t)tempD + (uint64_t)tempC +                       \
+      (uint64_t)tempB + (uint64_t)tempA;                                \
+                                                                        \
+    acc[0] = env->active_tc.LO[ac];                                     \
+    acc[1] = env->active_tc.HI[ac];                                     \
+                                                                        \
+    if (add_sub) {                                                      \
+        temp_sum = acc[0] + temp[0];                                    \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                  \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {                 \
+            acc[1] += 1;                                                \
+        }                                                               \
+        temp[0] = temp_sum;                                             \
+        temp[1] = acc[1] + temp[1];                                     \
+    } else {                                                            \
+        temp_sum = acc[0] - temp[0];                                    \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                    \
+            acc[1] -= 1;                                                \
+        }                                                               \
+        temp[0] = temp_sum;                                             \
+        temp[1] = acc[1] - temp[1];                                     \
+    }                                                                   \
+                                                                        \
+    env->active_tc.HI[ac] = temp[1];                                    \
+    env->active_tc.LO[ac] = temp[0];                                    \
+}
+
+DP_OB(dpau_h_obl, 1, 56, 48, 40, 32, 56, 48, 40, 32);
+DP_OB(dpau_h_obr, 1, 24, 16, 8, 0, 24, 16, 8, 0);
+DP_OB(dpsu_h_obl, 0, 56, 48, 40, 32, 56, 48, 40, 32);
+DP_OB(dpsu_h_obr, 0, 24, 16, 8, 0, 24, 16, 8, 0);
+
+#undef DP_OB
+#endif
+
+#define DP_NOFUNC_PH(name, is_add, rsmov1, rsmov2, rtmov1, rtmov2)             \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,              \
+                   CPUMIPSState *env)                                          \
+{                                                                              \
+    int16_t rsB, rsA, rtB, rtA;                                                \
+    int32_t  tempA, tempB;                                                     \
+    int64_t  acc;                                                              \
+                                                                               \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                                         \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                                         \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                                         \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                                         \
+                                                                               \
+    tempB = (int32_t)rsB * (int32_t)rtB;                                       \
+    tempA = (int32_t)rsA * (int32_t)rtA;                                       \
+                                                                               \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                            \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);                     \
+                                                                               \
+    if (is_add) {                                                              \
+        acc = acc + ((int64_t)tempB + (int64_t)tempA);                         \
+    } else {                                                                   \
+        acc = acc - ((int64_t)tempB + (int64_t)tempA);                         \
+    }                                                                          \
+                                                                               \
+    env->active_tc.HI[ac] = (target_long)(int32_t)((acc & MIPSDSP_LHI) >> 32); \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(acc & MIPSDSP_LLO);         \
+}
+
+DP_NOFUNC_PH(dpa_w_ph, 1, 16, 0, 16, 0);
+DP_NOFUNC_PH(dpax_w_ph, 1, 16, 0, 0, 16);
+DP_NOFUNC_PH(dps_w_ph, 0, 16, 0, 16, 0);
+DP_NOFUNC_PH(dpsx_w_ph, 0, 16, 0, 0, 16);
+#undef DP_NOFUNC_PH
+
+#define DP_HASFUNC_PH(name, is_add, rsmov1, rsmov2, rtmov1, rtmov2) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,   \
+                   CPUMIPSState *env)                      \
+{                                                          \
+    int16_t rsB, rsA, rtB, rtA;                            \
+    int32_t tempB, tempA;                                  \
+    int64_t acc, dotp;                                     \
+                                                           \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                     \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                     \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                     \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                     \
+                                                           \
+    tempB = mipsdsp_mul_q15_q15(ac, rsB, rtB, env);        \
+    tempA = mipsdsp_mul_q15_q15(ac, rsA, rtA, env);        \
+                                                           \
+    dotp = (int64_t)tempB + (int64_t)tempA;                \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |        \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO); \
+                                                           \
+    if (is_add) {                                          \
+        acc = acc + dotp;                                  \
+    } else {                                               \
+        acc = acc - dotp;                                  \
+    }                                                      \
+                                                           \
+    env->active_tc.HI[ac] = (target_long)(int32_t)         \
+        ((acc & MIPSDSP_LHI) >> 32);                       \
+    env->active_tc.LO[ac] = (target_long)(int32_t)         \
+        (acc & MIPSDSP_LLO);                               \
+}
+
+DP_HASFUNC_PH(dpaq_s_w_ph, 1, 16, 0, 16, 0);
+DP_HASFUNC_PH(dpaqx_s_w_ph, 1, 16, 0, 0, 16);
+DP_HASFUNC_PH(dpsq_s_w_ph, 0, 16, 0, 16, 0);
+DP_HASFUNC_PH(dpsqx_s_w_ph, 0, 16, 0, 0, 16);
+
+#undef DP_HASFUNC_PH
+
+#define DP_128OPERATION_PH(name, is_add) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt, \
+                          CPUMIPSState *env)                             \
+{                                                                        \
+    int16_t rsh, rsl, rth, rtl;                                          \
+    int32_t tempB, tempA, tempC62_31, tempC63;                           \
+    int64_t acc, dotp, tempC;                                            \
+                                                                         \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                                    \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                    \
+                                                                         \
+    tempB = mipsdsp_mul_q15_q15(ac, rsh, rtl, env);                      \
+    tempA = mipsdsp_mul_q15_q15(ac, rsl, rth, env);                      \
+                                                                         \
+    dotp = (int64_t)tempB + (int64_t)tempA;                              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                      \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);               \
+    if (is_add) {                                                        \
+        tempC = acc + dotp;                                              \
+    } else {                                                             \
+        tempC = acc - dotp;                                              \
+    }                                                                    \
+    tempC63 = (tempC >> 63) & 0x01;                                      \
+    tempC62_31 = (tempC >> 31) & 0xFFFFFFFF;                             \
+                                                                         \
+    if ((tempC63 == 0) && (tempC62_31 != 0x00000000)) {                  \
+        tempC = 0x7FFFFFFF;                                              \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                   \
+    }                                                                    \
+                                                                         \
+    if ((tempC63 == 1) && (tempC62_31 != 0xFFFFFFFF)) {                  \
+        tempC = (int64_t)(int32_t)0x80000000;                            \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                   \
+    }                                                                    \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+        ((tempC & MIPSDSP_LHI) >> 32);                                   \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                       \
+        (tempC & MIPSDSP_LLO);                                           \
+}
+
+DP_128OPERATION_PH(dpaqx_sa_w_ph, 1);
+DP_128OPERATION_PH(dpsqx_sa_w_ph, 0);
+
+#undef DP_128OPERATION_HP
+
+#if defined(TARGET_MIPS64)
+#define DP_QH(name, is_add, use_ac_env) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,    \
+                   CPUMIPSState *env)                                \
+{                                                                    \
+    int32_t rs3, rs2, rs1, rs0;                                      \
+    int32_t rt3, rt2, rt1, rt0;                                      \
+    int32_t tempD, tempC, tempB, tempA;                              \
+    int64_t acc[2];                                                  \
+    int64_t temp[2];                                                 \
+    int64_t temp_sum;                                                \
+                                                                     \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);                      \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                      \
+                                                                     \
+    if (use_ac_env) {                                                \
+        tempD = mipsdsp_mul_q15_q15(ac, rs3, rt3, env);              \
+        tempC = mipsdsp_mul_q15_q15(ac, rs2, rt2, env);              \
+        tempB = mipsdsp_mul_q15_q15(ac, rs1, rt1, env);              \
+        tempA = mipsdsp_mul_q15_q15(ac, rs0, rt0, env);              \
+    } else {                                                         \
+        tempD = mipsdsp_mul_u16_u16(rs3, rt3);                       \
+        tempC = mipsdsp_mul_u16_u16(rs2, rt2);                       \
+        tempB = mipsdsp_mul_u16_u16(rs1, rt1);                       \
+        tempA = mipsdsp_mul_u16_u16(rs0, rt0);                       \
+    }                                                                \
+                                                                     \
+    temp[0] = (int64_t)tempD + (int64_t)tempC +                      \
+              (int64_t)tempB + (int64_t)tempA;                       \
+                                                                     \
+    if (temp[0] >= 0) {                                              \
+        temp[1] = 0;                                                 \
+    } else {                                                         \
+        temp[1] = ~0ull;                                             \
+    }                                                                \
+                                                                     \
+    acc[1] = env->active_tc.HI[ac];                                  \
+    acc[0] = env->active_tc.LO[ac];                                  \
+                                                                     \
+    if (is_add) {                                                    \
+        temp_sum = acc[0] + temp[0];                                 \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&               \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {              \
+            acc[1] = acc[1] + 1;                                     \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] + temp[1];                                  \
+    } else {                                                         \
+        temp_sum = acc[0] - temp[0];                                 \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                 \
+            acc[1] = acc[1] - 1;                                     \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] - temp[1];                                  \
+    }                                                                \
+                                                                     \
+    env->active_tc.HI[ac] = temp[1];                                 \
+    env->active_tc.LO[ac] = temp[0];                                 \
+}
+
+DP_QH(dpa_w_qh, 1, 0);
+DP_QH(dpaq_s_w_qh, 1, 1);
+DP_QH(dps_w_qh, 0, 0);
+DP_QH(dpsq_s_w_qh, 0, 1);
+
+#undef DP_QH
+
+#endif
+
+#define DP_L_W(name, is_add) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,     \
+                   CPUMIPSState *env)                                 \
+{                                                                     \
+    int32_t temp63;                                                   \
+    int64_t dotp, acc;                                                \
+    uint64_t temp;                                                    \
+                                                                      \
+    dotp = mipsdsp_mul_q31_q31(ac, rs, rt, env);                      \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                   \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);            \
+    if (!is_add) {                                                    \
+        dotp = -dotp;                                                 \
+    }                                                                 \
+                                                                      \
+    temp = acc + dotp;                                                \
+    if (MIPSDSP_OVERFLOW((uint64_t)acc, (uint64_t)dotp, temp,         \
+                         (0x01ull << 63))) {                          \
+        temp63 = (temp >> 63) & 0x01;                                 \
+        if (temp63 == 1) {                                            \
+            temp = (0x01ull << 63) - 1;                               \
+        } else {                                                      \
+            temp = 0x01ull << 63;                                     \
+        }                                                             \
+                                                                      \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                \
+    }                                                                 \
+                                                                      \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                    \
+        ((temp & MIPSDSP_LHI) >> 32);                                 \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                    \
+        (temp & MIPSDSP_LLO);                                         \
+}
+
+DP_L_W(dpaq_sa_l_w, 1);
+DP_L_W(dpsq_sa_l_w, 0);
+
+#undef DP_L_W
+
+#if defined(TARGET_MIPS64)
+#define DP_L_PW(name, func) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int32_t rs1, rs0;                                             \
+    int32_t rt1, rt0;                                             \
+    int64_t tempB[2], tempA[2];                                   \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                             \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    tempB[0] = mipsdsp_mul_q31_q31(ac, rs1, rt1, env);            \
+    tempA[0] = mipsdsp_mul_q31_q31(ac, rs0, rt0, env);            \
+                                                                  \
+    if (tempB[0] >= 0) {                                          \
+        tempB[1] = 0x00;                                          \
+    } else {                                                      \
+        tempB[1] = ~0ull;                                         \
+    }                                                             \
+                                                                  \
+    if (tempA[0] >= 0) {                                          \
+        tempA[1] = 0x00;                                          \
+    } else {                                                      \
+        tempA[1] = ~0ull;                                         \
+    }                                                             \
+                                                                  \
+    temp_sum = tempB[0] + tempA[0];                               \
+    if (((uint64_t)temp_sum < (uint64_t)tempB[0]) &&              \
+        ((uint64_t)temp_sum < (uint64_t)tempA[0])) {              \
+        temp[1] += 1;                                             \
+    }                                                             \
+    temp[0] = temp_sum;                                           \
+    temp[1] += tempB[1] + tempA[1];                               \
+                                                                  \
+    mipsdsp_##func(acc, ac, temp, env);                           \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+DP_L_PW(dpaq_sa_l_pw, sat64_acc_add_q63);
+DP_L_PW(dpsq_sa_l_pw, sat64_acc_sub_q63);
+
+#undef DP_L_PW
+
+void helper_mulsaq_s_l_pw(target_ulong rs, target_ulong rt, uint32_t ac,
+                          CPUMIPSState *env)
+{
+    int32_t rs1, rs0;
+    int32_t rt1, rt0;
+    int64_t tempB[2], tempA[2];
+    int64_t temp[2];
+    int64_t acc[2];
+    int64_t temp_sum;
+
+    rs1 = (rs >> 32) & MIPSDSP_LLO;
+    rs0 = rs & MIPSDSP_LLO;
+    rt1 = (rt >> 32) & MIPSDSP_LLO;
+    rt0 = rt & MIPSDSP_LLO;
+
+    tempB[0] = mipsdsp_mul_q31_q31(ac, rs1, rt1, env);
+    tempA[0] = mipsdsp_mul_q31_q31(ac, rs0, rt0, env);
+
+    if (tempB[0] >= 0) {
+        tempB[1] = 0x00;
+    } else {
+        tempB[1] = ~0ull;
+    }
+
+    if (tempA[0] >= 0) {
+        tempA[1] = 0x00;
+    } else {
+        tempA[1] = ~0ull;
+    }
+
+    acc[0] = env->active_tc.LO[ac];
+    acc[1] = env->active_tc.HI[ac];
+
+    temp_sum = tempB[0] - tempA[0];
+    if ((uint64_t)temp_sum > (uint64_t)tempB[0]) {
+        tempB[1] -= 1;
+    }
+    temp[0] = temp_sum;
+    temp[1] = tempB[1] - tempA[1];
+
+    if ((temp[1] & 0x01) == 0) {
+        temp[1] = 0x00;
+    } else {
+        temp[1] = ~0ull;
+    }
+
+    temp_sum = acc[0] + temp[0];
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&
+       ((uint64_t)temp_sum < (uint64_t)temp[0])) {
+        acc[1] += 1;
+    }
+    acc[0] = temp_sum;
+    acc[1] += temp[1];
+
+    env->active_tc.HI[ac] = acc[1];
+    env->active_tc.LO[ac] = acc[0];
+}
+#endif
+
+#define MAQ_S_W(name, mov) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rsh, rth;                                             \
+    int32_t tempA;                                                \
+    int64_t tempL, acc;                                           \
+                                                                  \
+    rsh = (rs >> mov) & MIPSDSP_LO;                               \
+    rth = (rt >> mov) & MIPSDSP_LO;                               \
+    tempA  = mipsdsp_mul_q15_q15(ac, rsh, rth, env);              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |               \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);        \
+    tempL  = (int64_t)tempA + acc;                                \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                \
+        ((tempL & MIPSDSP_LHI) >> 32);                            \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                \
+        (tempL & MIPSDSP_LLO);                                    \
+}
+
+MAQ_S_W(maq_s_w_phl, 16);
+MAQ_S_W(maq_s_w_phr, 0);
+
+#undef MAQ_S_W
+
+#define MAQ_SA_W(name, mov) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                   CPUMIPSState *env)                                    \
+{                                                                        \
+    int16_t rsh, rth;                                                    \
+    int32_t tempA;                                                       \
+                                                                         \
+    rsh = (rs >> mov) & MIPSDSP_LO;                                      \
+    rth = (rt >> mov) & MIPSDSP_LO;                                      \
+    tempA = mipsdsp_mul_q15_q15(ac, rsh, rth, env);                      \
+    tempA = mipsdsp_sat32_acc_q31(ac, tempA, env);                       \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)(((int64_t)tempA &     \
+                                                    MIPSDSP_LHI) >> 32); \
+    env->active_tc.LO[ac] = (target_long)(int32_t)((int64_t)tempA &      \
+                                                   MIPSDSP_LLO);         \
+}
+
+MAQ_SA_W(maq_sa_w_phl, 16);
+MAQ_SA_W(maq_sa_w_phr, 0);
+
+#undef MAQ_SA_W
+
+#define MULQ_W(name, addvar) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,   \
+                           CPUMIPSState *env)                  \
+{                                                              \
+    uint32_t rs_t, rt_t;                                       \
+    int32_t tempI;                                             \
+    int64_t tempL;                                             \
+                                                               \
+    rs_t = rs & MIPSDSP_LLO;                                   \
+    rt_t = rt & MIPSDSP_LLO;                                   \
+                                                               \
+    if ((rs_t == 0x80000000) && (rt_t == 0x80000000)) {        \
+        tempL = 0x7FFFFFFF00000000ull;                         \
+        set_DSPControl_overflow_flag(1, 21, env);              \
+    } else {                                                   \
+        tempL  = ((int64_t)rs_t * (int64_t)rt_t) << 1;         \
+        tempL += addvar;                                       \
+    }                                                          \
+    tempI = (tempL & MIPSDSP_LHI) >> 32;                       \
+                                                               \
+    return (target_long)(int32_t)tempI;                        \
+}
+
+MULQ_W(mulq_s_w, 0);
+MULQ_W(mulq_rs_w, 0x80000000ull);
+
+#undef MULQ_W
+
+#if defined(TARGET_MIPS64)
+
+#define MAQ_S_W_QH(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rs_t, rt_t;                                           \
+    int32_t temp_mul;                                             \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LO;                              \
+    rt_t = (rt >> mov) & MIPSDSP_LO;                              \
+    temp_mul = mipsdsp_mul_q15_q15(ac, rs_t, rt_t, env);          \
+                                                                  \
+    temp[0] = (int64_t)temp_mul;                                  \
+    if (temp[0] >= 0) {                                           \
+        temp[1] = 0x00;                                           \
+    } else {                                                      \
+        temp[1] = ~0ull;                                          \
+    }                                                             \
+                                                                  \
+    acc[0] = env->active_tc.LO[ac];                               \
+    acc[1] = env->active_tc.HI[ac];                               \
+                                                                  \
+    temp_sum = acc[0] + temp[0];                                  \
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)temp[0])) {               \
+        acc[1] += 1;                                              \
+    }                                                             \
+    acc[0] = temp_sum;                                            \
+    acc[1] += temp[1];                                            \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_S_W_QH(maq_s_w_qhll, 48);
+MAQ_S_W_QH(maq_s_w_qhlr, 32);
+MAQ_S_W_QH(maq_s_w_qhrl, 16);
+MAQ_S_W_QH(maq_s_w_qhrr, 0);
+
+#undef MAQ_S_W_QH
+
+#define MAQ_SA_W(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rs_t, rt_t;                                           \
+    int32_t temp;                                                 \
+    int64_t acc[2];                                               \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LO;                              \
+    rt_t = (rt >> mov) & MIPSDSP_LO;                              \
+    temp = mipsdsp_mul_q15_q15(ac, rs_t, rt_t, env);              \
+    temp = mipsdsp_sat32_acc_q31(ac, temp, env);                  \
+                                                                  \
+    acc[0] = (int64_t)(int32_t)temp;                              \
+    if (acc[0] >= 0) {                                            \
+        acc[1] = 0x00;                                            \
+    } else {                                                      \
+        acc[1] = ~0ull;                                           \
+    }                                                             \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_SA_W(maq_sa_w_qhll, 48);
+MAQ_SA_W(maq_sa_w_qhlr, 32);
+MAQ_SA_W(maq_sa_w_qhrl, 16);
+MAQ_SA_W(maq_sa_w_qhrr, 0);
+
+#undef MAQ_SA_W
+
+#define MAQ_S_L_PW(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int32_t rs_t, rt_t;                                           \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LLO;                             \
+    rt_t = (rt >> mov) & MIPSDSP_LLO;                             \
+                                                                  \
+    temp[0] = mipsdsp_mul_q31_q31(ac, rs_t, rt_t, env);           \
+    if (temp[0] >= 0) {                                           \
+        temp[1] = 0x00;                                           \
+    } else {                                                      \
+        temp[1] = ~0ull;                                          \
+    }                                                             \
+                                                                  \
+    acc[0] = env->active_tc.LO[ac];                               \
+    acc[1] = env->active_tc.HI[ac];                               \
+                                                                  \
+    temp_sum = acc[0] + temp[0];                                  \
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)temp[0])) {               \
+        acc[1] += 1;                                              \
+    }                                                             \
+    acc[0] = temp_sum;                                            \
+    acc[1] += temp[1];                                            \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_S_L_PW(maq_s_l_pwl, 32);
+MAQ_S_L_PW(maq_s_l_pwr, 0);
+
+#undef MAQ_S_L_PW
+
+#define DM_OPERATE(name, func, is_add, sigext) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,    \
+                  CPUMIPSState *env)                                 \
+{                                                                    \
+    int32_t rs1, rs0;                                                \
+    int32_t rt1, rt0;                                                \
+    int64_t tempBL[2], tempAL[2];                                    \
+    int64_t acc[2];                                                  \
+    int64_t temp[2];                                                 \
+    int64_t temp_sum;                                                \
+                                                                     \
+    temp[0] = 0x00;                                                  \
+    temp[1] = 0x00;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                                \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                                \
+                                                                     \
+    if (sigext) {                                                    \
+        tempBL[0] = (int64_t)mipsdsp_##func(rs1, rt1);               \
+        tempAL[0] = (int64_t)mipsdsp_##func(rs0, rt0);               \
+                                                                     \
+        if (tempBL[0] >= 0) {                                        \
+            tempBL[1] = 0x0;                                         \
+        } else {                                                     \
+            tempBL[1] = ~0ull;                                       \
+        }                                                            \
+                                                                     \
+        if (tempAL[0] >= 0) {                                        \
+            tempAL[1] = 0x0;                                         \
+        } else {                                                     \
+            tempAL[1] = ~0ull;                                       \
+        }                                                            \
+    } else {                                                         \
+        tempBL[0] = mipsdsp_##func(rs1, rt1);                        \
+        tempAL[0] = mipsdsp_##func(rs0, rt0);                        \
+        tempBL[1] = 0;                                               \
+        tempAL[1] = 0;                                               \
+    }                                                                \
+                                                                     \
+    acc[1] = env->active_tc.HI[ac];                                  \
+    acc[0] = env->active_tc.LO[ac];                                  \
+                                                                     \
+    temp_sum = tempBL[0] + tempAL[0];                                \
+    if (((uint64_t)temp_sum < (uint64_t)tempBL[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)tempAL[0])) {                \
+        temp[1] += 1;                                                \
+    }                                                                \
+    temp[0] = temp_sum;                                              \
+    temp[1] += tempBL[1] + tempAL[1];                                \
+                                                                     \
+    if (is_add) {                                                    \
+        temp_sum = acc[0] + temp[0];                                 \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&               \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {              \
+            acc[1] += 1;                                             \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] + temp[1];                                  \
+    } else {                                                         \
+        temp_sum = acc[0] - temp[0];                                 \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                 \
+            acc[1] -= 1;                                             \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] - temp[1];                                  \
+    }                                                                \
+                                                                     \
+    env->active_tc.HI[ac] = temp[1];                                 \
+    env->active_tc.LO[ac] = temp[0];                                 \
+}
+
+DM_OPERATE(dmadd, mul_i32_i32, 1, 1);
+DM_OPERATE(dmaddu, mul_u32_u32, 1, 0);
+DM_OPERATE(dmsub, mul_i32_i32, 0, 1);
+DM_OPERATE(dmsubu, mul_u32_u32, 0, 0);
+#undef DM_OPERATE
+#endif
+
+/** DSP Bit/Manipulation Sub-class insns **/
+target_ulong helper_bitrev(target_ulong rt)
+{
+    int32_t temp;
+    uint32_t rd;
+    int i;
+
+    temp = rt & MIPSDSP_LO;
+    rd = 0;
+    for (i = 0; i < 16; i++) {
+        rd = (rd << 1) | (temp & 1);
+        temp = temp >> 1;
+    }
+
+    return (target_ulong)rd;
+}
+
+#define BIT_INSV(name, posfilter, sizefilter, ret_type)         \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong rs,  \
+                           target_ulong rt)                     \
+{                                                               \
+    uint32_t pos, size, msb, lsb;                               \
+    target_ulong filter;                                        \
+    target_ulong temp, temprs, temprt;                          \
+    target_ulong dspc;                                          \
+                                                                \
+    dspc = env->active_tc.DSPControl;                           \
+                                                                \
+    pos  = dspc & posfilter;                                    \
+    size = (dspc >> 7) & sizefilter;                            \
+                                                                \
+    msb  = pos + size - 1;                                      \
+    lsb  = pos;                                                 \
+                                                                \
+    if (lsb > msb || (msb > TARGET_LONG_BITS)) {                \
+        return rt;                                              \
+    }                                                           \
+                                                                \
+    filter = ((int32_t)0x01 << size) - 1;                       \
+    filter = filter << pos;                                     \
+    temprs = (rs << pos) & filter;                              \
+    temprt = rt & ~filter;                                      \
+    temp = temprs | temprt;                                     \
+                                                                \
+    return (target_long)(ret_type)temp;                         \
+}
+
+BIT_INSV(insv, 0x1F, 0x1F, int32_t);
+#ifdef TARGET_MIPS64
+BIT_INSV(dinsv, 0x7F, 0x3F, target_long);
+#endif
+
+#undef BIT_INSV
+
+
+/** DSP Compare-Pick Sub-class insns **/
+#define CMP_HAS_RET(name, func, split_num, filter, bit_size) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt) \
+{                                                       \
+    uint32_t rs_t, rt_t;                                \
+    uint8_t cc;                                         \
+    uint32_t temp = 0;                                  \
+    int i;                                              \
+                                                        \
+    for (i = 0; i < split_num; i++) {                   \
+        rs_t = (rs >> (bit_size * i)) & filter;         \
+        rt_t = (rt >> (bit_size * i)) & filter;         \
+        cc = mipsdsp_##func(rs_t, rt_t);                \
+        temp |= cc << i;                                \
+    }                                                   \
+                                                        \
+    return (target_ulong)temp;                          \
+}
+
+CMP_HAS_RET(cmpgu_eq_qb, cmpu_eq, 4, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_lt_qb, cmpu_lt, 4, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_le_qb, cmpu_le, 4, MIPSDSP_Q0, 8);
+
+#ifdef TARGET_MIPS64
+CMP_HAS_RET(cmpgu_eq_ob, cmpu_eq, 8, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_lt_ob, cmpu_lt, 8, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_le_ob, cmpu_le, 8, MIPSDSP_Q0, 8);
+#endif
+
+#undef CMP_HAS_RET
+
+
+#define CMP_NO_RET(name, func, split_num, filter, bit_size) \
+void helper_##name(target_ulong rs, target_ulong rt,        \
+                            CPUMIPSState *env)              \
+{                                                           \
+    int##bit_size##_t rs_t, rt_t;                           \
+    int##bit_size##_t flag = 0;                             \
+    int##bit_size##_t cc;                                   \
+    int i;                                                  \
+                                                            \
+    for (i = 0; i < split_num; i++) {                       \
+        rs_t = (rs >> (bit_size * i)) & filter;             \
+        rt_t = (rt >> (bit_size * i)) & filter;             \
+                                                            \
+        cc = mipsdsp_##func((int32_t)rs_t, (int32_t)rt_t);  \
+        flag |= cc << i;                                    \
+    }                                                       \
+                                                            \
+    set_DSPControl_24(flag, split_num, env);                \
+}
+
+CMP_NO_RET(cmpu_eq_qb, cmpu_eq, 4, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_lt_qb, cmpu_lt, 4, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_le_qb, cmpu_le, 4, MIPSDSP_Q0, 8);
+
+CMP_NO_RET(cmp_eq_ph, cmp_eq, 2, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_lt_ph, cmp_lt, 2, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_le_ph, cmp_le, 2, MIPSDSP_LO, 16);
+
+#ifdef TARGET_MIPS64
+CMP_NO_RET(cmpu_eq_ob, cmpu_eq, 8, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_lt_ob, cmpu_lt, 8, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_le_ob, cmpu_le, 8, MIPSDSP_Q0, 8);
+
+CMP_NO_RET(cmp_eq_qh, cmp_eq, 4, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_lt_qh, cmp_lt, 4, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_le_qh, cmp_le, 4, MIPSDSP_LO, 16);
+
+CMP_NO_RET(cmp_eq_pw, cmp_eq, 2, MIPSDSP_LLO, 32);
+CMP_NO_RET(cmp_lt_pw, cmp_lt, 2, MIPSDSP_LLO, 32);
+CMP_NO_RET(cmp_le_pw, cmp_le, 2, MIPSDSP_LLO, 32);
+#endif
+#undef CMP_NO_RET
+
+#if defined(TARGET_MIPS64)
+
+#define CMPGDU_OB(name) \
+target_ulong helper_cmpgdu_##name##_ob(target_ulong rs, target_ulong rt, \
+                                       CPUMIPSState *env)  \
+{                                                     \
+    int i;                                            \
+    uint8_t rs_t, rt_t;                               \
+    uint32_t cond;                                    \
+                                                      \
+    cond = 0;                                         \
+                                                      \
+    for (i = 0; i < 8; i++) {                         \
+        rs_t = (rs >> (8 * i)) & MIPSDSP_Q0;          \
+        rt_t = (rt >> (8 * i)) & MIPSDSP_Q0;          \
+                                                      \
+        if (mipsdsp_cmpu_##name(rs_t, rt_t)) {        \
+            cond |= 0x01 << i;                        \
+        }                                             \
+    }                                                 \
+                                                      \
+    set_DSPControl_24(cond, 8, env);                  \
+                                                      \
+    return (uint64_t)cond;                            \
+}
+
+CMPGDU_OB(eq)
+CMPGDU_OB(lt)
+CMPGDU_OB(le)
+#undef CMPGDU_OB
+#endif
+
+#define PICK_INSN(name, split_num, filter, bit_size, ret32bit) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,   \
+                            CPUMIPSState *env)                 \
+{                                                              \
+    uint32_t rs_t, rt_t;                                       \
+    uint32_t cc;                                               \
+    target_ulong dsp;                                          \
+    int i;                                                     \
+    target_ulong result = 0;                                   \
+                                                               \
+    dsp = env->active_tc.DSPControl;                           \
+    for (i = 0; i < split_num; i++) {                          \
+        rs_t = (rs >> (bit_size * i)) & filter;                \
+        rt_t = (rt >> (bit_size * i)) & filter;                \
+        cc = (dsp >> (24 + i)) & 0x01;                         \
+        cc = cc == 1 ? rs_t : rt_t;                            \
+                                                               \
+        result |= (target_ulong)cc << (bit_size * i);          \
+    }                                                          \
+                                                               \
+    if (ret32bit) {                                            \
+        result = (target_long)(int32_t)(result & MIPSDSP_LLO); \
+    }                                                          \
+                                                               \
+    return result;                                             \
+}
+
+PICK_INSN(pick_qb, 4, MIPSDSP_Q0, 8, 1);
+PICK_INSN(pick_ph, 2, MIPSDSP_LO, 16, 1);
+
+#ifdef TARGET_MIPS64
+PICK_INSN(pick_ob, 8, MIPSDSP_Q0, 8, 0);
+PICK_INSN(pick_qh, 4, MIPSDSP_LO, 16, 0);
+PICK_INSN(pick_pw, 2, MIPSDSP_LLO, 32, 0);
+#endif
+#undef PICK_INSN
+
+#define APPEND_INSN(name, ret_32) \
+target_ulong helper_##name(target_ulong rt, target_ulong rs, uint32_t sa) \
+{                                                                         \
+    target_ulong temp;                                                    \
+                                                                          \
+    if (ret_32) {                                                         \
+        temp = ((rt & MIPSDSP_LLO) << sa) |                               \
+               ((rs & MIPSDSP_LLO) & ((0x01 << sa) - 1));                 \
+        temp = (target_long)(int32_t)(temp & MIPSDSP_LLO);                \
+    } else {                                                              \
+        temp = (rt << sa) | (rs & ((0x01 << sa) - 1));                    \
+    }                                                                     \
+                                                                          \
+    return temp;                                                          \
+}
+
+APPEND_INSN(append, 1);
+#ifdef TARGET_MIPS64
+APPEND_INSN(dappend, 0);
+#endif
+#undef APPEND_INSN
+
+#define PREPEND_INSN(name, or_val, ret_32)                    \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,  \
+                           uint32_t sa)                       \
+{                                                             \
+    sa |= or_val;                                             \
+                                                              \
+    if (1) {                                                  \
+        return (target_long)(int32_t)(uint32_t)               \
+            (((rs & MIPSDSP_LLO) << (32 - sa)) |              \
+             ((rt & MIPSDSP_LLO) >> sa));                     \
+    } else {                                                  \
+        return (rs << (64 - sa)) | (rt >> sa);                \
+    }                                                         \
+}
+
+PREPEND_INSN(prepend, 0, 1);
+#ifdef TARGET_MIPS64
+PREPEND_INSN(prependw, 0, 0);
+PREPEND_INSN(prependd, 0x20, 0);
+#endif
+#undef PREPEND_INSN
+
+#define BALIGN_INSN(name, filter, ret32) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, uint32_t bp) \
+{                                                                         \
+    bp = bp & 0x03;                                                       \
+                                                                          \
+    if ((bp & 1) == 0) {                                                  \
+        return rt;                                                        \
+    } else {                                                              \
+        if (ret32) {                                                      \
+            return (target_long)(int32_t)((rt << (8 * bp)) |              \
+                                          (rs >> (8 * (4 - bp))));        \
+        } else {                                                          \
+            return (rt << (8 * bp)) | (rs >> (8 * (8 - bp)));             \
+        }                                                                 \
+    }                                                                     \
+}
+
+BALIGN_INSN(balign, 0x03, 1);
+#if defined(TARGET_MIPS64)
+BALIGN_INSN(dbalign, 0x07, 0);
+#endif
+#undef BALIGN_INSN
+
+target_ulong helper_packrl_ph(target_ulong rs, target_ulong rt)
+{
+    uint32_t rsl, rth;
+
+    rsl =  rs & MIPSDSP_LO;
+    rth = (rt & MIPSDSP_HI) >> 16;
+
+    return (target_long)(int32_t)((rsl << 16) | rth);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_packrl_pw(target_ulong rs, target_ulong rt)
+{
+    uint32_t rs0, rt1;
+
+    rs0 = rs & MIPSDSP_LLO;
+    rt1 = (rt >> 32) & MIPSDSP_LLO;
+
+    return ((uint64_t)rs0 << 32) | (uint64_t)rt1;
+}
+#endif
+
+/** DSP Accumulator and DSPControl Access Sub-class insns **/
+target_ulong helper_extr_w(target_ulong ac, target_ulong shift,
+                           CPUMIPSState *env)
+{
+    int32_t tempI;
+    int64_t tempDL[2];
+
+    shift = shift & 0x1F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    tempI = (tempDL[0] >> 1) & MIPSDSP_LLO;
+
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+
+    if ((!(tempDL[1] == 0 && (tempDL[0] & MIPSDSP_LHI) == 0x00)) &&
+        (!(tempDL[1] == 1 && (tempDL[0] & MIPSDSP_LHI) == MIPSDSP_LHI))) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)tempI;
+}
+
+target_ulong helper_extr_r_w(target_ulong ac, target_ulong shift,
+                             CPUMIPSState *env)
+{
+    int64_t tempDL[2];
+
+    shift = shift & 0x1F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 && (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)(int32_t)(tempDL[0] >> 1);
+}
+
+target_ulong helper_extr_rs_w(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    int32_t tempI, temp64;
+    int64_t tempDL[2];
+
+    shift = shift & 0x1F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+    tempI = tempDL[0] >> 1;
+
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        temp64 = tempDL[1];
+        if (temp64 == 0) {
+            tempI = 0x7FFFFFFF;
+        } else {
+            tempI = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)tempI;
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextr_w(target_ulong ac, target_ulong shift,
+                            CPUMIPSState *env)
+{
+    uint64_t temp[3];
+
+    shift = shift & 0x3F;
+
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_r_w(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_rs_w(target_ulong ac, target_ulong shift,
+                               CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        if (temp128 == 0) {
+            temp[0] = 0x0FFFFFFFF;
+        } else {
+            temp[0] = 0x0100000000ULL;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_l(target_ulong ac, target_ulong shift,
+                            CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    target_ulong result;
+
+    shift = shift & 0x3F;
+
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+
+target_ulong helper_dextr_r_l(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+    target_ulong result;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+
+target_ulong helper_dextr_rs_l(target_ulong ac, target_ulong shift,
+                               CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+    target_ulong result;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        if (temp128 == 0) {
+            temp[1] &= ~0x00ull - 1;
+            temp[0] |= ~0x00ull - 1;
+        } else {
+            temp[1] |= 0x01;
+            temp[0] &= 0x01;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+#endif
+
+target_ulong helper_extr_s_h(target_ulong ac, target_ulong shift,
+                             CPUMIPSState *env)
+{
+    int64_t temp, acc;
+
+    shift = shift & 0x1F;
+
+    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
+          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
+
+    temp = acc >> shift;
+
+    if (temp > (int64_t)0x7FFF) {
+        temp = 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 23, env);
+    } else if (temp < (int64_t)0xFFFFFFFFFFFF8000ULL) {
+        temp = 0xFFFF8000;
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)(int32_t)(temp & 0xFFFFFFFF);
+}
+
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextr_s_h(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    int64_t temp[2];
+    uint32_t temp127;
+
+    shift = shift & 0x1F;
+
+    mipsdsp_rashift_acc((uint64_t *)temp, ac, shift, env);
+
+    temp127 = (temp[1] >> 63) & 0x01;
+
+    if ((temp127 == 0) && (temp[1] > 0 || temp[0] > 32767)) {
+        temp[0] &= 0xFFFF0000;
+        temp[0] |= 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 23, env);
+    } else if ((temp127 == 1) &&
+            (temp[1] < 0xFFFFFFFFFFFFFFFFll
+             || temp[0] < 0xFFFFFFFFFFFF1000ll)) {
+        temp[0] &= 0xFFFF0000;
+        temp[0] |= 0x00008000;
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int16_t)(temp[0] & MIPSDSP_LO);
+}
+
+#endif
+
+target_ulong helper_extp(target_ulong ac, target_ulong size, CPUMIPSState *env)
+{
+    int32_t start_pos;
+    int sub;
+    uint32_t temp;
+    uint64_t acc;
+
+    size = size & 0x1F;
+
+    temp = 0;
+    start_pos = get_DSPControl_pos(env);
+    sub = start_pos - (size + 1);
+    if (sub >= -1) {
+        acc = ((uint64_t)env->active_tc.HI[ac] << 32) |
+              ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+        temp = (acc >> (start_pos - size)) &
+               (((uint32_t)0x01 << (size + 1)) - 1);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return (target_ulong)temp;
+}
+
+target_ulong helper_extpdp(target_ulong ac, target_ulong size,
+                           CPUMIPSState *env)
+{
+    int32_t start_pos;
+    int sub;
+    uint32_t temp;
+    uint64_t acc;
+
+    size = size & 0x1F;
+    temp = 0;
+    start_pos = get_DSPControl_pos(env);
+    sub = start_pos - (size + 1);
+    if (sub >= -1) {
+        acc  = ((uint64_t)env->active_tc.HI[ac] << 32) |
+               ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+        temp = (acc >> (start_pos - size)) &
+               (((uint32_t)0x01 << (size + 1)) - 1);
+
+        set_DSPControl_pos(start_pos - (size + 1), env);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return (target_ulong)temp;
+}
+
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextp(target_ulong ac, target_ulong size, CPUMIPSState *env)
+{
+    int start_pos;
+    int len;
+    int sub;
+    uint64_t tempB, tempA;
+    uint64_t temp;
+
+    temp = 0;
+
+    size = size & 0x3F;
+    start_pos = get_DSPControl_pos(env);
+    len = start_pos - size;
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    sub = start_pos - (size + 1);
+
+    if (sub >= -1) {
+        temp = (tempB << (64 - len)) | (tempA >> len);
+        temp = temp & ((0x01 << (size + 1)) - 1);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return temp;
+}
+
+target_ulong helper_dextpdp(target_ulong ac, target_ulong size,
+                            CPUMIPSState *env)
+{
+    int start_pos;
+    int len;
+    int sub;
+    uint64_t tempB, tempA;
+    uint64_t temp;
+
+    temp = 0;
+    size = size & 0x3F;
+    start_pos = get_DSPControl_pos(env);
+    len = start_pos - size;
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    sub = start_pos - (size + 1);
+
+    if (sub >= -1) {
+        temp = (tempB << (64 - len)) | (tempA >> len);
+        temp = temp & ((0x01 << (size + 1)) - 1);
+        set_DSPControl_pos(sub, env);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return temp;
+}
+
+#endif
+
+void helper_shilo(target_ulong ac, target_ulong rs, CPUMIPSState *env)
+{
+    int8_t  rs5_0;
+    uint64_t temp, acc;
+
+    rs5_0 = rs & 0x3F;
+    rs5_0 = (int8_t)(rs5_0 << 2) >> 2;
+
+    if (unlikely(rs5_0 == 0)) {
+        return;
+    }
+
+    acc   = (((uint64_t)env->active_tc.HI[ac] << 32) & MIPSDSP_LHI) |
+            ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+
+    if (rs5_0 > 0) {
+        temp = acc >> rs5_0;
+    } else {
+        temp = acc << -rs5_0;
+    }
+
+    env->active_tc.HI[ac] = (target_ulong)(int32_t)((temp & MIPSDSP_LHI) >> 32);
+    env->active_tc.LO[ac] = (target_ulong)(int32_t)(temp & MIPSDSP_LLO);
+}
+
+#if defined(TARGET_MIPS64)
+void helper_dshilo(target_ulong shift, target_ulong ac, CPUMIPSState *env)
+{
+    int8_t shift_t;
+    uint64_t tempB, tempA;
+
+    shift_t = (int8_t)(shift << 1) >> 1;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    if (shift_t != 0) {
+        if (shift_t >= 0) {
+            tempA = (tempB << (64 - shift_t)) | (tempA >> shift_t);
+            tempB = tempB >> shift_t;
+        } else {
+            shift_t = -shift_t;
+            tempB = (tempB << shift_t) | (tempA >> (64 - shift_t));
+            tempA = tempA << shift_t;
+        }
+    }
+
+    env->active_tc.HI[ac] = tempB;
+    env->active_tc.LO[ac] = tempA;
+}
+
+#endif
+void helper_mthlip(target_ulong ac, target_ulong rs, CPUMIPSState *env)
+{
+    int32_t tempA, tempB, pos;
+
+    tempA = rs;
+    tempB = env->active_tc.LO[ac];
+    env->active_tc.HI[ac] = (target_long)tempB;
+    env->active_tc.LO[ac] = (target_long)tempA;
+    pos = get_DSPControl_pos(env);
+
+    if (pos > 32) {
+        return;
+    } else {
+        set_DSPControl_pos(pos + 32, env);
+    }
+}
+
+#if defined(TARGET_MIPS64)
+void helper_dmthlip(target_ulong rs, target_ulong ac, CPUMIPSState *env)
+{
+    uint8_t ac_t;
+    uint8_t pos;
+    uint64_t tempB, tempA;
+
+    ac_t = ac & 0x3;
+
+    tempA = rs;
+    tempB = env->active_tc.LO[ac_t];
+
+    env->active_tc.HI[ac_t] = tempB;
+    env->active_tc.LO[ac_t] = tempA;
+
+    pos = get_DSPControl_pos(env);
+
+    if (pos <= 64) {
+        pos = pos + 64;
+        set_DSPControl_pos(pos, env);
+    }
+}
+#endif
+
+void helper_wrdsp(target_ulong rs, target_ulong mask_num, CPUMIPSState *env)
+{
+    uint8_t  mask[6];
+    uint8_t  i;
+    uint32_t newbits, overwrite;
+    target_ulong dsp;
+
+    newbits   = 0x00;
+    overwrite = 0xFFFFFFFF;
+    dsp = env->active_tc.DSPControl;
+
+    for (i = 0; i < 6; i++) {
+        mask[i] = (mask_num >> i) & 0x01;
+    }
+
+    if (mask[0] == 1) {
+#if defined(TARGET_MIPS64)
+        overwrite &= 0xFFFFFF80;
+        newbits   &= 0xFFFFFF80;
+        newbits   |= 0x0000007F & rs;
+#else
+        overwrite &= 0xFFFFFFC0;
+        newbits   &= 0xFFFFFFC0;
+        newbits   |= 0x0000003F & rs;
+#endif
+    }
+
+    if (mask[1] == 1) {
+        overwrite &= 0xFFFFE07F;
+        newbits   &= 0xFFFFE07F;
+        newbits   |= 0x00001F80 & rs;
+    }
+
+    if (mask[2] == 1) {
+        overwrite &= 0xFFFFDFFF;
+        newbits   &= 0xFFFFDFFF;
+        newbits   |= 0x00002000 & rs;
+    }
+
+    if (mask[3] == 1) {
+        overwrite &= 0xFF00FFFF;
+        newbits   &= 0xFF00FFFF;
+        newbits   |= 0x00FF0000 & rs;
+    }
+
+    if (mask[4] == 1) {
+        overwrite &= 0x00FFFFFF;
+        newbits   &= 0x00FFFFFF;
+#if defined(TARGET_MIPS64)
+        newbits   |= 0xFF000000 & rs;
+#else
+        newbits   |= 0x0F000000 & rs;
+#endif
+    }
+
+    if (mask[5] == 1) {
+        overwrite &= 0xFFFFBFFF;
+        newbits   &= 0xFFFFBFFF;
+        newbits   |= 0x00004000 & rs;
+    }
+
+    dsp = dsp & overwrite;
+    dsp = dsp | newbits;
+    env->active_tc.DSPControl = dsp;
+}
+
+target_ulong helper_rddsp(target_ulong masknum, CPUMIPSState *env)
+{
+    uint8_t  mask[6];
+    uint32_t ruler, i;
+    target_ulong temp;
+    target_ulong dsp;
+
+    ruler = 0x01;
+    for (i = 0; i < 6; i++) {
+        mask[i] = (masknum & ruler) >> i ;
+        ruler = ruler << 1;
+    }
+
+    temp  = 0x00;
+    dsp = env->active_tc.DSPControl;
+
+    if (mask[0] == 1) {
+#if defined(TARGET_MIPS64)
+        temp |= dsp & 0x7F;
+#else
+        temp |= dsp & 0x3F;
+#endif
+    }
+
+    if (mask[1] == 1) {
+        temp |= dsp & 0x1F80;
+    }
+
+    if (mask[2] == 1) {
+        temp |= dsp & 0x2000;
+    }
+
+    if (mask[3] == 1) {
+        temp |= dsp & 0x00FF0000;
+    }
+
+    if (mask[4] == 1) {
+#if defined(TARGET_MIPS64)
+        temp |= dsp & 0xFF000000;
+#else
+        temp |= dsp & 0x0F000000;
+#endif
+    }
+
+    if (mask[5] == 1) {
+        temp |= dsp & 0x4000;
+    }
+
+    return temp;
+}
+
+
+#undef MIPSDSP_LHI
+#undef MIPSDSP_LLO
+#undef MIPSDSP_HI
+#undef MIPSDSP_LO
+#undef MIPSDSP_Q3
+#undef MIPSDSP_Q2
+#undef MIPSDSP_Q1
+#undef MIPSDSP_Q0
+
+#undef MIPSDSP_SPLIT32_8
+#undef MIPSDSP_SPLIT32_16
+
+#undef MIPSDSP_RETURN32
+#undef MIPSDSP_RETURN32_8
+#undef MIPSDSP_RETURN32_16
+
+#ifdef TARGET_MIPS64
+#undef MIPSDSP_SPLIT64_16
+#undef MIPSDSP_SPLIT64_32
+#undef MIPSDSP_RETURN64_16
+#undef MIPSDSP_RETURN64_32
+#endif
diff --git a/target-mips/helper.c b/target-mips/helper.c
index 4208bb20c8b..e877b8db782 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -36,7 +36,7 @@ enum {
 #if !defined(CONFIG_USER_ONLY)
 
 /* no MMU emulation */
-int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int no_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                         target_ulong address, int rw, int access_type)
 {
     *physical = address;
@@ -45,7 +45,7 @@ int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *pr
 }
 
 /* fixed mapping MMU emulation */
-int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int fixed_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                            target_ulong address, int rw, int access_type)
 {
     if (address <= (int32_t)0x7FFFFFFFUL) {
@@ -63,7 +63,7 @@ int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int
 }
 
 /* MIPS32/MIPS64 R4000-style MMU emulation */
-int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                      target_ulong address, int rw, int access_type)
 {
     uint8_t ASID = env->CP0_EntryHi & 0xFF;
@@ -99,7 +99,7 @@ int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
     return TLBRET_NOMATCH;
 }
 
-static int get_physical_address (CPUMIPSState *env, target_phys_addr_t *physical,
+static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
                                 int *prot, target_ulong address,
                                 int rw, int access_type)
 {
@@ -254,9 +254,9 @@ static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-target_phys_addr_t cpu_get_phys_page_debug(CPUMIPSState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUMIPSState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int prot;
 
     if (get_physical_address(env, &phys_addr, &prot, addr, 0, ACCESS_INT) != 0)
@@ -269,7 +269,7 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
 #if !defined(CONFIG_USER_ONLY)
-    target_phys_addr_t physical;
+    hwaddr physical;
     int prot;
     int access_type;
 #endif
@@ -308,9 +308,9 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-target_phys_addr_t cpu_mips_translate_address(CPUMIPSState *env, target_ulong address, int rw)
+hwaddr cpu_mips_translate_address(CPUMIPSState *env, target_ulong address, int rw)
 {
-    target_phys_addr_t physical;
+    hwaddr physical;
     int prot;
     int access_type;
     int ret = 0;
@@ -592,6 +592,9 @@ void do_interrupt (CPUMIPSState *env)
     case EXCP_THREAD:
         cause = 25;
         goto set_EPC;
+    case EXCP_DSPDIS:
+        cause = 26;
+        goto set_EPC;
     case EXCP_CACHE:
         cause = 30;
         if (env->CP0_Status & (1 << CP0St_BEV)) {
diff --git a/target-mips/helper.h b/target-mips/helper.h
index 76fb451e776..9ea60ec1bb7 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,222 +1,218 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_2(raise_exception_err, noreturn, i32, int)
-DEF_HELPER_1(raise_exception, noreturn, i32)
+DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
+DEF_HELPER_2(raise_exception, noreturn, env, i32)
 
 #ifdef TARGET_MIPS64
-DEF_HELPER_3(ldl, tl, tl, tl, int)
-DEF_HELPER_3(ldr, tl, tl, tl, int)
-DEF_HELPER_3(sdl, void, tl, tl, int)
-DEF_HELPER_3(sdr, void, tl, tl, int)
+DEF_HELPER_4(sdl, void, env, tl, tl, int)
+DEF_HELPER_4(sdr, void, env, tl, tl, int)
 #endif
-DEF_HELPER_3(lwl, tl, tl, tl, int)
-DEF_HELPER_3(lwr, tl, tl, tl, int)
-DEF_HELPER_3(swl, void, tl, tl, int)
-DEF_HELPER_3(swr, void, tl, tl, int)
+DEF_HELPER_4(swl, void, env, tl, tl, int)
+DEF_HELPER_4(swr, void, env, tl, tl, int)
 
 #ifndef CONFIG_USER_ONLY
-DEF_HELPER_2(ll, tl, tl, int)
-DEF_HELPER_3(sc, tl, tl, tl, int)
+DEF_HELPER_3(ll, tl, env, tl, int)
+DEF_HELPER_4(sc, tl, env, tl, tl, int)
 #ifdef TARGET_MIPS64
-DEF_HELPER_2(lld, tl, tl, int)
-DEF_HELPER_3(scd, tl, tl, tl, int)
+DEF_HELPER_3(lld, tl, env, tl, int)
+DEF_HELPER_4(scd, tl, env, tl, tl, int)
 #endif
 #endif
 
-DEF_HELPER_FLAGS_1(clo, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(clz, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(clo, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, tl, tl)
 #ifdef TARGET_MIPS64
-DEF_HELPER_FLAGS_1(dclo, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(dclz, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_2(dmult, void, tl, tl)
-DEF_HELPER_2(dmultu, void, tl, tl)
-#endif
-
-DEF_HELPER_2(muls, tl, tl, tl)
-DEF_HELPER_2(mulsu, tl, tl, tl)
-DEF_HELPER_2(macc, tl, tl, tl)
-DEF_HELPER_2(maccu, tl, tl, tl)
-DEF_HELPER_2(msac, tl, tl, tl)
-DEF_HELPER_2(msacu, tl, tl, tl)
-DEF_HELPER_2(mulhi, tl, tl, tl)
-DEF_HELPER_2(mulhiu, tl, tl, tl)
-DEF_HELPER_2(mulshi, tl, tl, tl)
-DEF_HELPER_2(mulshiu, tl, tl, tl)
-DEF_HELPER_2(macchi, tl, tl, tl)
-DEF_HELPER_2(macchiu, tl, tl, tl)
-DEF_HELPER_2(msachi, tl, tl, tl)
-DEF_HELPER_2(msachiu, tl, tl, tl)
+DEF_HELPER_FLAGS_1(dclo, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(dclz, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_3(dmult, void, env, tl, tl)
+DEF_HELPER_3(dmultu, void, env, tl, tl)
+#endif
+
+DEF_HELPER_3(muls, tl, env, tl, tl)
+DEF_HELPER_3(mulsu, tl, env, tl, tl)
+DEF_HELPER_3(macc, tl, env, tl, tl)
+DEF_HELPER_3(maccu, tl, env, tl, tl)
+DEF_HELPER_3(msac, tl, env, tl, tl)
+DEF_HELPER_3(msacu, tl, env, tl, tl)
+DEF_HELPER_3(mulhi, tl, env, tl, tl)
+DEF_HELPER_3(mulhiu, tl, env, tl, tl)
+DEF_HELPER_3(mulshi, tl, env, tl, tl)
+DEF_HELPER_3(mulshiu, tl, env, tl, tl)
+DEF_HELPER_3(macchi, tl, env, tl, tl)
+DEF_HELPER_3(macchiu, tl, env, tl, tl)
+DEF_HELPER_3(msachi, tl, env, tl, tl)
+DEF_HELPER_3(msachiu, tl, env, tl, tl)
 
 #ifndef CONFIG_USER_ONLY
 /* CP0 helpers */
-DEF_HELPER_0(mfc0_mvpcontrol, tl)
-DEF_HELPER_0(mfc0_mvpconf0, tl)
-DEF_HELPER_0(mfc0_mvpconf1, tl)
-DEF_HELPER_0(mftc0_vpecontrol, tl)
-DEF_HELPER_0(mftc0_vpeconf0, tl)
-DEF_HELPER_0(mfc0_random, tl)
-DEF_HELPER_0(mfc0_tcstatus, tl)
-DEF_HELPER_0(mftc0_tcstatus, tl)
-DEF_HELPER_0(mfc0_tcbind, tl)
-DEF_HELPER_0(mftc0_tcbind, tl)
-DEF_HELPER_0(mfc0_tcrestart, tl)
-DEF_HELPER_0(mftc0_tcrestart, tl)
-DEF_HELPER_0(mfc0_tchalt, tl)
-DEF_HELPER_0(mftc0_tchalt, tl)
-DEF_HELPER_0(mfc0_tccontext, tl)
-DEF_HELPER_0(mftc0_tccontext, tl)
-DEF_HELPER_0(mfc0_tcschedule, tl)
-DEF_HELPER_0(mftc0_tcschedule, tl)
-DEF_HELPER_0(mfc0_tcschefback, tl)
-DEF_HELPER_0(mftc0_tcschefback, tl)
-DEF_HELPER_0(mfc0_count, tl)
-DEF_HELPER_0(mftc0_entryhi, tl)
-DEF_HELPER_0(mftc0_status, tl)
-DEF_HELPER_0(mftc0_cause, tl)
-DEF_HELPER_0(mftc0_epc, tl)
-DEF_HELPER_0(mftc0_ebase, tl)
-DEF_HELPER_1(mftc0_configx, tl, tl)
-DEF_HELPER_0(mfc0_lladdr, tl)
-DEF_HELPER_1(mfc0_watchlo, tl, i32)
-DEF_HELPER_1(mfc0_watchhi, tl, i32)
-DEF_HELPER_0(mfc0_debug, tl)
-DEF_HELPER_0(mftc0_debug, tl)
+DEF_HELPER_1(mfc0_mvpcontrol, tl, env)
+DEF_HELPER_1(mfc0_mvpconf0, tl, env)
+DEF_HELPER_1(mfc0_mvpconf1, tl, env)
+DEF_HELPER_1(mftc0_vpecontrol, tl, env)
+DEF_HELPER_1(mftc0_vpeconf0, tl, env)
+DEF_HELPER_1(mfc0_random, tl, env)
+DEF_HELPER_1(mfc0_tcstatus, tl, env)
+DEF_HELPER_1(mftc0_tcstatus, tl, env)
+DEF_HELPER_1(mfc0_tcbind, tl, env)
+DEF_HELPER_1(mftc0_tcbind, tl, env)
+DEF_HELPER_1(mfc0_tcrestart, tl, env)
+DEF_HELPER_1(mftc0_tcrestart, tl, env)
+DEF_HELPER_1(mfc0_tchalt, tl, env)
+DEF_HELPER_1(mftc0_tchalt, tl, env)
+DEF_HELPER_1(mfc0_tccontext, tl, env)
+DEF_HELPER_1(mftc0_tccontext, tl, env)
+DEF_HELPER_1(mfc0_tcschedule, tl, env)
+DEF_HELPER_1(mftc0_tcschedule, tl, env)
+DEF_HELPER_1(mfc0_tcschefback, tl, env)
+DEF_HELPER_1(mftc0_tcschefback, tl, env)
+DEF_HELPER_1(mfc0_count, tl, env)
+DEF_HELPER_1(mftc0_entryhi, tl, env)
+DEF_HELPER_1(mftc0_status, tl, env)
+DEF_HELPER_1(mftc0_cause, tl, env)
+DEF_HELPER_1(mftc0_epc, tl, env)
+DEF_HELPER_1(mftc0_ebase, tl, env)
+DEF_HELPER_2(mftc0_configx, tl, env, tl)
+DEF_HELPER_1(mfc0_lladdr, tl, env)
+DEF_HELPER_2(mfc0_watchlo, tl, env, i32)
+DEF_HELPER_2(mfc0_watchhi, tl, env, i32)
+DEF_HELPER_1(mfc0_debug, tl, env)
+DEF_HELPER_1(mftc0_debug, tl, env)
 #ifdef TARGET_MIPS64
-DEF_HELPER_0(dmfc0_tcrestart, tl)
-DEF_HELPER_0(dmfc0_tchalt, tl)
-DEF_HELPER_0(dmfc0_tccontext, tl)
-DEF_HELPER_0(dmfc0_tcschedule, tl)
-DEF_HELPER_0(dmfc0_tcschefback, tl)
-DEF_HELPER_0(dmfc0_lladdr, tl)
-DEF_HELPER_1(dmfc0_watchlo, tl, i32)
+DEF_HELPER_1(dmfc0_tcrestart, tl, env)
+DEF_HELPER_1(dmfc0_tchalt, tl, env)
+DEF_HELPER_1(dmfc0_tccontext, tl, env)
+DEF_HELPER_1(dmfc0_tcschedule, tl, env)
+DEF_HELPER_1(dmfc0_tcschefback, tl, env)
+DEF_HELPER_1(dmfc0_lladdr, tl, env)
+DEF_HELPER_2(dmfc0_watchlo, tl, env, i32)
 #endif /* TARGET_MIPS64 */
 
-DEF_HELPER_1(mtc0_index, void, tl)
-DEF_HELPER_1(mtc0_mvpcontrol, void, tl)
-DEF_HELPER_1(mtc0_vpecontrol, void, tl)
-DEF_HELPER_1(mttc0_vpecontrol, void, tl)
-DEF_HELPER_1(mtc0_vpeconf0, void, tl)
-DEF_HELPER_1(mttc0_vpeconf0, void, tl)
-DEF_HELPER_1(mtc0_vpeconf1, void, tl)
-DEF_HELPER_1(mtc0_yqmask, void, tl)
-DEF_HELPER_1(mtc0_vpeopt, void, tl)
-DEF_HELPER_1(mtc0_entrylo0, void, tl)
-DEF_HELPER_1(mtc0_tcstatus, void, tl)
-DEF_HELPER_1(mttc0_tcstatus, void, tl)
-DEF_HELPER_1(mtc0_tcbind, void, tl)
-DEF_HELPER_1(mttc0_tcbind, void, tl)
-DEF_HELPER_1(mtc0_tcrestart, void, tl)
-DEF_HELPER_1(mttc0_tcrestart, void, tl)
-DEF_HELPER_1(mtc0_tchalt, void, tl)
-DEF_HELPER_1(mttc0_tchalt, void, tl)
-DEF_HELPER_1(mtc0_tccontext, void, tl)
-DEF_HELPER_1(mttc0_tccontext, void, tl)
-DEF_HELPER_1(mtc0_tcschedule, void, tl)
-DEF_HELPER_1(mttc0_tcschedule, void, tl)
-DEF_HELPER_1(mtc0_tcschefback, void, tl)
-DEF_HELPER_1(mttc0_tcschefback, void, tl)
-DEF_HELPER_1(mtc0_entrylo1, void, tl)
-DEF_HELPER_1(mtc0_context, void, tl)
-DEF_HELPER_1(mtc0_pagemask, void, tl)
-DEF_HELPER_1(mtc0_pagegrain, void, tl)
-DEF_HELPER_1(mtc0_wired, void, tl)
-DEF_HELPER_1(mtc0_srsconf0, void, tl)
-DEF_HELPER_1(mtc0_srsconf1, void, tl)
-DEF_HELPER_1(mtc0_srsconf2, void, tl)
-DEF_HELPER_1(mtc0_srsconf3, void, tl)
-DEF_HELPER_1(mtc0_srsconf4, void, tl)
-DEF_HELPER_1(mtc0_hwrena, void, tl)
-DEF_HELPER_1(mtc0_count, void, tl)
-DEF_HELPER_1(mtc0_entryhi, void, tl)
-DEF_HELPER_1(mttc0_entryhi, void, tl)
-DEF_HELPER_1(mtc0_compare, void, tl)
-DEF_HELPER_1(mtc0_status, void, tl)
-DEF_HELPER_1(mttc0_status, void, tl)
-DEF_HELPER_1(mtc0_intctl, void, tl)
-DEF_HELPER_1(mtc0_srsctl, void, tl)
-DEF_HELPER_1(mtc0_cause, void, tl)
-DEF_HELPER_1(mttc0_cause, void, tl)
-DEF_HELPER_1(mtc0_ebase, void, tl)
-DEF_HELPER_1(mttc0_ebase, void, tl)
-DEF_HELPER_1(mtc0_config0, void, tl)
-DEF_HELPER_1(mtc0_config2, void, tl)
-DEF_HELPER_1(mtc0_lladdr, void, tl)
-DEF_HELPER_2(mtc0_watchlo, void, tl, i32)
-DEF_HELPER_2(mtc0_watchhi, void, tl, i32)
-DEF_HELPER_1(mtc0_xcontext, void, tl)
-DEF_HELPER_1(mtc0_framemask, void, tl)
-DEF_HELPER_1(mtc0_debug, void, tl)
-DEF_HELPER_1(mttc0_debug, void, tl)
-DEF_HELPER_1(mtc0_performance0, void, tl)
-DEF_HELPER_1(mtc0_taglo, void, tl)
-DEF_HELPER_1(mtc0_datalo, void, tl)
-DEF_HELPER_1(mtc0_taghi, void, tl)
-DEF_HELPER_1(mtc0_datahi, void, tl)
+DEF_HELPER_2(mtc0_index, void, env, tl)
+DEF_HELPER_2(mtc0_mvpcontrol, void, env, tl)
+DEF_HELPER_2(mtc0_vpecontrol, void, env, tl)
+DEF_HELPER_2(mttc0_vpecontrol, void, env, tl)
+DEF_HELPER_2(mtc0_vpeconf0, void, env, tl)
+DEF_HELPER_2(mttc0_vpeconf0, void, env, tl)
+DEF_HELPER_2(mtc0_vpeconf1, void, env, tl)
+DEF_HELPER_2(mtc0_yqmask, void, env, tl)
+DEF_HELPER_2(mtc0_vpeopt, void, env, tl)
+DEF_HELPER_2(mtc0_entrylo0, void, env, tl)
+DEF_HELPER_2(mtc0_tcstatus, void, env, tl)
+DEF_HELPER_2(mttc0_tcstatus, void, env, tl)
+DEF_HELPER_2(mtc0_tcbind, void, env, tl)
+DEF_HELPER_2(mttc0_tcbind, void, env, tl)
+DEF_HELPER_2(mtc0_tcrestart, void, env, tl)
+DEF_HELPER_2(mttc0_tcrestart, void, env, tl)
+DEF_HELPER_2(mtc0_tchalt, void, env, tl)
+DEF_HELPER_2(mttc0_tchalt, void, env, tl)
+DEF_HELPER_2(mtc0_tccontext, void, env, tl)
+DEF_HELPER_2(mttc0_tccontext, void, env, tl)
+DEF_HELPER_2(mtc0_tcschedule, void, env, tl)
+DEF_HELPER_2(mttc0_tcschedule, void, env, tl)
+DEF_HELPER_2(mtc0_tcschefback, void, env, tl)
+DEF_HELPER_2(mttc0_tcschefback, void, env, tl)
+DEF_HELPER_2(mtc0_entrylo1, void, env, tl)
+DEF_HELPER_2(mtc0_context, void, env, tl)
+DEF_HELPER_2(mtc0_pagemask, void, env, tl)
+DEF_HELPER_2(mtc0_pagegrain, void, env, tl)
+DEF_HELPER_2(mtc0_wired, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf0, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf1, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf2, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf3, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf4, void, env, tl)
+DEF_HELPER_2(mtc0_hwrena, void, env, tl)
+DEF_HELPER_2(mtc0_count, void, env, tl)
+DEF_HELPER_2(mtc0_entryhi, void, env, tl)
+DEF_HELPER_2(mttc0_entryhi, void, env, tl)
+DEF_HELPER_2(mtc0_compare, void, env, tl)
+DEF_HELPER_2(mtc0_status, void, env, tl)
+DEF_HELPER_2(mttc0_status, void, env, tl)
+DEF_HELPER_2(mtc0_intctl, void, env, tl)
+DEF_HELPER_2(mtc0_srsctl, void, env, tl)
+DEF_HELPER_2(mtc0_cause, void, env, tl)
+DEF_HELPER_2(mttc0_cause, void, env, tl)
+DEF_HELPER_2(mtc0_ebase, void, env, tl)
+DEF_HELPER_2(mttc0_ebase, void, env, tl)
+DEF_HELPER_2(mtc0_config0, void, env, tl)
+DEF_HELPER_2(mtc0_config2, void, env, tl)
+DEF_HELPER_2(mtc0_lladdr, void, env, tl)
+DEF_HELPER_3(mtc0_watchlo, void, env, tl, i32)
+DEF_HELPER_3(mtc0_watchhi, void, env, tl, i32)
+DEF_HELPER_2(mtc0_xcontext, void, env, tl)
+DEF_HELPER_2(mtc0_framemask, void, env, tl)
+DEF_HELPER_2(mtc0_debug, void, env, tl)
+DEF_HELPER_2(mttc0_debug, void, env, tl)
+DEF_HELPER_2(mtc0_performance0, void, env, tl)
+DEF_HELPER_2(mtc0_taglo, void, env, tl)
+DEF_HELPER_2(mtc0_datalo, void, env, tl)
+DEF_HELPER_2(mtc0_taghi, void, env, tl)
+DEF_HELPER_2(mtc0_datahi, void, env, tl)
 
 /* MIPS MT functions */
-DEF_HELPER_1(mftgpr, tl, i32);
-DEF_HELPER_1(mftlo, tl, i32)
-DEF_HELPER_1(mfthi, tl, i32)
-DEF_HELPER_1(mftacx, tl, i32)
-DEF_HELPER_0(mftdsp, tl)
-DEF_HELPER_2(mttgpr, void, tl, i32)
-DEF_HELPER_2(mttlo, void, tl, i32)
-DEF_HELPER_2(mtthi, void, tl, i32)
-DEF_HELPER_2(mttacx, void, tl, i32)
-DEF_HELPER_1(mttdsp, void, tl)
+DEF_HELPER_2(mftgpr, tl, env, i32);
+DEF_HELPER_2(mftlo, tl, env, i32)
+DEF_HELPER_2(mfthi, tl, env, i32)
+DEF_HELPER_2(mftacx, tl, env, i32)
+DEF_HELPER_1(mftdsp, tl, env)
+DEF_HELPER_3(mttgpr, void, env, tl, i32)
+DEF_HELPER_3(mttlo, void, env, tl, i32)
+DEF_HELPER_3(mtthi, void, env, tl, i32)
+DEF_HELPER_3(mttacx, void, env, tl, i32)
+DEF_HELPER_2(mttdsp, void, env, tl)
 DEF_HELPER_0(dmt, tl)
 DEF_HELPER_0(emt, tl)
-DEF_HELPER_0(dvpe, tl)
-DEF_HELPER_0(evpe, tl)
+DEF_HELPER_1(dvpe, tl, env)
+DEF_HELPER_1(evpe, tl, env)
 #endif /* !CONFIG_USER_ONLY */
 
 /* microMIPS functions */
-DEF_HELPER_3(lwm, void, tl, tl, i32);
-DEF_HELPER_3(swm, void, tl, tl, i32);
+DEF_HELPER_4(lwm, void, env, tl, tl, i32);
+DEF_HELPER_4(swm, void, env, tl, tl, i32);
 #ifdef TARGET_MIPS64
-DEF_HELPER_3(ldm, void, tl, tl, i32);
-DEF_HELPER_3(sdm, void, tl, tl, i32);
+DEF_HELPER_4(ldm, void, env, tl, tl, i32);
+DEF_HELPER_4(sdm, void, env, tl, tl, i32);
 #endif
 
 DEF_HELPER_2(fork, void, tl, tl)
-DEF_HELPER_1(yield, tl, tl)
+DEF_HELPER_2(yield, tl, env, tl)
 
 /* CP1 functions */
-DEF_HELPER_1(cfc1, tl, i32)
-DEF_HELPER_2(ctc1, void, tl, i32)
-
-DEF_HELPER_1(float_cvtd_s, i64, i32)
-DEF_HELPER_1(float_cvtd_w, i64, i32)
-DEF_HELPER_1(float_cvtd_l, i64, i64)
-DEF_HELPER_1(float_cvtl_d, i64, i64)
-DEF_HELPER_1(float_cvtl_s, i64, i32)
-DEF_HELPER_1(float_cvtps_pw, i64, i64)
-DEF_HELPER_1(float_cvtpw_ps, i64, i64)
-DEF_HELPER_1(float_cvts_d, i32, i64)
-DEF_HELPER_1(float_cvts_w, i32, i32)
-DEF_HELPER_1(float_cvts_l, i32, i64)
-DEF_HELPER_1(float_cvts_pl, i32, i32)
-DEF_HELPER_1(float_cvts_pu, i32, i32)
-DEF_HELPER_1(float_cvtw_s, i32, i32)
-DEF_HELPER_1(float_cvtw_d, i32, i64)
-
-DEF_HELPER_2(float_addr_ps, i64, i64, i64)
-DEF_HELPER_2(float_mulr_ps, i64, i64, i64)
+DEF_HELPER_2(cfc1, tl, env, i32)
+DEF_HELPER_3(ctc1, void, env, tl, i32)
 
-#define FOP_PROTO(op)                       \
-DEF_HELPER_1(float_ ## op ## l_s, i64, i32) \
-DEF_HELPER_1(float_ ## op ## l_d, i64, i64) \
-DEF_HELPER_1(float_ ## op ## w_s, i32, i32) \
-DEF_HELPER_1(float_ ## op ## w_d, i32, i64)
+DEF_HELPER_2(float_cvtd_s, i64, env, i32)
+DEF_HELPER_2(float_cvtd_w, i64, env, i32)
+DEF_HELPER_2(float_cvtd_l, i64, env, i64)
+DEF_HELPER_2(float_cvtl_d, i64, env, i64)
+DEF_HELPER_2(float_cvtl_s, i64, env, i32)
+DEF_HELPER_2(float_cvtps_pw, i64, env, i64)
+DEF_HELPER_2(float_cvtpw_ps, i64, env, i64)
+DEF_HELPER_2(float_cvts_d, i32, env, i64)
+DEF_HELPER_2(float_cvts_w, i32, env, i32)
+DEF_HELPER_2(float_cvts_l, i32, env, i64)
+DEF_HELPER_2(float_cvts_pl, i32, env, i32)
+DEF_HELPER_2(float_cvts_pu, i32, env, i32)
+DEF_HELPER_2(float_cvtw_s, i32, env, i32)
+DEF_HELPER_2(float_cvtw_d, i32, env, i64)
+
+DEF_HELPER_3(float_addr_ps, i64, env, i64, i64)
+DEF_HELPER_3(float_mulr_ps, i64, env, i64, i64)
+
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## l_s, i64, env, i32) \
+DEF_HELPER_2(float_ ## op ## l_d, i64, env, i64) \
+DEF_HELPER_2(float_ ## op ## w_s, i32, env, i32) \
+DEF_HELPER_2(float_ ## op ## w_d, i32, env, i64)
 FOP_PROTO(round)
 FOP_PROTO(trunc)
 FOP_PROTO(ceil)
 FOP_PROTO(floor)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                       \
-DEF_HELPER_1(float_ ## op ## _s, i32, i32)  \
-DEF_HELPER_1(float_ ## op ## _d, i64, i64)
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## _s, i32, env, i32)  \
+DEF_HELPER_2(float_ ## op ## _d, i64, env, i64)
 FOP_PROTO(sqrt)
 FOP_PROTO(rsqrt)
 FOP_PROTO(recip)
@@ -228,14 +224,20 @@ DEF_HELPER_1(float_ ## op ## _d, i64, i64)  \
 DEF_HELPER_1(float_ ## op ## _ps, i64, i64)
 FOP_PROTO(abs)
 FOP_PROTO(chs)
+#undef FOP_PROTO
+
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## _s, i32, env, i32)  \
+DEF_HELPER_2(float_ ## op ## _d, i64, env, i64)  \
+DEF_HELPER_2(float_ ## op ## _ps, i64, env, i64)
 FOP_PROTO(recip1)
 FOP_PROTO(rsqrt1)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                             \
-DEF_HELPER_2(float_ ## op ## _s, i32, i32, i32)   \
-DEF_HELPER_2(float_ ## op ## _d, i64, i64, i64)   \
-DEF_HELPER_2(float_ ## op ## _ps, i64, i64, i64)
+#define FOP_PROTO(op)                                  \
+DEF_HELPER_3(float_ ## op ## _s, i32, env, i32, i32)   \
+DEF_HELPER_3(float_ ## op ## _d, i64, env, i64, i64)   \
+DEF_HELPER_3(float_ ## op ## _ps, i64, env, i64, i64)
 FOP_PROTO(add)
 FOP_PROTO(sub)
 FOP_PROTO(mul)
@@ -244,23 +246,23 @@ FOP_PROTO(recip2)
 FOP_PROTO(rsqrt2)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                                 \
-DEF_HELPER_3(float_ ## op ## _s, i32, i32, i32, i32)  \
-DEF_HELPER_3(float_ ## op ## _d, i64, i64, i64, i64)  \
-DEF_HELPER_3(float_ ## op ## _ps, i64, i64, i64, i64)
-FOP_PROTO(muladd)
-FOP_PROTO(mulsub)
-FOP_PROTO(nmuladd)
-FOP_PROTO(nmulsub)
+#define FOP_PROTO(op)                                      \
+DEF_HELPER_4(float_ ## op ## _s, i32, env, i32, i32, i32)  \
+DEF_HELPER_4(float_ ## op ## _d, i64, env, i64, i64, i64)  \
+DEF_HELPER_4(float_ ## op ## _ps, i64, env, i64, i64, i64)
+FOP_PROTO(madd)
+FOP_PROTO(msub)
+FOP_PROTO(nmadd)
+FOP_PROTO(nmsub)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                               \
-DEF_HELPER_3(cmp_d_ ## op, void, i64, i64, int)     \
-DEF_HELPER_3(cmpabs_d_ ## op, void, i64, i64, int)  \
-DEF_HELPER_3(cmp_s_ ## op, void, i32, i32, int)     \
-DEF_HELPER_3(cmpabs_s_ ## op, void, i32, i32, int)  \
-DEF_HELPER_3(cmp_ps_ ## op, void, i64, i64, int)    \
-DEF_HELPER_3(cmpabs_ps_ ## op, void, i64, i64, int)
+#define FOP_PROTO(op)                                    \
+DEF_HELPER_4(cmp_d_ ## op, void, env, i64, i64, int)     \
+DEF_HELPER_4(cmpabs_d_ ## op, void, env, i64, i64, int)  \
+DEF_HELPER_4(cmp_s_ ## op, void, env, i32, i32, int)     \
+DEF_HELPER_4(cmpabs_s_ ## op, void, env, i32, i32, int)  \
+DEF_HELPER_4(cmp_ps_ ## op, void, env, i64, i64, int)    \
+DEF_HELPER_4(cmpabs_ps_ ## op, void, env, i64, i64, int)
 FOP_PROTO(f)
 FOP_PROTO(un)
 FOP_PROTO(eq)
@@ -281,20 +283,428 @@ FOP_PROTO(ngt)
 
 /* Special functions */
 #ifndef CONFIG_USER_ONLY
-DEF_HELPER_0(tlbwi, void)
-DEF_HELPER_0(tlbwr, void)
-DEF_HELPER_0(tlbp, void)
-DEF_HELPER_0(tlbr, void)
-DEF_HELPER_0(di, tl)
-DEF_HELPER_0(ei, tl)
-DEF_HELPER_0(eret, void)
-DEF_HELPER_0(deret, void)
+DEF_HELPER_1(tlbwi, void, env)
+DEF_HELPER_1(tlbwr, void, env)
+DEF_HELPER_1(tlbp, void, env)
+DEF_HELPER_1(tlbr, void, env)
+DEF_HELPER_1(di, tl, env)
+DEF_HELPER_1(ei, tl, env)
+DEF_HELPER_1(eret, void, env)
+DEF_HELPER_1(deret, void, env)
 #endif /* !CONFIG_USER_ONLY */
-DEF_HELPER_0(rdhwr_cpunum, tl)
-DEF_HELPER_0(rdhwr_synci_step, tl)
-DEF_HELPER_0(rdhwr_cc, tl)
-DEF_HELPER_0(rdhwr_ccres, tl)
-DEF_HELPER_1(pmon, void, int)
-DEF_HELPER_0(wait, void)
-
-#include "def-helper.h"
+DEF_HELPER_1(rdhwr_cpunum, tl, env)
+DEF_HELPER_1(rdhwr_synci_step, tl, env)
+DEF_HELPER_1(rdhwr_cc, tl, env)
+DEF_HELPER_1(rdhwr_ccres, tl, env)
+DEF_HELPER_2(pmon, void, env, int)
+DEF_HELPER_1(wait, void, env)
+
+/* Loongson multimedia functions.  */
+DEF_HELPER_FLAGS_2(paddsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddush, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddsb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddusb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(psubsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubush, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubsb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubusb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pshufh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packsswh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packsshb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packushb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(punpcklhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpcklbh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhbh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpcklwd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhwd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pavgh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pavgb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaxsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pminsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaxub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pminub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pcmpeqw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgtw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpeqh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgth, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpeqb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgtb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(psllw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psllh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrlw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrlh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psraw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrah, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pmullh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmulhh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmulhuh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaddhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pasubub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_1(biadd, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pmovmskb, TCG_CALL_NO_RWG_SE, i64, i64)
+
+/*** MIPS DSP ***/
+/* DSP Arithmetic Sub-class insns */
+DEF_HELPER_FLAGS_3(addq_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addq_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addq_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addq_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addu_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(adduh_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(adduh_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(addu_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(addqh_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addu_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(adduh_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(adduh_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(addu_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subq_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subq_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subq_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subq_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subu_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subuh_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subuh_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(subu_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subqh_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subu_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subuh_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subuh_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(subu_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addsc, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addwc, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(modsub, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(raddu_w_qb, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(raddu_l_ob, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(absq_s_qb, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_ph, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_w, 0, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(absq_s_ob, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_qh, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_pw, 0, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_2(precr_qb_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_qb_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_ph_w, TCG_CALL_NO_RWG_SE,
+                   tl, i32, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_r_ph_w, TCG_CALL_NO_RWG_SE,
+                   tl, i32, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_ph_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precrq_rs_ph_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(precr_ob_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_3(precr_sra_r_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_2(precrq_ob_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_qh_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precrq_rs_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(precrq_pw_l, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_3(precrqu_s_qb_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(precrqu_s_ob_qh,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, env)
+
+DEF_HELPER_FLAGS_1(preceq_pw_qhl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_1(precequ_ph_qbl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbra, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(precequ_qh_obl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_1(preceu_ph_qbl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbra, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(preceu_qh_obl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+
+/* DSP GPR-Based Shift Sub-class insns */
+DEF_HELPER_FLAGS_3(shll_qb, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_ob, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shll_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shll_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_2(shrl_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shrl_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shrl_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shrl_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(shra_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shra_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(shra_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shra_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+
+/* DSP Multiply Sub-class insns */
+DEF_HELPER_FLAGS_3(muleu_s_ph_qbl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleu_s_ph_qbr, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(muleu_s_qh_obl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleu_s_qh_obr, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(mulq_rs_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(mulq_rs_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(muleq_s_w_phl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleq_s_w_phr, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(muleq_s_pw_qhl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleq_s_pw_qhr, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_4(dpau_h_qbl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpau_h_qbr, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpau_h_obl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dpau_h_obr, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsu_h_qbl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsu_h_qbr, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsu_h_obl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dpsu_h_obr, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpa_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpa_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpax_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpaq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpaq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpaqx_s_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpaqx_sa_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dps_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dps_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsx_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsqx_s_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsqx_sa_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(mulsaq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(mulsaq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpaq_sa_l_w, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpaq_sa_l_pw, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsq_sa_l_w, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsq_sa_l_pw, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(mulsaq_s_l_pw, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(maq_s_w_phl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_s_w_phr, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_phl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_phr, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_3(mul_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mul_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_s_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_rs_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_4(mulsa_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(maq_s_w_qhll, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhlr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhrl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhrr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhll, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhlr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhrl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhrr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_l_pwl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_l_pwr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmadd, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmaddu, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmsub, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmsubu, 0, void, tl, tl, i32, env)
+#endif
+
+/* DSP Bit/Manipulation Sub-class insns */
+DEF_HELPER_FLAGS_1(bitrev, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_3(insv, 0, tl, env, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dinsv, 0, tl, env, tl, tl);
+#endif
+
+/* DSP Compare-Pick Sub-class insns */
+DEF_HELPER_FLAGS_3(cmpu_eq_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_lt_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_le_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_2(cmpgu_eq_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_lt_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_le_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(cmp_eq_ph, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_ph, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_ph, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(cmpu_eq_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_lt_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_le_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_eq_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_lt_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_le_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(cmpgu_eq_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_lt_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_le_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(cmp_eq_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_eq_pw, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_pw, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_pw, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(pick_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(pick_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(append, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dappend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_3(prepend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(prependd, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_3(prependw, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_3(balign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dbalign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_2(packrl_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(packrl_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+
+/* DSP Accumulator and DSPControl Access Sub-class insns */
+DEF_HELPER_FLAGS_3(extr_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extr_r_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extr_rs_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextr_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_r_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_rs_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_l, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_r_l, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_rs_l, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(extr_s_h, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextr_s_h, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(extp, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extpdp, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextp, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextpdp, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shilo, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dshilo, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(mthlip, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dmthlip, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(wrdsp, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_2(rddsp, 0, tl, tl, env)
+
+
+
+#include "exec/def-helper.h"
diff --git a/target-mips/lmi_helper.c b/target-mips/lmi_helper.c
new file mode 100644
index 00000000000..1b243535195
--- /dev/null
+++ b/target-mips/lmi_helper.c
@@ -0,0 +1,744 @@
+/*
+ *  Loongson Multimedia Instruction emulation helpers for QEMU.
+ *
+ *  Copyright (c) 2011  Richard Henderson <rth@twiddle.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/* If the byte ordering doesn't matter, i.e. all columns are treated
+   identically, then this union can be used directly.  If byte ordering
+   does matter, we generally ignore dumping to memory.  */
+typedef union {
+    uint8_t  ub[8];
+    int8_t   sb[8];
+    uint16_t uh[4];
+    int16_t  sh[4];
+    uint32_t uw[2];
+    int32_t  sw[2];
+    uint64_t d;
+} LMIValue;
+
+/* Some byte ordering issues can be mitigated by XORing in the following.  */
+#ifdef HOST_WORDS_BIGENDIAN
+# define BYTE_ORDER_XOR(N) N
+#else
+# define BYTE_ORDER_XOR(N) 0
+#endif
+
+#define SATSB(x)  (x < -0x80 ? -0x80 : x > 0x7f ? 0x7f : x)
+#define SATUB(x)  (x > 0xff ? 0xff : x)
+
+#define SATSH(x)  (x < -0x8000 ? -0x8000 : x > 0x7fff ? 0x7fff : x)
+#define SATUH(x)  (x > 0xffff ? 0xffff : x)
+
+#define SATSW(x) \
+    (x < -0x80000000ll ? -0x80000000ll : x > 0x7fffffff ? 0x7fffffff : x)
+#define SATUW(x)  (x > 0xffffffffull ? 0xffffffffull : x)
+
+uint64_t helper_paddsb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.sb[i] + vt.sb[i];
+        vs.sb[i] = SATSB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddusb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] + vt.ub[i];
+        vs.ub[i] = SATUB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.sh[i] + vt.sh[i];
+        vs.sh[i] = SATSH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddush(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.uh[i] + vt.uh[i];
+        vs.uh[i] = SATUH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        vs.ub[i] += vt.ub[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] += vt.uh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] += vt.uw[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubsb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.sb[i] - vt.sb[i];
+        vs.sb[i] = SATSB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubusb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] - vt.ub[i];
+        vs.ub[i] = SATUB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.sh[i] - vt.sh[i];
+        vs.sh[i] = SATSH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubush(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.uh[i] - vt.uh[i];
+        vs.uh[i] = SATUH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        vs.ub[i] -= vt.ub[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] -= vt.uh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] -= vt.uw[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_pshufh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs;
+    unsigned i;
+
+    vs.d = fs;
+    vd.d = 0;
+    for (i = 0; i < 4; i++, ft >>= 2) {
+        vd.uh[i ^ host] = vs.uh[(ft & 3) ^ host];
+    }
+    return vd.d;
+}
+
+uint64_t helper_packsswh(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    int64_t tmp;
+
+    tmp = (int32_t)(fs >> 0);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 0;
+
+    tmp = (int32_t)(fs >> 32);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 16;
+
+    tmp = (int32_t)(ft >> 0);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 32;
+
+    tmp = (int32_t)(ft >> 32);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 48;
+
+    return fd;
+}
+
+uint64_t helper_packsshb(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    unsigned int i;
+
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = fs >> (i * 16);
+        tmp = SATSB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8);
+    }
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = ft >> (i * 16);
+        tmp = SATSB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8 + 32);
+    }
+
+    return fd;
+}
+
+uint64_t helper_packushb(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    unsigned int i;
+
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = fs >> (i * 16);
+        tmp = SATUB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8);
+    }
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = ft >> (i * 16);
+        tmp = SATUB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8 + 32);
+    }
+
+    return fd;
+}
+
+uint64_t helper_punpcklwd(uint64_t fs, uint64_t ft)
+{
+    return (fs & 0xffffffff) | (ft << 32);
+}
+
+uint64_t helper_punpckhwd(uint64_t fs, uint64_t ft)
+{
+    return (fs >> 32) | (ft & ~0xffffffffull);
+}
+
+uint64_t helper_punpcklhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.uh[0 ^ host] = vs.uh[0 ^ host];
+    vd.uh[1 ^ host] = vt.uh[0 ^ host];
+    vd.uh[2 ^ host] = vs.uh[1 ^ host];
+    vd.uh[3 ^ host] = vt.uh[1 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpckhhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.uh[0 ^ host] = vs.uh[2 ^ host];
+    vd.uh[1 ^ host] = vt.uh[2 ^ host];
+    vd.uh[2 ^ host] = vs.uh[3 ^ host];
+    vd.uh[3 ^ host] = vt.uh[3 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpcklbh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(7);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.ub[0 ^ host] = vs.ub[0 ^ host];
+    vd.ub[1 ^ host] = vt.ub[0 ^ host];
+    vd.ub[2 ^ host] = vs.ub[1 ^ host];
+    vd.ub[3 ^ host] = vt.ub[1 ^ host];
+    vd.ub[4 ^ host] = vs.ub[2 ^ host];
+    vd.ub[5 ^ host] = vt.ub[2 ^ host];
+    vd.ub[6 ^ host] = vs.ub[3 ^ host];
+    vd.ub[7 ^ host] = vt.ub[3 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpckhbh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(7);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.ub[0 ^ host] = vs.ub[4 ^ host];
+    vd.ub[1 ^ host] = vt.ub[4 ^ host];
+    vd.ub[2 ^ host] = vs.ub[5 ^ host];
+    vd.ub[3 ^ host] = vt.ub[5 ^ host];
+    vd.ub[4 ^ host] = vs.ub[6 ^ host];
+    vd.ub[5 ^ host] = vt.ub[6 ^ host];
+    vd.ub[6 ^ host] = vs.ub[7 ^ host];
+    vd.ub[7 ^ host] = vt.ub[7 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_pavgh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = (vs.uh[i] + vt.uh[i] + 1) >> 1;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pavgb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = (vs.ub[i] + vt.ub[i] + 1) >> 1;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaxsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.sh[i] = (vs.sh[i] >= vt.sh[i] ? vs.sh[i] : vt.sh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pminsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.sh[i] = (vs.sh[i] <= vt.sh[i] ? vs.sh[i] : vt.sh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaxub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.ub[i] = (vs.ub[i] >= vt.ub[i] ? vs.ub[i] : vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pminub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.ub[i] = (vs.ub[i] <= vt.ub[i] ? vs.ub[i] : vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; i++) {
+        vs.uw[i] = -(vs.uw[i] == vt.uw[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgtw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; i++) {
+        vs.uw[i] = -(vs.uw[i] > vt.uw[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = -(vs.uh[i] == vt.uh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgth(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = -(vs.uh[i] > vt.uh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = -(vs.ub[i] == vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgtb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = -(vs.ub[i] > vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psllw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] <<= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrlw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psraw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        ft = 31;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.sw[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psllh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] <<= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrlh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrah(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        ft = 15;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.sh[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmullh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.sh[i] *= vt.sh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmulhh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int32_t r = vs.sh[i] * vt.sh[i];
+        vs.sh[i] = r >> 16;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmulhuh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        uint32_t r = vs.uh[i] * vt.uh[i];
+        vs.uh[i] = r >> 16;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaddhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vs, vt;
+    uint32_t p0, p1;
+
+    vs.d = fs;
+    vt.d = ft;
+    p0  = vs.sh[0 ^ host] * vt.sh[0 ^ host];
+    p0 += vs.sh[1 ^ host] * vt.sh[1 ^ host];
+    p1  = vs.sh[2 ^ host] * vt.sh[2 ^ host];
+    p1 += vs.sh[3 ^ host] * vt.sh[3 ^ host];
+
+    return ((uint64_t)p1 << 32) | p0;
+}
+
+uint64_t helper_pasubub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] - vt.ub[i];
+        vs.ub[i] = (r < 0 ? -r : r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_biadd(uint64_t fs)
+{
+    unsigned i, fd;
+
+    for (i = fd = 0; i < 8; ++i) {
+        fd += (fs >> (i * 8)) & 0xff;
+    }
+    return fd & 0xffff;
+}
+
+uint64_t helper_pmovmskb(uint64_t fs)
+{
+    unsigned fd = 0;
+
+    fd |= ((fs >>  7) & 1) << 0;
+    fd |= ((fs >> 15) & 1) << 1;
+    fd |= ((fs >> 23) & 1) << 2;
+    fd |= ((fs >> 31) & 1) << 3;
+    fd |= ((fs >> 39) & 1) << 4;
+    fd |= ((fs >> 47) & 1) << 5;
+    fd |= ((fs >> 55) & 1) << 6;
+    fd |= ((fs >> 63) & 1) << 7;
+
+    return fd & 0xff;
+}
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index e5bc93e2265..1bca4a159e7 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -18,116 +18,76 @@
  */
 #include <stdlib.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
-
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #ifndef CONFIG_USER_ONLY
 static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
 #endif
 
-static inline void compute_hflags(CPUMIPSState *env)
-{
-    env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
-                     MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
-                     MIPS_HFLAG_UX);
-    if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
-        !(env->CP0_Status & (1 << CP0St_ERL)) &&
-        !(env->hflags & MIPS_HFLAG_DM)) {
-        env->hflags |= (env->CP0_Status >> CP0St_KSU) & MIPS_HFLAG_KSU;
-    }
-#if defined(TARGET_MIPS64)
-    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
-        (env->CP0_Status & (1 << CP0St_PX)) ||
-        (env->CP0_Status & (1 << CP0St_UX))) {
-        env->hflags |= MIPS_HFLAG_64;
-    }
-    if (env->CP0_Status & (1 << CP0St_UX)) {
-        env->hflags |= MIPS_HFLAG_UX;
-    }
-#endif
-    if ((env->CP0_Status & (1 << CP0St_CU0)) ||
-        !(env->hflags & MIPS_HFLAG_KSU)) {
-        env->hflags |= MIPS_HFLAG_CP0;
-    }
-    if (env->CP0_Status & (1 << CP0St_CU1)) {
-        env->hflags |= MIPS_HFLAG_FPU;
-    }
-    if (env->CP0_Status & (1 << CP0St_FR)) {
-        env->hflags |= MIPS_HFLAG_F64;
-    }
-    if (env->insn_flags & ISA_MIPS32R2) {
-        if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    } else if (env->insn_flags & ISA_MIPS32) {
-        if (env->hflags & MIPS_HFLAG_64) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    } else if (env->insn_flags & ISA_MIPS4) {
-        /* All supported MIPS IV CPUs use the XX (CU3) to enable
-           and disable the MIPS IV extensions to the MIPS III ISA.
-           Some other MIPS IV CPUs ignore the bit, so the check here
-           would be too restrictive for them.  */
-        if (env->CP0_Status & (1 << CP0St_CU3)) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    }
-}
-
 /*****************************************************************************/
 /* Exceptions processing helpers */
 
-void helper_raise_exception_err (uint32_t exception, int error_code)
+static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
+                                                        uint32_t exception,
+                                                        int error_code,
+                                                        uintptr_t pc)
 {
-#if 1
-    if (exception < 0x100)
+    if (exception < EXCP_SC) {
         qemu_log("%s: %d %d\n", __func__, exception, error_code);
-#endif
+    }
     env->exception_index = exception;
     env->error_code = error_code;
+
+    if (pc) {
+        /* now we have a real cpu fault */
+        cpu_restore_state(env, pc);
+    }
+
     cpu_loop_exit(env);
 }
 
-void helper_raise_exception (uint32_t exception)
+static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
+                                                    uint32_t exception,
+                                                    uintptr_t pc)
 {
-    helper_raise_exception_err(exception, 0);
+    do_raise_exception_err(env, exception, 0, pc);
 }
 
-#if !defined(CONFIG_USER_ONLY)
-static void do_restore_state(uintptr_t pc)
+void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
+                                int error_code)
 {
-    TranslationBlock *tb;
+    do_raise_exception_err(env, exception, error_code, 0);
+}
 
-    tb = tb_find_pc (pc);
-    if (tb) {
-        cpu_restore_state(tb, env, pc);
-    }
+void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
+{
+    do_raise_exception(env, exception, 0);
 }
-#endif
 
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_LD(name, insn, type)                                     \
-static inline type do_##name(target_ulong addr, int mem_idx)            \
+static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             int mem_idx)                               \
 {                                                                       \
     return (type) insn##_raw(addr);                                     \
 }
 #else
 #define HELPER_LD(name, insn, type)                                     \
-static inline type do_##name(target_ulong addr, int mem_idx)            \
+static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             int mem_idx)                               \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: return (type) insn##_kernel(addr); break;                   \
-    case 1: return (type) insn##_super(addr); break;                    \
+    case 0: return (type) cpu_##insn##_kernel(env, addr); break;        \
+    case 1: return (type) cpu_##insn##_super(env, addr); break;         \
     default:                                                            \
-    case 2: return (type) insn##_user(addr); break;                     \
+    case 2: return (type) cpu_##insn##_user(env, addr); break;          \
     }                                                                   \
 }
 #endif
@@ -140,20 +100,22 @@ HELPER_LD(ld, ldq, int64_t)
 
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_ST(name, insn, type)                                     \
-static inline void do_##name(target_ulong addr, type val, int mem_idx)  \
+static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             type val, int mem_idx)                     \
 {                                                                       \
     insn##_raw(addr, val);                                              \
 }
 #else
 #define HELPER_ST(name, insn, type)                                     \
-static inline void do_##name(target_ulong addr, type val, int mem_idx)  \
+static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             type val, int mem_idx)                     \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: insn##_kernel(addr, val); break;                            \
-    case 1: insn##_super(addr, val); break;                             \
+    case 0: cpu_##insn##_kernel(env, addr, val); break;                 \
+    case 1: cpu_##insn##_super(env, addr, val); break;                  \
     default:                                                            \
-    case 2: insn##_user(addr, val); break;                              \
+    case 2: cpu_##insn##_user(env, addr, val); break;                   \
     }                                                                   \
 }
 #endif
@@ -187,12 +149,12 @@ target_ulong helper_dclz (target_ulong arg1)
 #endif /* TARGET_MIPS64 */
 
 /* 64 bits arithmetic for 32 bits hosts */
-static inline uint64_t get_HILO (void)
+static inline uint64_t get_HILO(CPUMIPSState *env)
 {
     return ((uint64_t)(env->active_tc.HI[0]) << 32) | (uint32_t)env->active_tc.LO[0];
 }
 
-static inline target_ulong set_HIT0_LO(uint64_t HILO)
+static inline target_ulong set_HIT0_LO(CPUMIPSState *env, uint64_t HILO)
 {
     target_ulong tmp;
     env->active_tc.LO[0] = (int32_t)(HILO & 0xFFFFFFFF);
@@ -200,7 +162,7 @@ static inline target_ulong set_HIT0_LO(uint64_t HILO)
     return tmp;
 }
 
-static inline target_ulong set_HI_LOT0(uint64_t HILO)
+static inline target_ulong set_HI_LOT0(CPUMIPSState *env, uint64_t HILO)
 {
     target_ulong tmp = env->active_tc.LO[0] = (int32_t)(HILO & 0xFFFFFFFF);
     env->active_tc.HI[0] = (int32_t)(HILO >> 32);
@@ -208,91 +170,110 @@ static inline target_ulong set_HI_LOT0(uint64_t HILO)
 }
 
 /* Multiplication variants of the vr54xx. */
-target_ulong helper_muls (target_ulong arg1, target_ulong arg2)
+target_ulong helper_muls(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0(0 - ((int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2));
+    return set_HI_LOT0(env, 0 - ((int64_t)(int32_t)arg1 *
+                                 (int64_t)(int32_t)arg2));
 }
 
-target_ulong helper_mulsu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulsu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0(0 - (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, 0 - (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_macc (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macc(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0((int64_t)get_HILO() + (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HI_LOT0(env, (int64_t)get_HILO(env) + (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_macchi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macchi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)get_HILO() + (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)get_HILO(env) + (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_maccu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_maccu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0((uint64_t)get_HILO() + (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, (uint64_t)get_HILO(env) +
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_macchiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macchiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)get_HILO() + (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)get_HILO(env) +
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_msac (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msac(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0((int64_t)get_HILO() - (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HI_LOT0(env, (int64_t)get_HILO(env) - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_msachi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msachi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)get_HILO() - (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)get_HILO(env) - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_msacu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msacu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0((uint64_t)get_HILO() - (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, (uint64_t)get_HILO(env) -
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_msachiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msachiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)get_HILO() - (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)get_HILO(env) -
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_mulhi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulhi(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_mulhiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulhiu(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_mulshi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulshi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO(0 - (int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, 0 - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_mulshiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulshiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO(0 - (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, 0 - (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
 #ifdef TARGET_MIPS64
-void helper_dmult (target_ulong arg1, target_ulong arg2)
+void helper_dmult(CPUMIPSState *env, target_ulong arg1, target_ulong arg2)
 {
     muls64(&(env->active_tc.LO[0]), &(env->active_tc.HI[0]), arg1, arg2);
 }
 
-void helper_dmultu (target_ulong arg1, target_ulong arg2)
+void helper_dmultu(CPUMIPSState *env, target_ulong arg1, target_ulong arg2)
 {
     mulu64(&(env->active_tc.LO[0]), &(env->active_tc.HI[0]), arg1, arg2);
 }
@@ -300,9 +281,11 @@ void helper_dmultu (target_ulong arg1, target_ulong arg2)
 
 #ifndef CONFIG_USER_ONLY
 
-static inline target_phys_addr_t do_translate_address(target_ulong address, int rw)
+static inline hwaddr do_translate_address(CPUMIPSState *env,
+                                                      target_ulong address,
+                                                      int rw)
 {
-    target_phys_addr_t lladdr;
+    hwaddr lladdr;
 
     lladdr = cpu_mips_translate_address(env, address, rw);
 
@@ -314,10 +297,10 @@ static inline target_phys_addr_t do_translate_address(target_ulong address, int
 }
 
 #define HELPER_LD_ATOMIC(name, insn)                                          \
-target_ulong helper_##name(target_ulong arg, int mem_idx)                     \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong arg, int mem_idx)  \
 {                                                                             \
-    env->lladdr = do_translate_address(arg, 0);                               \
-    env->llval = do_##insn(arg, mem_idx);                                     \
+    env->lladdr = do_translate_address(env, arg, 0);                          \
+    env->llval = do_##insn(env, arg, mem_idx);                                \
     return env->llval;                                                        \
 }
 HELPER_LD_ATOMIC(ll, lw)
@@ -327,18 +310,19 @@ HELPER_LD_ATOMIC(lld, ld)
 #undef HELPER_LD_ATOMIC
 
 #define HELPER_ST_ATOMIC(name, ld_insn, st_insn, almask)                      \
-target_ulong helper_##name(target_ulong arg1, target_ulong arg2, int mem_idx) \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong arg1,              \
+                           target_ulong arg2, int mem_idx)                    \
 {                                                                             \
     target_long tmp;                                                          \
                                                                               \
     if (arg2 & almask) {                                                      \
         env->CP0_BadVAddr = arg2;                                             \
-        helper_raise_exception(EXCP_AdES);                                    \
+        helper_raise_exception(env, EXCP_AdES);                               \
     }                                                                         \
-    if (do_translate_address(arg2, 1) == env->lladdr) {                       \
-        tmp = do_##ld_insn(arg2, mem_idx);                                    \
+    if (do_translate_address(env, arg2, 1) == env->lladdr) {                  \
+        tmp = do_##ld_insn(env, arg2, mem_idx);                               \
         if (tmp == env->llval) {                                              \
-            do_##st_insn(arg2, arg1, mem_idx);                                \
+            do_##st_insn(env, arg2, arg1, mem_idx);                           \
             return 1;                                                         \
         }                                                                     \
     }                                                                         \
@@ -359,80 +343,34 @@ HELPER_ST_ATOMIC(scd, ld, sd, 0x7)
 #define GET_OFFSET(addr, offset) (addr - (offset))
 #endif
 
-target_ulong helper_lwl(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    target_ulong tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0x00FFFFFF) | (tmp << 24);
-
-    if (GET_LMASK(arg2) <= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, 1), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFF) | (tmp << 16);
-    }
-
-    if (GET_LMASK(arg2) <= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, 2), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FF) | (tmp << 8);
-    }
-
-    if (GET_LMASK(arg2) == 0) {
-        tmp = do_lbu(GET_OFFSET(arg2, 3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00) | tmp;
-    }
-    return (int32_t)arg1;
-}
-
-target_ulong helper_lwr(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    target_ulong tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0xFFFFFF00) | tmp;
-
-    if (GET_LMASK(arg2) >= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, -1), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FF) | (tmp << 8);
-    }
-
-    if (GET_LMASK(arg2) >= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, -2), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFF) | (tmp << 16);
-    }
-
-    if (GET_LMASK(arg2) == 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, -3), mem_idx);
-        arg1 = (arg1 & 0x00FFFFFF) | (tmp << 24);
-    }
-    return (int32_t)arg1;
-}
-
-void helper_swl(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    do_sb(arg2, (uint8_t)(arg1 >> 24), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK(arg2) <= 2)
-        do_sb(GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK(arg2) <= 1)
-        do_sb(GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK(arg2) == 0)
-        do_sb(GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
 }
 
-void helper_swr(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
 
     if (GET_LMASK(arg2) >= 1)
-        do_sb(GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK(arg2) >= 2)
-        do_sb(GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK(arg2) == 3)
-        do_sb(GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
 }
 
 #if defined(TARGET_MIPS64)
@@ -445,292 +383,155 @@ void helper_swr(target_ulong arg1, target_ulong arg2, int mem_idx)
 #define GET_LMASK64(v) (((v) & 7) ^ 7)
 #endif
 
-target_ulong helper_ldl(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    uint64_t tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0x00FFFFFFFFFFFFFFULL) | (tmp << 56);
-
-    if (GET_LMASK64(arg2) <= 6) {
-        tmp = do_lbu(GET_OFFSET(arg2, 1), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFFFFFFFFFFULL) | (tmp << 48);
-    }
-
-    if (GET_LMASK64(arg2) <= 5) {
-        tmp = do_lbu(GET_OFFSET(arg2, 2), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FFFFFFFFFFULL) | (tmp << 40);
-    }
-
-    if (GET_LMASK64(arg2) <= 4) {
-        tmp = do_lbu(GET_OFFSET(arg2, 3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00FFFFFFFFULL) | (tmp << 32);
-    }
-
-    if (GET_LMASK64(arg2) <= 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, 4), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFF00FFFFFFULL) | (tmp << 24);
-    }
-
-    if (GET_LMASK64(arg2) <= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, 5), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFF00FFFFULL) | (tmp << 16);
-    }
-
-    if (GET_LMASK64(arg2) <= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, 6), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFF00FFULL) | (tmp << 8);
-    }
-
-    if (GET_LMASK64(arg2) == 0) {
-        tmp = do_lbu(GET_OFFSET(arg2, 7), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFFFF00ULL) | tmp;
-    }
-
-    return arg1;
-}
-
-target_ulong helper_ldr(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    uint64_t tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0xFFFFFFFFFFFFFF00ULL) | tmp;
-
-    if (GET_LMASK64(arg2) >= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, -1), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFF00FFULL) | (tmp  << 8);
-    }
-
-    if (GET_LMASK64(arg2) >= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, -2), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFF00FFFFULL) | (tmp << 16);
-    }
-
-    if (GET_LMASK64(arg2) >= 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, -3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFF00FFFFFFULL) | (tmp << 24);
-    }
-
-    if (GET_LMASK64(arg2) >= 4) {
-        tmp = do_lbu(GET_OFFSET(arg2, -4), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00FFFFFFFFULL) | (tmp << 32);
-    }
-
-    if (GET_LMASK64(arg2) >= 5) {
-        tmp = do_lbu(GET_OFFSET(arg2, -5), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FFFFFFFFFFULL) | (tmp << 40);
-    }
-
-    if (GET_LMASK64(arg2) >= 6) {
-        tmp = do_lbu(GET_OFFSET(arg2, -6), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFFFFFFFFFFULL) | (tmp << 48);
-    }
-
-    if (GET_LMASK64(arg2) == 7) {
-        tmp = do_lbu(GET_OFFSET(arg2, -7), mem_idx);
-        arg1 = (arg1 & 0x00FFFFFFFFFFFFFFULL) | (tmp << 56);
-    }
-
-    return arg1;
-}
-
-void helper_sdl(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)(arg1 >> 56), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx);
 
     if (GET_LMASK64(arg2) <= 6)
-        do_sb(GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
 
     if (GET_LMASK64(arg2) <= 5)
-        do_sb(GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
 
     if (GET_LMASK64(arg2) <= 4)
-        do_sb(GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
 
     if (GET_LMASK64(arg2) <= 3)
-        do_sb(GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK64(arg2) <= 2)
-        do_sb(GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK64(arg2) <= 1)
-        do_sb(GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK64(arg2) <= 0)
-        do_sb(GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
 }
 
-void helper_sdr(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
 
     if (GET_LMASK64(arg2) >= 1)
-        do_sb(GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK64(arg2) >= 2)
-        do_sb(GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK64(arg2) >= 3)
-        do_sb(GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK64(arg2) >= 4)
-        do_sb(GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
 
     if (GET_LMASK64(arg2) >= 5)
-        do_sb(GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
 
     if (GET_LMASK64(arg2) >= 6)
-        do_sb(GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
 
     if (GET_LMASK64(arg2) == 7)
-        do_sb(GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
 }
 #endif /* TARGET_MIPS64 */
 
 static const int multiple_regs[] = { 16, 17, 18, 19, 20, 21, 22, 23, 30 };
 
-void helper_lwm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_lwm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef ldfun
-#define ldfun ldl_raw
-#else
-    uint32_t (*ldfun)(target_ulong);
-
-    switch (mem_idx)
-    {
-    case 0: ldfun = ldl_kernel; break;
-    case 1: ldfun = ldl_super; break;
-    default:
-    case 2: ldfun = ldl_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = (target_long) ldfun(addr);
+            env->active_tc.gpr[multiple_regs[i]] =
+                (target_long)do_lw(env, addr, mem_idx);
             addr += 4;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = (target_long) ldfun(addr);
+        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx);
     }
 }
 
-void helper_swm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_swm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef stfun
-#define stfun stl_raw
-#else
-    void (*stfun)(target_ulong, uint32_t);
-
-    switch (mem_idx)
-    {
-    case 0: stfun = stl_kernel; break;
-    case 1: stfun = stl_super; break;
-     default:
-    case 2: stfun = stl_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            stfun(addr, env->active_tc.gpr[multiple_regs[i]]);
+            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
             addr += 4;
         }
     }
 
     if (do_r31) {
-        stfun(addr, env->active_tc.gpr[31]);
+        do_sw(env, addr, env->active_tc.gpr[31], mem_idx);
     }
 }
 
 #if defined(TARGET_MIPS64)
-void helper_ldm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_ldm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef ldfun
-#define ldfun ldq_raw
-#else
-    uint64_t (*ldfun)(target_ulong);
-
-    switch (mem_idx)
-    {
-    case 0: ldfun = ldq_kernel; break;
-    case 1: ldfun = ldq_super; break;
-    default:
-    case 2: ldfun = ldq_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = ldfun(addr);
+            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx);
             addr += 8;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = ldfun(addr);
+        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx);
     }
 }
 
-void helper_sdm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_sdm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef stfun
-#define stfun stq_raw
-#else
-    void (*stfun)(target_ulong, uint64_t);
-
-    switch (mem_idx)
-    {
-    case 0: stfun = stq_kernel; break;
-    case 1: stfun = stq_super; break;
-     default:
-    case 2: stfun = stq_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            stfun(addr, env->active_tc.gpr[multiple_regs[i]]);
+            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
             addr += 8;
         }
     }
 
     if (do_r31) {
-        stfun(addr, env->active_tc.gpr[31]);
+        do_sd(env, addr, env->active_tc.gpr[31], mem_idx);
     }
 }
 #endif
 
 #ifndef CONFIG_USER_ONLY
 /* SMP helpers.  */
-static int mips_vpe_is_wfi(CPUMIPSState *c)
+static bool mips_vpe_is_wfi(MIPSCPU *c)
 {
+    CPUMIPSState *env = &c->env;
+
     /* If the VPE is halted but otherwise active, it means it's waiting for
        an interrupt.  */
-    return c->halted && mips_vpe_active(c);
+    return env->halted && mips_vpe_active(env);
 }
 
 static inline void mips_vpe_wake(CPUMIPSState *c)
@@ -741,41 +542,53 @@ static inline void mips_vpe_wake(CPUMIPSState *c)
     cpu_interrupt(c, CPU_INTERRUPT_WAKE);
 }
 
-static inline void mips_vpe_sleep(CPUMIPSState *c)
+static inline void mips_vpe_sleep(MIPSCPU *cpu)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* The VPE was shut off, really go to bed.
        Reset any old _WAKE requests.  */
     c->halted = 1;
     cpu_reset_interrupt(c, CPU_INTERRUPT_WAKE);
 }
 
-static inline void mips_tc_wake(CPUMIPSState *c, int tc)
+static inline void mips_tc_wake(MIPSCPU *cpu, int tc)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* FIXME: TC reschedule.  */
-    if (mips_vpe_active(c) && !mips_vpe_is_wfi(c)) {
+    if (mips_vpe_active(c) && !mips_vpe_is_wfi(cpu)) {
         mips_vpe_wake(c);
     }
 }
 
-static inline void mips_tc_sleep(CPUMIPSState *c, int tc)
+static inline void mips_tc_sleep(MIPSCPU *cpu, int tc)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* FIXME: TC reschedule.  */
     if (!mips_vpe_active(c)) {
-        mips_vpe_sleep(c);
+        mips_vpe_sleep(cpu);
     }
 }
 
-/* tc should point to an int with the value of the global TC index.
-   This function will transform it into a local index within the
-   returned CPUMIPSState.
-
-   FIXME: This code assumes that all VPEs have the same number of TCs,
+/**
+ * mips_cpu_map_tc:
+ * @env: CPU from which mapping is performed.
+ * @tc: Should point to an int with the value of the global TC index.
+ *
+ * This function will transform @tc into a local index within the
+ * returned #CPUMIPSState.
+ */
+/* FIXME: This code assumes that all VPEs have the same number of TCs,
           which depends on runtime setup. Can probably be fixed by
           walking the list of CPUMIPSStates.  */
-static CPUMIPSState *mips_cpu_map_tc(int *tc)
+static CPUMIPSState *mips_cpu_map_tc(CPUMIPSState *env, int *tc)
 {
-    CPUMIPSState *other;
-    int vpe_idx, nr_threads = env->nr_threads;
+    MIPSCPU *cpu;
+    CPUState *cs;
+    CPUState *other_cs;
+    int vpe_idx;
     int tc_idx = *tc;
 
     if (!(env->CP0_VPEConf0 & (1 << CP0VPEC0_MVP))) {
@@ -784,10 +597,15 @@ static CPUMIPSState *mips_cpu_map_tc(int *tc)
         return env;
     }
 
-    vpe_idx = tc_idx / nr_threads;
-    *tc = tc_idx % nr_threads;
-    other = qemu_get_cpu(vpe_idx);
-    return other ? other : env;
+    cs = CPU(mips_env_get_cpu(env));
+    vpe_idx = tc_idx / cs->nr_threads;
+    *tc = tc_idx % cs->nr_threads;
+    other_cs = qemu_get_cpu(vpe_idx);
+    if (other_cs == NULL) {
+        return env;
+    }
+    cpu = MIPS_CPU(other_cs);
+    return &cpu->env;
 }
 
 /* The per VPE CP0_Status register shares some fields with the per TC
@@ -799,7 +617,7 @@ static CPUMIPSState *mips_cpu_map_tc(int *tc)
    These helper call synchronizes the regs for a given cpu.  */
 
 /* Called for updates to CP0_Status.  */
-static void sync_c0_status(CPUMIPSState *cpu, int tc)
+static void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 {
     int32_t tcstatus, *tcst;
     uint32_t v = cpu->CP0_Status;
@@ -834,7 +652,8 @@ static void sync_c0_status(CPUMIPSState *cpu, int tc)
 }
 
 /* Called for updates to CP0_TCStatus.  */
-static void sync_c0_tcstatus(CPUMIPSState *cpu, int tc, target_ulong v)
+static void sync_c0_tcstatus(CPUMIPSState *cpu, int tc,
+                             target_ulong v)
 {
     uint32_t status;
     uint32_t tcu, tmx, tasid, tksu;
@@ -883,35 +702,35 @@ static void sync_c0_entryhi(CPUMIPSState *cpu, int tc)
 }
 
 /* CP0 helpers */
-target_ulong helper_mfc0_mvpcontrol (void)
+target_ulong helper_mfc0_mvpcontrol(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPControl;
 }
 
-target_ulong helper_mfc0_mvpconf0 (void)
+target_ulong helper_mfc0_mvpconf0(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPConf0;
 }
 
-target_ulong helper_mfc0_mvpconf1 (void)
+target_ulong helper_mfc0_mvpconf1(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPConf1;
 }
 
-target_ulong helper_mfc0_random (void)
+target_ulong helper_mfc0_random(CPUMIPSState *env)
 {
     return (int32_t)cpu_mips_get_random(env);
 }
 
-target_ulong helper_mfc0_tcstatus (void)
+target_ulong helper_mfc0_tcstatus(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCStatus;
 }
 
-target_ulong helper_mftc0_tcstatus(void)
+target_ulong helper_mftc0_tcstatus(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCStatus;
@@ -919,15 +738,15 @@ target_ulong helper_mftc0_tcstatus(void)
         return other->tcs[other_tc].CP0_TCStatus;
 }
 
-target_ulong helper_mfc0_tcbind (void)
+target_ulong helper_mfc0_tcbind(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCBind;
 }
 
-target_ulong helper_mftc0_tcbind(void)
+target_ulong helper_mftc0_tcbind(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCBind;
@@ -935,15 +754,15 @@ target_ulong helper_mftc0_tcbind(void)
         return other->tcs[other_tc].CP0_TCBind;
 }
 
-target_ulong helper_mfc0_tcrestart (void)
+target_ulong helper_mfc0_tcrestart(CPUMIPSState *env)
 {
     return env->active_tc.PC;
 }
 
-target_ulong helper_mftc0_tcrestart(void)
+target_ulong helper_mftc0_tcrestart(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.PC;
@@ -951,15 +770,15 @@ target_ulong helper_mftc0_tcrestart(void)
         return other->tcs[other_tc].PC;
 }
 
-target_ulong helper_mfc0_tchalt (void)
+target_ulong helper_mfc0_tchalt(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCHalt;
 }
 
-target_ulong helper_mftc0_tchalt(void)
+target_ulong helper_mftc0_tchalt(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCHalt;
@@ -967,15 +786,15 @@ target_ulong helper_mftc0_tchalt(void)
         return other->tcs[other_tc].CP0_TCHalt;
 }
 
-target_ulong helper_mfc0_tccontext (void)
+target_ulong helper_mfc0_tccontext(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCContext;
 }
 
-target_ulong helper_mftc0_tccontext(void)
+target_ulong helper_mftc0_tccontext(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCContext;
@@ -983,15 +802,15 @@ target_ulong helper_mftc0_tccontext(void)
         return other->tcs[other_tc].CP0_TCContext;
 }
 
-target_ulong helper_mfc0_tcschedule (void)
+target_ulong helper_mfc0_tcschedule(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCSchedule;
 }
 
-target_ulong helper_mftc0_tcschedule(void)
+target_ulong helper_mftc0_tcschedule(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCSchedule;
@@ -999,15 +818,15 @@ target_ulong helper_mftc0_tcschedule(void)
         return other->tcs[other_tc].CP0_TCSchedule;
 }
 
-target_ulong helper_mfc0_tcschefback (void)
+target_ulong helper_mfc0_tcschefback(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCScheFBack;
 }
 
-target_ulong helper_mftc0_tcschefback(void)
+target_ulong helper_mftc0_tcschefback(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCScheFBack;
@@ -1015,24 +834,24 @@ target_ulong helper_mftc0_tcschefback(void)
         return other->tcs[other_tc].CP0_TCScheFBack;
 }
 
-target_ulong helper_mfc0_count (void)
+target_ulong helper_mfc0_count(CPUMIPSState *env)
 {
     return (int32_t)cpu_mips_get_count(env);
 }
 
-target_ulong helper_mftc0_entryhi(void)
+target_ulong helper_mftc0_entryhi(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EntryHi;
 }
 
-target_ulong helper_mftc0_cause(void)
+target_ulong helper_mftc0_cause(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     int32_t tccause;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc) {
         tccause = other->CP0_Cause;
@@ -1043,30 +862,30 @@ target_ulong helper_mftc0_cause(void)
     return tccause;
 }
 
-target_ulong helper_mftc0_status(void)
+target_ulong helper_mftc0_status(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_Status;
 }
 
-target_ulong helper_mfc0_lladdr (void)
+target_ulong helper_mfc0_lladdr(CPUMIPSState *env)
 {
     return (int32_t)(env->lladdr >> env->CP0_LLAddr_shift);
 }
 
-target_ulong helper_mfc0_watchlo (uint32_t sel)
+target_ulong helper_mfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return (int32_t)env->CP0_WatchLo[sel];
 }
 
-target_ulong helper_mfc0_watchhi (uint32_t sel)
+target_ulong helper_mfc0_watchhi(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchHi[sel];
 }
 
-target_ulong helper_mfc0_debug (void)
+target_ulong helper_mfc0_debug(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Debug;
     if (env->hflags & MIPS_HFLAG_DM)
@@ -1075,11 +894,11 @@ target_ulong helper_mfc0_debug (void)
     return t0;
 }
 
-target_ulong helper_mftc0_debug(void)
+target_ulong helper_mftc0_debug(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     int32_t tcstatus;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         tcstatus = other->active_tc.CP0_Debug_tcstatus;
@@ -1092,43 +911,43 @@ target_ulong helper_mftc0_debug(void)
 }
 
 #if defined(TARGET_MIPS64)
-target_ulong helper_dmfc0_tcrestart (void)
+target_ulong helper_dmfc0_tcrestart(CPUMIPSState *env)
 {
     return env->active_tc.PC;
 }
 
-target_ulong helper_dmfc0_tchalt (void)
+target_ulong helper_dmfc0_tchalt(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCHalt;
 }
 
-target_ulong helper_dmfc0_tccontext (void)
+target_ulong helper_dmfc0_tccontext(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCContext;
 }
 
-target_ulong helper_dmfc0_tcschedule (void)
+target_ulong helper_dmfc0_tcschedule(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCSchedule;
 }
 
-target_ulong helper_dmfc0_tcschefback (void)
+target_ulong helper_dmfc0_tcschefback(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCScheFBack;
 }
 
-target_ulong helper_dmfc0_lladdr (void)
+target_ulong helper_dmfc0_lladdr(CPUMIPSState *env)
 {
     return env->lladdr >> env->CP0_LLAddr_shift;
 }
 
-target_ulong helper_dmfc0_watchlo (uint32_t sel)
+target_ulong helper_dmfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchLo[sel];
 }
 #endif /* TARGET_MIPS64 */
 
-void helper_mtc0_index (target_ulong arg1)
+void helper_mtc0_index(CPUMIPSState *env, target_ulong arg1)
 {
     int num = 1;
     unsigned int tmp = env->tlb->nb_tlb;
@@ -1140,7 +959,7 @@ void helper_mtc0_index (target_ulong arg1)
     env->CP0_Index = (env->CP0_Index & 0x80000000) | (arg1 & (num - 1));
 }
 
-void helper_mtc0_mvpcontrol (target_ulong arg1)
+void helper_mtc0_mvpcontrol(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1157,7 +976,7 @@ void helper_mtc0_mvpcontrol (target_ulong arg1)
     env->mvp->CP0_MVPControl = newval;
 }
 
-void helper_mtc0_vpecontrol (target_ulong arg1)
+void helper_mtc0_vpecontrol(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask;
     uint32_t newval;
@@ -1174,10 +993,10 @@ void helper_mtc0_vpecontrol (target_ulong arg1)
     env->CP0_VPEControl = newval;
 }
 
-void helper_mttc0_vpecontrol(target_ulong arg1)
+void helper_mttc0_vpecontrol(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     uint32_t mask;
     uint32_t newval;
 
@@ -1190,23 +1009,23 @@ void helper_mttc0_vpecontrol(target_ulong arg1)
     other->CP0_VPEControl = newval;
 }
 
-target_ulong helper_mftc0_vpecontrol(void)
+target_ulong helper_mftc0_vpecontrol(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     /* FIXME: Mask away return zero on read bits.  */
     return other->CP0_VPEControl;
 }
 
-target_ulong helper_mftc0_vpeconf0(void)
+target_ulong helper_mftc0_vpeconf0(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_VPEConf0;
 }
 
-void helper_mtc0_vpeconf0 (target_ulong arg1)
+void helper_mtc0_vpeconf0(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1223,10 +1042,10 @@ void helper_mtc0_vpeconf0 (target_ulong arg1)
     env->CP0_VPEConf0 = newval;
 }
 
-void helper_mttc0_vpeconf0(target_ulong arg1)
+void helper_mttc0_vpeconf0(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     uint32_t mask = 0;
     uint32_t newval;
 
@@ -1237,7 +1056,7 @@ void helper_mttc0_vpeconf0(target_ulong arg1)
     other->CP0_VPEConf0 = newval;
 }
 
-void helper_mtc0_vpeconf1 (target_ulong arg1)
+void helper_mtc0_vpeconf1(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1255,25 +1074,25 @@ void helper_mtc0_vpeconf1 (target_ulong arg1)
     env->CP0_VPEConf1 = newval;
 }
 
-void helper_mtc0_yqmask (target_ulong arg1)
+void helper_mtc0_yqmask(CPUMIPSState *env, target_ulong arg1)
 {
     /* Yield qualifier inputs not implemented. */
     env->CP0_YQMask = 0x00000000;
 }
 
-void helper_mtc0_vpeopt (target_ulong arg1)
+void helper_mtc0_vpeopt(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_VPEOpt = arg1 & 0x0000ffff;
 }
 
-void helper_mtc0_entrylo0 (target_ulong arg1)
+void helper_mtc0_entrylo0(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     env->CP0_EntryLo0 = arg1 & 0x3FFFFFFF;
 }
 
-void helper_mtc0_tcstatus (target_ulong arg1)
+void helper_mtc0_tcstatus(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = env->CP0_TCStatus_rw_bitmask;
     uint32_t newval;
@@ -1284,10 +1103,10 @@ void helper_mtc0_tcstatus (target_ulong arg1)
     sync_c0_tcstatus(env, env->current_tc, newval);
 }
 
-void helper_mttc0_tcstatus (target_ulong arg1)
+void helper_mttc0_tcstatus(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCStatus = arg1;
@@ -1296,7 +1115,7 @@ void helper_mttc0_tcstatus (target_ulong arg1)
     sync_c0_tcstatus(other, other_tc, arg1);
 }
 
-void helper_mtc0_tcbind (target_ulong arg1)
+void helper_mtc0_tcbind(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = (1 << CP0TCBd_TBE);
     uint32_t newval;
@@ -1307,12 +1126,12 @@ void helper_mtc0_tcbind (target_ulong arg1)
     env->active_tc.CP0_TCBind = newval;
 }
 
-void helper_mttc0_tcbind (target_ulong arg1)
+void helper_mttc0_tcbind(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     uint32_t mask = (1 << CP0TCBd_TBE);
     uint32_t newval;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other->mvp->CP0_MVPControl & (1 << CP0MVPCo_VPC))
         mask |= (1 << CP0TCBd_CurVPE);
@@ -1325,7 +1144,7 @@ void helper_mttc0_tcbind (target_ulong arg1)
     }
 }
 
-void helper_mtc0_tcrestart (target_ulong arg1)
+void helper_mtc0_tcrestart(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.PC = arg1;
     env->active_tc.CP0_TCStatus &= ~(1 << CP0TCSt_TDS);
@@ -1333,10 +1152,10 @@ void helper_mtc0_tcrestart (target_ulong arg1)
     /* MIPS16 not implemented. */
 }
 
-void helper_mttc0_tcrestart (target_ulong arg1)
+void helper_mttc0_tcrestart(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc) {
         other->active_tc.PC = arg1;
@@ -1351,22 +1170,25 @@ void helper_mttc0_tcrestart (target_ulong arg1)
     }
 }
 
-void helper_mtc0_tchalt (target_ulong arg1)
+void helper_mtc0_tchalt(CPUMIPSState *env, target_ulong arg1)
 {
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+
     env->active_tc.CP0_TCHalt = arg1 & 0x1;
 
     // TODO: Halt TC / Restart (if allocated+active) TC.
     if (env->active_tc.CP0_TCHalt & 1) {
-        mips_tc_sleep(env, env->current_tc);
+        mips_tc_sleep(cpu, env->current_tc);
     } else {
-        mips_tc_wake(env, env->current_tc);
+        mips_tc_wake(cpu, env->current_tc);
     }
 }
 
-void helper_mttc0_tchalt (target_ulong arg1)
+void helper_mttc0_tchalt(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
+    MIPSCPU *other_cpu = mips_env_get_cpu(other);
 
     // TODO: Halt TC / Restart (if allocated+active) TC.
 
@@ -1376,21 +1198,21 @@ void helper_mttc0_tchalt (target_ulong arg1)
         other->tcs[other_tc].CP0_TCHalt = arg1;
 
     if (arg1 & 1) {
-        mips_tc_sleep(other, other_tc);
+        mips_tc_sleep(other_cpu, other_tc);
     } else {
-        mips_tc_wake(other, other_tc);
+        mips_tc_wake(other_cpu, other_tc);
     }
 }
 
-void helper_mtc0_tccontext (target_ulong arg1)
+void helper_mtc0_tccontext(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCContext = arg1;
 }
 
-void helper_mttc0_tccontext (target_ulong arg1)
+void helper_mttc0_tccontext(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCContext = arg1;
@@ -1398,15 +1220,15 @@ void helper_mttc0_tccontext (target_ulong arg1)
         other->tcs[other_tc].CP0_TCContext = arg1;
 }
 
-void helper_mtc0_tcschedule (target_ulong arg1)
+void helper_mtc0_tcschedule(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCSchedule = arg1;
 }
 
-void helper_mttc0_tcschedule (target_ulong arg1)
+void helper_mttc0_tcschedule(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCSchedule = arg1;
@@ -1414,15 +1236,15 @@ void helper_mttc0_tcschedule (target_ulong arg1)
         other->tcs[other_tc].CP0_TCSchedule = arg1;
 }
 
-void helper_mtc0_tcschefback (target_ulong arg1)
+void helper_mtc0_tcschefback(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCScheFBack = arg1;
 }
 
-void helper_mttc0_tcschefback (target_ulong arg1)
+void helper_mttc0_tcschefback(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCScheFBack = arg1;
@@ -1430,25 +1252,25 @@ void helper_mttc0_tcschefback (target_ulong arg1)
         other->tcs[other_tc].CP0_TCScheFBack = arg1;
 }
 
-void helper_mtc0_entrylo1 (target_ulong arg1)
+void helper_mtc0_entrylo1(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     env->CP0_EntryLo1 = arg1 & 0x3FFFFFFF;
 }
 
-void helper_mtc0_context (target_ulong arg1)
+void helper_mtc0_context(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Context = (env->CP0_Context & 0x007FFFFF) | (arg1 & ~0x007FFFFF);
 }
 
-void helper_mtc0_pagemask (target_ulong arg1)
+void helper_mtc0_pagemask(CPUMIPSState *env, target_ulong arg1)
 {
     /* 1k pages not implemented */
     env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));
 }
 
-void helper_mtc0_pagegrain (target_ulong arg1)
+void helper_mtc0_pagegrain(CPUMIPSState *env, target_ulong arg1)
 {
     /* SmartMIPS not implemented */
     /* Large physaddr (PABITS) not implemented */
@@ -1456,47 +1278,47 @@ void helper_mtc0_pagegrain (target_ulong arg1)
     env->CP0_PageGrain = 0;
 }
 
-void helper_mtc0_wired (target_ulong arg1)
+void helper_mtc0_wired(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Wired = arg1 % env->tlb->nb_tlb;
 }
 
-void helper_mtc0_srsconf0 (target_ulong arg1)
+void helper_mtc0_srsconf0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf0 |= arg1 & env->CP0_SRSConf0_rw_bitmask;
 }
 
-void helper_mtc0_srsconf1 (target_ulong arg1)
+void helper_mtc0_srsconf1(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf1 |= arg1 & env->CP0_SRSConf1_rw_bitmask;
 }
 
-void helper_mtc0_srsconf2 (target_ulong arg1)
+void helper_mtc0_srsconf2(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf2 |= arg1 & env->CP0_SRSConf2_rw_bitmask;
 }
 
-void helper_mtc0_srsconf3 (target_ulong arg1)
+void helper_mtc0_srsconf3(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf3 |= arg1 & env->CP0_SRSConf3_rw_bitmask;
 }
 
-void helper_mtc0_srsconf4 (target_ulong arg1)
+void helper_mtc0_srsconf4(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf4 |= arg1 & env->CP0_SRSConf4_rw_bitmask;
 }
 
-void helper_mtc0_hwrena (target_ulong arg1)
+void helper_mtc0_hwrena(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_HWREna = arg1 & 0x0000000F;
 }
 
-void helper_mtc0_count (target_ulong arg1)
+void helper_mtc0_count(CPUMIPSState *env, target_ulong arg1)
 {
     cpu_mips_store_count(env, arg1);
 }
 
-void helper_mtc0_entryhi (target_ulong arg1)
+void helper_mtc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
     target_ulong old, val;
 
@@ -1515,21 +1337,21 @@ void helper_mtc0_entryhi (target_ulong arg1)
         cpu_mips_tlb_flush(env, 1);
 }
 
-void helper_mttc0_entryhi(target_ulong arg1)
+void helper_mttc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     other->CP0_EntryHi = arg1;
     sync_c0_entryhi(other, other_tc);
 }
 
-void helper_mtc0_compare (target_ulong arg1)
+void helper_mtc0_compare(CPUMIPSState *env, target_ulong arg1)
 {
     cpu_mips_store_compare(env, arg1);
 }
 
-void helper_mtc0_status (target_ulong arg1)
+void helper_mtc0_status(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t val, old;
     uint32_t mask = env->CP0_Status_rw_bitmask;
@@ -1538,7 +1360,7 @@ void helper_mtc0_status (target_ulong arg1)
     old = env->CP0_Status;
     env->CP0_Status = (env->CP0_Status & ~mask) | val;
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
-        sync_c0_status(env, env->current_tc);
+        sync_c0_status(env, env, env->current_tc);
     } else {
         compute_hflags(env);
     }
@@ -1557,22 +1379,22 @@ void helper_mtc0_status (target_ulong arg1)
     }
 }
 
-void helper_mttc0_status(target_ulong arg1)
+void helper_mttc0_status(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     other->CP0_Status = arg1 & ~0xf1000018;
-    sync_c0_status(other, other_tc);
+    sync_c0_status(env, other, other_tc);
 }
 
-void helper_mtc0_intctl (target_ulong arg1)
+void helper_mtc0_intctl(CPUMIPSState *env, target_ulong arg1)
 {
     /* vectored interrupts not implemented, no performance counters. */
     env->CP0_IntCtl = (env->CP0_IntCtl & ~0x000003e0) | (arg1 & 0x000003e0);
 }
 
-void helper_mtc0_srsctl (target_ulong arg1)
+void helper_mtc0_srsctl(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = (0xf << CP0SRSCtl_ESS) | (0xf << CP0SRSCtl_PSS);
     env->CP0_SRSCtl = (env->CP0_SRSCtl & ~mask) | (arg1 & mask);
@@ -1606,52 +1428,52 @@ static void mtc0_cause(CPUMIPSState *cpu, target_ulong arg1)
     }
 }
 
-void helper_mtc0_cause(target_ulong arg1)
+void helper_mtc0_cause(CPUMIPSState *env, target_ulong arg1)
 {
     mtc0_cause(env, arg1);
 }
 
-void helper_mttc0_cause(target_ulong arg1)
+void helper_mttc0_cause(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     mtc0_cause(other, arg1);
 }
 
-target_ulong helper_mftc0_epc(void)
+target_ulong helper_mftc0_epc(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EPC;
 }
 
-target_ulong helper_mftc0_ebase(void)
+target_ulong helper_mftc0_ebase(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EBase;
 }
 
-void helper_mtc0_ebase (target_ulong arg1)
+void helper_mtc0_ebase(CPUMIPSState *env, target_ulong arg1)
 {
     /* vectored interrupts not implemented */
     env->CP0_EBase = (env->CP0_EBase & ~0x3FFFF000) | (arg1 & 0x3FFFF000);
 }
 
-void helper_mttc0_ebase(target_ulong arg1)
+void helper_mttc0_ebase(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     other->CP0_EBase = (other->CP0_EBase & ~0x3FFFF000) | (arg1 & 0x3FFFF000);
 }
 
-target_ulong helper_mftc0_configx(target_ulong idx)
+target_ulong helper_mftc0_configx(CPUMIPSState *env, target_ulong idx)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     switch (idx) {
     case 0: return other->CP0_Config0;
@@ -1667,49 +1489,49 @@ target_ulong helper_mftc0_configx(target_ulong idx)
     return 0;
 }
 
-void helper_mtc0_config0 (target_ulong arg1)
+void helper_mtc0_config0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Config0 = (env->CP0_Config0 & 0x81FFFFF8) | (arg1 & 0x00000007);
 }
 
-void helper_mtc0_config2 (target_ulong arg1)
+void helper_mtc0_config2(CPUMIPSState *env, target_ulong arg1)
 {
     /* tertiary/secondary caches not implemented */
     env->CP0_Config2 = (env->CP0_Config2 & 0x8FFF0FFF);
 }
 
-void helper_mtc0_lladdr (target_ulong arg1)
+void helper_mtc0_lladdr(CPUMIPSState *env, target_ulong arg1)
 {
     target_long mask = env->CP0_LLAddr_rw_bitmask;
     arg1 = arg1 << env->CP0_LLAddr_shift;
     env->lladdr = (env->lladdr & ~mask) | (arg1 & mask);
 }
 
-void helper_mtc0_watchlo (target_ulong arg1, uint32_t sel)
+void helper_mtc0_watchlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     /* Watch exceptions for instructions, data loads, data stores
        not implemented. */
     env->CP0_WatchLo[sel] = (arg1 & ~0x7);
 }
 
-void helper_mtc0_watchhi (target_ulong arg1, uint32_t sel)
+void helper_mtc0_watchhi(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     env->CP0_WatchHi[sel] = (arg1 & 0x40FF0FF8);
     env->CP0_WatchHi[sel] &= ~(env->CP0_WatchHi[sel] & arg1 & 0x7);
 }
 
-void helper_mtc0_xcontext (target_ulong arg1)
+void helper_mtc0_xcontext(CPUMIPSState *env, target_ulong arg1)
 {
     target_ulong mask = (1ULL << (env->SEGBITS - 7)) - 1;
     env->CP0_XContext = (env->CP0_XContext & mask) | (arg1 & ~mask);
 }
 
-void helper_mtc0_framemask (target_ulong arg1)
+void helper_mtc0_framemask(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Framemask = arg1; /* XXX */
 }
 
-void helper_mtc0_debug (target_ulong arg1)
+void helper_mtc0_debug(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Debug = (env->CP0_Debug & 0x8C03FC1F) | (arg1 & 0x13300120);
     if (arg1 & (1 << CP0DB_DM))
@@ -1718,11 +1540,11 @@ void helper_mtc0_debug (target_ulong arg1)
         env->hflags &= ~MIPS_HFLAG_DM;
 }
 
-void helper_mttc0_debug(target_ulong arg1)
+void helper_mttc0_debug(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     uint32_t val = arg1 & ((1 << CP0DB_SSt) | (1 << CP0DB_Halt));
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     /* XXX: Might be wrong, check with EJTAG spec. */
     if (other_tc == other->current_tc)
@@ -1734,36 +1556,36 @@ void helper_mttc0_debug(target_ulong arg1)
                      (arg1 & ~((1 << CP0DB_SSt) | (1 << CP0DB_Halt)));
 }
 
-void helper_mtc0_performance0 (target_ulong arg1)
+void helper_mtc0_performance0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Performance0 = arg1 & 0x000007ff;
 }
 
-void helper_mtc0_taglo (target_ulong arg1)
+void helper_mtc0_taglo(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_TagLo = arg1 & 0xFFFFFCF6;
 }
 
-void helper_mtc0_datalo (target_ulong arg1)
+void helper_mtc0_datalo(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_DataLo = arg1; /* XXX */
 }
 
-void helper_mtc0_taghi (target_ulong arg1)
+void helper_mtc0_taghi(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_TagHi = arg1; /* XXX */
 }
 
-void helper_mtc0_datahi (target_ulong arg1)
+void helper_mtc0_datahi(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_DataHi = arg1; /* XXX */
 }
 
 /* MIPS MT functions */
-target_ulong helper_mftgpr(uint32_t sel)
+target_ulong helper_mftgpr(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.gpr[sel];
@@ -1771,10 +1593,10 @@ target_ulong helper_mftgpr(uint32_t sel)
         return other->tcs[other_tc].gpr[sel];
 }
 
-target_ulong helper_mftlo(uint32_t sel)
+target_ulong helper_mftlo(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.LO[sel];
@@ -1782,10 +1604,10 @@ target_ulong helper_mftlo(uint32_t sel)
         return other->tcs[other_tc].LO[sel];
 }
 
-target_ulong helper_mfthi(uint32_t sel)
+target_ulong helper_mfthi(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.HI[sel];
@@ -1793,10 +1615,10 @@ target_ulong helper_mfthi(uint32_t sel)
         return other->tcs[other_tc].HI[sel];
 }
 
-target_ulong helper_mftacx(uint32_t sel)
+target_ulong helper_mftacx(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.ACX[sel];
@@ -1804,10 +1626,10 @@ target_ulong helper_mftacx(uint32_t sel)
         return other->tcs[other_tc].ACX[sel];
 }
 
-target_ulong helper_mftdsp(void)
+target_ulong helper_mftdsp(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.DSPControl;
@@ -1815,10 +1637,10 @@ target_ulong helper_mftdsp(void)
         return other->tcs[other_tc].DSPControl;
 }
 
-void helper_mttgpr(target_ulong arg1, uint32_t sel)
+void helper_mttgpr(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.gpr[sel] = arg1;
@@ -1826,10 +1648,10 @@ void helper_mttgpr(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].gpr[sel] = arg1;
 }
 
-void helper_mttlo(target_ulong arg1, uint32_t sel)
+void helper_mttlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.LO[sel] = arg1;
@@ -1837,10 +1659,10 @@ void helper_mttlo(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].LO[sel] = arg1;
 }
 
-void helper_mtthi(target_ulong arg1, uint32_t sel)
+void helper_mtthi(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.HI[sel] = arg1;
@@ -1848,10 +1670,10 @@ void helper_mtthi(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].HI[sel] = arg1;
 }
 
-void helper_mttacx(target_ulong arg1, uint32_t sel)
+void helper_mttacx(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.ACX[sel] = arg1;
@@ -1859,10 +1681,10 @@ void helper_mttacx(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].ACX[sel] = arg1;
 }
 
-void helper_mttdsp(target_ulong arg1)
+void helper_mttdsp(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.DSPControl = arg1;
@@ -1883,37 +1705,41 @@ target_ulong helper_emt(void)
     return 0;
 }
 
-target_ulong helper_dvpe(void)
+target_ulong helper_dvpe(CPUMIPSState *env)
 {
-    CPUMIPSState *other_cpu = first_cpu;
+    CPUMIPSState *other_cpu_env = first_cpu;
     target_ulong prev = env->mvp->CP0_MVPControl;
 
     do {
         /* Turn off all VPEs except the one executing the dvpe.  */
-        if (other_cpu != env) {
-            other_cpu->mvp->CP0_MVPControl &= ~(1 << CP0MVPCo_EVP);
+        if (other_cpu_env != env) {
+            MIPSCPU *other_cpu = mips_env_get_cpu(other_cpu_env);
+
+            other_cpu_env->mvp->CP0_MVPControl &= ~(1 << CP0MVPCo_EVP);
             mips_vpe_sleep(other_cpu);
         }
-        other_cpu = other_cpu->next_cpu;
-    } while (other_cpu);
+        other_cpu_env = other_cpu_env->next_cpu;
+    } while (other_cpu_env);
     return prev;
 }
 
-target_ulong helper_evpe(void)
+target_ulong helper_evpe(CPUMIPSState *env)
 {
-    CPUMIPSState *other_cpu = first_cpu;
+    CPUMIPSState *other_cpu_env = first_cpu;
     target_ulong prev = env->mvp->CP0_MVPControl;
 
     do {
-        if (other_cpu != env
-           /* If the VPE is WFI, don't disturb its sleep.  */
-           && !mips_vpe_is_wfi(other_cpu)) {
+        MIPSCPU *other_cpu = mips_env_get_cpu(other_cpu_env);
+
+        if (other_cpu_env != env
+            /* If the VPE is WFI, don't disturb its sleep.  */
+            && !mips_vpe_is_wfi(other_cpu)) {
             /* Enable the VPE.  */
-            other_cpu->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
-            mips_vpe_wake(other_cpu); /* And wake it up.  */
+            other_cpu_env->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
+            mips_vpe_wake(other_cpu_env); /* And wake it up.  */
         }
-        other_cpu = other_cpu->next_cpu;
-    } while (other_cpu);
+        other_cpu_env = other_cpu_env->next_cpu;
+    } while (other_cpu_env);
     return prev;
 }
 #endif /* !CONFIG_USER_ONLY */
@@ -1925,7 +1751,7 @@ void helper_fork(target_ulong arg1, target_ulong arg2)
     // TODO: store to TC register
 }
 
-target_ulong helper_yield(target_ulong arg)
+target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
 {
     target_long arg1 = arg;
 
@@ -1936,13 +1762,13 @@ target_ulong helper_yield(target_ulong arg)
                 env->active_tc.CP0_TCStatus & (1 << CP0TCSt_DT)) {
                 env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
                 env->CP0_VPEControl |= 4 << CP0VPECo_EXCPT;
-                helper_raise_exception(EXCP_THREAD);
+                helper_raise_exception(env, EXCP_THREAD);
             }
         }
     } else if (arg1 == 0) {
         if (0 /* TODO: TC underflow */) {
             env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
-            helper_raise_exception(EXCP_THREAD);
+            helper_raise_exception(env, EXCP_THREAD);
         } else {
             // TODO: Deallocate TC
         }
@@ -1950,7 +1776,7 @@ target_ulong helper_yield(target_ulong arg)
         /* Yield qualifier inputs not implemented. */
         env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
         env->CP0_VPEControl |= 2 << CP0VPECo_EXCPT;
-        helper_raise_exception(EXCP_THREAD);
+        helper_raise_exception(env, EXCP_THREAD);
     }
     return env->CP0_YQMask;
 }
@@ -1972,7 +1798,7 @@ static void r4k_mips_tlb_flush_extra (CPUMIPSState *env, int first)
     }
 }
 
-static void r4k_fill_tlb (int idx)
+static void r4k_fill_tlb(CPUMIPSState *env, int idx)
 {
     r4k_tlb_t *tlb;
 
@@ -1995,30 +1821,48 @@ static void r4k_fill_tlb (int idx)
     tlb->PFN[1] = (env->CP0_EntryLo1 >> 6) << 12;
 }
 
-void r4k_helper_tlbwi (void)
+void r4k_helper_tlbwi(CPUMIPSState *env)
 {
+    r4k_tlb_t *tlb;
     int idx;
+    target_ulong VPN;
+    uint8_t ASID;
+    bool G, V0, D0, V1, D1;
 
     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
+    tlb = &env->tlb->mmu.r4k.tlb[idx];
+    VPN = env->CP0_EntryHi & (TARGET_PAGE_MASK << 1);
+#if defined(TARGET_MIPS64)
+    VPN &= env->SEGMask;
+#endif
+    ASID = env->CP0_EntryHi & 0xff;
+    G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
+    V0 = (env->CP0_EntryLo0 & 2) != 0;
+    D0 = (env->CP0_EntryLo0 & 4) != 0;
+    V1 = (env->CP0_EntryLo1 & 2) != 0;
+    D1 = (env->CP0_EntryLo1 & 4) != 0;
 
-    /* Discard cached TLB entries.  We could avoid doing this if the
-       tlbwi is just upgrading access permissions on the current entry;
-       that might be a further win.  */
-    r4k_mips_tlb_flush_extra (env, env->tlb->nb_tlb);
+    /* Discard cached TLB entries, unless tlbwi is just upgrading access
+       permissions on the current entry. */
+    if (tlb->VPN != VPN || tlb->ASID != ASID || tlb->G != G ||
+        (tlb->V0 && !V0) || (tlb->D0 && !D0) ||
+        (tlb->V1 && !V1) || (tlb->D1 && !D1)) {
+        r4k_mips_tlb_flush_extra(env, env->tlb->nb_tlb);
+    }
 
     r4k_invalidate_tlb(env, idx, 0);
-    r4k_fill_tlb(idx);
+    r4k_fill_tlb(env, idx);
 }
 
-void r4k_helper_tlbwr (void)
+void r4k_helper_tlbwr(CPUMIPSState *env)
 {
     int r = cpu_mips_get_random(env);
 
     r4k_invalidate_tlb(env, r, 1);
-    r4k_fill_tlb(r);
+    r4k_fill_tlb(env, r);
 }
 
-void r4k_helper_tlbp (void)
+void r4k_helper_tlbp(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
     target_ulong mask;
@@ -2034,6 +1878,9 @@ void r4k_helper_tlbp (void)
         mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
         tag = env->CP0_EntryHi & ~mask;
         VPN = tlb->VPN & ~mask;
+#if defined(TARGET_MIPS64)
+        tag &= env->SEGMask;
+#endif
         /* Check ASID, virtual page number & size */
         if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
             /* TLB match */
@@ -2049,6 +1896,9 @@ void r4k_helper_tlbp (void)
             mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
             tag = env->CP0_EntryHi & ~mask;
             VPN = tlb->VPN & ~mask;
+#if defined(TARGET_MIPS64)
+            tag &= env->SEGMask;
+#endif
             /* Check ASID, virtual page number & size */
             if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
                 r4k_mips_tlb_flush_extra (env, i);
@@ -2060,7 +1910,7 @@ void r4k_helper_tlbp (void)
     }
 }
 
-void r4k_helper_tlbr (void)
+void r4k_helper_tlbr(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
     uint8_t ASID;
@@ -2084,28 +1934,28 @@ void r4k_helper_tlbr (void)
                         (tlb->C1 << 3) | (tlb->PFN[1] >> 6);
 }
 
-void helper_tlbwi(void)
+void helper_tlbwi(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbwi();
+    env->tlb->helper_tlbwi(env);
 }
 
-void helper_tlbwr(void)
+void helper_tlbwr(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbwr();
+    env->tlb->helper_tlbwr(env);
 }
 
-void helper_tlbp(void)
+void helper_tlbp(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbp();
+    env->tlb->helper_tlbp(env);
 }
 
-void helper_tlbr(void)
+void helper_tlbr(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbr();
+    env->tlb->helper_tlbr(env);
 }
 
 /* Specials */
-target_ulong helper_di (void)
+target_ulong helper_di(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Status;
 
@@ -2113,7 +1963,7 @@ target_ulong helper_di (void)
     return t0;
 }
 
-target_ulong helper_ei (void)
+target_ulong helper_ei(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Status;
 
@@ -2121,7 +1971,7 @@ target_ulong helper_ei (void)
     return t0;
 }
 
-static void debug_pre_eret (void)
+static void debug_pre_eret(CPUMIPSState *env)
 {
     if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
         qemu_log("ERET: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx,
@@ -2134,7 +1984,7 @@ static void debug_pre_eret (void)
     }
 }
 
-static void debug_post_eret (void)
+static void debug_post_eret(CPUMIPSState *env)
 {
     if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
         qemu_log("  =>  PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx,
@@ -2152,7 +2002,7 @@ static void debug_post_eret (void)
     }
 }
 
-static void set_pc (target_ulong error_pc)
+static void set_pc(CPUMIPSState *env, target_ulong error_pc)
 {
     env->active_tc.PC = error_pc & ~(target_ulong)1;
     if (error_pc & 1) {
@@ -2162,78 +2012,78 @@ static void set_pc (target_ulong error_pc)
     }
 }
 
-void helper_eret (void)
+void helper_eret(CPUMIPSState *env)
 {
-    debug_pre_eret();
+    debug_pre_eret(env);
     if (env->CP0_Status & (1 << CP0St_ERL)) {
-        set_pc(env->CP0_ErrorEPC);
+        set_pc(env, env->CP0_ErrorEPC);
         env->CP0_Status &= ~(1 << CP0St_ERL);
     } else {
-        set_pc(env->CP0_EPC);
+        set_pc(env, env->CP0_EPC);
         env->CP0_Status &= ~(1 << CP0St_EXL);
     }
     compute_hflags(env);
-    debug_post_eret();
+    debug_post_eret(env);
     env->lladdr = 1;
 }
 
-void helper_deret (void)
+void helper_deret(CPUMIPSState *env)
 {
-    debug_pre_eret();
-    set_pc(env->CP0_DEPC);
+    debug_pre_eret(env);
+    set_pc(env, env->CP0_DEPC);
 
     env->hflags &= MIPS_HFLAG_DM;
     compute_hflags(env);
-    debug_post_eret();
+    debug_post_eret(env);
     env->lladdr = 1;
 }
 #endif /* !CONFIG_USER_ONLY */
 
-target_ulong helper_rdhwr_cpunum(void)
+target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 0)))
         return env->CP0_EBase & 0x3ff;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_synci_step(void)
+target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 1)))
         return env->SYNCI_Step;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_cc(void)
+target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 2)))
         return env->CP0_Count;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_ccres(void)
+target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 3)))
         return env->CCRes;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-void helper_pmon (int function)
+void helper_pmon(CPUMIPSState *env, int function)
 {
     function /= 2;
     switch (function) {
@@ -2259,88 +2109,69 @@ void helper_pmon (int function)
     }
 }
 
-void helper_wait (void)
+void helper_wait(CPUMIPSState *env)
 {
     env->halted = 1;
     cpu_reset_interrupt(env, CPU_INTERRUPT_WAKE);
-    helper_raise_exception(EXCP_HLT);
+    helper_raise_exception(env, EXCP_HLT);
 }
 
 #if !defined(CONFIG_USER_ONLY)
 
-static void QEMU_NORETURN do_unaligned_access(target_ulong addr, int is_write,
+static void QEMU_NORETURN do_unaligned_access(CPUMIPSState *env,
+                                              target_ulong addr, int is_write,
                                               int is_user, uintptr_t retaddr);
 
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
-static void do_unaligned_access(target_ulong addr, int is_write,
-                                int is_user, uintptr_t retaddr)
+static void do_unaligned_access(CPUMIPSState *env, target_ulong addr,
+                                int is_write, int is_user, uintptr_t retaddr)
 {
     env->CP0_BadVAddr = addr;
-    do_restore_state (retaddr);
-    helper_raise_exception ((is_write == 1) ? EXCP_AdES : EXCP_AdEL);
+    do_raise_exception(env, (is_write == 1) ? EXCP_AdES : EXCP_AdEL, retaddr);
 }
 
-void tlb_fill(CPUMIPSState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUMIPSState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUMIPSState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_mips_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
-        if (retaddr) {
-            /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
-        }
-        helper_raise_exception_err(env->exception_index, env->error_code);
+        do_raise_exception_err(env, env->exception_index,
+                               env->error_code, retaddr);
     }
-    env = saved_env;
 }
 
-void cpu_unassigned_access(CPUMIPSState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMIPSState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size)
 {
-    env = env1;
-
     if (is_exec)
-        helper_raise_exception(EXCP_IBE);
+        helper_raise_exception(env, EXCP_IBE);
     else
-        helper_raise_exception(EXCP_DBE);
+        helper_raise_exception(env, EXCP_DBE);
 }
 #endif /* !CONFIG_USER_ONLY */
 
 /* Complex FPU operations which may need stack space. */
 
-#define FLOAT_ONE32 make_float32(0x3f8 << 20)
-#define FLOAT_ONE64 make_float64(0x3ffULL << 52)
 #define FLOAT_TWO32 make_float32(1 << 30)
 #define FLOAT_TWO64 make_float64(1ULL << 62)
-#define FLOAT_QNAN32 0x7fbfffff
-#define FLOAT_QNAN64 0x7ff7ffffffffffffULL
-#define FLOAT_SNAN32 0x7fffffff
-#define FLOAT_SNAN64 0x7fffffffffffffffULL
+#define FP_TO_INT32_OVERFLOW 0x7fffffff
+#define FP_TO_INT64_OVERFLOW 0x7fffffffffffffffULL
 
 /* convert MIPS rounding mode in FCR31 to IEEE library */
 static unsigned int ieee_rm[] = {
@@ -2350,13 +2181,19 @@ static unsigned int ieee_rm[] = {
     float_round_down
 };
 
-#define RESTORE_ROUNDING_MODE \
-    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3], &env->active_fpu.fp_status)
+static inline void restore_rounding_mode(CPUMIPSState *env)
+{
+    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3],
+                            &env->active_fpu.fp_status);
+}
 
-#define RESTORE_FLUSH_MODE \
-    set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0, &env->active_fpu.fp_status);
+static inline void restore_flush_mode(CPUMIPSState *env)
+{
+    set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0,
+                      &env->active_fpu.fp_status);
+}
 
-target_ulong helper_cfc1 (uint32_t reg)
+target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
 {
     target_ulong arg1;
 
@@ -2381,7 +2218,7 @@ target_ulong helper_cfc1 (uint32_t reg)
     return arg1;
 }
 
-void helper_ctc1 (target_ulong arg1, uint32_t reg)
+void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t reg)
 {
     switch(reg) {
     case 25:
@@ -2410,12 +2247,12 @@ void helper_ctc1 (target_ulong arg1, uint32_t reg)
         return;
     }
     /* set rounding mode */
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     /* set flush-to-zero mode */
-    RESTORE_FLUSH_MODE;
+    restore_flush_mode(env);
     set_float_exception_flags(0, &env->active_fpu.fp_status);
     if ((GET_FP_ENABLE(env->active_fpu.fcr31) | 0x20) & GET_FP_CAUSE(env->active_fpu.fcr31))
-        helper_raise_exception(EXCP_FPE);
+        do_raise_exception(env, EXCP_FPE, GETPC());
 }
 
 static inline int ieee_ex_to_mips(int xcpt)
@@ -2441,15 +2278,21 @@ static inline int ieee_ex_to_mips(int xcpt)
     return ret;
 }
 
-static inline void update_fcr31(void)
+static inline void update_fcr31(CPUMIPSState *env, uintptr_t pc)
 {
     int tmp = ieee_ex_to_mips(get_float_exception_flags(&env->active_fpu.fp_status));
 
     SET_FP_CAUSE(env->active_fpu.fcr31, tmp);
-    if (GET_FP_ENABLE(env->active_fpu.fcr31) & tmp)
-        helper_raise_exception(EXCP_FPE);
-    else
-        UPDATE_FP_FLAGS(env->active_fpu.fcr31, tmp);
+
+    if (tmp) {
+        set_float_exception_flags(0, &env->active_fpu.fp_status);
+
+        if (GET_FP_ENABLE(env->active_fpu.fcr31) & tmp) {
+            do_raise_exception(env, EXCP_FPE, pc);
+        } else {
+            UPDATE_FP_FLAGS(env->active_fpu.fcr31, tmp);
+        }
+    }
 }
 
 /* Float support.
@@ -2458,385 +2301,409 @@ static inline void update_fcr31(void)
    paired single lower "pl", paired single upper "pu".  */
 
 /* unary operations, modifying fp status  */
-uint64_t helper_float_sqrt_d(uint64_t fdt0)
+uint64_t helper_float_sqrt_d(CPUMIPSState *env, uint64_t fdt0)
 {
-    return float64_sqrt(fdt0, &env->active_fpu.fp_status);
+    fdt0 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+    return fdt0;
 }
 
-uint32_t helper_float_sqrt_s(uint32_t fst0)
+uint32_t helper_float_sqrt_s(CPUMIPSState *env, uint32_t fst0)
 {
-    return float32_sqrt(fst0, &env->active_fpu.fp_status);
+    fst0 = float32_sqrt(fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+    return fst0;
 }
 
-uint64_t helper_float_cvtd_s(uint32_t fst0)
+uint64_t helper_float_cvtd_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float32_to_float64(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtd_w(uint32_t wt0)
+uint64_t helper_float_cvtd_w(CPUMIPSState *env, uint32_t wt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = int32_to_float64(wt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtd_l(uint64_t dt0)
+uint64_t helper_float_cvtd_l(CPUMIPSState *env, uint64_t dt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = int64_to_float64(dt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtl_d(uint64_t fdt0)
+uint64_t helper_float_cvtl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_cvtl_s(uint32_t fst0)
+uint64_t helper_float_cvtl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_cvtps_pw(uint64_t dt0)
+uint64_t helper_float_cvtps_pw(CPUMIPSState *env, uint64_t dt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int32_to_float32(dt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
     fsth2 = int32_to_float32(dt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_cvtpw_ps(uint64_t fdt0)
+uint64_t helper_float_cvtpw_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
     uint32_t wth2;
+    int excp, excph;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
+    excp = get_float_exception_flags(&env->active_fpu.fp_status);
+    if (excp & (float_flag_overflow | float_flag_invalid)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+
+    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wth2 = float32_to_int32(fdt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID)) {
-        wt2 = FLOAT_SNAN32;
-        wth2 = FLOAT_SNAN32;
+    excph = get_float_exception_flags(&env->active_fpu.fp_status);
+    if (excph & (float_flag_overflow | float_flag_invalid)) {
+        wth2 = FP_TO_INT32_OVERFLOW;
     }
+
+    set_float_exception_flags(excp | excph, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+
     return ((uint64_t)wth2 << 32) | wt2;
 }
 
-uint32_t helper_float_cvts_d(uint64_t fdt0)
+uint32_t helper_float_cvts_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float64_to_float32(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_w(uint32_t wt0)
+uint32_t helper_float_cvts_w(CPUMIPSState *env, uint32_t wt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int32_to_float32(wt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_l(uint64_t dt0)
+uint32_t helper_float_cvts_l(CPUMIPSState *env, uint64_t dt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int64_to_float32(dt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_pl(uint32_t wt0)
+uint32_t helper_float_cvts_pl(CPUMIPSState *env, uint32_t wt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = wt0;
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_cvts_pu(uint32_t wth0)
+uint32_t helper_float_cvts_pu(CPUMIPSState *env, uint32_t wth0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = wth0;
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_cvtw_s(uint32_t fst0)
+uint32_t helper_float_cvtw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    update_fcr31(env, GETPC());
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
     return wt2;
 }
 
-uint32_t helper_float_cvtw_d(uint64_t fdt0)
+uint32_t helper_float_cvtw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_roundl_d(uint64_t fdt0)
+uint64_t helper_float_roundl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_roundl_s(uint32_t fst0)
+uint64_t helper_float_roundl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_roundw_d(uint64_t fdt0)
+uint32_t helper_float_roundw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_roundw_s(uint32_t fst0)
+uint32_t helper_float_roundw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_truncl_d(uint64_t fdt0)
+uint64_t helper_float_truncl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float64_to_int64_round_to_zero(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_truncl_s(uint32_t fst0)
+uint64_t helper_float_truncl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float32_to_int64_round_to_zero(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_truncw_d(uint64_t fdt0)
+uint32_t helper_float_truncw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float64_to_int32_round_to_zero(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_truncw_s(uint32_t fst0)
+uint32_t helper_float_truncw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32_round_to_zero(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_ceill_d(uint64_t fdt0)
+uint64_t helper_float_ceill_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_ceill_s(uint32_t fst0)
+uint64_t helper_float_ceill_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_ceilw_d(uint64_t fdt0)
+uint32_t helper_float_ceilw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_ceilw_s(uint32_t fst0)
+uint32_t helper_float_ceilw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_floorl_d(uint64_t fdt0)
+uint64_t helper_float_floorl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_floorl_s(uint32_t fst0)
+uint64_t helper_float_floorl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_floorw_d(uint64_t fdt0)
+uint32_t helper_float_floorw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_floorw_s(uint32_t fst0)
+uint32_t helper_float_floorw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
@@ -2864,145 +2731,133 @@ FLOAT_UNOP(chs)
 #undef FLOAT_UNOP
 
 /* MIPS specific unary operations */
-uint64_t helper_float_recip_d(uint64_t fdt0)
+uint64_t helper_float_recip_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip_s(uint32_t fst0)
+uint32_t helper_float_recip_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt_d(uint64_t fdt0)
+uint64_t helper_float_rsqrt_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt_s(uint32_t fst0)
+uint32_t helper_float_rsqrt_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fst0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip1_d(uint64_t fdt0)
+uint64_t helper_float_recip1_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip1_s(uint32_t fst0)
+uint32_t helper_float_recip1_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip1_ps(uint64_t fdt0)
+uint64_t helper_float_recip1_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
-    fsth2 = float32_div(FLOAT_ONE32, fdt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
+    fsth2 = float32_div(float32_one, fdt0 >> 32, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_rsqrt1_d(uint64_t fdt0)
+uint64_t helper_float_rsqrt1_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt1_s(uint32_t fst0)
+uint32_t helper_float_rsqrt1_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fst0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt1_ps(uint64_t fdt0)
+uint64_t helper_float_rsqrt1_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
     fsth2 = float32_sqrt(fdt0 >> 32, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    fsth2 = float32_div(FLOAT_ONE32, fsth2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    fsth2 = float32_div(float32_one, fsth2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-#define FLOAT_OP(name, p) void helper_float_##name##_##p(void)
+#define FLOAT_OP(name, p) void helper_float_##name##_##p(CPUMIPSState *env)
 
 /* binary operations */
 #define FLOAT_BINOP(name)                                          \
-uint64_t helper_float_ ## name ## _d(uint64_t fdt0, uint64_t fdt1)     \
+uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,            \
+                                     uint64_t fdt0, uint64_t fdt1) \
 {                                                                  \
     uint64_t dt2;                                                  \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     dt2 = float64_ ## name (fdt0, fdt1, &env->active_fpu.fp_status);     \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID)                \
-        dt2 = FLOAT_QNAN64;                                        \
+    update_fcr31(env, GETPC());                                    \
     return dt2;                                                    \
 }                                                                  \
                                                                    \
-uint32_t helper_float_ ## name ## _s(uint32_t fst0, uint32_t fst1)     \
+uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,            \
+                                     uint32_t fst0, uint32_t fst1) \
 {                                                                  \
     uint32_t wt2;                                                  \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     wt2 = float32_ ## name (fst0, fst1, &env->active_fpu.fp_status);     \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID)                \
-        wt2 = FLOAT_QNAN32;                                        \
+    update_fcr31(env, GETPC());                                    \
     return wt2;                                                    \
 }                                                                  \
                                                                    \
-uint64_t helper_float_ ## name ## _ps(uint64_t fdt0, uint64_t fdt1)    \
+uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,           \
+                                      uint64_t fdt0,               \
+                                      uint64_t fdt1)               \
 {                                                                  \
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;                             \
     uint32_t fsth0 = fdt0 >> 32;                                   \
@@ -3011,14 +2866,9 @@ uint64_t helper_float_ ## name ## _ps(uint64_t fdt0, uint64_t fdt1)    \
     uint32_t wt2;                                                  \
     uint32_t wth2;                                                 \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     wt2 = float32_ ## name (fst0, fst1, &env->active_fpu.fp_status);     \
     wth2 = float32_ ## name (fsth0, fsth1, &env->active_fpu.fp_status);  \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID) {              \
-        wt2 = FLOAT_QNAN32;                                        \
-        wth2 = FLOAT_QNAN32;                                       \
-    }                                                              \
+    update_fcr31(env, GETPC());                                    \
     return ((uint64_t)wth2 << 32) | wt2;                           \
 }
 
@@ -3028,158 +2878,120 @@ FLOAT_BINOP(mul)
 FLOAT_BINOP(div)
 #undef FLOAT_BINOP
 
-/* ternary operations */
-#define FLOAT_TERNOP(name1, name2)                                        \
-uint64_t helper_float_ ## name1 ## name2 ## _d(uint64_t fdt0, uint64_t fdt1,  \
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    fdt0 = float64_ ## name1 (fdt0, fdt1, &env->active_fpu.fp_status);          \
-    return float64_ ## name2 (fdt0, fdt2, &env->active_fpu.fp_status);          \
-}                                                                         \
-                                                                          \
-uint32_t helper_float_ ## name1 ## name2 ## _s(uint32_t fst0, uint32_t fst1,  \
-                                           uint32_t fst2)                 \
-{                                                                         \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    return float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-}                                                                         \
-                                                                          \
-uint64_t helper_float_ ## name1 ## name2 ## _ps(uint64_t fdt0, uint64_t fdt1, \
-                                            uint64_t fdt2)                \
-{                                                                         \
-    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                                    \
-    uint32_t fsth0 = fdt0 >> 32;                                          \
-    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                                    \
-    uint32_t fsth1 = fdt1 >> 32;                                          \
-    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                                    \
-    uint32_t fsth2 = fdt2 >> 32;                                          \
-                                                                          \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fsth0 = float32_ ## name1 (fsth0, fsth1, &env->active_fpu.fp_status);       \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    fsth2 = float32_ ## name2 (fsth0, fsth2, &env->active_fpu.fp_status);       \
-    return ((uint64_t)fsth2 << 32) | fst2;                                \
-}
-
-FLOAT_TERNOP(mul, add)
-FLOAT_TERNOP(mul, sub)
-#undef FLOAT_TERNOP
-
-/* negated ternary operations */
-#define FLOAT_NTERNOP(name1, name2)                                       \
-uint64_t helper_float_n ## name1 ## name2 ## _d(uint64_t fdt0, uint64_t fdt1, \
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    fdt0 = float64_ ## name1 (fdt0, fdt1, &env->active_fpu.fp_status);          \
-    fdt2 = float64_ ## name2 (fdt0, fdt2, &env->active_fpu.fp_status);          \
-    return float64_chs(fdt2);                                             \
-}                                                                         \
-                                                                          \
-uint32_t helper_float_n ## name1 ## name2 ## _s(uint32_t fst0, uint32_t fst1, \
-                                           uint32_t fst2)                 \
-{                                                                         \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    return float32_chs(fst2);                                             \
-}                                                                         \
-                                                                          \
-uint64_t helper_float_n ## name1 ## name2 ## _ps(uint64_t fdt0, uint64_t fdt1,\
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                                    \
-    uint32_t fsth0 = fdt0 >> 32;                                          \
-    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                                    \
-    uint32_t fsth1 = fdt1 >> 32;                                          \
-    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                                    \
-    uint32_t fsth2 = fdt2 >> 32;                                          \
-                                                                          \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fsth0 = float32_ ## name1 (fsth0, fsth1, &env->active_fpu.fp_status);       \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    fsth2 = float32_ ## name2 (fsth0, fsth2, &env->active_fpu.fp_status);       \
-    fst2 = float32_chs(fst2);                                             \
-    fsth2 = float32_chs(fsth2);                                           \
-    return ((uint64_t)fsth2 << 32) | fst2;                                \
-}
-
-FLOAT_NTERNOP(mul, add)
-FLOAT_NTERNOP(mul, sub)
-#undef FLOAT_NTERNOP
+/* FMA based operations */
+#define FLOAT_FMA(name, type)                                        \
+uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,              \
+                                     uint64_t fdt0, uint64_t fdt1,   \
+                                     uint64_t fdt2)                  \
+{                                                                    \
+    fdt0 = float64_muladd(fdt0, fdt1, fdt2, type,                    \
+                         &env->active_fpu.fp_status);                \
+    update_fcr31(env, GETPC());                                      \
+    return fdt0;                                                     \
+}                                                                    \
+                                                                     \
+uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,              \
+                                     uint32_t fst0, uint32_t fst1,   \
+                                     uint32_t fst2)                  \
+{                                                                    \
+    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
+                         &env->active_fpu.fp_status);                \
+    update_fcr31(env, GETPC());                                      \
+    return fst0;                                                     \
+}                                                                    \
+                                                                     \
+uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,             \
+                                      uint64_t fdt0, uint64_t fdt1,  \
+                                      uint64_t fdt2)                 \
+{                                                                    \
+    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                               \
+    uint32_t fsth0 = fdt0 >> 32;                                     \
+    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                               \
+    uint32_t fsth1 = fdt1 >> 32;                                     \
+    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                               \
+    uint32_t fsth2 = fdt2 >> 32;                                     \
+                                                                     \
+    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
+                          &env->active_fpu.fp_status);               \
+    fsth0 = float32_muladd(fsth0, fsth1, fsth2, type,                \
+                           &env->active_fpu.fp_status);              \
+    update_fcr31(env, GETPC());                                      \
+    return ((uint64_t)fsth0 << 32) | fst0;                           \
+}
+FLOAT_FMA(madd, 0)
+FLOAT_FMA(msub, float_muladd_negate_c)
+FLOAT_FMA(nmadd, float_muladd_negate_result)
+FLOAT_FMA(nmsub, float_muladd_negate_result | float_muladd_negate_c)
+#undef FLOAT_FMA
 
 /* MIPS specific binary operations */
-uint64_t helper_float_recip2_d(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_recip2_d(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_mul(fdt0, fdt2, &env->active_fpu.fp_status);
-    fdt2 = float64_chs(float64_sub(fdt2, FLOAT_ONE64, &env->active_fpu.fp_status));
-    update_fcr31();
+    fdt2 = float64_chs(float64_sub(fdt2, float64_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip2_s(uint32_t fst0, uint32_t fst2)
+uint32_t helper_float_recip2_s(CPUMIPSState *env, uint32_t fst0, uint32_t fst2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
-    fst2 = float32_chs(float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    update_fcr31();
+    fst2 = float32_chs(float32_sub(fst2, float32_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip2_ps(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_recip2_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
     uint32_t fst2 = fdt2 & 0XFFFFFFFF;
     uint32_t fsth2 = fdt2 >> 32;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
     fsth2 = float32_mul(fsth0, fsth2, &env->active_fpu.fp_status);
-    fst2 = float32_chs(float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    fsth2 = float32_chs(float32_sub(fsth2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    update_fcr31();
+    fst2 = float32_chs(float32_sub(fst2, float32_one, &env->active_fpu.fp_status));
+    fsth2 = float32_chs(float32_sub(fsth2, float32_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_rsqrt2_d(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_rsqrt2_d(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_mul(fdt0, fdt2, &env->active_fpu.fp_status);
-    fdt2 = float64_sub(fdt2, FLOAT_ONE64, &env->active_fpu.fp_status);
+    fdt2 = float64_sub(fdt2, float64_one, &env->active_fpu.fp_status);
     fdt2 = float64_chs(float64_div(fdt2, FLOAT_TWO64, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt2_s(uint32_t fst0, uint32_t fst2)
+uint32_t helper_float_rsqrt2_s(CPUMIPSState *env, uint32_t fst0, uint32_t fst2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
-    fst2 = float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status);
+    fst2 = float32_sub(fst2, float32_one, &env->active_fpu.fp_status);
     fst2 = float32_chs(float32_div(fst2, FLOAT_TWO32, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt2_ps(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_rsqrt2_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
     uint32_t fst2 = fdt2 & 0XFFFFFFFF;
     uint32_t fsth2 = fdt2 >> 32;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
     fsth2 = float32_mul(fsth0, fsth2, &env->active_fpu.fp_status);
-    fst2 = float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status);
-    fsth2 = float32_sub(fsth2, FLOAT_ONE32, &env->active_fpu.fp_status);
+    fst2 = float32_sub(fst2, float32_one, &env->active_fpu.fp_status);
+    fsth2 = float32_sub(fsth2, float32_one, &env->active_fpu.fp_status);
     fst2 = float32_chs(float32_div(fst2, FLOAT_TWO32, &env->active_fpu.fp_status));
     fsth2 = float32_chs(float32_div(fsth2, FLOAT_TWO32, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_addr_ps(uint64_t fdt0, uint64_t fdt1)
+uint64_t helper_float_addr_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt1)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
@@ -3188,14 +3000,13 @@ uint64_t helper_float_addr_ps(uint64_t fdt0, uint64_t fdt1)
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_add (fst0, fsth0, &env->active_fpu.fp_status);
     fsth2 = float32_add (fst1, fsth1, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_mulr_ps(uint64_t fdt0, uint64_t fdt1)
+uint64_t helper_float_mulr_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt1)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
@@ -3204,34 +3015,33 @@ uint64_t helper_float_mulr_ps(uint64_t fdt0, uint64_t fdt1)
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul (fst0, fsth0, &env->active_fpu.fp_status);
     fsth2 = float32_mul (fst1, fsth1, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
 /* compare operations */
 #define FOP_COND_D(op, cond)                                   \
-void helper_cmp_d_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
+void helper_cmp_d_ ## op(CPUMIPSState *env, uint64_t fdt0,     \
+                         uint64_t fdt1, int cc)                \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
         CLEAR_FP_COND(cc, env->active_fpu);                    \
 }                                                              \
-void helper_cmpabs_d_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
+void helper_cmpabs_d_ ## op(CPUMIPSState *env, uint64_t fdt0,  \
+                            uint64_t fdt1, int cc)             \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     fdt0 = float64_abs(fdt0);                                  \
     fdt1 = float64_abs(fdt1);                                  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
@@ -3260,25 +3070,25 @@ FOP_COND_D(le,  float64_le(fdt0, fdt1, &env->active_fpu.fp_status))
 FOP_COND_D(ngt, float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)  || float64_le(fdt0, fdt1, &env->active_fpu.fp_status))
 
 #define FOP_COND_S(op, cond)                                   \
-void helper_cmp_s_ ## op (uint32_t fst0, uint32_t fst1, int cc)    \
+void helper_cmp_s_ ## op(CPUMIPSState *env, uint32_t fst0,     \
+                         uint32_t fst1, int cc)                \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
         CLEAR_FP_COND(cc, env->active_fpu);                    \
 }                                                              \
-void helper_cmpabs_s_ ## op (uint32_t fst0, uint32_t fst1, int cc) \
+void helper_cmpabs_s_ ## op(CPUMIPSState *env, uint32_t fst0,  \
+                            uint32_t fst1, int cc)             \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     fst0 = float32_abs(fst0);                                  \
     fst1 = float32_abs(fst1);                                  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
@@ -3307,18 +3117,18 @@ FOP_COND_S(le,  float32_le(fst0, fst1, &env->active_fpu.fp_status))
 FOP_COND_S(ngt, float32_unordered(fst1, fst0, &env->active_fpu.fp_status)  || float32_le(fst0, fst1, &env->active_fpu.fp_status))
 
 #define FOP_COND_PS(op, condl, condh)                           \
-void helper_cmp_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
+void helper_cmp_ps_ ## op(CPUMIPSState *env, uint64_t fdt0,     \
+                          uint64_t fdt1, int cc)                \
 {                                                               \
     uint32_t fst0, fsth0, fst1, fsth1;                          \
     int ch, cl;                                                 \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);   \
     fst0 = fdt0 & 0XFFFFFFFF;                                   \
     fsth0 = fdt0 >> 32;                                         \
     fst1 = fdt1 & 0XFFFFFFFF;                                   \
     fsth1 = fdt1 >> 32;                                         \
     cl = condl;                                                 \
     ch = condh;                                                 \
-    update_fcr31();                                             \
+    update_fcr31(env, GETPC());                                 \
     if (cl)                                                     \
         SET_FP_COND(cc, env->active_fpu);                       \
     else                                                        \
@@ -3328,7 +3138,8 @@ void helper_cmp_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
     else                                                        \
         CLEAR_FP_COND(cc + 1, env->active_fpu);                 \
 }                                                               \
-void helper_cmpabs_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
+void helper_cmpabs_ps_ ## op(CPUMIPSState *env, uint64_t fdt0,  \
+                             uint64_t fdt1, int cc)             \
 {                                                               \
     uint32_t fst0, fsth0, fst1, fsth1;                          \
     int ch, cl;                                                 \
@@ -3338,7 +3149,7 @@ void helper_cmpabs_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
     fsth1 = float32_abs(fdt1 >> 32);                            \
     cl = condl;                                                 \
     ch = condh;                                                 \
-    update_fcr31();                                             \
+    update_fcr31(env, GETPC());                                 \
     if (cl)                                                     \
         SET_FP_COND(cc, env->active_fpu);                       \
     else                                                        \
diff --git a/target-mips/translate.c b/target-mips/translate.c
index b2934195362..206ba83401b 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -5,6 +5,7 @@
  *  Copyright (c) 2006 Marius Groeger (FPU operations)
  *  Copyright (c) 2006 Thiemo Seufer (MIPS32R2 support)
  *  Copyright (c) 2009 CodeSourcery (MIPS16 and microMIPS support)
+ *  Copyright (c) 2012 Jia Liu & Dongxue Zhang (MIPS ASE DSP support)
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -21,14 +22,14 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
 #include "helper.h"
 
-//#define MIPS_DEBUG_DISAS
+#define MIPS_DEBUG_DISAS 0
 //#define MIPS_DEBUG_SIGN_EXTENSIONS
 
 /* MIPS major opcodes */
@@ -312,6 +313,35 @@ enum {
     OPC_MODU_G_2E   = 0x23 | OPC_SPECIAL3,
     OPC_DMOD_G_2E   = 0x26 | OPC_SPECIAL3,
     OPC_DMODU_G_2E  = 0x27 | OPC_SPECIAL3,
+
+    /* MIPS DSP Load */
+    OPC_LX_DSP         = 0x0A | OPC_SPECIAL3,
+    /* MIPS DSP Arithmetic */
+    OPC_ADDU_QB_DSP    = 0x10 | OPC_SPECIAL3,
+    OPC_ADDU_OB_DSP    = 0x14 | OPC_SPECIAL3,
+    OPC_ABSQ_S_PH_DSP  = 0x12 | OPC_SPECIAL3,
+    OPC_ABSQ_S_QH_DSP  = 0x16 | OPC_SPECIAL3,
+    /* OPC_ADDUH_QB_DSP is same as OPC_MULT_G_2E.  */
+    /* OPC_ADDUH_QB_DSP   = 0x18 | OPC_SPECIAL3,  */
+    OPC_CMPU_EQ_QB_DSP = 0x11 | OPC_SPECIAL3,
+    OPC_CMPU_EQ_OB_DSP = 0x15 | OPC_SPECIAL3,
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_QB_DSP    = 0x13 | OPC_SPECIAL3,
+    OPC_SHLL_OB_DSP    = 0x17 | OPC_SPECIAL3,
+    /* MIPS DSP Multiply Sub-class insns */
+    /* OPC_MUL_PH_DSP is same as OPC_ADDUH_QB_DSP.  */
+    /* OPC_MUL_PH_DSP     = 0x18 | OPC_SPECIAL3,  */
+    OPC_DPA_W_PH_DSP   = 0x30 | OPC_SPECIAL3,
+    OPC_DPAQ_W_QH_DSP  = 0x34 | OPC_SPECIAL3,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_INSV_DSP       = 0x0C | OPC_SPECIAL3,
+    OPC_DINSV_DSP      = 0x0D | OPC_SPECIAL3,
+    /* MIPS DSP Compare-Pick Sub-class */
+    OPC_APPEND_DSP     = 0x31 | OPC_SPECIAL3,
+    OPC_DAPPEND_DSP    = 0x35 | OPC_SPECIAL3,
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_EXTR_W_DSP     = 0x38 | OPC_SPECIAL3,
+    OPC_DEXTR_W_DSP    = 0x3C | OPC_SPECIAL3,
 };
 
 /* BSHFL opcodes */
@@ -331,6 +361,413 @@ enum {
     OPC_DSHD     = (0x05 << 6) | OPC_DBSHFL,
 };
 
+/* MIPS DSP REGIMM opcodes */
+enum {
+    OPC_BPOSGE32 = (0x1C << 16) | OPC_REGIMM,
+    OPC_BPOSGE64 = (0x1D << 16) | OPC_REGIMM,
+};
+
+#define MASK_LX(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+/* MIPS DSP Load */
+enum {
+    OPC_LBUX = (0x06 << 6) | OPC_LX_DSP,
+    OPC_LHX  = (0x04 << 6) | OPC_LX_DSP,
+    OPC_LWX  = (0x00 << 6) | OPC_LX_DSP,
+    OPC_LDX = (0x08 << 6) | OPC_LX_DSP,
+};
+
+#define MASK_ADDU_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ADDQ_PH        = (0x0A << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDQ_S_PH      = (0x0E << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDQ_S_W       = (0x16 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_QB        = (0x00 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_S_QB      = (0x04 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_PH        = (0x08 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_S_PH      = (0x0C << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_PH        = (0x0B << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_S_PH      = (0x0F << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_S_W       = (0x17 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_QB        = (0x01 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_S_QB      = (0x05 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_PH        = (0x09 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_S_PH      = (0x0D << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDSC          = (0x10 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDWC          = (0x11 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MODSUB         = (0x12 << 6) | OPC_ADDU_QB_DSP,
+    OPC_RADDU_W_QB     = (0x14 << 6) | OPC_ADDU_QB_DSP,
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MULEU_S_PH_QBL = (0x06 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEU_S_PH_QBR = (0x07 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULQ_RS_PH     = (0x1F << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEQ_S_W_PHL  = (0x1C << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEQ_S_W_PHR  = (0x1D << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULQ_S_PH      = (0x1E << 6) | OPC_ADDU_QB_DSP,
+};
+
+#define OPC_ADDUH_QB_DSP OPC_MULT_G_2E
+#define MASK_ADDUH_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ADDUH_QB   = (0x00 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDUH_R_QB = (0x02 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_PH   = (0x08 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_R_PH = (0x0A << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_W    = (0x10 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_R_W  = (0x12 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBUH_QB   = (0x01 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBUH_R_QB = (0x03 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_PH   = (0x09 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_R_PH = (0x0B << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_W    = (0x11 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_R_W  = (0x13 << 6) | OPC_ADDUH_QB_DSP,
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MUL_PH     = (0x0C << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MUL_S_PH   = (0x0E << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MULQ_S_W   = (0x16 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MULQ_RS_W  = (0x17 << 6) | OPC_ADDUH_QB_DSP,
+};
+
+#define MASK_ABSQ_S_PH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ABSQ_S_QB       = (0x01 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_ABSQ_S_PH       = (0x09 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_ABSQ_S_W        = (0x11 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQ_W_PHL    = (0x0C << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQ_W_PHR    = (0x0D << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBL  = (0x04 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBR  = (0x05 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBLA = (0x06 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBRA = (0x07 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBL   = (0x1C << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBR   = (0x1D << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBLA  = (0x1E << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBRA  = (0x1F << 6) | OPC_ABSQ_S_PH_DSP,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_BITREV          = (0x1B << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPL_QB         = (0x02 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPLV_QB        = (0x03 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPL_PH         = (0x0A << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPLV_PH        = (0x0B << 6) | OPC_ABSQ_S_PH_DSP,
+};
+
+#define MASK_CMPU_EQ_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECR_QB_PH      = (0x0D << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_QB_PH     = (0x0C << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECR_SRA_PH_W   = (0x1E << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECR_SRA_R_PH_W = (0x1F << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_PH_W      = (0x14 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_RS_PH_W   = (0x15 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQU_S_QB_PH  = (0x0F << 6) | OPC_CMPU_EQ_QB_DSP,
+    /* DSP Compare-Pick Sub-class */
+    OPC_CMPU_EQ_QB       = (0x00 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPU_LT_QB       = (0x01 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPU_LE_QB       = (0x02 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_EQ_QB      = (0x04 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_LT_QB      = (0x05 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_LE_QB      = (0x06 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_EQ_QB     = (0x18 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_LT_QB     = (0x19 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_LE_QB     = (0x1A << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_EQ_PH        = (0x08 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_LT_PH        = (0x09 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_LE_PH        = (0x0A << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PICK_QB          = (0x03 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PICK_PH          = (0x0B << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PACKRL_PH        = (0x0E << 6) | OPC_CMPU_EQ_QB_DSP,
+};
+
+#define MASK_SHLL_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_QB    = (0x00 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_QB   = (0x02 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_PH    = (0x08 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_PH   = (0x0A << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_S_PH  = (0x0C << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_S_PH = (0x0E << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_S_W   = (0x14 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_S_W  = (0x16 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRL_QB    = (0x01 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRLV_QB   = (0x03 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRL_PH    = (0x19 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRLV_PH   = (0x1B << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_QB    = (0x04 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_QB  = (0x05 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_QB   = (0x06 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_QB = (0x07 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_PH    = (0x09 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_PH   = (0x0B << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_PH  = (0x0D << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_PH = (0x0F << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_W   = (0x15 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_W  = (0x17 << 6) | OPC_SHLL_QB_DSP,
+};
+
+#define MASK_DPA_W_PH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_DPAU_H_QBL    = (0x03 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAU_H_QBR    = (0x07 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSU_H_QBL    = (0x0B << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSU_H_QBR    = (0x0F << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPA_W_PH      = (0x00 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAX_W_PH     = (0x08 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQ_S_W_PH   = (0x04 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQX_S_W_PH  = (0x18 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQX_SA_W_PH = (0x1A << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPS_W_PH      = (0x01 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSX_W_PH     = (0x09 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQ_S_W_PH   = (0x05 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQX_S_W_PH  = (0x19 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQX_SA_W_PH = (0x1B << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MULSAQ_S_W_PH = (0x06 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQ_SA_L_W   = (0x0C << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQ_SA_L_W   = (0x0D << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_S_W_PHL   = (0x14 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_S_W_PHR   = (0x16 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_SA_W_PHL  = (0x10 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_SA_W_PHR  = (0x12 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MULSA_W_PH    = (0x02 << 6) | OPC_DPA_W_PH_DSP,
+};
+
+#define MASK_INSV(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_INSV = (0x00 << 6) | OPC_INSV_DSP,
+};
+
+#define MASK_APPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Compare-Pick Sub-class */
+    OPC_APPEND  = (0x00 << 6) | OPC_APPEND_DSP,
+    OPC_PREPEND = (0x01 << 6) | OPC_APPEND_DSP,
+    OPC_BALIGN  = (0x10 << 6) | OPC_APPEND_DSP,
+};
+
+#define MASK_EXTR_W(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_EXTR_W     = (0x00 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_R_W   = (0x04 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_RS_W  = (0x06 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_S_H   = (0x0E << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_S_H  = (0x0F << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_W    = (0x01 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_R_W  = (0x05 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_RS_W = (0x07 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTP       = (0x02 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPV      = (0x03 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPDP     = (0x0A << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPDPV    = (0x0B << 6) | OPC_EXTR_W_DSP,
+    OPC_SHILO      = (0x1A << 6) | OPC_EXTR_W_DSP,
+    OPC_SHILOV     = (0x1B << 6) | OPC_EXTR_W_DSP,
+    OPC_MTHLIP     = (0x1F << 6) | OPC_EXTR_W_DSP,
+    OPC_WRDSP      = (0x13 << 6) | OPC_EXTR_W_DSP,
+    OPC_RDDSP      = (0x12 << 6) | OPC_EXTR_W_DSP,
+};
+
+#define MASK_ABSQ_S_QH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECEQ_L_PWL    = (0x14 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_L_PWR    = (0x15 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHL   = (0x0C << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHR   = (0x0D << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHLA  = (0x0E << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHRA  = (0x0F << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBL  = (0x04 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBR  = (0x05 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBLA = (0x06 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBRA = (0x07 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBL   = (0x1C << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBR   = (0x1D << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBLA  = (0x1E << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBRA  = (0x1F << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_OB       = (0x01 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_PW       = (0x11 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_QH       = (0x09 << 6) | OPC_ABSQ_S_QH_DSP,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_REPL_OB         = (0x02 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPL_PW         = (0x12 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPL_QH         = (0x0A << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_OB        = (0x03 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_PW        = (0x13 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_QH        = (0x0B << 6) | OPC_ABSQ_S_QH_DSP,
+};
+
+#define MASK_ADDU_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MULEQ_S_PW_QHL = (0x1C << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEQ_S_PW_QHR = (0x1D << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEU_S_QH_OBL = (0x06 << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEU_S_QH_OBR = (0x07 << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULQ_RS_QH     = (0x1F << 6) | OPC_ADDU_OB_DSP,
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_RADDU_L_OB     = (0x14 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_PW        = (0x13 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_S_PW      = (0x17 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_QH        = (0x0B << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_S_QH      = (0x0F << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_OB        = (0x01 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_S_OB      = (0x05 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_QH        = (0x09 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_S_QH      = (0x0D << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBUH_OB       = (0x19 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBUH_R_OB     = (0x1B << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_PW        = (0x12 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_S_PW      = (0x16 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_QH        = (0x0A << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_S_QH      = (0x0E << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_OB        = (0x00 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_S_OB      = (0x04 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_QH        = (0x08 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_S_QH      = (0x0C << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDUH_OB       = (0x18 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDUH_R_OB     = (0x1A << 6) | OPC_ADDU_OB_DSP,
+};
+
+#define MASK_CMPU_EQ_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Compare-Pick Sub-class */
+    OPC_CMP_EQ_PW         = (0x10 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LT_PW         = (0x11 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LE_PW         = (0x12 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_EQ_QH         = (0x08 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LT_QH         = (0x09 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LE_QH         = (0x0A << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_EQ_OB      = (0x18 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_LT_OB      = (0x19 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_LE_OB      = (0x1A << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_EQ_OB       = (0x04 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_LT_OB       = (0x05 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_LE_OB       = (0x06 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_EQ_OB        = (0x00 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_LT_OB        = (0x01 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_LE_OB        = (0x02 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PACKRL_PW         = (0x0E << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_OB           = (0x03 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_PW           = (0x13 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_QH           = (0x0B << 6) | OPC_CMPU_EQ_OB_DSP,
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECR_OB_QH       = (0x0D << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECR_SRA_QH_PW   = (0x1E << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECR_SRA_R_QH_PW = (0x1F << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_OB_QH      = (0x0C << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_PW_L       = (0x1C << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_QH_PW      = (0x14 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_RS_QH_PW   = (0x15 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQU_S_OB_QH   = (0x0F << 6) | OPC_CMPU_EQ_OB_DSP,
+};
+
+#define MASK_DAPPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Compare-Pick Sub-class */
+    OPC_DAPPEND  = (0x00 << 6) | OPC_DAPPEND_DSP,
+    OPC_PREPENDD = (0x03 << 6) | OPC_DAPPEND_DSP,
+    OPC_PREPENDW = (0x01 << 6) | OPC_DAPPEND_DSP,
+    OPC_DBALIGN  = (0x10 << 6) | OPC_DAPPEND_DSP,
+};
+
+#define MASK_DEXTR_W(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_DMTHLIP     = (0x1F << 6) | OPC_DEXTR_W_DSP,
+    OPC_DSHILO      = (0x1A << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTP       = (0x02 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPDP     = (0x0A << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPDPV    = (0x0B << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPV      = (0x03 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_L     = (0x10 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_R_L   = (0x14 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_RS_L  = (0x16 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_W     = (0x00 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_R_W   = (0x04 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_RS_W  = (0x06 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_S_H   = (0x0E << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_L    = (0x11 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_R_L  = (0x15 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_RS_L = (0x17 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_S_H  = (0x0F << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_W    = (0x01 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_R_W  = (0x05 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_RS_W = (0x07 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DSHILOV     = (0x1B << 6) | OPC_DEXTR_W_DSP,
+};
+
+#define MASK_DINSV(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_DINSV = (0x00 << 6) | OPC_DINSV_DSP,
+};
+
+#define MASK_DPAQ_W_QH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_DMADD         = (0x19 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMADDU        = (0x1D << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMSUB         = (0x1B << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMSUBU        = (0x1F << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPA_W_QH      = (0x00 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAQ_S_W_QH   = (0x04 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAQ_SA_L_PW  = (0x0C << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAU_H_OBL    = (0x03 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAU_H_OBR    = (0x07 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPS_W_QH      = (0x01 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSQ_S_W_QH   = (0x05 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSQ_SA_L_PW  = (0x0D << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSU_H_OBL    = (0x0B << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSU_H_OBR    = (0x0F << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_L_PWL   = (0x1C << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_L_PWR   = (0x1E << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHLL  = (0x14 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHLL = (0x10 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHLR  = (0x15 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHLR = (0x11 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHRL  = (0x16 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHRL = (0x12 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHRR  = (0x17 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHRR = (0x13 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MULSAQ_S_L_PW = (0x0E << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MULSAQ_S_W_QH = (0x06 << 6) | OPC_DPAQ_W_QH_DSP,
+};
+
+#define MASK_SHLL_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_PW    = (0x10 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_S_PW  = (0x14 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_OB   = (0x02 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_PW   = (0x12 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_S_PW = (0x16 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_QH   = (0x0A << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_S_QH = (0x0E << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_PW    = (0x11 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_PW  = (0x15 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_OB   = (0x06 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_OB = (0x07 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_PW   = (0x13 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_PW = (0x17 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_QH   = (0x0B << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_QH = (0x0F << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRLV_OB   = (0x03 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRLV_QH   = (0x1B << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_OB    = (0x00 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_QH    = (0x08 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_S_QH  = (0x0C << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_OB    = (0x04 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_OB  = (0x05 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_QH    = (0x09 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_QH  = (0x0D << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRL_OB    = (0x01 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRL_QH    = (0x19 << 6) | OPC_SHLL_OB_DSP,
+};
+
 /* Coprocessor 0 (rs field) */
 #define MASK_CP0(op)       MASK_OP_MAJOR(op) | (op & (0x1F << 21))
 
@@ -446,6 +883,103 @@ enum {
     OPC_BC2     = (0x08 << 21) | OPC_CP2,
 };
 
+#define MASK_LMI(op)  (MASK_OP_MAJOR(op) | (op & (0x1F << 21)) | (op & 0x1F))
+
+enum {
+    OPC_PADDSH  = (24 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDUSH = (25 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDH   = (26 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDW   = (27 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDSB  = (28 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDUSB = (29 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDB   = (30 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDD   = (31 << 21) | (0x00) | OPC_CP2,
+
+    OPC_PSUBSH  = (24 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBUSH = (25 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBH   = (26 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBW   = (27 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBSB  = (28 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBUSB = (29 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBB   = (30 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBD   = (31 << 21) | (0x01) | OPC_CP2,
+
+    OPC_PSHUFH   = (24 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKSSWH = (25 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKSSHB = (26 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKUSHB = (27 << 21) | (0x02) | OPC_CP2,
+    OPC_XOR_CP2  = (28 << 21) | (0x02) | OPC_CP2,
+    OPC_NOR_CP2  = (29 << 21) | (0x02) | OPC_CP2,
+    OPC_AND_CP2  = (30 << 21) | (0x02) | OPC_CP2,
+    OPC_PANDN    = (31 << 21) | (0x02) | OPC_CP2,
+
+    OPC_PUNPCKLHW = (24 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKHHW = (25 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKLBH = (26 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKHBH = (27 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_0  = (28 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_1  = (29 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_2  = (30 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_3  = (31 << 21) | (0x03) | OPC_CP2,
+
+    OPC_PAVGH   = (24 << 21) | (0x08) | OPC_CP2,
+    OPC_PAVGB   = (25 << 21) | (0x08) | OPC_CP2,
+    OPC_PMAXSH  = (26 << 21) | (0x08) | OPC_CP2,
+    OPC_PMINSH  = (27 << 21) | (0x08) | OPC_CP2,
+    OPC_PMAXUB  = (28 << 21) | (0x08) | OPC_CP2,
+    OPC_PMINUB  = (29 << 21) | (0x08) | OPC_CP2,
+
+    OPC_PCMPEQW = (24 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTW = (25 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPEQH = (26 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTH = (27 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPEQB = (28 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTB = (29 << 21) | (0x09) | OPC_CP2,
+
+    OPC_PSLLW   = (24 << 21) | (0x0A) | OPC_CP2,
+    OPC_PSLLH   = (25 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULLH  = (26 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULHH  = (27 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULUW  = (28 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULHUH = (29 << 21) | (0x0A) | OPC_CP2,
+
+    OPC_PSRLW     = (24 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRLH     = (25 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRAW     = (26 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRAH     = (27 << 21) | (0x0B) | OPC_CP2,
+    OPC_PUNPCKLWD = (28 << 21) | (0x0B) | OPC_CP2,
+    OPC_PUNPCKHWD = (29 << 21) | (0x0B) | OPC_CP2,
+
+    OPC_ADDU_CP2 = (24 << 21) | (0x0C) | OPC_CP2,
+    OPC_OR_CP2   = (25 << 21) | (0x0C) | OPC_CP2,
+    OPC_ADD_CP2  = (26 << 21) | (0x0C) | OPC_CP2,
+    OPC_DADD_CP2 = (27 << 21) | (0x0C) | OPC_CP2,
+    OPC_SEQU_CP2 = (28 << 21) | (0x0C) | OPC_CP2,
+    OPC_SEQ_CP2  = (29 << 21) | (0x0C) | OPC_CP2,
+
+    OPC_SUBU_CP2 = (24 << 21) | (0x0D) | OPC_CP2,
+    OPC_PASUBUB  = (25 << 21) | (0x0D) | OPC_CP2,
+    OPC_SUB_CP2  = (26 << 21) | (0x0D) | OPC_CP2,
+    OPC_DSUB_CP2 = (27 << 21) | (0x0D) | OPC_CP2,
+    OPC_SLTU_CP2 = (28 << 21) | (0x0D) | OPC_CP2,
+    OPC_SLT_CP2  = (29 << 21) | (0x0D) | OPC_CP2,
+
+    OPC_SLL_CP2  = (24 << 21) | (0x0E) | OPC_CP2,
+    OPC_DSLL_CP2 = (25 << 21) | (0x0E) | OPC_CP2,
+    OPC_PEXTRH   = (26 << 21) | (0x0E) | OPC_CP2,
+    OPC_PMADDHW  = (27 << 21) | (0x0E) | OPC_CP2,
+    OPC_SLEU_CP2 = (28 << 21) | (0x0E) | OPC_CP2,
+    OPC_SLE_CP2  = (29 << 21) | (0x0E) | OPC_CP2,
+
+    OPC_SRL_CP2  = (24 << 21) | (0x0F) | OPC_CP2,
+    OPC_DSRL_CP2 = (25 << 21) | (0x0F) | OPC_CP2,
+    OPC_SRA_CP2  = (26 << 21) | (0x0F) | OPC_CP2,
+    OPC_DSRA_CP2 = (27 << 21) | (0x0F) | OPC_CP2,
+    OPC_BIADD    = (28 << 21) | (0x0F) | OPC_CP2,
+    OPC_PMOVMSKB = (29 << 21) | (0x0F) | OPC_CP2,
+};
+
+
 #define MASK_CP3(op)       MASK_OP_MAJOR(op) | (op & 0x3F)
 
 enum {
@@ -478,32 +1012,52 @@ static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC], cpu_ACX[MIPS_DSP_ACC];
 static TCGv cpu_dspctrl, btarget, bcond;
 static TCGv_i32 hflags;
 static TCGv_i32 fpu_fcr0, fpu_fcr31;
+static TCGv_i64 fpu_f64[32];
 
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
+static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
-#define gen_helper_0i(name, arg) do {                             \
+#define gen_helper_0e0i(name, arg) do {                           \
     TCGv_i32 helper_tmp = tcg_const_i32(arg);                     \
-    gen_helper_##name(helper_tmp);                                \
+    gen_helper_##name(cpu_env, helper_tmp);                       \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_0e1i(name, arg1, arg2) do {                    \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg2);                    \
+    gen_helper_##name(cpu_env, arg1, helper_tmp);                 \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_1i(name, arg1, arg2) do {                      \
+#define gen_helper_1e0i(name, ret, arg1) do {                     \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg1);                    \
+    gen_helper_##name(ret, cpu_env, helper_tmp);                  \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_1e1i(name, ret, arg1, arg2) do {               \
     TCGv_i32 helper_tmp = tcg_const_i32(arg2);                    \
-    gen_helper_##name(arg1, helper_tmp);                          \
+    gen_helper_##name(ret, cpu_env, arg1, helper_tmp);            \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_2i(name, arg1, arg2, arg3) do {                \
+#define gen_helper_0e2i(name, arg1, arg2, arg3) do {              \
     TCGv_i32 helper_tmp = tcg_const_i32(arg3);                    \
-    gen_helper_##name(arg1, arg2, helper_tmp);                    \
+    gen_helper_##name(cpu_env, arg1, arg2, helper_tmp);           \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_3i(name, arg1, arg2, arg3, arg4) do {          \
+#define gen_helper_1e2i(name, ret, arg1, arg2, arg3) do {         \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg3);                    \
+    gen_helper_##name(ret, cpu_env, arg1, arg2, helper_tmp);      \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_0e3i(name, arg1, arg2, arg3, arg4) do {        \
     TCGv_i32 helper_tmp = tcg_const_i32(arg4);                    \
-    gen_helper_##name(arg1, arg2, arg3, helper_tmp);              \
+    gen_helper_##name(cpu_env, arg1, arg2, arg3, helper_tmp);     \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
@@ -527,43 +1081,51 @@ enum {
     BS_EXCP     = 3, /* We reached an exception condition */
 };
 
-static const char *regnames[] =
-    { "r0", "at", "v0", "v1", "a0", "a1", "a2", "a3",
-      "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
-      "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
-      "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra", };
+static const char * const regnames[] = {
+    "r0", "at", "v0", "v1", "a0", "a1", "a2", "a3",
+    "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
+    "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+    "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra",
+};
 
-static const char *regnames_HI[] =
-    { "HI0", "HI1", "HI2", "HI3", };
+static const char * const regnames_HI[] = {
+    "HI0", "HI1", "HI2", "HI3",
+};
 
-static const char *regnames_LO[] =
-    { "LO0", "LO1", "LO2", "LO3", };
+static const char * const regnames_LO[] = {
+    "LO0", "LO1", "LO2", "LO3",
+};
 
-static const char *regnames_ACX[] =
-    { "ACX0", "ACX1", "ACX2", "ACX3", };
+static const char * const regnames_ACX[] = {
+    "ACX0", "ACX1", "ACX2", "ACX3",
+};
 
-static const char *fregnames[] =
-    { "f0",  "f1",  "f2",  "f3",  "f4",  "f5",  "f6",  "f7",
-      "f8",  "f9",  "f10", "f11", "f12", "f13", "f14", "f15",
-      "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
-      "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", };
+static const char * const fregnames[] = {
+    "f0",  "f1",  "f2",  "f3",  "f4",  "f5",  "f6",  "f7",
+    "f8",  "f9",  "f10", "f11", "f12", "f13", "f14", "f15",
+    "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
+    "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",
+};
 
-#ifdef MIPS_DEBUG_DISAS
-#define MIPS_DEBUG(fmt, ...)                         \
-        qemu_log_mask(CPU_LOG_TB_IN_ASM,                \
-                       TARGET_FMT_lx ": %08x " fmt "\n", \
-                       ctx->pc, ctx->opcode , ## __VA_ARGS__)
-#define LOG_DISAS(...) qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__)
-#else
-#define MIPS_DEBUG(fmt, ...) do { } while(0)
-#define LOG_DISAS(...) do { } while (0)
-#endif
+#define MIPS_DEBUG(fmt, ...)                                                  \
+    do {                                                                      \
+        if (MIPS_DEBUG_DISAS) {                                               \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
+                          TARGET_FMT_lx ": %08x " fmt "\n",                   \
+                          ctx->pc, ctx->opcode , ## __VA_ARGS__);             \
+        }                                                                     \
+    } while (0)
+
+#define LOG_DISAS(...)                                                        \
+    do {                                                                      \
+        if (MIPS_DEBUG_DISAS) {                                               \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
+        }                                                                     \
+    } while (0)
 
 #define MIPS_INVAL(op)                                                        \
-do {                                                                          \
     MIPS_DEBUG("Invalid %s %03x %03x %03x", op, ctx->opcode >> 26,            \
-               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F));             \
-} while (0)
+               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F))
 
 /* General purpose registers moves. */
 static inline void gen_load_gpr (TCGv t, int reg)
@@ -640,54 +1202,54 @@ static inline void gen_store_srsgpr (int from, int to)
 }
 
 /* Floating point register moves. */
-static inline void gen_load_fpr32 (TCGv_i32 t, int reg)
+static void gen_load_fpr32(TCGv_i32 t, int reg)
 {
-    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[FP_ENDIAN_IDX]));
+    tcg_gen_trunc_i64_i32(t, fpu_f64[reg]);
 }
 
-static inline void gen_store_fpr32 (TCGv_i32 t, int reg)
+static void gen_store_fpr32(TCGv_i32 t, int reg)
 {
-    tcg_gen_st_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_extu_i32_i64(t64, t);
+    tcg_gen_deposit_i64(fpu_f64[reg], fpu_f64[reg], t64, 0, 32);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_load_fpr32h (TCGv_i32 t, int reg)
+static void gen_load_fpr32h(TCGv_i32 t, int reg)
 {
-    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[!FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(t64, fpu_f64[reg], 32);
+    tcg_gen_trunc_i64_i32(t, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_store_fpr32h (TCGv_i32 t, int reg)
+static void gen_store_fpr32h(TCGv_i32 t, int reg)
 {
-    tcg_gen_st_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[!FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_extu_i32_i64(t64, t);
+    tcg_gen_deposit_i64(fpu_f64[reg], fpu_f64[reg], t64, 32, 32);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_load_fpr64 (DisasContext *ctx, TCGv_i64 t, int reg)
+static void gen_load_fpr64(DisasContext *ctx, TCGv_i64 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        tcg_gen_ld_i64(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].d));
+        tcg_gen_mov_i64(t, fpu_f64[reg]);
     } else {
-        TCGv_i32 t0 = tcg_temp_new_i32();
-        TCGv_i32 t1 = tcg_temp_new_i32();
-        gen_load_fpr32(t0, reg & ~1);
-        gen_load_fpr32(t1, reg | 1);
-        tcg_gen_concat_i32_i64(t, t0, t1);
-        tcg_temp_free_i32(t0);
-        tcg_temp_free_i32(t1);
+        tcg_gen_concat32_i64(t, fpu_f64[reg & ~1], fpu_f64[reg | 1]);
     }
 }
 
-static inline void gen_store_fpr64 (DisasContext *ctx, TCGv_i64 t, int reg)
+static void gen_store_fpr64(DisasContext *ctx, TCGv_i64 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        tcg_gen_st_i64(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].d));
+        tcg_gen_mov_i64(fpu_f64[reg], t);
     } else {
-        TCGv_i64 t0 = tcg_temp_new_i64();
-        TCGv_i32 t1 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(t1, t);
-        gen_store_fpr32(t1, reg & ~1);
+        TCGv_i64 t0;
+        tcg_gen_deposit_i64(fpu_f64[reg & ~1], fpu_f64[reg & ~1], t, 0, 32);
+        t0 = tcg_temp_new_i64();
         tcg_gen_shri_i64(t0, t, 32);
-        tcg_gen_trunc_i64_i32(t1, t0);
-        gen_store_fpr32(t1, reg | 1);
-        tcg_temp_free_i32(t1);
+        tcg_gen_deposit_i64(fpu_f64[reg | 1], fpu_f64[reg | 1], t0, 0, 32);
         tcg_temp_free_i64(t0);
     }
 }
@@ -748,7 +1310,7 @@ generate_exception_err (DisasContext *ctx, int excp, int err)
     TCGv_i32 texcp = tcg_const_i32(excp);
     TCGv_i32 terr = tcg_const_i32(err);
     save_cpu_state(ctx, 1);
-    gen_helper_raise_exception_err(texcp, terr);
+    gen_helper_raise_exception_err(cpu_env, texcp, terr);
     tcg_temp_free_i32(terr);
     tcg_temp_free_i32(texcp);
 }
@@ -757,7 +1319,7 @@ static inline void
 generate_exception (DisasContext *ctx, int excp)
 {
     save_cpu_state(ctx, 1);
-    gen_helper_0i(raise_exception, excp);
+    gen_helper_0e0i(raise_exception, excp);
 }
 
 /* Addresses computation */
@@ -824,6 +1386,24 @@ static inline void check_cp1_registers(DisasContext *ctx, int regs)
         generate_exception(ctx, EXCP_RI);
 }
 
+/* Verify that the processor is running with DSP instructions enabled.
+   This is enabled by CP0 Status register MX(24) bit.
+ */
+
+static inline void check_dsp(DisasContext *ctx)
+{
+    if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
+        generate_exception(ctx, EXCP_DSPDIS);
+    }
+}
+
+static inline void check_dspr2(DisasContext *ctx)
+{
+    if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
+        generate_exception(ctx, EXCP_DSPDIS);
+    }
+}
+
 /* This code generates a "reserved instruction" exception if the
    CPU does not support the instruction set corresponding to flags. */
 static inline void check_insn(CPUMIPSState *env, DisasContext *ctx, int flags)
@@ -871,22 +1451,22 @@ static inline void gen_cmp ## type ## _ ## fmt(DisasContext *ctx, int n,      \
     gen_ldcmp_fpr##bits (ctx, fp0, fs);                                       \
     gen_ldcmp_fpr##bits (ctx, fp1, ft);                                       \
     switch (n) {                                                              \
-    case  0: gen_helper_2i(cmp ## type ## _ ## fmt ## _f, fp0, fp1, cc);    break;\
-    case  1: gen_helper_2i(cmp ## type ## _ ## fmt ## _un, fp0, fp1, cc);   break;\
-    case  2: gen_helper_2i(cmp ## type ## _ ## fmt ## _eq, fp0, fp1, cc);   break;\
-    case  3: gen_helper_2i(cmp ## type ## _ ## fmt ## _ueq, fp0, fp1, cc);  break;\
-    case  4: gen_helper_2i(cmp ## type ## _ ## fmt ## _olt, fp0, fp1, cc);  break;\
-    case  5: gen_helper_2i(cmp ## type ## _ ## fmt ## _ult, fp0, fp1, cc);  break;\
-    case  6: gen_helper_2i(cmp ## type ## _ ## fmt ## _ole, fp0, fp1, cc);  break;\
-    case  7: gen_helper_2i(cmp ## type ## _ ## fmt ## _ule, fp0, fp1, cc);  break;\
-    case  8: gen_helper_2i(cmp ## type ## _ ## fmt ## _sf, fp0, fp1, cc);   break;\
-    case  9: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngle, fp0, fp1, cc); break;\
-    case 10: gen_helper_2i(cmp ## type ## _ ## fmt ## _seq, fp0, fp1, cc);  break;\
-    case 11: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngl, fp0, fp1, cc);  break;\
-    case 12: gen_helper_2i(cmp ## type ## _ ## fmt ## _lt, fp0, fp1, cc);   break;\
-    case 13: gen_helper_2i(cmp ## type ## _ ## fmt ## _nge, fp0, fp1, cc);  break;\
-    case 14: gen_helper_2i(cmp ## type ## _ ## fmt ## _le, fp0, fp1, cc);   break;\
-    case 15: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngt, fp0, fp1, cc);  break;\
+    case  0: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _f, fp0, fp1, cc);    break;\
+    case  1: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _un, fp0, fp1, cc);   break;\
+    case  2: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _eq, fp0, fp1, cc);   break;\
+    case  3: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ueq, fp0, fp1, cc);  break;\
+    case  4: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _olt, fp0, fp1, cc);  break;\
+    case  5: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ult, fp0, fp1, cc);  break;\
+    case  6: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ole, fp0, fp1, cc);  break;\
+    case  7: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ule, fp0, fp1, cc);  break;\
+    case  8: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _sf, fp0, fp1, cc);   break;\
+    case  9: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngle, fp0, fp1, cc); break;\
+    case 10: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _seq, fp0, fp1, cc);  break;\
+    case 11: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngl, fp0, fp1, cc);  break;\
+    case 12: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _lt, fp0, fp1, cc);   break;\
+    case 13: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _nge, fp0, fp1, cc);  break;\
+    case 14: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _le, fp0, fp1, cc);   break;\
+    case 15: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngt, fp0, fp1, cc);  break;\
     default: abort();                                                         \
     }                                                                         \
     tcg_temp_free_i##bits (fp0);                                              \
@@ -904,35 +1484,6 @@ FOP_CONDS(abs, 1, ps, FMT_PS, 64)
 #undef gen_ldcmp_fpr64
 
 /* load/store instructions. */
-#define OP_LD(insn,fname)                                                 \
-static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)   \
-{                                                                         \
-    tcg_gen_qemu_##fname(ret, arg1, ctx->mem_idx);                        \
-}
-OP_LD(lb,ld8s);
-OP_LD(lbu,ld8u);
-OP_LD(lh,ld16s);
-OP_LD(lhu,ld16u);
-OP_LD(lw,ld32s);
-#if defined(TARGET_MIPS64)
-OP_LD(lwu,ld32u);
-OP_LD(ld,ld64);
-#endif
-#undef OP_LD
-
-#define OP_ST(insn,fname)                                                  \
-static inline void op_st_##insn(TCGv arg1, TCGv arg2, DisasContext *ctx)   \
-{                                                                          \
-    tcg_gen_qemu_##fname(arg1, arg2, ctx->mem_idx);                        \
-}
-OP_ST(sb,st8);
-OP_ST(sh,st16);
-OP_ST(sw,st32);
-#if defined(TARGET_MIPS64)
-OP_ST(sd,st64);
-#endif
-#undef OP_ST
-
 #ifdef CONFIG_USER_ONLY
 #define OP_LD_ATOMIC(insn,fname)                                           \
 static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
@@ -948,7 +1499,7 @@ static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
 #define OP_LD_ATOMIC(insn,fname)                                           \
 static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
 {                                                                          \
-    gen_helper_2i(insn, ret, arg1, ctx->mem_idx);                          \
+    gen_helper_1e1i(insn, ret, arg1, ctx->mem_idx);                        \
 }
 #endif
 OP_LD_ATOMIC(ll,ld32s);
@@ -975,7 +1526,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
     tcg_gen_movi_tl(t0, rt | ((almask << 3) & 0x20));                        \
     tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, llreg));                   \
     tcg_gen_st_tl(arg1, cpu_env, offsetof(CPUMIPSState, llnewval));              \
-    gen_helper_0i(raise_exception, EXCP_SC);                                 \
+    gen_helper_0e0i(raise_exception, EXCP_SC);                               \
     gen_set_label(l2);                                                       \
     tcg_gen_movi_tl(t0, 0);                                                  \
     gen_store_gpr(t0, rt);                                                   \
@@ -986,7 +1537,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
 static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx) \
 {                                                                            \
     TCGv t0 = tcg_temp_new();                                                \
-    gen_helper_3i(insn, t0, arg1, arg2, ctx->mem_idx);                       \
+    gen_helper_1e2i(insn, t0, arg1, arg2, ctx->mem_idx);                     \
     gen_store_gpr(t0, rt);                                                   \
     tcg_temp_free(t0);                                                       \
 }
@@ -1029,7 +1580,7 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
                     int rt, int base, int16_t offset)
 {
     const char *opn = "ld";
-    TCGv t0, t1;
+    TCGv t0, t1, t2;
 
     if (rt == 0 && env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
@@ -1040,20 +1591,17 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
     }
 
     t0 = tcg_temp_new();
-    t1 = tcg_temp_new();
     gen_base_offset_addr(ctx, t0, base, offset);
 
     switch (opc) {
 #if defined(TARGET_MIPS64)
     case OPC_LWU:
-        save_cpu_state(ctx, 0);
-        op_ld_lwu(t0, t0, ctx);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lwu";
         break;
     case OPC_LD:
-        save_cpu_state(ctx, 0);
-        op_ld_ld(t0, t0, ctx);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "ld";
         break;
@@ -1064,78 +1612,130 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
         opn = "lld";
         break;
     case OPC_LDL:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 7);
+#ifndef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 7);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~7);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        tcg_gen_shl_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 63);
+        t2 = tcg_const_tl(0x7fffffffffffffffull);
+        tcg_gen_shr_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(ldl, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "ldl";
         break;
     case OPC_LDR:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 7);
+#ifdef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 7);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~7);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        tcg_gen_shr_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 63);
+        t2 = tcg_const_tl(0xfffffffffffffffeull);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(ldr, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "ldr";
         break;
     case OPC_LDPC:
-        save_cpu_state(ctx, 0);
-        tcg_gen_movi_tl(t1, pc_relative_pc(ctx));
+        t1 = tcg_const_tl(pc_relative_pc(ctx));
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_ld(t0, t0, ctx);
+        tcg_temp_free(t1);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "ldpc";
         break;
 #endif
     case OPC_LWPC:
-        save_cpu_state(ctx, 0);
-        tcg_gen_movi_tl(t1, pc_relative_pc(ctx));
+        t1 = tcg_const_tl(pc_relative_pc(ctx));
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_lw(t0, t0, ctx);
+        tcg_temp_free(t1);
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lwpc";
         break;
     case OPC_LW:
-        save_cpu_state(ctx, 0);
-        op_ld_lw(t0, t0, ctx);
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lw";
         break;
     case OPC_LH:
-        save_cpu_state(ctx, 0);
-        op_ld_lh(t0, t0, ctx);
+        tcg_gen_qemu_ld16s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lh";
         break;
     case OPC_LHU:
-        save_cpu_state(ctx, 0);
-        op_ld_lhu(t0, t0, ctx);
+        tcg_gen_qemu_ld16u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lhu";
         break;
     case OPC_LB:
-        save_cpu_state(ctx, 0);
-        op_ld_lb(t0, t0, ctx);
+        tcg_gen_qemu_ld8s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lb";
         break;
     case OPC_LBU:
-        save_cpu_state(ctx, 0);
-        op_ld_lbu(t0, t0, ctx);
+        tcg_gen_qemu_ld8u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lbu";
         break;
     case OPC_LWL:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 3);
+#ifndef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 3);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~3);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
+        tcg_gen_shl_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 31);
+        t2 = tcg_const_tl(0x7fffffffull);
+        tcg_gen_shr_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(lwl, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        tcg_gen_ext32s_tl(t0, t0);
+        gen_store_gpr(t0, rt);
         opn = "lwl";
         break;
     case OPC_LWR:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 3);
+#ifdef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 3);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~3);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
+        tcg_gen_shr_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 31);
+        t2 = tcg_const_tl(0xfffffffeull);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(lwr, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "lwr";
         break;
     case OPC_LL:
@@ -1148,7 +1748,6 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
     (void)opn; /* avoid a compiler warning */
     MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
-    tcg_temp_free(t1);
 }
 
 /* Store */
@@ -1164,44 +1763,40 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
     switch (opc) {
 #if defined(TARGET_MIPS64)
     case OPC_SD:
-        save_cpu_state(ctx, 0);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         opn = "sd";
         break;
     case OPC_SDL:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(sdl, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
         opn = "sdl";
         break;
     case OPC_SDR:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(sdr, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
         opn = "sdr";
         break;
 #endif
     case OPC_SW:
-        save_cpu_state(ctx, 0);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         opn = "sw";
         break;
     case OPC_SH:
-        save_cpu_state(ctx, 0);
-        op_st_sh(t1, t0, ctx);
+        tcg_gen_qemu_st16(t1, t0, ctx->mem_idx);
         opn = "sh";
         break;
     case OPC_SB:
-        save_cpu_state(ctx, 0);
-        op_st_sb(t1, t0, ctx);
+        tcg_gen_qemu_st8(t1, t0, ctx->mem_idx);
         opn = "sb";
         break;
     case OPC_SWL:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(swl, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
         opn = "swl";
         break;
     case OPC_SWR:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(swr, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
         opn = "swr";
         break;
     }
@@ -1219,13 +1814,14 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
     const char *opn = "st_cond";
     TCGv t0, t1;
 
+#ifdef CONFIG_USER_ONLY
     t0 = tcg_temp_local_new();
-
-    gen_base_offset_addr(ctx, t0, base, offset);
-    /* Don't do NOP if destination is zero: we must perform the actual
-       memory access. */
-
     t1 = tcg_temp_local_new();
+#else
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+#endif
+    gen_base_offset_addr(ctx, t0, base, offset);
     gen_load_gpr(t1, rt);
     switch (opc) {
 #if defined(TARGET_MIPS64)
@@ -1413,10 +2009,10 @@ static void gen_arith_imm (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Logic with immediate operand */
-static void gen_logic_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int16_t imm)
+static void gen_logic_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                          int rt, int rs, int16_t imm)
 {
     target_ulong uimm;
-    const char *opn = "imm logic";
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
@@ -1430,33 +2026,39 @@ static void gen_logic_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int1
             tcg_gen_andi_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], 0);
-        opn = "andi";
+        MIPS_DEBUG("andi %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_ORI:
         if (rs != 0)
             tcg_gen_ori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        opn = "ori";
+        MIPS_DEBUG("ori %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_XORI:
         if (likely(rs != 0))
             tcg_gen_xori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        opn = "xori";
+        MIPS_DEBUG("xori %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_LUI:
         tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-        opn = "lui";
+        MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
+        break;
+
+    default:
+        MIPS_DEBUG("Unknown logical immediate opcode %08x", opc);
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
 }
 
 /* Set on less than with immediate operand */
-static void gen_slt_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int16_t imm)
+static void gen_slt_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                        int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
     const char *opn = "imm arith";
@@ -1757,45 +2359,44 @@ static void gen_arith (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Conditional move */
-static void gen_cond_move (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_cond_move(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                          int rd, int rs, int rt)
 {
     const char *opn = "cond move";
-    int l1;
+    TCGv t0, t1, t2;
 
     if (rd == 0) {
-        /* If no destination, treat it as a NOP.
-           For add & sub, we must generate the overflow exception when needed. */
+        /* If no destination, treat it as a NOP. */
         MIPS_DEBUG("NOP");
         return;
     }
 
-    l1 = gen_new_label();
+    t0 = tcg_temp_new();
+    gen_load_gpr(t0, rt);
+    t1 = tcg_const_tl(0);
+    t2 = tcg_temp_new();
+    gen_load_gpr(t2, rs);
     switch (opc) {
     case OPC_MOVN:
-        if (likely(rt != 0))
-            tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr[rt], 0, l1);
-        else
-            tcg_gen_br(l1);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
         opn = "movn";
         break;
     case OPC_MOVZ:
-        if (likely(rt != 0))
-            tcg_gen_brcondi_tl(TCG_COND_NE, cpu_gpr[rt], 0, l1);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
         opn = "movz";
         break;
     }
-    if (rs != 0)
-        tcg_gen_mov_tl(cpu_gpr[rd], cpu_gpr[rs]);
-    else
-        tcg_gen_movi_tl(cpu_gpr[rd], 0);
-    gen_set_label(l1);
+    tcg_temp_free(t2);
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
 
     (void)opn; /* avoid a compiler warning */
     MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Logic */
-static void gen_logic (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_logic(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                      int rd, int rs, int rt)
 {
     const char *opn = "logic";
 
@@ -1856,7 +2457,8 @@ static void gen_logic (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
 }
 
 /* Set on lower than */
-static void gen_slt (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_slt(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                    int rd, int rs, int rt)
 {
     const char *opn = "slt";
     TCGv t0, t1;
@@ -1972,33 +2574,75 @@ static void gen_shift (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 static void gen_HILO (DisasContext *ctx, uint32_t opc, int reg)
 {
     const char *opn = "hilo";
+    unsigned int acc;
 
     if (reg == 0 && (opc == OPC_MFHI || opc == OPC_MFLO)) {
         /* Treat as NOP. */
         MIPS_DEBUG("NOP");
         return;
     }
+
+    if (opc == OPC_MFHI || opc == OPC_MFLO) {
+        acc = ((ctx->opcode) >> 21) & 0x03;
+    } else {
+        acc = ((ctx->opcode) >> 11) & 0x03;
+    }
+
+    if (acc != 0) {
+        check_dsp(ctx);
+    }
+
     switch (opc) {
     case OPC_MFHI:
-        tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[0]);
+#if defined(TARGET_MIPS64)
+        if (acc != 0) {
+            tcg_gen_ext32s_tl(cpu_gpr[reg], cpu_HI[acc]);
+        } else
+#endif
+        {
+            tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[acc]);
+        }
         opn = "mfhi";
         break;
     case OPC_MFLO:
-        tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[0]);
+#if defined(TARGET_MIPS64)
+        if (acc != 0) {
+            tcg_gen_ext32s_tl(cpu_gpr[reg], cpu_LO[acc]);
+        } else
+#endif
+        {
+            tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[acc]);
+        }
         opn = "mflo";
         break;
     case OPC_MTHI:
-        if (reg != 0)
-            tcg_gen_mov_tl(cpu_HI[0], cpu_gpr[reg]);
-        else
-            tcg_gen_movi_tl(cpu_HI[0], 0);
+        if (reg != 0) {
+#if defined(TARGET_MIPS64)
+            if (acc != 0) {
+                tcg_gen_ext32s_tl(cpu_HI[acc], cpu_gpr[reg]);
+            } else
+#endif
+            {
+                tcg_gen_mov_tl(cpu_HI[acc], cpu_gpr[reg]);
+            }
+        } else {
+            tcg_gen_movi_tl(cpu_HI[acc], 0);
+        }
         opn = "mthi";
         break;
     case OPC_MTLO:
-        if (reg != 0)
-            tcg_gen_mov_tl(cpu_LO[0], cpu_gpr[reg]);
-        else
-            tcg_gen_movi_tl(cpu_LO[0], 0);
+        if (reg != 0) {
+#if defined(TARGET_MIPS64)
+            if (acc != 0) {
+                tcg_gen_ext32s_tl(cpu_LO[acc], cpu_gpr[reg]);
+            } else
+#endif
+            {
+                tcg_gen_mov_tl(cpu_LO[acc], cpu_gpr[reg]);
+            }
+        } else {
+            tcg_gen_movi_tl(cpu_LO[acc], 0);
+        }
         opn = "mtlo";
         break;
     }
@@ -2011,61 +2655,50 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
 {
     const char *opn = "mul/div";
     TCGv t0, t1;
+    unsigned int acc;
 
-    switch (opc) {
-    case OPC_DIV:
-    case OPC_DIVU:
-#if defined(TARGET_MIPS64)
-    case OPC_DDIV:
-    case OPC_DDIVU:
-#endif
-        t0 = tcg_temp_local_new();
-        t1 = tcg_temp_local_new();
-        break;
-    default:
-        t0 = tcg_temp_new();
-        t1 = tcg_temp_new();
-        break;
-    }
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
 
     gen_load_gpr(t0, rs);
     gen_load_gpr(t1, rt);
+
     switch (opc) {
     case OPC_DIV:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
             tcg_gen_ext32s_tl(t0, t0);
             tcg_gen_ext32s_tl(t1, t1);
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t0, INT_MIN, l2);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t1, -1, l2);
-
-            tcg_gen_mov_tl(cpu_LO[0], t0);
-            tcg_gen_movi_tl(cpu_HI[0], 0);
-            tcg_gen_br(l1);
-            gen_set_label(l2);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, INT_MIN);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
             tcg_gen_div_tl(cpu_LO[0], t0, t1);
             tcg_gen_rem_tl(cpu_HI[0], t0, t1);
             tcg_gen_ext32s_tl(cpu_LO[0], cpu_LO[0]);
             tcg_gen_ext32s_tl(cpu_HI[0], cpu_HI[0]);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "div";
         break;
     case OPC_DIVU:
         {
-            int l1 = gen_new_label();
-
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
             tcg_gen_divu_tl(cpu_LO[0], t0, t1);
             tcg_gen_remu_tl(cpu_HI[0], t0, t1);
             tcg_gen_ext32s_tl(cpu_LO[0], cpu_LO[0]);
             tcg_gen_ext32s_tl(cpu_HI[0], cpu_HI[0]);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "divu";
         break;
@@ -2073,6 +2706,10 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
@@ -2082,8 +2719,8 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "mult";
         break;
@@ -2091,6 +2728,10 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
@@ -2102,47 +2743,48 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "multu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DDIV:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t0, -1LL << 63, l2);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t1, -1LL, l2);
-            tcg_gen_mov_tl(cpu_LO[0], t0);
-            tcg_gen_movi_tl(cpu_HI[0], 0);
-            tcg_gen_br(l1);
-            gen_set_label(l2);
-            tcg_gen_div_i64(cpu_LO[0], t0, t1);
-            tcg_gen_rem_i64(cpu_HI[0], t0, t1);
-            gen_set_label(l1);
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, -1LL << 63);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1LL);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_div_tl(cpu_LO[0], t0, t1);
+            tcg_gen_rem_tl(cpu_HI[0], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "ddiv";
         break;
     case OPC_DDIVU:
         {
-            int l1 = gen_new_label();
-
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
             tcg_gen_divu_i64(cpu_LO[0], t0, t1);
             tcg_gen_remu_i64(cpu_HI[0], t0, t1);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "ddivu";
         break;
     case OPC_DMULT:
-        gen_helper_dmult(t0, t1);
+        gen_helper_dmult(cpu_env, t0, t1);
         opn = "dmult";
         break;
     case OPC_DMULTU:
-        gen_helper_dmultu(t0, t1);
+        gen_helper_dmultu(cpu_env, t0, t1);
         opn = "dmultu";
         break;
 #endif
@@ -2150,41 +2792,49 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "madd";
         break;
     case OPC_MADDU:
-       {
+        {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_extu_tl_i64(t2, t0);
             tcg_gen_extu_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "maddu";
         break;
@@ -2192,19 +2842,23 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "msub";
         break;
@@ -2212,21 +2866,25 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_extu_tl_i64(t2, t0);
             tcg_gen_extu_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "msubu";
         break;
@@ -2254,59 +2912,59 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
 
     switch (opc) {
     case OPC_VR54XX_MULS:
-        gen_helper_muls(t0, t0, t1);
+        gen_helper_muls(t0, cpu_env, t0, t1);
         opn = "muls";
         break;
     case OPC_VR54XX_MULSU:
-        gen_helper_mulsu(t0, t0, t1);
+        gen_helper_mulsu(t0, cpu_env, t0, t1);
         opn = "mulsu";
         break;
     case OPC_VR54XX_MACC:
-        gen_helper_macc(t0, t0, t1);
+        gen_helper_macc(t0, cpu_env, t0, t1);
         opn = "macc";
         break;
     case OPC_VR54XX_MACCU:
-        gen_helper_maccu(t0, t0, t1);
+        gen_helper_maccu(t0, cpu_env, t0, t1);
         opn = "maccu";
         break;
     case OPC_VR54XX_MSAC:
-        gen_helper_msac(t0, t0, t1);
+        gen_helper_msac(t0, cpu_env, t0, t1);
         opn = "msac";
         break;
     case OPC_VR54XX_MSACU:
-        gen_helper_msacu(t0, t0, t1);
+        gen_helper_msacu(t0, cpu_env, t0, t1);
         opn = "msacu";
         break;
     case OPC_VR54XX_MULHI:
-        gen_helper_mulhi(t0, t0, t1);
+        gen_helper_mulhi(t0, cpu_env, t0, t1);
         opn = "mulhi";
         break;
     case OPC_VR54XX_MULHIU:
-        gen_helper_mulhiu(t0, t0, t1);
+        gen_helper_mulhiu(t0, cpu_env, t0, t1);
         opn = "mulhiu";
         break;
     case OPC_VR54XX_MULSHI:
-        gen_helper_mulshi(t0, t0, t1);
+        gen_helper_mulshi(t0, cpu_env, t0, t1);
         opn = "mulshi";
         break;
     case OPC_VR54XX_MULSHIU:
-        gen_helper_mulshiu(t0, t0, t1);
+        gen_helper_mulshiu(t0, cpu_env, t0, t1);
         opn = "mulshiu";
         break;
     case OPC_VR54XX_MACCHI:
-        gen_helper_macchi(t0, t0, t1);
+        gen_helper_macchi(t0, cpu_env, t0, t1);
         opn = "macchi";
         break;
     case OPC_VR54XX_MACCHIU:
-        gen_helper_macchiu(t0, t0, t1);
+        gen_helper_macchiu(t0, cpu_env, t0, t1);
         opn = "macchiu";
         break;
     case OPC_VR54XX_MSACHI:
-        gen_helper_msachi(t0, t0, t1);
+        gen_helper_msachi(t0, cpu_env, t0, t1);
         opn = "msachi";
         break;
     case OPC_VR54XX_MSACHIU:
-        gen_helper_msachiu(t0, t0, t1);
+        gen_helper_msachiu(t0, cpu_env, t0, t1);
         opn = "msachiu";
         break;
     default:
@@ -2362,8 +3020,8 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
 }
 
 /* Godson integer instructions */
-static void gen_loongson_integer (DisasContext *ctx, uint32_t opc,
-                                int rd, int rs, int rt)
+static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
+                                 int rd, int rs, int rt)
 {
     const char *opn = "loongson";
     TCGv t0, t1;
@@ -2576,6 +3234,278 @@ static void gen_loongson_integer (DisasContext *ctx, uint32_t opc,
     tcg_temp_free(t1);
 }
 
+/* Loongson multimedia instructions */
+static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
+{
+    const char *opn = "loongson_cp2";
+    uint32_t opc, shift_max;
+    TCGv_i64 t0, t1;
+
+    opc = MASK_LMI(ctx->opcode);
+    switch (opc) {
+    case OPC_ADD_CP2:
+    case OPC_SUB_CP2:
+    case OPC_DADD_CP2:
+    case OPC_DSUB_CP2:
+        t0 = tcg_temp_local_new_i64();
+        t1 = tcg_temp_local_new_i64();
+        break;
+    default:
+        t0 = tcg_temp_new_i64();
+        t1 = tcg_temp_new_i64();
+        break;
+    }
+
+    gen_load_fpr64(ctx, t0, rs);
+    gen_load_fpr64(ctx, t1, rt);
+
+#define LMI_HELPER(UP, LO) \
+    case OPC_##UP: gen_helper_##LO(t0, t0, t1); opn = #LO; break
+#define LMI_HELPER_1(UP, LO) \
+    case OPC_##UP: gen_helper_##LO(t0, t0); opn = #LO; break
+#define LMI_DIRECT(UP, LO, OP) \
+    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); opn = #LO; break
+
+    switch (opc) {
+    LMI_HELPER(PADDSH, paddsh);
+    LMI_HELPER(PADDUSH, paddush);
+    LMI_HELPER(PADDH, paddh);
+    LMI_HELPER(PADDW, paddw);
+    LMI_HELPER(PADDSB, paddsb);
+    LMI_HELPER(PADDUSB, paddusb);
+    LMI_HELPER(PADDB, paddb);
+
+    LMI_HELPER(PSUBSH, psubsh);
+    LMI_HELPER(PSUBUSH, psubush);
+    LMI_HELPER(PSUBH, psubh);
+    LMI_HELPER(PSUBW, psubw);
+    LMI_HELPER(PSUBSB, psubsb);
+    LMI_HELPER(PSUBUSB, psubusb);
+    LMI_HELPER(PSUBB, psubb);
+
+    LMI_HELPER(PSHUFH, pshufh);
+    LMI_HELPER(PACKSSWH, packsswh);
+    LMI_HELPER(PACKSSHB, packsshb);
+    LMI_HELPER(PACKUSHB, packushb);
+
+    LMI_HELPER(PUNPCKLHW, punpcklhw);
+    LMI_HELPER(PUNPCKHHW, punpckhhw);
+    LMI_HELPER(PUNPCKLBH, punpcklbh);
+    LMI_HELPER(PUNPCKHBH, punpckhbh);
+    LMI_HELPER(PUNPCKLWD, punpcklwd);
+    LMI_HELPER(PUNPCKHWD, punpckhwd);
+
+    LMI_HELPER(PAVGH, pavgh);
+    LMI_HELPER(PAVGB, pavgb);
+    LMI_HELPER(PMAXSH, pmaxsh);
+    LMI_HELPER(PMINSH, pminsh);
+    LMI_HELPER(PMAXUB, pmaxub);
+    LMI_HELPER(PMINUB, pminub);
+
+    LMI_HELPER(PCMPEQW, pcmpeqw);
+    LMI_HELPER(PCMPGTW, pcmpgtw);
+    LMI_HELPER(PCMPEQH, pcmpeqh);
+    LMI_HELPER(PCMPGTH, pcmpgth);
+    LMI_HELPER(PCMPEQB, pcmpeqb);
+    LMI_HELPER(PCMPGTB, pcmpgtb);
+
+    LMI_HELPER(PSLLW, psllw);
+    LMI_HELPER(PSLLH, psllh);
+    LMI_HELPER(PSRLW, psrlw);
+    LMI_HELPER(PSRLH, psrlh);
+    LMI_HELPER(PSRAW, psraw);
+    LMI_HELPER(PSRAH, psrah);
+
+    LMI_HELPER(PMULLH, pmullh);
+    LMI_HELPER(PMULHH, pmulhh);
+    LMI_HELPER(PMULHUH, pmulhuh);
+    LMI_HELPER(PMADDHW, pmaddhw);
+
+    LMI_HELPER(PASUBUB, pasubub);
+    LMI_HELPER_1(BIADD, biadd);
+    LMI_HELPER_1(PMOVMSKB, pmovmskb);
+
+    LMI_DIRECT(PADDD, paddd, add);
+    LMI_DIRECT(PSUBD, psubd, sub);
+    LMI_DIRECT(XOR_CP2, xor, xor);
+    LMI_DIRECT(NOR_CP2, nor, nor);
+    LMI_DIRECT(AND_CP2, and, and);
+    LMI_DIRECT(PANDN, pandn, andc);
+    LMI_DIRECT(OR, or, or);
+
+    case OPC_PINSRH_0:
+        tcg_gen_deposit_i64(t0, t0, t1, 0, 16);
+        opn = "pinsrh_0";
+        break;
+    case OPC_PINSRH_1:
+        tcg_gen_deposit_i64(t0, t0, t1, 16, 16);
+        opn = "pinsrh_1";
+        break;
+    case OPC_PINSRH_2:
+        tcg_gen_deposit_i64(t0, t0, t1, 32, 16);
+        opn = "pinsrh_2";
+        break;
+    case OPC_PINSRH_3:
+        tcg_gen_deposit_i64(t0, t0, t1, 48, 16);
+        opn = "pinsrh_3";
+        break;
+
+    case OPC_PEXTRH:
+        tcg_gen_andi_i64(t1, t1, 3);
+        tcg_gen_shli_i64(t1, t1, 4);
+        tcg_gen_shr_i64(t0, t0, t1);
+        tcg_gen_ext16u_i64(t0, t0);
+        opn = "pextrh";
+        break;
+
+    case OPC_ADDU_CP2:
+        tcg_gen_add_i64(t0, t0, t1);
+        tcg_gen_ext32s_i64(t0, t0);
+        opn = "addu";
+        break;
+    case OPC_SUBU_CP2:
+        tcg_gen_sub_i64(t0, t0, t1);
+        tcg_gen_ext32s_i64(t0, t0);
+        opn = "addu";
+        break;
+
+    case OPC_SLL_CP2:
+        opn = "sll";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_SRL_CP2:
+        opn = "srl";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_SRA_CP2:
+        opn = "sra";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_DSLL_CP2:
+        opn = "dsll";
+        shift_max = 64;
+        goto do_shift;
+    case OPC_DSRL_CP2:
+        opn = "dsrl";
+        shift_max = 64;
+        goto do_shift;
+    case OPC_DSRA_CP2:
+        opn = "dsra";
+        shift_max = 64;
+        goto do_shift;
+    do_shift:
+        /* Make sure shift count isn't TCG undefined behaviour.  */
+        tcg_gen_andi_i64(t1, t1, shift_max - 1);
+
+        switch (opc) {
+        case OPC_SLL_CP2:
+        case OPC_DSLL_CP2:
+            tcg_gen_shl_i64(t0, t0, t1);
+            break;
+        case OPC_SRA_CP2:
+        case OPC_DSRA_CP2:
+            /* Since SRA is UndefinedResult without sign-extended inputs,
+               we can treat SRA and DSRA the same.  */
+            tcg_gen_sar_i64(t0, t0, t1);
+            break;
+        case OPC_SRL_CP2:
+            /* We want to shift in zeros for SRL; zero-extend first.  */
+            tcg_gen_ext32u_i64(t0, t0);
+            /* FALLTHRU */
+        case OPC_DSRL_CP2:
+            tcg_gen_shr_i64(t0, t0, t1);
+            break;
+        }
+
+        if (shift_max == 32) {
+            tcg_gen_ext32s_i64(t0, t0);
+        }
+
+        /* Shifts larger than MAX produce zero.  */
+        tcg_gen_setcondi_i64(TCG_COND_LTU, t1, t1, shift_max);
+        tcg_gen_neg_i64(t1, t1);
+        tcg_gen_and_i64(t0, t0, t1);
+        break;
+
+    case OPC_ADD_CP2:
+    case OPC_DADD_CP2:
+        {
+            TCGv_i64 t2 = tcg_temp_new_i64();
+            int lab = gen_new_label();
+
+            tcg_gen_mov_i64(t2, t0);
+            tcg_gen_add_i64(t0, t1, t2);
+            if (opc == OPC_ADD_CP2) {
+                tcg_gen_ext32s_i64(t0, t0);
+            }
+            tcg_gen_xor_i64(t1, t1, t2);
+            tcg_gen_xor_i64(t2, t2, t0);
+            tcg_gen_andc_i64(t1, t2, t1);
+            tcg_temp_free_i64(t2);
+            tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
+            generate_exception(ctx, EXCP_OVERFLOW);
+            gen_set_label(lab);
+
+            opn = (opc == OPC_ADD_CP2 ? "add" : "dadd");
+            break;
+        }
+
+    case OPC_SUB_CP2:
+    case OPC_DSUB_CP2:
+        {
+            TCGv_i64 t2 = tcg_temp_new_i64();
+            int lab = gen_new_label();
+
+            tcg_gen_mov_i64(t2, t0);
+            tcg_gen_sub_i64(t0, t1, t2);
+            if (opc == OPC_SUB_CP2) {
+                tcg_gen_ext32s_i64(t0, t0);
+            }
+            tcg_gen_xor_i64(t1, t1, t2);
+            tcg_gen_xor_i64(t2, t2, t0);
+            tcg_gen_and_i64(t1, t1, t2);
+            tcg_temp_free_i64(t2);
+            tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
+            generate_exception(ctx, EXCP_OVERFLOW);
+            gen_set_label(lab);
+
+            opn = (opc == OPC_SUB_CP2 ? "sub" : "dsub");
+            break;
+        }
+
+    case OPC_PMULUW:
+        tcg_gen_ext32u_i64(t0, t0);
+        tcg_gen_ext32u_i64(t1, t1);
+        tcg_gen_mul_i64(t0, t0, t1);
+        opn = "pmuluw";
+        break;
+
+    case OPC_SEQU_CP2:
+    case OPC_SEQ_CP2:
+    case OPC_SLTU_CP2:
+    case OPC_SLT_CP2:
+    case OPC_SLEU_CP2:
+    case OPC_SLE_CP2:
+        /* ??? Document is unclear: Set FCC[CC].  Does that mean the
+           FD field is the CC field?  */
+    default:
+        MIPS_INVAL(opn);
+        generate_exception(ctx, EXCP_RI);
+        return;
+    }
+
+#undef LMI_HELPER
+#undef LMI_DIRECT
+
+    gen_store_fpr64(ctx, t0, rd);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s %s, %s, %s", opn,
+               fregnames[rd], fregnames[rs], fregnames[rt]);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
 /* Traps */
 static void gen_trap (DisasContext *ctx, uint32_t opc,
                       int rs, int rt, int16_t imm)
@@ -2683,7 +3613,7 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
             save_cpu_state(ctx, 0);
-            gen_helper_0i(raise_exception, EXCP_DEBUG);
+            gen_helper_0e0i(raise_exception, EXCP_DEBUG);
         }
         tcg_gen_exit_tb(0);
     }
@@ -2743,6 +3673,16 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         }
         btgt = ctx->pc + insn_bytes + offset;
         break;
+    case OPC_BPOSGE32:
+#if defined(TARGET_MIPS64)
+    case OPC_BPOSGE64:
+        tcg_gen_andi_tl(t0, cpu_dspctrl, 0x7F);
+#else
+        tcg_gen_andi_tl(t0, cpu_dspctrl, 0x3F);
+#endif
+        bcond_compute = 1;
+        btgt = ctx->pc + insn_bytes + offset;
+        break;
     case OPC_J:
     case OPC_JAL:
     case OPC_JALX:
@@ -2931,6 +3871,16 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             MIPS_DEBUG("bltzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
+        case OPC_BPOSGE32:
+            tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 32);
+            MIPS_DEBUG("bposge32 " TARGET_FMT_lx, btgt);
+            goto not_likely;
+#if defined(TARGET_MIPS64)
+        case OPC_BPOSGE64:
+            tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 64);
+            MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
+            goto not_likely;
+#endif
         case OPC_BLTZALS:
         case OPC_BLTZAL:
             ctx->hflags |= (opc == OPC_BLTZALS
@@ -2982,7 +3932,6 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
 {
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
-    target_ulong mask;
 
     gen_load_gpr(t1, rs);
     switch (opc) {
@@ -3015,45 +3964,22 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
     case OPC_INS:
         if (lsb > msb)
             goto fail;
-        mask = ((msb - lsb + 1 < 32) ? ((1 << (msb - lsb + 1)) - 1) : ~0) << lsb;
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         tcg_gen_ext32s_tl(t0, t0);
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DINSM:
-        if (lsb > msb)
-            goto fail;
-        mask = ((msb - lsb + 1 + 32 < 64) ? ((1ULL << (msb - lsb + 1 + 32)) - 1) : ~0ULL) << lsb;
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb + 32 - lsb + 1);
         break;
     case OPC_DINSU:
-        if (lsb > msb)
-            goto fail;
-        mask = ((1ULL << (msb - lsb + 1)) - 1) << (lsb + 32);
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb + 32);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb + 32, msb - lsb + 1);
         break;
     case OPC_DINS:
-        if (lsb > msb)
-            goto fail;
         gen_load_gpr(t0, rt);
-        mask = ((1ULL << (msb - lsb + 1)) - 1) << lsb;
-        gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         break;
 #endif
     default:
@@ -3187,17 +4113,17 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpcontrol(arg);
+            gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf0(arg);
+            gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf1(arg);
+            gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
@@ -3207,7 +4133,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 1:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_random(arg);
+            gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
@@ -3258,37 +4184,37 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcstatus(arg);
+            gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcbind(arg);
+            gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcrestart(arg);
+            gen_helper_mfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tchalt(arg);
+            gen_helper_mfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tccontext(arg);
+            gen_helper_mfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcschedule(arg);
+            gen_helper_mfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcschefback(arg);
+            gen_helper_mfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
@@ -3399,7 +4325,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             /* Mark as an IO operation because we read the time.  */
             if (use_icount)
                 gen_io_start();
-            gen_helper_mfc0_count(arg);
+            gen_helper_mfc0_count(arg, cpu_env);
             if (use_icount) {
                 gen_io_end();
             }
@@ -3531,7 +4457,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_lladdr(arg);
+            gen_helper_mfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
         default:
@@ -3541,7 +4467,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mfc0_watchlo, arg, sel);
+            gen_helper_1e0i(mfc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -3551,7 +4477,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 19:
         switch (sel) {
         case 0 ...7:
-            gen_helper_1i(mfc0_watchhi, arg, sel);
+            gen_helper_1e0i(mfc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -3590,7 +4516,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_debug(arg); /* EJTAG support */
+            gen_helper_mfc0_debug(arg, cpu_env); /* EJTAG support */
             rn = "Debug";
             break;
         case 1:
@@ -3765,12 +4691,12 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 0:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_index(arg);
+            gen_helper_mtc0_index(cpu_env, arg);
             rn = "Index";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_mvpcontrol(arg);
+            gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
@@ -3795,22 +4721,22 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpecontrol(arg);
+            gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf0(arg);
+            gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf1(arg);
+            gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_yqmask(arg);
+            gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
@@ -3825,7 +4751,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeopt(arg);
+            gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
@@ -3835,42 +4761,42 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 2:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo0(arg);
+            gen_helper_mtc0_entrylo0(cpu_env, arg);
             rn = "EntryLo0";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcstatus(arg);
+            gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcbind(arg);
+            gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcrestart(arg);
+            gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tchalt(arg);
+            gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tccontext(arg);
+            gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschedule(arg);
+            gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschefback(arg);
+            gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
@@ -3880,7 +4806,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 3:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo1(arg);
+            gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
         default:
@@ -3890,11 +4816,11 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 4:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_context(arg);
+            gen_helper_mtc0_context(cpu_env, arg);
             rn = "Context";
             break;
         case 1:
-//            gen_helper_mtc0_contextconfig(arg); /* SmartMIPS ASE */
+//            gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
 //            break;
         default:
@@ -3904,12 +4830,12 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 5:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_pagemask(arg);
+            gen_helper_mtc0_pagemask(cpu_env, arg);
             rn = "PageMask";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_pagegrain(arg);
+            gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
         default:
@@ -3919,32 +4845,32 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 6:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_wired(arg);
+            gen_helper_mtc0_wired(cpu_env, arg);
             rn = "Wired";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf0(arg);
+            gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf1(arg);
+            gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf2(arg);
+            gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf3(arg);
+            gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf4(arg);
+            gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
         default:
@@ -3955,7 +4881,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_hwrena(arg);
+            gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
         default:
@@ -3969,7 +4895,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 9:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_count(arg);
+            gen_helper_mtc0_count(cpu_env, arg);
             rn = "Count";
             break;
         /* 6,7 are implementation dependent */
@@ -3980,7 +4906,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 10:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entryhi(arg);
+            gen_helper_mtc0_entryhi(cpu_env, arg);
             rn = "EntryHi";
             break;
         default:
@@ -3990,7 +4916,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 11:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_compare(arg);
+            gen_helper_mtc0_compare(cpu_env, arg);
             rn = "Compare";
             break;
         /* 6,7 are implementation dependent */
@@ -4002,7 +4928,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_status(arg);
+            gen_helper_mtc0_status(cpu_env, arg);
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
@@ -4010,14 +4936,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_intctl(arg);
+            gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsctl(arg);
+            gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
@@ -4037,7 +4963,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_cause(arg);
+            gen_helper_mtc0_cause(cpu_env, arg);
             rn = "Cause";
             break;
         default:
@@ -4062,7 +4988,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_ebase(arg);
+            gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
         default:
@@ -4072,7 +4998,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 16:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_config0(arg);
+            gen_helper_mtc0_config0(cpu_env, arg);
             rn = "Config";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4082,7 +5008,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Config1";
             break;
         case 2:
-            gen_helper_mtc0_config2(arg);
+            gen_helper_mtc0_config2(cpu_env, arg);
             rn = "Config2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4109,7 +5035,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_lladdr(arg);
+            gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
         default:
@@ -4119,7 +5045,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchlo, arg, sel);
+            gen_helper_0e1i(mtc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -4129,7 +5055,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchhi, arg, sel);
+            gen_helper_0e1i(mtc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -4141,7 +5067,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 0:
 #if defined(TARGET_MIPS64)
             check_insn(env, ctx, ISA_MIPS3);
-            gen_helper_mtc0_xcontext(arg);
+            gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
 #endif
@@ -4153,7 +5079,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
         switch (sel) {
         case 0:
-            gen_helper_mtc0_framemask(arg);
+            gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
@@ -4167,20 +5093,20 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_debug(arg); /* EJTAG support */
+            gen_helper_mtc0_debug(cpu_env, arg); /* EJTAG support */
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_mtc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol(cpu_env, arg); /* PDtrace support */
             rn = "TraceControl";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
 //            break;
         case 2:
-//            gen_helper_mtc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol2(cpu_env, arg); /* PDtrace support */
             rn = "TraceControl2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4188,13 +5114,13 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 3:
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
-//            gen_helper_mtc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_mtc0_usertracedata(cpu_env, arg); /* PDtrace support */
             rn = "UserTraceData";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
 //            break;
         case 4:
-//            gen_helper_mtc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracebpc(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceBPC";
@@ -4217,7 +5143,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 25:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_performance0(arg);
+            gen_helper_mtc0_performance0(cpu_env, arg);
             rn = "Performance0";
             break;
         case 1:
@@ -4272,14 +5198,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taglo(arg);
+            gen_helper_mtc0_taglo(cpu_env, arg);
             rn = "TagLo";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datalo(arg);
+            gen_helper_mtc0_datalo(cpu_env, arg);
             rn = "DataLo";
             break;
         default:
@@ -4292,14 +5218,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taghi(arg);
+            gen_helper_mtc0_taghi(cpu_env, arg);
             rn = "TagHi";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datahi(arg);
+            gen_helper_mtc0_datahi(cpu_env, arg);
             rn = "DataHi";
             break;
         default:
@@ -4364,17 +5290,17 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpcontrol(arg);
+            gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf0(arg);
+            gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf1(arg);
+            gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
@@ -4384,7 +5310,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 1:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_random(arg);
+            gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
@@ -4434,37 +5360,37 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcstatus(arg);
+            gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcbind(arg);
+            gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcrestart(arg);
+            gen_helper_dmfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tchalt(arg);
+            gen_helper_dmfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tccontext(arg);
+            gen_helper_dmfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcschedule(arg);
+            gen_helper_dmfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcschefback(arg);
+            gen_helper_dmfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
@@ -4572,7 +5498,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             /* Mark as an IO operation because we read the time.  */
             if (use_icount)
                 gen_io_start();
-            gen_helper_mfc0_count(arg);
+            gen_helper_mfc0_count(arg, cpu_env);
             if (use_icount) {
                 gen_io_end();
             }
@@ -4701,7 +5627,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_dmfc0_lladdr(arg);
+            gen_helper_dmfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
         default:
@@ -4711,7 +5637,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(dmfc0_watchlo, arg, sel);
+            gen_helper_1e0i(dmfc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -4721,7 +5647,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mfc0_watchhi, arg, sel);
+            gen_helper_1e0i(mfc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -4757,23 +5683,23 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_debug(arg); /* EJTAG support */
+            gen_helper_mfc0_debug(arg, cpu_env); /* EJTAG support */
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_dmfc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracecontrol(arg, cpu_env); /* PDtrace support */
             rn = "TraceControl";
 //            break;
         case 2:
-//            gen_helper_dmfc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracecontrol2(arg, cpu_env); /* PDtrace support */
             rn = "TraceControl2";
 //            break;
         case 3:
-//            gen_helper_dmfc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_dmfc0_usertracedata(arg, cpu_env); /* PDtrace support */
             rn = "UserTraceData";
 //            break;
         case 4:
-//            gen_helper_dmfc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracebpc(arg, cpu_env); /* PDtrace support */
             rn = "TraceBPC";
 //            break;
         default:
@@ -4931,12 +5857,12 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 0:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_index(arg);
+            gen_helper_mtc0_index(cpu_env, arg);
             rn = "Index";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_mvpcontrol(arg);
+            gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
@@ -4961,22 +5887,22 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpecontrol(arg);
+            gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf0(arg);
+            gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf1(arg);
+            gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_yqmask(arg);
+            gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
@@ -4991,7 +5917,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeopt(arg);
+            gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
@@ -5001,42 +5927,42 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 2:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo0(arg);
+            gen_helper_mtc0_entrylo0(cpu_env, arg);
             rn = "EntryLo0";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcstatus(arg);
+            gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcbind(arg);
+            gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcrestart(arg);
+            gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tchalt(arg);
+            gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tccontext(arg);
+            gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschedule(arg);
+            gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschefback(arg);
+            gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
@@ -5046,7 +5972,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 3:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo1(arg);
+            gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
         default:
@@ -5056,11 +5982,11 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 4:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_context(arg);
+            gen_helper_mtc0_context(cpu_env, arg);
             rn = "Context";
             break;
         case 1:
-//           gen_helper_mtc0_contextconfig(arg); /* SmartMIPS ASE */
+//           gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
 //           break;
         default:
@@ -5070,12 +5996,12 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 5:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_pagemask(arg);
+            gen_helper_mtc0_pagemask(cpu_env, arg);
             rn = "PageMask";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_pagegrain(arg);
+            gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
         default:
@@ -5085,32 +6011,32 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 6:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_wired(arg);
+            gen_helper_mtc0_wired(cpu_env, arg);
             rn = "Wired";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf0(arg);
+            gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf1(arg);
+            gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf2(arg);
+            gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf3(arg);
+            gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf4(arg);
+            gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
         default:
@@ -5121,7 +6047,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_hwrena(arg);
+            gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
         default:
@@ -5135,7 +6061,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 9:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_count(arg);
+            gen_helper_mtc0_count(cpu_env, arg);
             rn = "Count";
             break;
         /* 6,7 are implementation dependent */
@@ -5148,7 +6074,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 10:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entryhi(arg);
+            gen_helper_mtc0_entryhi(cpu_env, arg);
             rn = "EntryHi";
             break;
         default:
@@ -5158,7 +6084,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 11:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_compare(arg);
+            gen_helper_mtc0_compare(cpu_env, arg);
             rn = "Compare";
             break;
         /* 6,7 are implementation dependent */
@@ -5172,7 +6098,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_status(arg);
+            gen_helper_mtc0_status(cpu_env, arg);
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
@@ -5180,14 +6106,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_intctl(arg);
+            gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsctl(arg);
+            gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
@@ -5212,7 +6138,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             if (use_icount) {
                 gen_io_start();
             }
-            gen_helper_mtc0_cause(arg);
+            gen_helper_mtc0_cause(cpu_env, arg);
             if (use_icount) {
                 gen_io_end();
             }
@@ -5242,7 +6168,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_ebase(arg);
+            gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
         default:
@@ -5252,7 +6178,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 16:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_config0(arg);
+            gen_helper_mtc0_config0(cpu_env, arg);
             rn = "Config";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -5262,7 +6188,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Config1";
             break;
         case 2:
-            gen_helper_mtc0_config2(arg);
+            gen_helper_mtc0_config2(cpu_env, arg);
             rn = "Config2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -5280,7 +6206,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_lladdr(arg);
+            gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
         default:
@@ -5290,7 +6216,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchlo, arg, sel);
+            gen_helper_0e1i(mtc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -5300,7 +6226,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchhi, arg, sel);
+            gen_helper_0e1i(mtc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -5311,7 +6237,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS3);
-            gen_helper_mtc0_xcontext(arg);
+            gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
         default:
@@ -5322,7 +6248,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
         switch (sel) {
         case 0:
-            gen_helper_mtc0_framemask(arg);
+            gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
@@ -5336,32 +6262,32 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_debug(arg); /* EJTAG support */
+            gen_helper_mtc0_debug(cpu_env, arg); /* EJTAG support */
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_mtc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceControl";
 //            break;
         case 2:
-//            gen_helper_mtc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol2(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceControl2";
 //            break;
         case 3:
-//            gen_helper_mtc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_mtc0_usertracedata(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "UserTraceData";
 //            break;
         case 4:
-//            gen_helper_mtc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracebpc(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceBPC";
@@ -5384,35 +6310,35 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 25:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_performance0(arg);
+            gen_helper_mtc0_performance0(cpu_env, arg);
             rn = "Performance0";
             break;
         case 1:
-//            gen_helper_mtc0_performance1(arg);
+//            gen_helper_mtc0_performance1(cpu_env, arg);
             rn = "Performance1";
 //            break;
         case 2:
-//            gen_helper_mtc0_performance2(arg);
+//            gen_helper_mtc0_performance2(cpu_env, arg);
             rn = "Performance2";
 //            break;
         case 3:
-//            gen_helper_mtc0_performance3(arg);
+//            gen_helper_mtc0_performance3(cpu_env, arg);
             rn = "Performance3";
 //            break;
         case 4:
-//            gen_helper_mtc0_performance4(arg);
+//            gen_helper_mtc0_performance4(cpu_env, arg);
             rn = "Performance4";
 //            break;
         case 5:
-//            gen_helper_mtc0_performance5(arg);
+//            gen_helper_mtc0_performance5(cpu_env, arg);
             rn = "Performance5";
 //            break;
         case 6:
-//            gen_helper_mtc0_performance6(arg);
+//            gen_helper_mtc0_performance6(cpu_env, arg);
             rn = "Performance6";
 //            break;
         case 7:
-//            gen_helper_mtc0_performance7(arg);
+//            gen_helper_mtc0_performance7(cpu_env, arg);
             rn = "Performance7";
 //            break;
         default:
@@ -5439,14 +6365,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taglo(arg);
+            gen_helper_mtc0_taglo(cpu_env, arg);
             rn = "TagLo";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datalo(arg);
+            gen_helper_mtc0_datalo(cpu_env, arg);
             rn = "DataLo";
             break;
         default:
@@ -5459,14 +6385,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taghi(arg);
+            gen_helper_mtc0_taghi(cpu_env, arg);
             rn = "TagHi";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datahi(arg);
+            gen_helper_mtc0_datahi(cpu_env, arg);
             rn = "DataHi";
             break;
         default:
@@ -5533,10 +6459,10 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 1:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_vpecontrol(t0);
+                gen_helper_mftc0_vpecontrol(t0, cpu_env);
                 break;
             case 2:
-                gen_helper_mftc0_vpeconf0(t0);
+                gen_helper_mftc0_vpeconf0(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5546,25 +6472,25 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 2:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_tcstatus(t0);
+                gen_helper_mftc0_tcstatus(t0, cpu_env);
                 break;
             case 2:
-                gen_helper_mftc0_tcbind(t0);
+                gen_helper_mftc0_tcbind(t0, cpu_env);
                 break;
             case 3:
-                gen_helper_mftc0_tcrestart(t0);
+                gen_helper_mftc0_tcrestart(t0, cpu_env);
                 break;
             case 4:
-                gen_helper_mftc0_tchalt(t0);
+                gen_helper_mftc0_tchalt(t0, cpu_env);
                 break;
             case 5:
-                gen_helper_mftc0_tccontext(t0);
+                gen_helper_mftc0_tccontext(t0, cpu_env);
                 break;
             case 6:
-                gen_helper_mftc0_tcschedule(t0);
+                gen_helper_mftc0_tcschedule(t0, cpu_env);
                 break;
             case 7:
-                gen_helper_mftc0_tcschefback(t0);
+                gen_helper_mftc0_tcschefback(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5574,7 +6500,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 10:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_entryhi(t0);
+                gen_helper_mftc0_entryhi(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5583,7 +6509,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 12:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_status(t0);
+                gen_helper_mftc0_status(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5592,7 +6518,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 13:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_cause(t0);
+                gen_helper_mftc0_cause(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5602,7 +6528,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 14:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_epc(t0);
+                gen_helper_mftc0_epc(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5612,7 +6538,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 15:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_ebase(t0);
+                gen_helper_mftc0_ebase(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5622,7 +6548,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 16:
             switch (sel) {
             case 0 ... 7:
-                gen_helper_mftc0_configx(t0, tcg_const_tl(sel));
+                gen_helper_mftc0_configx(t0, cpu_env, tcg_const_tl(sel));
                 break;
             default:
                 goto die;
@@ -5632,7 +6558,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 23:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_debug(t0);
+                gen_helper_mftc0_debug(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5645,49 +6571,49 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
     } else switch (sel) {
     /* GPR registers. */
     case 0:
-        gen_helper_1i(mftgpr, t0, rt);
+        gen_helper_1e0i(mftgpr, t0, rt);
         break;
     /* Auxiliary CPU registers */
     case 1:
         switch (rt) {
         case 0:
-            gen_helper_1i(mftlo, t0, 0);
+            gen_helper_1e0i(mftlo, t0, 0);
             break;
         case 1:
-            gen_helper_1i(mfthi, t0, 0);
+            gen_helper_1e0i(mfthi, t0, 0);
             break;
         case 2:
-            gen_helper_1i(mftacx, t0, 0);
+            gen_helper_1e0i(mftacx, t0, 0);
             break;
         case 4:
-            gen_helper_1i(mftlo, t0, 1);
+            gen_helper_1e0i(mftlo, t0, 1);
             break;
         case 5:
-            gen_helper_1i(mfthi, t0, 1);
+            gen_helper_1e0i(mfthi, t0, 1);
             break;
         case 6:
-            gen_helper_1i(mftacx, t0, 1);
+            gen_helper_1e0i(mftacx, t0, 1);
             break;
         case 8:
-            gen_helper_1i(mftlo, t0, 2);
+            gen_helper_1e0i(mftlo, t0, 2);
             break;
         case 9:
-            gen_helper_1i(mfthi, t0, 2);
+            gen_helper_1e0i(mfthi, t0, 2);
             break;
         case 10:
-            gen_helper_1i(mftacx, t0, 2);
+            gen_helper_1e0i(mftacx, t0, 2);
             break;
         case 12:
-            gen_helper_1i(mftlo, t0, 3);
+            gen_helper_1e0i(mftlo, t0, 3);
             break;
         case 13:
-            gen_helper_1i(mfthi, t0, 3);
+            gen_helper_1e0i(mfthi, t0, 3);
             break;
         case 14:
-            gen_helper_1i(mftacx, t0, 3);
+            gen_helper_1e0i(mftacx, t0, 3);
             break;
         case 16:
-            gen_helper_mftdsp(t0);
+            gen_helper_mftdsp(t0, cpu_env);
             break;
         default:
             goto die;
@@ -5712,7 +6638,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        gen_helper_1i(cfc1, t0, rt);
+        gen_helper_1e0i(cfc1, t0, rt);
         break;
     /* COP2: Not implemented. */
     case 4:
@@ -5751,10 +6677,10 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 1:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_vpecontrol(t0);
+                gen_helper_mttc0_vpecontrol(cpu_env, t0);
                 break;
             case 2:
-                gen_helper_mttc0_vpeconf0(t0);
+                gen_helper_mttc0_vpeconf0(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5764,25 +6690,25 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 2:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_tcstatus(t0);
+                gen_helper_mttc0_tcstatus(cpu_env, t0);
                 break;
             case 2:
-                gen_helper_mttc0_tcbind(t0);
+                gen_helper_mttc0_tcbind(cpu_env, t0);
                 break;
             case 3:
-                gen_helper_mttc0_tcrestart(t0);
+                gen_helper_mttc0_tcrestart(cpu_env, t0);
                 break;
             case 4:
-                gen_helper_mttc0_tchalt(t0);
+                gen_helper_mttc0_tchalt(cpu_env, t0);
                 break;
             case 5:
-                gen_helper_mttc0_tccontext(t0);
+                gen_helper_mttc0_tccontext(cpu_env, t0);
                 break;
             case 6:
-                gen_helper_mttc0_tcschedule(t0);
+                gen_helper_mttc0_tcschedule(cpu_env, t0);
                 break;
             case 7:
-                gen_helper_mttc0_tcschefback(t0);
+                gen_helper_mttc0_tcschefback(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5792,7 +6718,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 10:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_entryhi(t0);
+                gen_helper_mttc0_entryhi(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5801,7 +6727,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 12:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_status(t0);
+                gen_helper_mttc0_status(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5810,7 +6736,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 13:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_cause(t0);
+                gen_helper_mttc0_cause(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5820,7 +6746,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 15:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_ebase(t0);
+                gen_helper_mttc0_ebase(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5830,7 +6756,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 23:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_debug(t0);
+                gen_helper_mttc0_debug(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5843,49 +6769,49 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
     } else switch (sel) {
     /* GPR registers. */
     case 0:
-        gen_helper_1i(mttgpr, t0, rd);
+        gen_helper_0e1i(mttgpr, t0, rd);
         break;
     /* Auxiliary CPU registers */
     case 1:
         switch (rd) {
         case 0:
-            gen_helper_1i(mttlo, t0, 0);
+            gen_helper_0e1i(mttlo, t0, 0);
             break;
         case 1:
-            gen_helper_1i(mtthi, t0, 0);
+            gen_helper_0e1i(mtthi, t0, 0);
             break;
         case 2:
-            gen_helper_1i(mttacx, t0, 0);
+            gen_helper_0e1i(mttacx, t0, 0);
             break;
         case 4:
-            gen_helper_1i(mttlo, t0, 1);
+            gen_helper_0e1i(mttlo, t0, 1);
             break;
         case 5:
-            gen_helper_1i(mtthi, t0, 1);
+            gen_helper_0e1i(mtthi, t0, 1);
             break;
         case 6:
-            gen_helper_1i(mttacx, t0, 1);
+            gen_helper_0e1i(mttacx, t0, 1);
             break;
         case 8:
-            gen_helper_1i(mttlo, t0, 2);
+            gen_helper_0e1i(mttlo, t0, 2);
             break;
         case 9:
-            gen_helper_1i(mtthi, t0, 2);
+            gen_helper_0e1i(mtthi, t0, 2);
             break;
         case 10:
-            gen_helper_1i(mttacx, t0, 2);
+            gen_helper_0e1i(mttacx, t0, 2);
             break;
         case 12:
-            gen_helper_1i(mttlo, t0, 3);
+            gen_helper_0e1i(mttlo, t0, 3);
             break;
         case 13:
-            gen_helper_1i(mtthi, t0, 3);
+            gen_helper_0e1i(mtthi, t0, 3);
             break;
         case 14:
-            gen_helper_1i(mttacx, t0, 3);
+            gen_helper_0e1i(mttacx, t0, 3);
             break;
         case 16:
-            gen_helper_mttdsp(t0);
+            gen_helper_mttdsp(cpu_env, t0);
             break;
         default:
             goto die;
@@ -5910,7 +6836,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        gen_helper_1i(ctc1, t0, rd);
+        gen_helper_0e1i(ctc1, t0, rd);
         break;
     /* COP2: Not implemented. */
     case 4:
@@ -5995,30 +6921,30 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         opn = "tlbwi";
         if (!env->tlb->helper_tlbwi)
             goto die;
-        gen_helper_tlbwi();
+        gen_helper_tlbwi(cpu_env);
         break;
     case OPC_TLBWR:
         opn = "tlbwr";
         if (!env->tlb->helper_tlbwr)
             goto die;
-        gen_helper_tlbwr();
+        gen_helper_tlbwr(cpu_env);
         break;
     case OPC_TLBP:
         opn = "tlbp";
         if (!env->tlb->helper_tlbp)
             goto die;
-        gen_helper_tlbp();
+        gen_helper_tlbp(cpu_env);
         break;
     case OPC_TLBR:
         opn = "tlbr";
         if (!env->tlb->helper_tlbr)
             goto die;
-        gen_helper_tlbr();
+        gen_helper_tlbr(cpu_env);
         break;
     case OPC_ERET:
         opn = "eret";
         check_insn(env, ctx, ISA_MIPS2);
-        gen_helper_eret();
+        gen_helper_eret(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     case OPC_DERET:
@@ -6028,7 +6954,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
             MIPS_INVAL(opn);
             generate_exception(ctx, EXCP_RI);
         } else {
-            gen_helper_deret();
+            gen_helper_deret(cpu_env);
             ctx->bstate = BS_EXCP;
         }
         break;
@@ -6039,7 +6965,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         ctx->pc += 4;
         save_cpu_state(ctx, 1);
         ctx->pc -= 4;
-        gen_helper_wait();
+        gen_helper_wait(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     default:
@@ -6340,13 +7266,13 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
         opn = "mtc1";
         break;
     case OPC_CFC1:
-        gen_helper_1i(cfc1, t0, fs);
+        gen_helper_1e0i(cfc1, t0, fs);
         gen_store_gpr(t0, rt);
         opn = "cfc1";
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
-        gen_helper_1i(ctc1, t0, fs);
+        gen_helper_0e1i(ctc1, t0, fs);
         opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
@@ -6543,7 +7469,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_add_s(fp0, fp0, fp1);
+            gen_helper_float_add_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6558,7 +7484,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_sub_s(fp0, fp0, fp1);
+            gen_helper_float_sub_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6573,7 +7499,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_mul_s(fp0, fp0, fp1);
+            gen_helper_float_mul_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6588,7 +7514,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_div_s(fp0, fp0, fp1);
+            gen_helper_float_div_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6601,7 +7527,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_sqrt_s(fp0, fp0);
+            gen_helper_float_sqrt_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6646,7 +7572,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_roundl_s(fp64, fp32);
+            gen_helper_float_roundl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6660,7 +7586,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_truncl_s(fp64, fp32);
+            gen_helper_float_truncl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6674,7 +7600,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_ceill_s(fp64, fp32);
+            gen_helper_float_ceill_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6688,7 +7614,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_floorl_s(fp64, fp32);
+            gen_helper_float_floorl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6700,7 +7626,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_roundw_s(fp0, fp0);
+            gen_helper_float_roundw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6711,7 +7637,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_truncw_s(fp0, fp0);
+            gen_helper_float_truncw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6722,7 +7648,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_ceilw_s(fp0, fp0);
+            gen_helper_float_ceilw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6733,7 +7659,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_floorw_s(fp0, fp0);
+            gen_helper_float_floorw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6781,7 +7707,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_recip_s(fp0, fp0);
+            gen_helper_float_recip_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6793,7 +7719,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_rsqrt_s(fp0, fp0);
+            gen_helper_float_rsqrt_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6807,7 +7733,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_recip2_s(fp0, fp0, fp1);
+            gen_helper_float_recip2_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6820,7 +7746,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_recip1_s(fp0, fp0);
+            gen_helper_float_recip1_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6832,7 +7758,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_rsqrt1_s(fp0, fp0);
+            gen_helper_float_rsqrt1_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6846,7 +7772,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_rsqrt2_s(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6860,7 +7786,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtd_s(fp64, fp32);
+            gen_helper_float_cvtd_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6872,7 +7798,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvtw_s(fp0, fp0);
+            gen_helper_float_cvtw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6885,7 +7811,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtl_s(fp64, fp32);
+            gen_helper_float_cvtl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6941,7 +7867,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_add_d(fp0, fp0, fp1);
+            gen_helper_float_add_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6957,7 +7883,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_sub_d(fp0, fp0, fp1);
+            gen_helper_float_sub_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6973,7 +7899,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_mul_d(fp0, fp0, fp1);
+            gen_helper_float_mul_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6989,7 +7915,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_div_d(fp0, fp0, fp1);
+            gen_helper_float_div_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7003,7 +7929,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_sqrt_d(fp0, fp0);
+            gen_helper_float_sqrt_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7050,7 +7976,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_roundl_d(fp0, fp0);
+            gen_helper_float_roundl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7062,7 +7988,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_truncl_d(fp0, fp0);
+            gen_helper_float_truncl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7074,7 +8000,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_ceill_d(fp0, fp0);
+            gen_helper_float_ceill_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7086,7 +8012,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_floorl_d(fp0, fp0);
+            gen_helper_float_floorl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7099,7 +8025,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_roundw_d(fp32, fp64);
+            gen_helper_float_roundw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7113,7 +8039,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_truncw_d(fp32, fp64);
+            gen_helper_float_truncw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7127,7 +8053,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_ceilw_d(fp32, fp64);
+            gen_helper_float_ceilw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7141,7 +8067,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_floorw_d(fp32, fp64);
+            gen_helper_float_floorw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7190,7 +8116,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip_d(fp0, fp0);
+            gen_helper_float_recip_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7202,7 +8128,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt_d(fp0, fp0);
+            gen_helper_float_rsqrt_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7216,7 +8142,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_recip2_d(fp0, fp0, fp1);
+            gen_helper_float_recip2_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7229,7 +8155,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip1_d(fp0, fp0);
+            gen_helper_float_recip1_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7241,7 +8167,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt1_d(fp0, fp0);
+            gen_helper_float_rsqrt1_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7255,7 +8181,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_rsqrt2_d(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7293,7 +8219,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvts_d(fp32, fp64);
+            gen_helper_float_cvts_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7307,7 +8233,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvtw_d(fp32, fp64);
+            gen_helper_float_cvtw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7320,7 +8246,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtl_d(fp0, fp0);
+            gen_helper_float_cvtl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7331,7 +8257,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvts_w(fp0, fp0);
+            gen_helper_float_cvts_w(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7344,7 +8270,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtd_w(fp64, fp32);
+            gen_helper_float_cvtd_w(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -7358,7 +8284,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvts_l(fp32, fp64);
+            gen_helper_float_cvts_l(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7371,7 +8297,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtd_l(fp0, fp0);
+            gen_helper_float_cvtd_l(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7383,7 +8309,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtps_pw(fp0, fp0);
+            gen_helper_float_cvtps_pw(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7397,7 +8323,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_add_ps(fp0, fp0, fp1);
+            gen_helper_float_add_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7412,7 +8338,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_sub_ps(fp0, fp0, fp1);
+            gen_helper_float_sub_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7427,7 +8353,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_mul_ps(fp0, fp0, fp1);
+            gen_helper_float_mul_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7515,7 +8441,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, ft);
             gen_load_fpr64(ctx, fp1, fs);
-            gen_helper_float_addr_ps(fp0, fp0, fp1);
+            gen_helper_float_addr_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7530,7 +8456,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, ft);
             gen_load_fpr64(ctx, fp1, fs);
-            gen_helper_float_mulr_ps(fp0, fp0, fp1);
+            gen_helper_float_mulr_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7545,7 +8471,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_recip2_ps(fp0, fp0, fp1);
+            gen_helper_float_recip2_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7558,7 +8484,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip1_ps(fp0, fp0);
+            gen_helper_float_recip1_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7570,7 +8496,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt1_ps(fp0, fp0);
+            gen_helper_float_rsqrt1_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7584,7 +8510,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_rsqrt2_ps(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7597,7 +8523,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32h(fp0, fs);
-            gen_helper_float_cvts_pu(fp0, fp0);
+            gen_helper_float_cvts_pu(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7609,7 +8535,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtpw_ps(fp0, fp0);
+            gen_helper_float_cvtpw_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7621,7 +8547,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvts_pl(fp0, fp0);
+            gen_helper_float_cvts_pl(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7747,7 +8673,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
     }
     /* Don't do NOP if destination is zero: we must perform the actual
        memory access. */
-    save_cpu_state(ctx, 0);
     switch (opc) {
     case OPC_LWXC1:
         check_cop1x(ctx);
@@ -7887,7 +8812,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_muladd_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -7906,7 +8831,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_muladd_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7924,7 +8849,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_muladd_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7942,7 +8867,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_mulsub_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -7961,7 +8886,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_mulsub_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7979,7 +8904,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_mulsub_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7997,7 +8922,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_nmuladd_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -8016,7 +8941,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmuladd_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8034,7 +8959,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmuladd_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8052,7 +8977,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_nmulsub_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -8071,7 +8996,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmulsub_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8089,7 +9014,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmulsub_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8122,22 +9047,22 @@ gen_rdhwr (CPUMIPSState *env, DisasContext *ctx, int rt, int rd)
     switch (rd) {
     case 0:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_cpunum(t0);
+        gen_helper_rdhwr_cpunum(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 1:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_synci_step(t0);
+        gen_helper_rdhwr_synci_step(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 2:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_cc(t0);
+        gen_helper_rdhwr_cc(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 3:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_ccres(t0);
+        gen_helper_rdhwr_ccres(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 29:
@@ -8214,7 +9139,7 @@ static void handle_delay_slot (CPUMIPSState *env, DisasContext *ctx,
             }
             if (ctx->singlestep_enabled) {
                 save_cpu_state(ctx, 0);
-                gen_helper_0i(raise_exception, EXCP_DEBUG);
+                gen_helper_0e0i(raise_exception, EXCP_DEBUG);
             }
             tcg_gen_exit_tb(0);
             break;
@@ -8390,22 +9315,22 @@ static void gen_mips16_save (DisasContext *ctx,
     case 4:
         gen_base_offset_addr(ctx, t0, 29, 12);
         gen_load_gpr(t1, 7);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 3:
         gen_base_offset_addr(ctx, t0, 29, 8);
         gen_load_gpr(t1, 6);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 2:
         gen_base_offset_addr(ctx, t0, 29, 4);
         gen_load_gpr(t1, 5);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 1:
         gen_base_offset_addr(ctx, t0, 29, 0);
         gen_load_gpr(t1, 4);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
     }
 
     gen_load_gpr(t0, 29);
@@ -8413,7 +9338,7 @@ static void gen_mips16_save (DisasContext *ctx,
 #define DECR_AND_STORE(reg) do {                \
         tcg_gen_subi_tl(t0, t0, 4);             \
         gen_load_gpr(t1, reg);                  \
-        op_st_sw(t1, t0, ctx);                  \
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);                  \
     } while (0)
 
     if (do_ra) {
@@ -8511,10 +9436,10 @@ static void gen_mips16_restore (DisasContext *ctx,
 
     tcg_gen_addi_tl(t0, cpu_gpr[29], framesize);
 
-#define DECR_AND_LOAD(reg) do {                 \
-        tcg_gen_subi_tl(t0, t0, 4);             \
-        op_ld_lw(t1, t0, ctx);                  \
-        gen_store_gpr(t1, reg);                 \
+#define DECR_AND_LOAD(reg) do {                   \
+        tcg_gen_subi_tl(t0, t0, 4);               \
+        tcg_gen_qemu_ld32u(t1, t0, ctx->mem_idx); \
+        gen_store_gpr(t1, reg);                   \
     } while (0)
 
     if (do_ra) {
@@ -8678,7 +9603,7 @@ static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
 static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
                                        int *is_branch)
 {
-    int extend = lduw_code(ctx->pc + 2);
+    int extend = cpu_lduw_code(env, ctx->pc + 2);
     int op, rx, ry, funct, sa;
     int16_t imm, offset;
 
@@ -8760,10 +9685,10 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         gen_arith_imm(env, ctx, OPC_ADDIU, rx, rx, imm);
         break;
     case M16_OPC_SLTI:
-        gen_slt_imm(env, OPC_SLTI, 24, rx, imm);
+        gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
         break;
     case M16_OPC_SLTIU:
-        gen_slt_imm(env, OPC_SLTIU, 24, rx, imm);
+        gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
         break;
     case M16_OPC_I8:
         switch (funct) {
@@ -8904,7 +9829,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_JAL:
-        offset = lduw_code(ctx->pc + 2);
+        offset = cpu_lduw_code(env, ctx->pc + 2);
         offset = (((ctx->opcode & 0x1f) << 21)
                   | ((ctx->opcode >> 5) & 0x1f) << 16
                   | offset) << 2;
@@ -8974,15 +9899,13 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_SLTI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_slt_imm(env, OPC_SLTI, 24, rx, imm);
+            gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
         }
         break;
     case M16_OPC_SLTIU:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_slt_imm(env, OPC_SLTIU, 24, rx, imm);
+            gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
         }
         break;
     case M16_OPC_I8:
@@ -9057,8 +9980,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_CMPI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_logic_imm(env, OPC_XORI, 24, rx, imm);
+            gen_logic_imm(env, ctx, OPC_XORI, 24, rx, imm);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -9170,10 +10092,10 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             }
             break;
         case RR_SLT:
-            gen_slt(env, OPC_SLT, 24, rx, ry);
+            gen_slt(env, ctx, OPC_SLT, 24, rx, ry);
             break;
         case RR_SLTU:
-            gen_slt(env, OPC_SLTU, 24, rx, ry);
+            gen_slt(env, ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
             generate_exception(ctx, EXCP_BREAK);
@@ -9194,22 +10116,22 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             break;
 #endif
         case RR_CMP:
-            gen_logic(env, OPC_XOR, 24, rx, ry);
+            gen_logic(env, ctx, OPC_XOR, 24, rx, ry);
             break;
         case RR_NEG:
             gen_arith(env, ctx, OPC_SUBU, rx, 0, ry);
             break;
         case RR_AND:
-            gen_logic(env, OPC_AND, rx, rx, ry);
+            gen_logic(env, ctx, OPC_AND, rx, rx, ry);
             break;
         case RR_OR:
-            gen_logic(env, OPC_OR, rx, rx, ry);
+            gen_logic(env, ctx, OPC_OR, rx, rx, ry);
             break;
         case RR_XOR:
-            gen_logic(env, OPC_XOR, rx, rx, ry);
+            gen_logic(env, ctx, OPC_XOR, rx, rx, ry);
             break;
         case RR_NOT:
-            gen_logic(env, OPC_NOR, rx, ry, 0);
+            gen_logic(env, ctx, OPC_NOR, rx, ry, 0);
             break;
         case RR_MFHI:
             gen_HILO(ctx, OPC_MFHI, rx);
@@ -9317,9 +10239,19 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     return n_bytes;
 }
 
-/* microMIPS extension to MIPS32 */
+/* microMIPS extension to MIPS32/MIPS64 */
 
-/* microMIPS32 major opcodes */
+/*
+ * microMIPS32/microMIPS64 major opcodes
+ *
+ * 1. MIPS Architecture for Programmers Volume II-B:
+ *      The microMIPS32 Instruction Set (Revision 3.05)
+ *
+ *    Table 6.2 microMIPS32 Encoding of Major Opcode Field
+ *
+ * 2. MIPS Architecture For Programmers Volume II-A:
+ *      The MIPS64 Instruction Set (Revision 3.51)
+ */
 
 enum {
     POOL32A = 0x00,
@@ -9346,9 +10278,10 @@ enum {
     POOL16D = 0x13,
     ORI32 = 0x14,
     POOL32F = 0x15,
-    POOL32S = 0x16,
-    DADDIU32 = 0x17,
+    POOL32S = 0x16,  /* MIPS64 */
+    DADDIU32 = 0x17, /* MIPS64 */
 
+    /* 0x1f is reserved */
     POOL32C = 0x18,
     LWGP16 = 0x19,
     LW16 = 0x1a,
@@ -9356,7 +10289,6 @@ enum {
     XORI32 = 0x1c,
     JALS32 = 0x1d,
     ADDIUPC = 0x1e,
-    POOL48A = 0x1f,
 
     /* 0x20 is reserved */
     RES_20 = 0x20,
@@ -9385,8 +10317,8 @@ enum {
     B16 = 0x33,
     ANDI32 = 0x34,
     J32 = 0x35,
-    SD32 = 0x36,
-    LD32 = 0x37,
+    SD32 = 0x36, /* MIPS64 */
+    LD32 = 0x37, /* MIPS64 */
 
     /* 0x38 and 0x39 are reserved */
     RES_38 = 0x38,
@@ -9437,6 +10369,19 @@ enum {
 
 /* POOL32AXF encoding of minor opcode field extension */
 
+/*
+ * 1. MIPS Architecture for Programmers Volume II-B:
+ *      The microMIPS32 Instruction Set (Revision 3.05)
+ *
+ *    Table 6.5 POOL32Axf Encoding of Minor Opcode Extension Field
+ *
+ * 2. MIPS Architecture for Programmers VolumeIV-e:
+ *      The MIPS DSP Application-Specific Extension
+ *        to the microMIPS32 Architecture (Revision 2.34)
+ *
+ *    Table 5.5 POOL32Axf Encoding of Minor Opcode Extension Field
+ */
+
 enum {
     /* bits 11..6 */
     TEQ = 0x00,
@@ -9449,6 +10394,8 @@ enum {
     MFC0 = 0x03,
     MTC0 = 0x0b,
 
+    /* begin of microMIPS32 DSP */
+
     /* bits 13..12 for 0x01 */
     MFHI_ACC = 0x0,
     MFLO_ACC = 0x1,
@@ -9463,7 +10410,9 @@ enum {
 
     /* bits 13..12 for 0x32 */
     MULT_ACC = 0x0,
-    MULTU_ACC = 0x0,
+    MULTU_ACC = 0x1,
+
+    /* end of microMIPS32 DSP */
 
     /* bits 15..12 for 0x2c */
     SEB = 0x2,
@@ -9831,12 +10780,13 @@ static void gen_andi16 (CPUMIPSState *env, DisasContext *ctx)
     int rs = mmreg(uMIPS_RS(ctx->opcode));
     int encoded = ZIMM(ctx->opcode, 0, 4);
 
-    gen_logic_imm(env, OPC_ANDI, rd, rs, decoded_imm[encoded]);
+    gen_logic_imm(env, ctx, OPC_ANDI, rd, rs, decoded_imm[encoded]);
 }
 
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
                                int base, int16_t offset)
 {
+    const char *opn = "ldst_multiple";
     TCGv t0, t1;
     TCGv_i32 t2;
 
@@ -9855,20 +10805,25 @@ static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
     save_cpu_state(ctx, 1);
     switch (opc) {
     case LWM32:
-        gen_helper_lwm(t0, t1, t2);
+        gen_helper_lwm(cpu_env, t0, t1, t2);
+        opn = "lwm";
         break;
     case SWM32:
-        gen_helper_swm(t0, t1, t2);
+        gen_helper_swm(cpu_env, t0, t1, t2);
+        opn = "swm";
         break;
 #ifdef TARGET_MIPS64
     case LDM:
-        gen_helper_ldm(t0, t1, t2);
+        gen_helper_ldm(cpu_env, t0, t1, t2);
+        opn = "ldm";
         break;
     case SDM:
-        gen_helper_sdm(t0, t1, t2);
+        gen_helper_sdm(cpu_env, t0, t1, t2);
+        opn = "sdm";
         break;
 #endif
     }
+    (void)opn;
     MIPS_DEBUG("%s, %x, %d(%s)", opn, reglist, offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -9887,25 +10842,25 @@ static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_bran
     case NOT16 + 1:
     case NOT16 + 2:
     case NOT16 + 3:
-        gen_logic(env, OPC_NOR, rd, rs, 0);
+        gen_logic(env, ctx, OPC_NOR, rd, rs, 0);
         break;
     case XOR16 + 0:
     case XOR16 + 1:
     case XOR16 + 2:
     case XOR16 + 3:
-        gen_logic(env, OPC_XOR, rd, rd, rs);
+        gen_logic(env, ctx, OPC_XOR, rd, rd, rs);
         break;
     case AND16 + 0:
     case AND16 + 1:
     case AND16 + 2:
     case AND16 + 3:
-        gen_logic(env, OPC_AND, rd, rd, rs);
+        gen_logic(env, ctx, OPC_AND, rd, rd, rs);
         break;
     case OR16 + 0:
     case OR16 + 1:
     case OR16 + 2:
     case OR16 + 3:
-        gen_logic(env, OPC_OR, rd, rd, rs);
+        gen_logic(env, ctx, OPC_OR, rd, rd, rs);
         break;
     case LWM16 + 0:
     case LWM16 + 1:
@@ -10017,8 +10972,7 @@ static void gen_ldxs (DisasContext *ctx, int base, int index, int rd)
         gen_op_addr_add(ctx, t0, t1, t0);
     }
 
-    save_cpu_state(ctx, 0);
-    op_ld_lw(t1, t0, ctx);
+    tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
     gen_store_gpr(t1, rd);
 
     tcg_temp_free(t0);
@@ -10047,23 +11001,21 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
             generate_exception(ctx, EXCP_RI);
             return;
         }
-        save_cpu_state(ctx, 0);
-        op_ld_lw(t1, t0, ctx);
+        tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd);
         tcg_gen_movi_tl(t1, 4);
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_lw(t1, t0, ctx);
+        tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd+1);
         opn = "lwp";
         break;
     case SWP:
-        save_cpu_state(ctx, 0);
         gen_load_gpr(t1, rd);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         tcg_gen_movi_tl(t1, 4);
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         opn = "swp";
         break;
 #ifdef TARGET_MIPS64
@@ -10072,23 +11024,21 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
             generate_exception(ctx, EXCP_RI);
             return;
         }
-        save_cpu_state(ctx, 0);
-        op_ld_ld(t1, t0, ctx);
+        tcg_gen_qemu_ld64(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd);
         tcg_gen_movi_tl(t1, 8);
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_ld(t1, t0, ctx);
+        tcg_gen_qemu_ld64(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd+1);
         opn = "ldp";
         break;
     case SDP:
-        save_cpu_state(ctx, 0);
         gen_load_gpr(t1, rd);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         tcg_gen_movi_tl(t1, 8);
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         opn = "sdp";
         break;
 #endif
@@ -10287,7 +11237,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
                 TCGv t0 = tcg_temp_new();
 
                 save_cpu_state(ctx, 1);
-                gen_helper_di(t0);
+                gen_helper_di(t0, cpu_env);
                 gen_store_gpr(t0, rs);
                 /* Stop translation as we may have switched the execution mode */
                 ctx->bstate = BS_STOP;
@@ -10300,7 +11250,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
                 TCGv t0 = tcg_temp_new();
 
                 save_cpu_state(ctx, 1);
-                gen_helper_ei(t0);
+                gen_helper_ei(t0, cpu_env);
                 gen_store_gpr(t0, rs);
                 /* Stop translation as we may have switched the execution mode */
                 ctx->bstate = BS_STOP;
@@ -10635,7 +11585,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     uint32_t op, minor, mips32_op;
     uint32_t cond, fmt, cc;
 
-    insn = lduw_code(ctx->pc + 2);
+    insn = cpu_lduw_code(env, ctx->pc + 2);
     ctx->opcode = (ctx->opcode << 16) | insn;
 
     rt = (ctx->opcode >> 21) & 0x1f;
@@ -10719,7 +11669,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case XOR32:
                 mips32_op = OPC_XOR;
             do_logic:
-                gen_logic(env, mips32_op, rd, rs, rt);
+                gen_logic(env, ctx, mips32_op, rd, rs, rt);
                 break;
                 /* Set less than */
             case SLT:
@@ -10728,7 +11678,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case SLTU:
                 mips32_op = OPC_SLTU;
             do_slt:
-                gen_slt(env, mips32_op, rd, rs, rt);
+                gen_slt(env, ctx, mips32_op, rd, rs, rt);
                 break;
             default:
                 goto pool32a_invalid;
@@ -10744,7 +11694,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case MOVZ:
                 mips32_op = OPC_MOVZ;
             do_cmov:
-                gen_cond_move(env, mips32_op, rd, rs, rt);
+                gen_cond_move(env, ctx, mips32_op, rd, rs, rt);
                 break;
             case LWXS:
                 gen_ldxs(ctx, rs, rt, rd);
@@ -11157,7 +12107,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
                target. */
             break;
         case LUI:
-            gen_logic_imm(env, OPC_LUI, rs, -1, imm);
+            gen_logic_imm(env, ctx, OPC_LUI, rs, -1, imm);
             break;
         case SYNCI:
             break;
@@ -11276,7 +12226,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case ANDI32:
         mips32_op = OPC_ANDI;
     do_logici:
-        gen_logic_imm(env, mips32_op, rt, rs, imm);
+        gen_logic_imm(env, ctx, mips32_op, rt, rs, imm);
         break;
 
         /* Set less than immediate */
@@ -11286,7 +12236,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case SLTIU32:
         mips32_op = OPC_SLTIU;
     do_slti:
-        gen_slt_imm(env, mips32_op, rt, rs, imm);
+        gen_slt_imm(env, ctx, mips32_op, rt, rs, imm);
         break;
     case JALX32:
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -11433,7 +12383,6 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
         case LB32:
         case LH32:
         case DADDIU32:
-        case POOL48A:           /* ??? */
         case LWC132:
         case LDC132:
         case LD32:
@@ -11701,6 +12650,1687 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
 
 #endif
 
+/* MIPSDSP functions. */
+static void gen_mipsdsp_ld(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                           int rd, int base, int offset)
+{
+    const char *opn = "ldx";
+    TCGv t0;
+
+    if (rd == 0) {
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    check_dsp(ctx);
+    t0 = tcg_temp_new();
+
+    if (base == 0) {
+        gen_load_gpr(t0, offset);
+    } else if (offset == 0) {
+        gen_load_gpr(t0, base);
+    } else {
+        gen_op_addr_add(ctx, t0, cpu_gpr[base], cpu_gpr[offset]);
+    }
+
+    switch (opc) {
+    case OPC_LBUX:
+        tcg_gen_qemu_ld8u(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lbux";
+        break;
+    case OPC_LHX:
+        tcg_gen_qemu_ld16s(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lhx";
+        break;
+    case OPC_LWX:
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lwx";
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_LDX:
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "ldx";
+        break;
+#endif
+    }
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s %s, %s(%s)", opn,
+               regnames[rd], regnames[offset], regnames[base]);
+    tcg_temp_free(t0);
+}
+
+static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                              int ret, int v1, int v2)
+{
+    const char *opn = "mipsdsp arith";
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    /* OPC_MULT_G_2E is equal OPC_ADDUH_QB_DSP */
+    case OPC_MULT_G_2E:
+        check_dspr2(ctx);
+        switch (op2) {
+        case OPC_ADDUH_QB:
+            gen_helper_adduh_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDUH_R_QB:
+            gen_helper_adduh_r_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_PH:
+            gen_helper_addqh_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_R_PH:
+            gen_helper_addqh_r_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_W:
+            gen_helper_addqh_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_R_W:
+            gen_helper_addqh_r_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_QB:
+            gen_helper_subuh_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_R_QB:
+            gen_helper_subuh_r_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_PH:
+            gen_helper_subqh_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_R_PH:
+            gen_helper_subqh_r_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_W:
+            gen_helper_subqh_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_R_W:
+            gen_helper_subqh_r_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+    case OPC_ABSQ_S_PH_DSP:
+        switch (op2) {
+        case OPC_ABSQ_S_QB:
+            check_dspr2(ctx);
+            gen_helper_absq_s_qb(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_absq_s_ph(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_W:
+            check_dsp(ctx);
+            gen_helper_absq_s_w(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_PRECEQ_W_PHL:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0xFFFF0000);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_PRECEQ_W_PHR:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0x0000FFFF);
+            tcg_gen_shli_tl(cpu_gpr[ret], cpu_gpr[ret], 16);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_PRECEQU_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBLA:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBRA:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBLA:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBRA:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbra(cpu_gpr[ret], v2_t);
+            break;
+        }
+        break;
+    case OPC_ADDU_QB_DSP:
+        switch (op2) {
+        case OPC_ADDQ_PH:
+            check_dsp(ctx);
+            gen_helper_addq_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_addq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_W:
+            check_dsp(ctx);
+            gen_helper_addq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_QB:
+            check_dsp(ctx);
+            gen_helper_addu_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_QB:
+            check_dsp(ctx);
+            gen_helper_addu_s_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_PH:
+            check_dspr2(ctx);
+            gen_helper_addu_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_PH:
+            check_dspr2(ctx);
+            gen_helper_addu_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_PH:
+            check_dsp(ctx);
+            gen_helper_subq_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_subq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_W:
+            check_dsp(ctx);
+            gen_helper_subq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_QB:
+            check_dsp(ctx);
+            gen_helper_subu_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_QB:
+            check_dsp(ctx);
+            gen_helper_subu_s_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_PH:
+            check_dspr2(ctx);
+            gen_helper_subu_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_PH:
+            check_dspr2(ctx);
+            gen_helper_subu_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDSC:
+            check_dsp(ctx);
+            gen_helper_addsc(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDWC:
+            check_dsp(ctx);
+            gen_helper_addwc(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MODSUB:
+            check_dsp(ctx);
+            gen_helper_modsub(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_RADDU_W_QB:
+            check_dsp(ctx);
+            gen_helper_raddu_w_qb(cpu_gpr[ret], v1_t);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_QB_DSP:
+        switch (op2) {
+        case OPC_PRECR_QB_PH:
+            check_dspr2(ctx);
+            gen_helper_precr_qb_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_QB_PH:
+            check_dsp(ctx);
+            gen_helper_precrq_qb_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECR_SRA_PH_W:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_t = tcg_const_i32(v2);
+                gen_helper_precr_sra_ph_w(cpu_gpr[ret], sa_t, v1_t,
+                                          cpu_gpr[ret]);
+                tcg_temp_free_i32(sa_t);
+                break;
+            }
+        case OPC_PRECR_SRA_R_PH_W:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_t = tcg_const_i32(v2);
+                gen_helper_precr_sra_r_ph_w(cpu_gpr[ret], sa_t, v1_t,
+                                            cpu_gpr[ret]);
+                tcg_temp_free_i32(sa_t);
+                break;
+            }
+        case OPC_PRECRQ_PH_W:
+            check_dsp(ctx);
+            gen_helper_precrq_ph_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_RS_PH_W:
+            check_dsp(ctx);
+            gen_helper_precrq_rs_ph_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PRECRQU_S_QB_PH:
+            check_dsp(ctx);
+            gen_helper_precrqu_s_qb_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ABSQ_S_QH_DSP:
+        switch (op2) {
+        case OPC_PRECEQ_L_PWL:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0xFFFFFFFF00000000ull);
+            break;
+        case OPC_PRECEQ_L_PWR:
+            check_dsp(ctx);
+            tcg_gen_shli_tl(cpu_gpr[ret], v2_t, 32);
+            break;
+        case OPC_PRECEQ_PW_QHL:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHR:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHLA:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHRA:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBLA:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBRA:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBLA:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBRA:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_ABSQ_S_OB:
+            check_dspr2(ctx);
+            gen_helper_absq_s_ob(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_absq_s_pw(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_absq_s_qh(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_ADDU_OB_DSP:
+        switch (op2) {
+        case OPC_RADDU_L_OB:
+            check_dsp(ctx);
+            gen_helper_raddu_l_ob(cpu_gpr[ret], v1_t);
+            break;
+        case OPC_SUBQ_PW:
+            check_dsp(ctx);
+            gen_helper_subq_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_subq_s_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_QH:
+            check_dsp(ctx);
+            gen_helper_subq_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_subq_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_OB:
+            check_dsp(ctx);
+            gen_helper_subu_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_OB:
+            check_dsp(ctx);
+            gen_helper_subu_s_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_QH:
+            check_dspr2(ctx);
+            gen_helper_subu_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_QH:
+            check_dspr2(ctx);
+            gen_helper_subu_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBUH_OB:
+            check_dspr2(ctx);
+            gen_helper_subuh_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_R_OB:
+            check_dspr2(ctx);
+            gen_helper_subuh_r_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQ_PW:
+            check_dsp(ctx);
+            gen_helper_addq_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_addq_s_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_QH:
+            check_dsp(ctx);
+            gen_helper_addq_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_addq_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_OB:
+            check_dsp(ctx);
+            gen_helper_addu_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_OB:
+            check_dsp(ctx);
+            gen_helper_addu_s_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_QH:
+            check_dspr2(ctx);
+            gen_helper_addu_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_QH:
+            check_dspr2(ctx);
+            gen_helper_addu_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDUH_OB:
+            check_dspr2(ctx);
+            gen_helper_adduh_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDUH_R_OB:
+            check_dspr2(ctx);
+            gen_helper_adduh_r_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_OB_DSP:
+        switch (op2) {
+        case OPC_PRECR_OB_QH:
+            check_dspr2(ctx);
+            gen_helper_precr_ob_qh(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECR_SRA_QH_PW:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 ret_t = tcg_const_i32(ret);
+                gen_helper_precr_sra_qh_pw(v2_t, v1_t, v2_t, ret_t);
+                tcg_temp_free_i32(ret_t);
+                break;
+            }
+        case OPC_PRECR_SRA_R_QH_PW:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_v = tcg_const_i32(ret);
+                gen_helper_precr_sra_r_qh_pw(v2_t, v1_t, v2_t, sa_v);
+                tcg_temp_free_i32(sa_v);
+                break;
+            }
+        case OPC_PRECRQ_OB_QH:
+            check_dsp(ctx);
+            gen_helper_precrq_ob_qh(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_PW_L:
+            check_dsp(ctx);
+            gen_helper_precrq_pw_l(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_QH_PW:
+            check_dsp(ctx);
+            gen_helper_precrq_qh_pw(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_RS_QH_PW:
+            check_dsp(ctx);
+            gen_helper_precrq_rs_qh_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PRECRQU_S_OB_QH:
+            check_dsp(ctx);
+            gen_helper_precrqu_s_ob_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
+                              int ret, int v1, int v2)
+{
+    uint32_t op2;
+    const char *opn = "mipsdsp shift";
+    TCGv t0;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    tcg_gen_movi_tl(t0, v1);
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (opc) {
+    case OPC_SHLL_QB_DSP:
+        {
+            op2 = MASK_SHLL_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_SHLL_QB:
+                check_dsp(ctx);
+                gen_helper_shll_qb(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_QB:
+                check_dsp(ctx);
+                gen_helper_shll_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_PH:
+                check_dsp(ctx);
+                gen_helper_shll_ph(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_PH:
+                check_dsp(ctx);
+                gen_helper_shll_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_S_PH:
+                check_dsp(ctx);
+                gen_helper_shll_s_ph(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_S_PH:
+                check_dsp(ctx);
+                gen_helper_shll_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_S_W:
+                check_dsp(ctx);
+                gen_helper_shll_s_w(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_S_W:
+                check_dsp(ctx);
+                gen_helper_shll_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHRL_QB:
+                check_dsp(ctx);
+                gen_helper_shrl_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRLV_QB:
+                check_dsp(ctx);
+                gen_helper_shrl_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRL_PH:
+                check_dspr2(ctx);
+                gen_helper_shrl_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRLV_PH:
+                check_dspr2(ctx);
+                gen_helper_shrl_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRA_R_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_r_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRAV_R_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_r_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_PH:
+                check_dsp(ctx);
+                gen_helper_shra_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRA_R_PH:
+                check_dsp(ctx);
+                gen_helper_shra_r_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_PH:
+                check_dsp(ctx);
+                gen_helper_shra_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRAV_R_PH:
+                check_dsp(ctx);
+                gen_helper_shra_r_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_R_W:
+                check_dsp(ctx);
+                gen_helper_shra_r_w(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_R_W:
+                check_dsp(ctx);
+                gen_helper_shra_r_w(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK SHLL.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        }
+#ifdef TARGET_MIPS64
+    case OPC_SHLL_OB_DSP:
+        op2 = MASK_SHLL_OB(ctx->opcode);
+        switch (op2) {
+        case OPC_SHLL_PW:
+            check_dsp(ctx);
+            gen_helper_shll_pw(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_PW:
+            check_dsp(ctx);
+            gen_helper_shll_pw(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_S_PW:
+            check_dsp(ctx);
+            gen_helper_shll_s_pw(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_S_PW:
+            check_dsp(ctx);
+            gen_helper_shll_s_pw(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_OB:
+            check_dsp(ctx);
+            gen_helper_shll_ob(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_OB:
+            check_dsp(ctx);
+            gen_helper_shll_ob(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_QH:
+            check_dsp(ctx);
+            gen_helper_shll_qh(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_QH:
+            check_dsp(ctx);
+            gen_helper_shll_qh(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_S_QH:
+            check_dsp(ctx);
+            gen_helper_shll_s_qh(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_S_QH:
+            check_dsp(ctx);
+            gen_helper_shll_s_qh(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHRA_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_r_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_r_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_PW:
+            check_dsp(ctx);
+            gen_helper_shra_pw(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_PW:
+            check_dsp(ctx);
+            gen_helper_shra_pw(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_PW:
+            check_dsp(ctx);
+            gen_helper_shra_r_pw(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_PW:
+            check_dsp(ctx);
+            gen_helper_shra_r_pw(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_QH:
+            check_dsp(ctx);
+            gen_helper_shra_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_QH:
+            check_dsp(ctx);
+            gen_helper_shra_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_QH:
+            check_dsp(ctx);
+            gen_helper_shra_r_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_QH:
+            check_dsp(ctx);
+            gen_helper_shra_r_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRL_OB:
+            check_dsp(ctx);
+            gen_helper_shrl_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRLV_OB:
+            check_dsp(ctx);
+            gen_helper_shrl_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRL_QH:
+            check_dspr2(ctx);
+            gen_helper_shrl_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRLV_QH:
+            check_dspr2(ctx);
+            gen_helper_shrl_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK SHLL.OB");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(t0);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                                 int ret, int v1, int v2, int check_ret)
+{
+    const char *opn = "mipsdsp multiply";
+    TCGv_i32 t0;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new_i32();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    tcg_gen_movi_i32(t0, ret);
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
+     * the same mask and op1. */
+    case OPC_MULT_G_2E:
+        switch (op2) {
+        case  OPC_MUL_PH:
+            gen_helper_mul_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case  OPC_MUL_S_PH:
+            gen_helper_mul_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_S_W:
+            gen_helper_mulq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_W:
+            gen_helper_mulq_rs_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_DPA_W_PH_DSP:
+        switch (op2) {
+        case OPC_DPAU_H_QBL:
+            check_dsp(ctx);
+            gen_helper_dpau_h_qbl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAU_H_QBR:
+            check_dsp(ctx);
+            gen_helper_dpau_h_qbr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSU_H_QBL:
+            check_dsp(ctx);
+            gen_helper_dpsu_h_qbl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSU_H_QBR:
+            check_dsp(ctx);
+            gen_helper_dpsu_h_qbr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAX_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpax_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_dpaq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQX_S_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpaqx_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQX_SA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpaqx_sa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPS_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dps_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSX_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsx_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_dpsq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQX_S_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsqx_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQX_SA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsqx_sa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULSAQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_mulsaq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQ_SA_L_W:
+            check_dsp(ctx);
+            gen_helper_dpaq_sa_l_w(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQ_SA_L_W:
+            check_dsp(ctx);
+            gen_helper_dpsq_sa_l_w(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_S_W_PHL:
+            check_dsp(ctx);
+            gen_helper_maq_s_w_phl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_S_W_PHR:
+            check_dsp(ctx);
+            gen_helper_maq_s_w_phr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_SA_W_PHL:
+            check_dsp(ctx);
+            gen_helper_maq_sa_w_phl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_SA_W_PHR:
+            check_dsp(ctx);
+            gen_helper_maq_sa_w_phr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULSA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_mulsa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_DPAQ_W_QH_DSP:
+        {
+            int ac = ret & 0x03;
+            tcg_gen_movi_i32(t0, ac);
+
+            switch (op2) {
+            case OPC_DMADD:
+                check_dsp(ctx);
+                gen_helper_dmadd(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMADDU:
+                check_dsp(ctx);
+                gen_helper_dmaddu(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMSUB:
+                check_dsp(ctx);
+                gen_helper_dmsub(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMSUBU:
+                check_dsp(ctx);
+                gen_helper_dmsubu(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPA_W_QH:
+                check_dspr2(ctx);
+                gen_helper_dpa_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_dpaq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAQ_SA_L_PW:
+                check_dsp(ctx);
+                gen_helper_dpaq_sa_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAU_H_OBL:
+                check_dsp(ctx);
+                gen_helper_dpau_h_obl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAU_H_OBR:
+                check_dsp(ctx);
+                gen_helper_dpau_h_obr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPS_W_QH:
+                check_dspr2(ctx);
+                gen_helper_dps_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_dpsq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSQ_SA_L_PW:
+                check_dsp(ctx);
+                gen_helper_dpsq_sa_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSU_H_OBL:
+                check_dsp(ctx);
+                gen_helper_dpsu_h_obl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSU_H_OBR:
+                check_dsp(ctx);
+                gen_helper_dpsu_h_obr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_L_PWL:
+                check_dsp(ctx);
+                gen_helper_maq_s_l_pwl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_L_PWR:
+                check_dsp(ctx);
+                gen_helper_maq_s_l_pwr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHLL:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhll(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHLL:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhll(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHLR:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhlr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHLR:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhlr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHRL:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhrl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHRL:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhrl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHRR:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhrr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHRR:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhrr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MULSAQ_S_L_PW:
+                check_dsp(ctx);
+                gen_helper_mulsaq_s_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MULSAQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_mulsaq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            }
+        }
+        break;
+#endif
+    case OPC_ADDU_QB_DSP:
+        switch (op2) {
+        case OPC_MULEU_S_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_muleu_s_ph_qbl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_muleu_s_ph_qbr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_PH:
+            check_dsp(ctx);
+            gen_helper_mulq_rs_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_W_PHL:
+            check_dsp(ctx);
+            gen_helper_muleq_s_w_phl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_W_PHR:
+            check_dsp(ctx);
+            gen_helper_muleq_s_w_phr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_S_PH:
+            check_dspr2(ctx);
+            gen_helper_mulq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ADDU_OB_DSP:
+        switch (op2) {
+        case OPC_MULEQ_S_PW_QHL:
+            check_dsp(ctx);
+            gen_helper_muleq_s_pw_qhl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_PW_QHR:
+            check_dsp(ctx);
+            gen_helper_muleq_s_pw_qhr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_muleu_s_qh_obl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_muleu_s_qh_obr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_QH:
+            check_dsp(ctx);
+            gen_helper_mulq_rs_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free_i32(t0);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+
+}
+
+static void gen_mipsdsp_bitinsn(CPUMIPSState *env, DisasContext *ctx,
+                                uint32_t op1, uint32_t op2,
+                                int ret, int val)
+{
+    const char *opn = "mipsdsp Bit/ Manipulation";
+    int16_t imm;
+    TCGv t0;
+    TCGv val_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    val_t = tcg_temp_new();
+    gen_load_gpr(val_t, val);
+
+    switch (op1) {
+    case OPC_ABSQ_S_PH_DSP:
+        switch (op2) {
+        case OPC_BITREV:
+            check_dsp(ctx);
+            gen_helper_bitrev(cpu_gpr[ret], val_t);
+            break;
+        case OPC_REPL_QB:
+            check_dsp(ctx);
+            {
+                target_long result;
+                imm = (ctx->opcode >> 16) & 0xFF;
+                result = (uint32_t)imm << 24 |
+                         (uint32_t)imm << 16 |
+                         (uint32_t)imm << 8  |
+                         (uint32_t)imm;
+                result = (int32_t)result;
+                tcg_gen_movi_tl(cpu_gpr[ret], result);
+            }
+            break;
+        case OPC_REPLV_QB:
+            check_dsp(ctx);
+            tcg_gen_ext8u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 8);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_REPL_PH:
+            check_dsp(ctx);
+            {
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
+                tcg_gen_movi_tl(cpu_gpr[ret], \
+                                (target_long)((int32_t)imm << 16 | \
+                                (uint16_t)imm));
+            }
+            break;
+        case OPC_REPLV_PH:
+            check_dsp(ctx);
+            tcg_gen_ext16u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ABSQ_S_QH_DSP:
+        switch (op2) {
+        case OPC_REPL_OB:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0xFF;
+                temp = ((uint64_t)imm << 8) | (uint64_t)imm;
+                temp = (temp << 16) | temp;
+                temp = (temp << 32) | temp;
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPL_PW:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
+                temp = ((target_long)imm << 32) \
+                       | ((target_long)imm & 0xFFFFFFFF);
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPL_QH:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
+
+                temp = ((uint64_t)(uint16_t)imm << 48) |
+                       ((uint64_t)(uint16_t)imm << 32) |
+                       ((uint64_t)(uint16_t)imm << 16) |
+                       (uint64_t)(uint16_t)imm;
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPLV_OB:
+            check_dsp(ctx);
+            tcg_gen_ext8u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 8);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        case OPC_REPLV_PW:
+            check_dsp(ctx);
+            tcg_gen_ext32u_i64(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        case OPC_REPLV_QH:
+            check_dsp(ctx);
+            tcg_gen_ext16u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        }
+        break;
+#endif
+    }
+    tcg_temp_free(t0);
+    tcg_temp_free(val_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
+                                     uint32_t op1, uint32_t op2,
+                                     int ret, int v1, int v2, int check_ret)
+{
+    const char *opn = "mipsdsp add compare pick";
+    TCGv_i32 t0;
+    TCGv t1;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    case OPC_APPEND_DSP:
+        switch (op2) {
+        case OPC_APPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_append(cpu_gpr[ret], cpu_gpr[ret], v1_t, t0);
+            break;
+        case OPC_PREPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prepend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_BALIGN:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_balign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        default:            /* Invid */
+            MIPS_INVAL("MASK APPEND");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_QB_DSP:
+        switch (op2) {
+        case OPC_CMPU_EQ_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_eq_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LT_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_lt_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LE_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_le_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGU_EQ_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_eq_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LT_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_lt_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LE_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_le_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGDU_EQ_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_eq_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMPGDU_LT_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_lt_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMPGDU_LE_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_le_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMP_EQ_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_le_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_QB:
+            check_dsp(ctx);
+            gen_helper_pick_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_PH:
+            check_dsp(ctx);
+            gen_helper_pick_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PACKRL_PH:
+            check_dsp(ctx);
+            gen_helper_packrl_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_CMPU_EQ_OB_DSP:
+        switch (op2) {
+        case OPC_CMP_EQ_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_le_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_EQ_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_le_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_EQ_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_eq_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_LT_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_lt_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_LE_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_le_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGU_EQ_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_eq_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LT_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_lt_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LE_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_le_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPU_EQ_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_eq_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LT_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_lt_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LE_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_le_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PACKRL_PW:
+            check_dsp(ctx);
+            gen_helper_packrl_pw(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PICK_OB:
+            check_dsp(ctx);
+            gen_helper_pick_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_PW:
+            check_dsp(ctx);
+            gen_helper_pick_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_QH:
+            check_dsp(ctx);
+            gen_helper_pick_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_DAPPEND_DSP:
+        switch (op2) {
+        case OPC_DAPPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_dappend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_PREPENDD:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prependd(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_PREPENDW:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prependw(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_DBALIGN:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_dbalign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK DAPPEND");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free_i32(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                                int ret, int v1, int v2, int check_ret)
+
+{
+    const char *opn = "mipsdsp accumulator";
+    TCGv t0;
+    TCGv t1;
+    TCGv v1_t;
+    TCGv v2_t;
+    int16_t imm;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    case OPC_EXTR_W_DSP:
+        check_dsp(ctx);
+        switch (op2) {
+        case OPC_EXTR_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_r_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_rs_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTRV_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_s_h(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_r_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_rs_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTPDP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extpdp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTPDPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extpdp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_SHILO:
+            imm = (ctx->opcode >> 20) & 0x3F;
+            tcg_gen_movi_tl(t0, ret);
+            tcg_gen_movi_tl(t1, imm);
+            gen_helper_shilo(t0, t1, cpu_env);
+            break;
+        case OPC_SHILOV:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_shilo(t0, v1_t, cpu_env);
+            break;
+        case OPC_MTHLIP:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_mthlip(t0, v1_t, cpu_env);
+            break;
+        case OPC_WRDSP:
+            imm = (ctx->opcode >> 11) & 0x3FF;
+            tcg_gen_movi_tl(t0, imm);
+            gen_helper_wrdsp(v1_t, t0, cpu_env);
+            break;
+        case OPC_RDDSP:
+            imm = (ctx->opcode >> 16) & 0x03FF;
+            tcg_gen_movi_tl(t0, imm);
+            gen_helper_rddsp(cpu_gpr[ret], t0, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_DEXTR_W_DSP:
+        check_dsp(ctx);
+        switch (op2) {
+        case OPC_DMTHLIP:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_dmthlip(v1_t, t0, cpu_env);
+            break;
+        case OPC_DSHILO:
+            {
+                int shift = (ctx->opcode >> 19) & 0x7F;
+                int ac = (ctx->opcode >> 11) & 0x03;
+                tcg_gen_movi_tl(t0, shift);
+                tcg_gen_movi_tl(t1, ac);
+                gen_helper_dshilo(t0, t1, cpu_env);
+                break;
+            }
+        case OPC_DSHILOV:
+            {
+                int ac = (ctx->opcode >> 11) & 0x03;
+                tcg_gen_movi_tl(t0, ac);
+                gen_helper_dshilo(v1_t, t0, cpu_env);
+                break;
+            }
+        case OPC_DEXTP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+
+            gen_helper_dextp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTPDP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextpdp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTPDPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextpdp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTR_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_R_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_r_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_RS_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_rs_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_r_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_rs_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTRV_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTRV_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_R_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_r_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_RS_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_rs_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_r_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_rs_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+/* End MIPSDSP functions. */
+
 static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 {
     int32_t offset;
@@ -11726,8 +14356,9 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         gen_set_label(l1);
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(ctx->pc);
+    }
 
     op = MASK_OP_MAJOR(ctx->opcode);
     rs = (ctx->opcode >> 21) & 0x1f;
@@ -11763,7 +14394,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_MOVZ:
             check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32 |
                                  INSN_LOONGSON2E | INSN_LOONGSON2F);
-            gen_cond_move(env, op1, rd, rs, rt);
+            gen_cond_move(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_ADD ... OPC_SUBU:
             gen_arith(env, ctx, op1, rd, rs, rt);
@@ -11790,13 +14421,13 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             break;
         case OPC_SLT:          /* Set on less than */
         case OPC_SLTU:
-            gen_slt(env, op1, rd, rs, rt);
+            gen_slt(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_AND:          /* Logic*/
         case OPC_OR:
         case OPC_NOR:
         case OPC_XOR:
-            gen_logic(env, op1, rd, rs, rt);
+            gen_logic(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_MULT ... OPC_DIVU:
             if (sa) {
@@ -11827,7 +14458,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             MIPS_INVAL("PMON / selsl");
             generate_exception(ctx, EXCP_RI);
 #else
-            gen_helper_0i(pmon, sa);
+            gen_helper_0e0i(pmon, sa);
 #endif
             break;
         case OPC_SYSCALL:
@@ -12045,16 +14676,278 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 
                 save_cpu_state(ctx, 1);
                 gen_load_gpr(t0, rs);
-                gen_helper_yield(t0, t0);
+                gen_helper_yield(t0, cpu_env, t0);
                 gen_store_gpr(t0, rd);
                 tcg_temp_free(t0);
             }
             break;
         case OPC_DIV_G_2E ... OPC_DIVU_G_2E:
-        case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
         case OPC_MOD_G_2E ... OPC_MODU_G_2E:
-            check_insn(env, ctx, INSN_LOONGSON2E);
-            gen_loongson_integer(ctx, op1, rd, rs, rt);
+        case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
+        /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
+         * the same mask and op1. */
+            if ((env->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
+                op2 = MASK_ADDUH_QB(ctx->opcode);
+                switch (op2) {
+                case OPC_ADDUH_QB:
+                case OPC_ADDUH_R_QB:
+                case OPC_ADDQH_PH:
+                case OPC_ADDQH_R_PH:
+                case OPC_ADDQH_W:
+                case OPC_ADDQH_R_W:
+                case OPC_SUBUH_QB:
+                case OPC_SUBUH_R_QB:
+                case OPC_SUBQH_PH:
+                case OPC_SUBQH_R_PH:
+                case OPC_SUBQH_W:
+                case OPC_SUBQH_R_W:
+                    gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                    break;
+                case OPC_MUL_PH:
+                case OPC_MUL_S_PH:
+                case OPC_MULQ_S_W:
+                case OPC_MULQ_RS_W:
+                    gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                    break;
+                default:
+                    MIPS_INVAL("MASK ADDUH.QB");
+                    generate_exception(ctx, EXCP_RI);
+                    break;
+                }
+            } else if (env->insn_flags & INSN_LOONGSON2E) {
+                gen_loongson_integer(ctx, op1, rd, rs, rt);
+            } else {
+                generate_exception(ctx, EXCP_RI);
+            }
+            break;
+        case OPC_LX_DSP:
+            op2 = MASK_LX(ctx->opcode);
+            switch (op2) {
+#if defined(TARGET_MIPS64)
+            case OPC_LDX:
+#endif
+            case OPC_LBUX:
+            case OPC_LHX:
+            case OPC_LWX:
+                gen_mipsdsp_ld(env, ctx, op2, rd, rs, rt);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK LX");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ABSQ_S_PH_DSP:
+            op2 = MASK_ABSQ_S_PH(ctx->opcode);
+            switch (op2) {
+            case OPC_ABSQ_S_QB:
+            case OPC_ABSQ_S_PH:
+            case OPC_ABSQ_S_W:
+            case OPC_PRECEQ_W_PHL:
+            case OPC_PRECEQ_W_PHR:
+            case OPC_PRECEQU_PH_QBL:
+            case OPC_PRECEQU_PH_QBR:
+            case OPC_PRECEQU_PH_QBLA:
+            case OPC_PRECEQU_PH_QBRA:
+            case OPC_PRECEU_PH_QBL:
+            case OPC_PRECEU_PH_QBR:
+            case OPC_PRECEU_PH_QBLA:
+            case OPC_PRECEU_PH_QBRA:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_BITREV:
+            case OPC_REPL_QB:
+            case OPC_REPLV_QB:
+            case OPC_REPL_PH:
+            case OPC_REPLV_PH:
+                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                break;
+            default:
+                MIPS_INVAL("MASK ABSQ_S.PH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ADDU_QB_DSP:
+            op2 = MASK_ADDU_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_ADDQ_PH:
+            case OPC_ADDQ_S_PH:
+            case OPC_ADDQ_S_W:
+            case OPC_ADDU_QB:
+            case OPC_ADDU_S_QB:
+            case OPC_ADDU_PH:
+            case OPC_ADDU_S_PH:
+            case OPC_SUBQ_PH:
+            case OPC_SUBQ_S_PH:
+            case OPC_SUBQ_S_W:
+            case OPC_SUBU_QB:
+            case OPC_SUBU_S_QB:
+            case OPC_SUBU_PH:
+            case OPC_SUBU_S_PH:
+            case OPC_ADDSC:
+            case OPC_ADDWC:
+            case OPC_MODSUB:
+            case OPC_RADDU_W_QB:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_MULEU_S_PH_QBL:
+            case OPC_MULEU_S_PH_QBR:
+            case OPC_MULQ_RS_PH:
+            case OPC_MULEQ_S_W_PHL:
+            case OPC_MULEQ_S_W_PHR:
+            case OPC_MULQ_S_PH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ADDU.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+
+            }
+            break;
+        case OPC_CMPU_EQ_QB_DSP:
+            op2 = MASK_CMPU_EQ_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECR_SRA_PH_W:
+            case OPC_PRECR_SRA_R_PH_W:
+                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
+                break;
+            case OPC_PRECR_QB_PH:
+            case OPC_PRECRQ_QB_PH:
+            case OPC_PRECRQ_PH_W:
+            case OPC_PRECRQ_RS_PH_W:
+            case OPC_PRECRQU_S_QB_PH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_CMPU_EQ_QB:
+            case OPC_CMPU_LT_QB:
+            case OPC_CMPU_LE_QB:
+            case OPC_CMP_EQ_PH:
+            case OPC_CMP_LT_PH:
+            case OPC_CMP_LE_PH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_CMPGU_EQ_QB:
+            case OPC_CMPGU_LT_QB:
+            case OPC_CMPGU_LE_QB:
+            case OPC_CMPGDU_EQ_QB:
+            case OPC_CMPGDU_LT_QB:
+            case OPC_CMPGDU_LE_QB:
+            case OPC_PICK_QB:
+            case OPC_PICK_PH:
+            case OPC_PACKRL_PH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK CMPU.EQ.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_SHLL_QB_DSP:
+            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+            break;
+        case OPC_DPA_W_PH_DSP:
+            op2 = MASK_DPA_W_PH(ctx->opcode);
+            switch (op2) {
+            case OPC_DPAU_H_QBL:
+            case OPC_DPAU_H_QBR:
+            case OPC_DPSU_H_QBL:
+            case OPC_DPSU_H_QBR:
+            case OPC_DPA_W_PH:
+            case OPC_DPAX_W_PH:
+            case OPC_DPAQ_S_W_PH:
+            case OPC_DPAQX_S_W_PH:
+            case OPC_DPAQX_SA_W_PH:
+            case OPC_DPS_W_PH:
+            case OPC_DPSX_W_PH:
+            case OPC_DPSQ_S_W_PH:
+            case OPC_DPSQX_S_W_PH:
+            case OPC_DPSQX_SA_W_PH:
+            case OPC_MULSAQ_S_W_PH:
+            case OPC_DPAQ_SA_L_W:
+            case OPC_DPSQ_SA_L_W:
+            case OPC_MAQ_S_W_PHL:
+            case OPC_MAQ_S_W_PHR:
+            case OPC_MAQ_SA_W_PHL:
+            case OPC_MAQ_SA_W_PHR:
+            case OPC_MULSA_W_PH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DPAW.PH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_INSV_DSP:
+            op2 = MASK_INSV(ctx->opcode);
+            switch (op2) {
+            case OPC_INSV:
+                check_dsp(ctx);
+                {
+                    TCGv t0, t1;
+
+                    if (rt == 0) {
+                        MIPS_DEBUG("NOP");
+                        break;
+                    }
+
+                    t0 = tcg_temp_new();
+                    t1 = tcg_temp_new();
+
+                    gen_load_gpr(t0, rt);
+                    gen_load_gpr(t1, rs);
+
+                    gen_helper_insv(cpu_gpr[rt], cpu_env, t1, t0);
+
+                    tcg_temp_free(t0);
+                    tcg_temp_free(t1);
+                    break;
+                }
+            default:            /* Invalid */
+                MIPS_INVAL("MASK INSV");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_APPEND_DSP:
+            check_dspr2(ctx);
+            op2 = MASK_APPEND(ctx->opcode);
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            break;
+        case OPC_EXTR_W_DSP:
+            op2 = MASK_EXTR_W(ctx->opcode);
+            switch (op2) {
+            case OPC_EXTR_W:
+            case OPC_EXTR_R_W:
+            case OPC_EXTR_RS_W:
+            case OPC_EXTR_S_H:
+            case OPC_EXTRV_S_H:
+            case OPC_EXTRV_W:
+            case OPC_EXTRV_R_W:
+            case OPC_EXTRV_RS_W:
+            case OPC_EXTP:
+            case OPC_EXTPV:
+            case OPC_EXTPDP:
+            case OPC_EXTPDPV:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
+                break;
+            case OPC_RDDSP:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            case OPC_SHILO:
+            case OPC_SHILOV:
+            case OPC_MTHLIP:
+            case OPC_WRDSP:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK EXTR.W");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
             break;
 #if defined(TARGET_MIPS64)
         case OPC_DEXTM ... OPC_DEXT:
@@ -12075,6 +14968,235 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             check_insn(env, ctx, INSN_LOONGSON2E);
             gen_loongson_integer(ctx, op1, rd, rs, rt);
             break;
+        case OPC_ABSQ_S_QH_DSP:
+            op2 = MASK_ABSQ_S_QH(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECEQ_L_PWL:
+            case OPC_PRECEQ_L_PWR:
+            case OPC_PRECEQ_PW_QHL:
+            case OPC_PRECEQ_PW_QHR:
+            case OPC_PRECEQ_PW_QHLA:
+            case OPC_PRECEQ_PW_QHRA:
+            case OPC_PRECEQU_QH_OBL:
+            case OPC_PRECEQU_QH_OBR:
+            case OPC_PRECEQU_QH_OBLA:
+            case OPC_PRECEQU_QH_OBRA:
+            case OPC_PRECEU_QH_OBL:
+            case OPC_PRECEU_QH_OBR:
+            case OPC_PRECEU_QH_OBLA:
+            case OPC_PRECEU_QH_OBRA:
+            case OPC_ABSQ_S_OB:
+            case OPC_ABSQ_S_PW:
+            case OPC_ABSQ_S_QH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_REPL_OB:
+            case OPC_REPL_PW:
+            case OPC_REPL_QH:
+            case OPC_REPLV_OB:
+            case OPC_REPLV_PW:
+            case OPC_REPLV_QH:
+                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ABSQ_S.QH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ADDU_OB_DSP:
+            op2 = MASK_ADDU_OB(ctx->opcode);
+            switch (op2) {
+            case OPC_RADDU_L_OB:
+            case OPC_SUBQ_PW:
+            case OPC_SUBQ_S_PW:
+            case OPC_SUBQ_QH:
+            case OPC_SUBQ_S_QH:
+            case OPC_SUBU_OB:
+            case OPC_SUBU_S_OB:
+            case OPC_SUBU_QH:
+            case OPC_SUBU_S_QH:
+            case OPC_SUBUH_OB:
+            case OPC_SUBUH_R_OB:
+            case OPC_ADDQ_PW:
+            case OPC_ADDQ_S_PW:
+            case OPC_ADDQ_QH:
+            case OPC_ADDQ_S_QH:
+            case OPC_ADDU_OB:
+            case OPC_ADDU_S_OB:
+            case OPC_ADDU_QH:
+            case OPC_ADDU_S_QH:
+            case OPC_ADDUH_OB:
+            case OPC_ADDUH_R_OB:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_MULEQ_S_PW_QHL:
+            case OPC_MULEQ_S_PW_QHR:
+            case OPC_MULEU_S_QH_OBL:
+            case OPC_MULEU_S_QH_OBR:
+            case OPC_MULQ_RS_QH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ADDU.OB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_CMPU_EQ_OB_DSP:
+            op2 = MASK_CMPU_EQ_OB(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECR_SRA_QH_PW:
+            case OPC_PRECR_SRA_R_QH_PW:
+                /* Return value is rt. */
+                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
+                break;
+            case OPC_PRECR_OB_QH:
+            case OPC_PRECRQ_OB_QH:
+            case OPC_PRECRQ_PW_L:
+            case OPC_PRECRQ_QH_PW:
+            case OPC_PRECRQ_RS_QH_PW:
+            case OPC_PRECRQU_S_OB_QH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_CMPU_EQ_OB:
+            case OPC_CMPU_LT_OB:
+            case OPC_CMPU_LE_OB:
+            case OPC_CMP_EQ_QH:
+            case OPC_CMP_LT_QH:
+            case OPC_CMP_LE_QH:
+            case OPC_CMP_EQ_PW:
+            case OPC_CMP_LT_PW:
+            case OPC_CMP_LE_PW:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_CMPGDU_EQ_OB:
+            case OPC_CMPGDU_LT_OB:
+            case OPC_CMPGDU_LE_OB:
+            case OPC_CMPGU_EQ_OB:
+            case OPC_CMPGU_LT_OB:
+            case OPC_CMPGU_LE_OB:
+            case OPC_PACKRL_PW:
+            case OPC_PICK_OB:
+            case OPC_PICK_PW:
+            case OPC_PICK_QH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK CMPU_EQ.OB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DAPPEND_DSP:
+            check_dspr2(ctx);
+            op2 = MASK_DAPPEND(ctx->opcode);
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            break;
+        case OPC_DEXTR_W_DSP:
+            op2 = MASK_DEXTR_W(ctx->opcode);
+            switch (op2) {
+            case OPC_DEXTP:
+            case OPC_DEXTPDP:
+            case OPC_DEXTPDPV:
+            case OPC_DEXTPV:
+            case OPC_DEXTR_L:
+            case OPC_DEXTR_R_L:
+            case OPC_DEXTR_RS_L:
+            case OPC_DEXTR_W:
+            case OPC_DEXTR_R_W:
+            case OPC_DEXTR_RS_W:
+            case OPC_DEXTR_S_H:
+            case OPC_DEXTRV_L:
+            case OPC_DEXTRV_R_L:
+            case OPC_DEXTRV_RS_L:
+            case OPC_DEXTRV_S_H:
+            case OPC_DEXTRV_W:
+            case OPC_DEXTRV_R_W:
+            case OPC_DEXTRV_RS_W:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
+                break;
+            case OPC_DMTHLIP:
+            case OPC_DSHILO:
+            case OPC_DSHILOV:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK EXTR.W");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DPAQ_W_QH_DSP:
+            op2 = MASK_DPAQ_W_QH(ctx->opcode);
+            switch (op2) {
+            case OPC_DPAU_H_OBL:
+            case OPC_DPAU_H_OBR:
+            case OPC_DPSU_H_OBL:
+            case OPC_DPSU_H_OBR:
+            case OPC_DPA_W_QH:
+            case OPC_DPAQ_S_W_QH:
+            case OPC_DPS_W_QH:
+            case OPC_DPSQ_S_W_QH:
+            case OPC_MULSAQ_S_W_QH:
+            case OPC_DPAQ_SA_L_PW:
+            case OPC_DPSQ_SA_L_PW:
+            case OPC_MULSAQ_S_L_PW:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_MAQ_S_W_QHLL:
+            case OPC_MAQ_S_W_QHLR:
+            case OPC_MAQ_S_W_QHRL:
+            case OPC_MAQ_S_W_QHRR:
+            case OPC_MAQ_SA_W_QHLL:
+            case OPC_MAQ_SA_W_QHLR:
+            case OPC_MAQ_SA_W_QHRL:
+            case OPC_MAQ_SA_W_QHRR:
+            case OPC_MAQ_S_L_PWL:
+            case OPC_MAQ_S_L_PWR:
+            case OPC_DMADD:
+            case OPC_DMADDU:
+            case OPC_DMSUB:
+            case OPC_DMSUBU:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DPAQ.W.QH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DINSV_DSP:
+            op2 = MASK_INSV(ctx->opcode);
+            switch (op2) {
+            case OPC_DINSV:
+                {
+                    TCGv t0, t1;
+
+                    if (rt == 0) {
+                        MIPS_DEBUG("NOP");
+                        break;
+                    }
+                    check_dsp(ctx);
+
+                    t0 = tcg_temp_new();
+                    t1 = tcg_temp_new();
+
+                    gen_load_gpr(t0, rt);
+                    gen_load_gpr(t1, rs);
+
+                    gen_helper_dinsv(cpu_gpr[rt], cpu_env, t1, t0);
+                    break;
+                }
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DINSV");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_SHLL_OB_DSP:
+            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+            break;
 #endif
         default:            /* Invalid */
             MIPS_INVAL("special3");
@@ -12098,6 +15220,14 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             check_insn(env, ctx, ISA_MIPS32R2);
             /* Treat as NOP. */
             break;
+        case OPC_BPOSGE32:    /* MIPS DSP branch */
+#if defined(TARGET_MIPS64)
+        case OPC_BPOSGE64:
+#endif
+            check_dsp(ctx);
+            gen_compute_branch(ctx, op1, 4, -1, -2, (int32_t)imm << 2);
+            *is_branch = 1;
+            break;
         default:            /* Invalid */
             MIPS_INVAL("regimm");
             generate_exception(ctx, EXCP_RI);
@@ -12144,18 +15274,18 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                     break;
                 case OPC_DVPE:
                     check_insn(env, ctx, ASE_MT);
-                    gen_helper_dvpe(t0);
+                    gen_helper_dvpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_EVPE:
                     check_insn(env, ctx, ASE_MT);
-                    gen_helper_evpe(t0);
+                    gen_helper_evpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_DI:
                     check_insn(env, ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
-                    gen_helper_di(t0);
+                    gen_helper_di(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     /* Stop translation as we may have switched the execution mode */
                     ctx->bstate = BS_STOP;
@@ -12163,7 +15293,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                 case OPC_EI:
                     check_insn(env, ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
-                    gen_helper_ei(t0);
+                    gen_helper_ei(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     /* Stop translation as we may have switched the execution mode */
                     ctx->bstate = BS_STOP;
@@ -12197,13 +15327,13 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
          break;
     case OPC_SLTI: /* Set on less than with immediate opcode */
     case OPC_SLTIU:
-         gen_slt_imm(env, op, rt, rs, imm);
+         gen_slt_imm(env, ctx, op, rt, rs, imm);
          break;
     case OPC_ANDI: /* Arithmetic with immediate opcode */
     case OPC_LUI:
     case OPC_ORI:
     case OPC_XORI:
-         gen_logic_imm(env, op, rt, rs, imm);
+         gen_logic_imm(env, ctx, op, rt, rs, imm);
          break;
     case OPC_J ... OPC_JAL: /* Jump */
          offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -12298,10 +15428,14 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
     case OPC_LDC2:
     case OPC_SWC2:
     case OPC_SDC2:
-    case OPC_CP2:
         /* COP2: Not implemented. */
         generate_exception_err(ctx, EXCP_CpU, 2);
         break;
+    case OPC_CP2:
+        check_insn(env, ctx, INSN_LOONGSON2F);
+        /* Note that these instructions use different fields.  */
+        gen_loongson_multimedia(ctx, sa, rd, rt);
+        break;
 
     case OPC_CP3:
         if (env->CP0_Config1 & (1 << CP0C1_FP)) {
@@ -12406,7 +15540,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         qemu_log("search pc %d\n", search_pc);
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.singlestep_enabled = env->singlestep_enabled;
@@ -12432,7 +15566,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
                 if (bp->pc == ctx.pc) {
                     save_cpu_state(&ctx, 1);
                     ctx.bstate = BS_BRANCH;
-                    gen_helper_0i(raise_exception, EXCP_DEBUG);
+                    gen_helper_0e0i(raise_exception, EXCP_DEBUG);
                     /* Include the breakpoint location or the tb won't
                      * be flushed when it must be.  */
                     ctx.pc += 4;
@@ -12442,30 +15576,31 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         }
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.pc;
+            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
             gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            gen_opc_btarget[lj] = ctx.btarget;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
         is_branch = 0;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
-            ctx.opcode = ldl_code(ctx.pc);
+            ctx.opcode = cpu_ldl_code(env, ctx.pc);
             insn_bytes = 4;
             decode_opc(env, &ctx, &is_branch);
         } else if (env->insn_flags & ASE_MICROMIPS) {
-            ctx.opcode = lduw_code(ctx.pc);
+            ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_micromips_opc(env, &ctx, &is_branch);
         } else if (env->insn_flags & ASE_MIPS16) {
-            ctx.opcode = lduw_code(ctx.pc);
+            ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx, &is_branch);
         } else {
             generate_exception(&ctx, EXCP_RI);
@@ -12489,8 +15624,9 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
             break;
 
-        if (gen_opc_ptr >= gen_opc_end)
+        if (tcg_ctx.gen_opc_ptr >= gen_opc_end) {
             break;
+        }
 
         if (num_insns >= max_insns)
             break;
@@ -12502,7 +15638,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         gen_io_end();
     if (env->singlestep_enabled && ctx.bstate != BS_BRANCH) {
         save_cpu_state(&ctx, ctx.bstate == BS_NONE);
-        gen_helper_0i(raise_exception, EXCP_DEBUG);
+        gen_helper_0e0i(raise_exception, EXCP_DEBUG);
     } else {
         switch (ctx.bstate) {
         case BS_STOP:
@@ -12522,12 +15658,12 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
     }
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
@@ -12536,7 +15672,7 @@ done_generating:
     LOG_DISAS("\n");
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
@@ -12670,6 +15806,12 @@ static void mips_tcg_init(void)
         cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
                                         offsetof(CPUMIPSState, active_tc.gpr[i]),
                                         regnames[i]);
+
+    for (i = 0; i < 32; i++) {
+        int off = offsetof(CPUMIPSState, active_fpu.fpr[i]);
+        fpu_f64[i] = tcg_global_mem_new_i64(TCG_AREG0, off, fregnames[i]);
+    }
+
     cpu_PC = tcg_global_mem_new(TCG_AREG0,
                                 offsetof(CPUMIPSState, active_tc.PC), "PC");
     for (i = 0; i < MIPS_DSP_ACC; i++) {
@@ -12736,13 +15878,10 @@ MIPSCPU *cpu_mips_init(const char *cpu_model)
 
 void cpu_state_reset(CPUMIPSState *env)
 {
-    if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
-        log_cpu_state(env, 0);
-    }
-
-    memset(env, 0, offsetof(CPUMIPSState, breakpoints));
-    tlb_flush(env, 1);
+#ifndef CONFIG_USER_ONLY
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+#endif
 
     /* Reset registers to their default values */
     env->CP0_PRid = env->cpu_model->CP0_PRid;
@@ -12787,18 +15926,18 @@ void cpu_state_reset(CPUMIPSState *env)
     env->insn_flags = env->cpu_model->insn_flags;
 
 #if defined(CONFIG_USER_ONLY)
-    env->hflags = MIPS_HFLAG_UM;
+    env->CP0_Status = (MIPS_HFLAG_UM << CP0St_KSU);
     /* Enable access to the CPUNum, SYNCI_Step, CC, and CCRes RDHWR
        hardware registers.  */
     env->CP0_HWREna |= 0x0000000F;
     if (env->CP0_Config1 & (1 << CP0C1_FP)) {
-        env->hflags |= MIPS_HFLAG_FPU;
+        env->CP0_Status |= (1 << CP0St_CU1);
     }
-#ifdef TARGET_MIPS64
-    if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
-        env->hflags |= MIPS_HFLAG_F64;
+    if (env->cpu_model->insn_flags & ASE_DSPR2) {
+        env->hflags |= MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2;
+    } else if (env->cpu_model->insn_flags & ASE_DSP) {
+        env->hflags |= MIPS_HFLAG_DSP;
     }
-#endif
 #else
     if (env->hflags & MIPS_HFLAG_BMASK) {
         /* If the exception was raised from a delay slot,
@@ -12811,7 +15950,7 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_Random = env->tlb->nb_tlb - 1;
     env->tlb->tlb_in_use = env->tlb->nb_tlb;
     env->CP0_Wired = 0;
-    env->CP0_EBase = 0x80000000 | (env->cpu_index & 0x3FF);
+    env->CP0_EBase = 0x80000000 | (cs->cpu_index & 0x3FF);
     env->CP0_Status = (1 << CP0St_BEV) | (1 << CP0St_ERL);
     /* vectored interrupts not implemented, timer on int 7,
        no performance counters. */
@@ -12828,20 +15967,19 @@ void cpu_state_reset(CPUMIPSState *env)
     }
     /* Count register increments in debug mode, EJTAG version 1 */
     env->CP0_Debug = (1 << CP0DB_CNT) | (0x1 << CP0DB_VER);
-    env->hflags = MIPS_HFLAG_CP0;
 
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         int i;
 
         /* Only TC0 on VPE 0 starts as active.  */
         for (i = 0; i < ARRAY_SIZE(env->tcs); i++) {
-            env->tcs[i].CP0_TCBind = env->cpu_index << CP0TCBd_CurVPE;
+            env->tcs[i].CP0_TCBind = cs->cpu_index << CP0TCBd_CurVPE;
             env->tcs[i].CP0_TCHalt = 1;
         }
         env->active_tc.CP0_TCHalt = 1;
         env->halted = 1;
 
-        if (!env->cpu_index) {
+        if (cs->cpu_index == 0) {
             /* VPE0 starts up enabled.  */
             env->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
             env->CP0_VPEConf0 |= (1 << CP0VPEC0_MVP) | (1 << CP0VPEC0_VPA);
@@ -12856,17 +15994,22 @@ void cpu_state_reset(CPUMIPSState *env)
         }
     }
 #endif
-#if defined(TARGET_MIPS64)
-    if (env->cpu_model->insn_flags & ISA_MIPS3) {
-        env->hflags |= MIPS_HFLAG_64;
-    }
-#endif
+    compute_hflags(env);
     env->exception_index = EXCP_NONE;
 }
 
 void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->active_tc.PC = gen_opc_pc[pc_pos];
+    env->active_tc.PC = tcg_ctx.gen_opc_pc[pc_pos];
     env->hflags &= ~MIPS_HFLAG_BMASK;
     env->hflags |= gen_opc_hflags[pc_pos];
+    switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
+    case MIPS_HFLAG_BR:
+        break;
+    case MIPS_HFLAG_BC:
+    case MIPS_HFLAG_BL:
+    case MIPS_HFLAG_B:
+        env->btarget = gen_opc_btarget[pc_pos];
+        break;
+    }
 }
diff --git a/target-mips/translate_init.c b/target-mips/translate_init.c
index c39138f3c53..7cf238f34b2 100644
--- a/target-mips/translate_init.c
+++ b/target-mips/translate_init.c
@@ -311,6 +311,29 @@ static const mips_def_t mips_defs[] =
         .insn_flags = CPU_MIPS32R2 | ASE_MIPS16 | ASE_DSP | ASE_MT,
         .mmu_type = MMU_TYPE_R4000,
     },
+    {
+        .name = "74Kf",
+        .CP0_PRid = 0x00019700,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                    (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
+                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
+                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
+                       (1 << CP0C1_CA),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (0 << CP0C3_VInt) | (1 << CP0C3_DSPP),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x3778FF1F,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) | (0x93 << FCR0_PRID),
+        .SEGBITS = 32,
+        .PABITS = 32,
+        .insn_flags = CPU_MIPS32R2 | ASE_MIPS16 | ASE_DSP | ASE_DSPR2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
 #if defined(TARGET_MIPS64)
     {
         .name = "R4000",
@@ -484,6 +507,35 @@ static const mips_def_t mips_defs[] =
       .insn_flags = CPU_LOONGSON2F,
       .mmu_type = MMU_TYPE_R4000,
     },
+    {
+        /* A generic CPU providing MIPS64 ASE DSP 2 features.
+           FIXME: Eventually this should be replaced by a real CPU model. */
+        .name = "mips64dspr2",
+        .CP0_PRid = 0x00010000,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) | (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (63 << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_LPA),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 0,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x37FBFFFF,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_3D) | (1 << FCR0_PS) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .SEGBITS = 42,
+        /* The architectural limit is 59, but we have hardcoded 36 bit
+           in some places...
+        .PABITS = 59, */ /* the architectural limit */
+        .PABITS = 36,
+        .insn_flags = CPU_MIPS64R2 | ASE_DSP | ASE_DSPR2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
 
 #endif
 };
diff --git a/target-openrisc/cpu.c b/target-openrisc/cpu.c
index ba35b175811..54876d904ba 100644
--- a/target-openrisc/cpu.c
+++ b/target-openrisc/cpu.c
@@ -27,7 +27,7 @@ static void openrisc_cpu_reset(CPUState *s)
     OpenRISCCPUClass *occ = OPENRISC_CPU_GET_CLASS(cpu);
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", cpu->env.cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(&cpu->env, 0);
     }
 
@@ -88,6 +88,23 @@ static void openrisc_cpu_initfn(Object *obj)
 }
 
 /* CPU models */
+
+static ObjectClass *openrisc_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && (!object_class_dynamic_cast(oc, TYPE_OPENRISC_CPU) ||
+                       object_class_is_abstract(oc))) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void or1200_initfn(Object *obj)
 {
     OpenRISCCPU *cpu = OPENRISC_CPU(obj);
@@ -120,6 +137,8 @@ static void openrisc_cpu_class_init(ObjectClass *oc, void *data)
 
     occ->parent_reset = cc->reset;
     cc->reset = openrisc_cpu_reset;
+
+    cc->class_by_name = openrisc_cpu_class_by_name;
 }
 
 static void cpu_register(const OpenRISCCPUInfo *info)
@@ -132,7 +151,7 @@ static void cpu_register(const OpenRISCCPUInfo *info)
         .class_size = sizeof(OpenRISCCPUClass),
     };
 
-    type_register_static(&type_info);
+    type_register(&type_info);
 }
 
 static const TypeInfo openrisc_cpu_type_info = {
@@ -158,11 +177,13 @@ static void openrisc_cpu_register_types(void)
 OpenRISCCPU *cpu_openrisc_init(const char *cpu_model)
 {
     OpenRISCCPU *cpu;
+    ObjectClass *oc;
 
-    if (!object_class_by_name(cpu_model)) {
+    oc = openrisc_cpu_class_by_name(cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = OPENRISC_CPU(object_new(cpu_model));
+    cpu = OPENRISC_CPU(object_new(object_class_get_name(oc)));
     cpu->env.cpu_model_str = cpu_model;
 
     openrisc_cpu_realize(OBJECT(cpu), NULL);
@@ -170,11 +191,6 @@ OpenRISCCPU *cpu_openrisc_init(const char *cpu_model)
     return cpu;
 }
 
-typedef struct OpenRISCCPUList {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} OpenRISCCPUList;
-
 /* Sort alphabetically by type name, except for "any". */
 static gint openrisc_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -196,7 +212,7 @@ static gint openrisc_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void openrisc_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *oc = data;
-    OpenRISCCPUList *s = user_data;
+    CPUListState *s = user_data;
 
     (*s->cpu_fprintf)(s->file, "  %s\n",
                       object_class_get_name(oc));
@@ -204,7 +220,7 @@ static void openrisc_cpu_list_entry(gpointer data, gpointer user_data)
 
 void cpu_openrisc_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    OpenRISCCPUList s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index de21a877d32..3beab45c3c4 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -30,10 +30,10 @@ struct OpenRISCCPU;
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
-#include "qemu/cpu.h"
-#include "error.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
+#include "qom/cpu.h"
+#include "qapi/error.h"
 
 #define TYPE_OPENRISC_CPU "or32-cpu"
 
@@ -89,24 +89,6 @@ enum {
 /* Interrupt */
 #define NR_IRQS  32
 
-/* Registers */
-enum {
-    R0 = 0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10,
-    R11, R12, R13, R14, R15, R16, R17, R18, R19, R20,
-    R21, R22, R23, R24, R25, R26, R27, R28, R29, R30,
-    R31
-};
-
-/* Register aliases */
-enum {
-    R_ZERO = R0,
-    R_SP = R1,
-    R_FP = R2,
-    R_LR = R9,
-    R_RV = R11,
-    R_RVH = R12
-};
-
 /* Unit presece register */
 enum {
     UPR_UP = (1 << 0),
@@ -279,11 +261,11 @@ typedef struct CPUOpenRISCTLBContext {
     OpenRISCTLBEntry dtlb[DTLB_WAYS][DTLB_SIZE];
 
     int (*cpu_openrisc_map_address_code)(struct OpenRISCCPU *cpu,
-                                         target_phys_addr_t *physical,
+                                         hwaddr *physical,
                                          int *prot,
                                          target_ulong address, int rw);
     int (*cpu_openrisc_map_address_data)(struct OpenRISCCPU *cpu,
-                                         target_phys_addr_t *physical,
+                                         hwaddr *physical,
                                          int *prot,
                                          target_ulong address, int rw);
 } CPUOpenRISCTLBContext;
@@ -387,13 +369,13 @@ void cpu_openrisc_count_stop(OpenRISCCPU *cpu);
 
 void cpu_openrisc_mmu_init(OpenRISCCPU *cpu);
 int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
-                                target_phys_addr_t *physical,
+                                hwaddr *physical,
                                 int *prot, target_ulong address, int rw);
 int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw);
 int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw);
 #endif
 
@@ -416,7 +398,7 @@ static inline void cpu_clone_regs(CPUOpenRISCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
                                         target_ulong *pc,
@@ -437,13 +419,15 @@ static inline int cpu_mmu_index(CPUOpenRISCState *env)
 }
 
 #define CPU_INTERRUPT_TIMER   CPU_INTERRUPT_TGT_INT_0
-static inline bool cpu_has_work(CPUOpenRISCState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUOpenRISCState *env = &OPENRISC_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD |
                                      CPU_INTERRUPT_TIMER);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline target_ulong cpu_get_pc(CPUOpenRISCState *env)
 {
diff --git a/target-openrisc/exception_helper.c b/target-openrisc/exception_helper.c
index dab41481519..0c53b7755b5 100644
--- a/target-openrisc/exception_helper.c
+++ b/target-openrisc/exception_helper.c
@@ -23,7 +23,7 @@
 
 void HELPER(exception)(CPUOpenRISCState *env, uint32_t excp)
 {
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     raise_exception(cpu, excp);
 }
diff --git a/target-openrisc/fpu_helper.c b/target-openrisc/fpu_helper.c
index b184d5ef73f..4615a366d19 100644
--- a/target-openrisc/fpu_helper.c
+++ b/target-openrisc/fpu_helper.c
@@ -68,7 +68,7 @@ static inline void update_fpcsr(OpenRISCCPU *cpu)
 uint64_t HELPER(itofd)(CPUOpenRISCState *env, uint64_t val)
 {
     uint64_t itofd;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     itofd = int32_to_float64(val, &cpu->env.fp_status);
@@ -80,7 +80,7 @@ uint64_t HELPER(itofd)(CPUOpenRISCState *env, uint64_t val)
 uint32_t HELPER(itofs)(CPUOpenRISCState *env, uint32_t val)
 {
     uint32_t itofs;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     itofs = int32_to_float32(val, &cpu->env.fp_status);
@@ -92,7 +92,7 @@ uint32_t HELPER(itofs)(CPUOpenRISCState *env, uint32_t val)
 uint64_t HELPER(ftoid)(CPUOpenRISCState *env, uint64_t val)
 {
     uint64_t ftoid;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     ftoid = float32_to_int64(val, &cpu->env.fp_status);
@@ -104,7 +104,7 @@ uint64_t HELPER(ftoid)(CPUOpenRISCState *env, uint64_t val)
 uint32_t HELPER(ftois)(CPUOpenRISCState *env, uint32_t val)
 {
     uint32_t ftois;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     ftois = float32_to_int32(val, &cpu->env.fp_status);
@@ -120,7 +120,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     uint64_t result;                                                      \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     result = float64_ ## name(fdt0, fdt1, &cpu->env.fp_status);           \
     update_fpcsr(cpu);                                                    \
@@ -131,7 +131,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     uint32_t result;                                                      \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     result = float32_ ## name(fdt0, fdt1, &cpu->env.fp_status);           \
     update_fpcsr(cpu);                                                    \
@@ -152,7 +152,7 @@ uint64_t helper_float_ ## name1 ## name2 ## _d(CPUOpenRISCState *env,     \
 {                                                                         \
     uint64_t result, temp, hi, lo;                                        \
     uint32_t val1, val2;                                                  \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     hi = env->fpmaddhi;                                                   \
     lo = env->fpmaddlo;                                                   \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
@@ -174,7 +174,7 @@ uint32_t helper_float_ ## name1 ## name2 ## _s(CPUOpenRISCState *env,     \
 {                                                                         \
     uint64_t result, temp, hi, lo;                                        \
     uint32_t val1, val2;                                                  \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     hi = cpu->env.fpmaddhi;                                               \
     lo = cpu->env.fpmaddlo;                                               \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
@@ -198,7 +198,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = float64_ ## name(fdt0, fdt1, &cpu->env.fp_status);              \
     update_fpcsr(cpu);                                                    \
@@ -209,7 +209,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                              uint32_t fdt0, uint32_t fdt1)\
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = float32_ ## name(fdt0, fdt1, &cpu->env.fp_status);              \
     update_fpcsr(cpu);                                                    \
@@ -227,7 +227,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_eq_quiet(fdt0, fdt1, &cpu->env.fp_status);             \
     update_fpcsr(cpu);                                                    \
@@ -238,7 +238,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_eq_quiet(fdt0, fdt1, &cpu->env.fp_status);             \
     update_fpcsr(cpu);                                                    \
@@ -253,7 +253,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_le(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -264,7 +264,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_le(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -278,7 +278,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_lt(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -289,7 +289,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_lt(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
diff --git a/target-openrisc/helper.h b/target-openrisc/helper.h
index 404d46447fa..2af97901ce8 100644
--- a/target-openrisc/helper.h
+++ b/target-openrisc/helper.h
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 /* exception */
 DEF_HELPER_FLAGS_2(exception, 0, void, env, i32)
@@ -67,4 +67,4 @@ DEF_HELPER_FLAGS_1(rfe, 0, void, env)
 DEF_HELPER_FLAGS_4(mtspr, 0, void, env, tl, tl, tl)
 DEF_HELPER_FLAGS_4(mfspr, 0, tl, env, tl, tl, tl)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-openrisc/int_helper.c b/target-openrisc/int_helper.c
index 2fdfd27712c..16cb5abfcd3 100644
--- a/target-openrisc/int_helper.c
+++ b/target-openrisc/int_helper.c
@@ -21,7 +21,7 @@
 #include "cpu.h"
 #include "helper.h"
 #include "exception.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 target_ulong HELPER(ff1)(target_ulong x)
 {
@@ -48,7 +48,7 @@ uint32_t HELPER(mul32)(CPUOpenRISCState *env,
     uint64_t result;
     uint32_t high, cy;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     result = (uint64_t)ra * rb;
     /* regisiers in or32 is 32bit, so 32 is NOT a magic number.
diff --git a/target-openrisc/interrupt.c b/target-openrisc/interrupt.c
index 642da7de49e..7f2c025da24 100644
--- a/target-openrisc/interrupt.c
+++ b/target-openrisc/interrupt.c
@@ -19,8 +19,8 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #ifndef CONFIG_USER_ONLY
 #include "hw/loader.h"
 #endif
diff --git a/target-openrisc/interrupt_helper.c b/target-openrisc/interrupt_helper.c
index 79f5afed440..a176441b01e 100644
--- a/target-openrisc/interrupt_helper.c
+++ b/target-openrisc/interrupt_helper.c
@@ -23,7 +23,7 @@
 
 void HELPER(rfe)(CPUOpenRISCState *env)
 {
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 #ifndef CONFIG_USER_ONLY
     int need_flush_tlb = (cpu->env.sr & (SR_SM | SR_IME | SR_DME)) ^
                          (cpu->env.esr & (SR_SM | SR_IME | SR_DME));
diff --git a/target-openrisc/mmu.c b/target-openrisc/mmu.c
index 0be1d413c94..d354e1f8b2d 100644
--- a/target-openrisc/mmu.c
+++ b/target-openrisc/mmu.c
@@ -20,15 +20,15 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #ifndef CONFIG_USER_ONLY
 #include "hw/loader.h"
 #endif
 
 #ifndef CONFIG_USER_ONLY
 int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
-                                target_phys_addr_t *physical,
+                                hwaddr *physical,
                                 int *prot, target_ulong address, int rw)
 {
     *physical = address;
@@ -37,7 +37,7 @@ int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
 }
 
 int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw)
 {
     int vpn = address >> TARGET_PAGE_BITS;
@@ -72,7 +72,7 @@ int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
 }
 
 int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw)
 {
     int vpn = address >> TARGET_PAGE_BITS;
@@ -116,7 +116,7 @@ int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
 }
 
 static int cpu_openrisc_get_phys_addr(OpenRISCCPU *cpu,
-                                      target_phys_addr_t *physical,
+                                      hwaddr *physical,
                                       int *prot, target_ulong address,
                                       int rw)
 {
@@ -185,9 +185,9 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
                                   target_ulong address, int rw, int mmu_idx)
 {
     int ret = 0;
-    target_phys_addr_t physical = 0;
+    hwaddr physical = 0;
     int prot = 0;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     ret = cpu_openrisc_get_phys_addr(cpu, &physical, &prot,
                                      address, rw);
@@ -209,7 +209,7 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
                                   target_ulong address, int rw, int mmu_idx)
 {
     int ret = 0;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     cpu_openrisc_raise_mmu_exception(cpu, address, rw, ret);
     ret = 1;
@@ -219,12 +219,12 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
 #endif
 
 #ifndef CONFIG_USER_ONLY
-target_phys_addr_t cpu_get_phys_page_debug(CPUOpenRISCState *env,
+hwaddr cpu_get_phys_page_debug(CPUOpenRISCState *env,
                                            target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int prot;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     if (cpu_openrisc_get_phys_addr(cpu, &phys_addr, &prot, addr, 0)) {
         return -1;
diff --git a/target-openrisc/mmu_helper.c b/target-openrisc/mmu_helper.c
index 59ed371ae05..e46b0929847 100644
--- a/target-openrisc/mmu_helper.c
+++ b/target-openrisc/mmu_helper.c
@@ -21,26 +21,24 @@
 #include "cpu.h"
 
 #ifndef CONFIG_USER_ONLY
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUOpenRISCState *env, target_ulong addr, int is_write,
               int mmu_idx, uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    unsigned long pc;
     int ret;
 
     ret = cpu_openrisc_handle_mmu_fault(env, addr, is_write, mmu_idx);
@@ -48,13 +46,7 @@ void tlb_fill(CPUOpenRISCState *env, target_ulong addr, int is_write,
     if (ret) {
         if (retaddr) {
             /* now we have a real cpu fault.  */
-            pc = (unsigned long)retaddr;
-            tb = tb_find_pc(pc);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we
-                   have a virtual CPU fault.  */
-                cpu_restore_state(tb, env, pc);
-            }
+            cpu_restore_state(env, retaddr);
         }
         /* Raise Exception.  */
         cpu_loop_exit(env);
diff --git a/target-openrisc/sys_helper.c b/target-openrisc/sys_helper.c
index f160dc397cd..3c5f45ab755 100644
--- a/target-openrisc/sys_helper.c
+++ b/target-openrisc/sys_helper.c
@@ -30,7 +30,7 @@ void HELPER(mtspr)(CPUOpenRISCState *env,
     int spr = (ra | offset);
     int idx;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     switch (spr) {
     case TO_SPR(0, 0): /* VR */
@@ -177,7 +177,7 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env,
     int spr = (ra | offset);
     int idx;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     switch (spr) {
     case TO_SPR(0, 0): /* VR */
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 325ba09cb50..1e1b30cdcb1 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -19,13 +19,13 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "disas.h"
+#include "exec/exec-all.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu-common.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "config.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -61,7 +61,7 @@ static TCGv_i32 fpcsr;
 static TCGv machi, maclo;
 static TCGv fpmaddhi, fpmaddlo;
 static TCGv_i32 env_flags;
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void openrisc_translate_init(void)
 {
@@ -1675,7 +1675,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     pc_start = tb->pc;
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     dc->is_jmp = DISAS_NEXT;
     dc->ppc = pc_start;
     dc->pc = pc_start;
@@ -1703,19 +1703,19 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     do {
         check_breakpoint(cpu, dc);
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (k < j) {
                 k++;
                 while (k < j) {
-                    gen_opc_instr_start[k++] = 0;
+                    tcg_ctx.gen_opc_instr_start[k++] = 0;
                 }
             }
-            gen_opc_pc[k] = dc->pc;
-            gen_opc_instr_start[k] = 1;
-            gen_opc_icount[k] = num_insns;
+            tcg_ctx.gen_opc_pc[k] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[k] = 1;
+            tcg_ctx.gen_opc_icount[k] = num_insns;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc->pc);
         }
 
@@ -1744,7 +1744,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
             }
         }
     } while (!dc->is_jmp
-             && gen_opc_ptr < gen_opc_end
+             && tcg_ctx.gen_opc_ptr < gen_opc_end
              && !cpu->env.singlestep_enabled
              && !singlestep
              && (dc->pc < next_page_start)
@@ -1782,12 +1782,12 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         k++;
         while (k <= j) {
-            gen_opc_instr_start[k++] = 0;
+            tcg_ctx.gen_opc_instr_start[k++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -1797,9 +1797,10 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(&cpu->env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 }
@@ -1831,5 +1832,5 @@ void cpu_dump_state(CPUOpenRISCState *env, FILE *f,
 void restore_state_to_opc(CPUOpenRISCState *env, TranslationBlock *tb,
                           int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-ppc/Makefile.objs b/target-ppc/Makefile.objs
index 237a0ed4f7f..a028dcdcd00 100644
--- a/target-ppc/Makefile.objs
+++ b/target-ppc/Makefile.objs
@@ -1,7 +1,6 @@
-obj-y += translate.o helper.o
+obj-y += translate.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
-obj-y += helper.o
 obj-y += excp_helper.o
 obj-y += fpu_helper.o
 obj-y += int_helper.o
@@ -9,4 +8,3 @@ obj-y += mmu_helper.o
 obj-y += timebase_helper.o
 obj-y += misc_helper.o
 obj-y += mem_helper.o
-obj-y += mpic_helper.o
diff --git a/target-ppc/cpu-qom.h b/target-ppc/cpu-qom.h
index fef6f95a04d..b338f8fb561 100644
--- a/target-ppc/cpu-qom.h
+++ b/target-ppc/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_PPC_CPU_QOM_H
 #define QEMU_PPC_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #ifdef TARGET_PPC64
@@ -50,6 +50,9 @@ typedef struct PowerPCCPUClass {
     /*< public >*/
 
     void (*parent_reset)(CPUState *cpu);
+
+    /* TODO inline fields here */
+    ppc_def_t *info;
 } PowerPCCPUClass;
 
 /**
@@ -73,5 +76,7 @@ static inline PowerPCCPU *ppc_env_get_cpu(CPUPPCState *env)
 
 #define ENV_GET_CPU(e) CPU(ppc_env_get_cpu(e))
 
+PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr);
+
 
 #endif
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index ca2fc2198ee..953146eeba3 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -73,9 +73,9 @@
 
 #define CPUArchState struct CPUPPCState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -355,7 +355,7 @@ struct ppc6xx_tlb_t {
 
 typedef struct ppcemb_tlb_t ppcemb_tlb_t;
 struct ppcemb_tlb_t {
-    target_phys_addr_t RPN;
+    uint64_t RPN;
     target_ulong EPN;
     target_ulong PID;
     target_ulong size;
@@ -963,7 +963,7 @@ struct CPUPPCState {
     /* floating point registers */
     float64 fpr[32];
     /* floating point status and control register */
-    uint32_t fpscr;
+    target_ulong fpscr;
 
     /* Next instruction pointer */
     target_ulong nip;
@@ -983,8 +983,8 @@ struct CPUPPCState {
     int slb_nr;
 #endif
     /* segment registers */
-    target_phys_addr_t htab_base;
-    target_phys_addr_t htab_mask;
+    hwaddr htab_base;
+    hwaddr htab_mask;
     target_ulong sr[32];
     /* externally stored hash table */
     uint8_t *external_htab;
@@ -1014,6 +1014,8 @@ struct CPUPPCState {
     /* Altivec registers */
     ppc_avr_t avr[32];
     uint32_t vscr;
+    /* VSX registers */
+    uint64_t vsr[32];
     /* SPE registers */
     uint64_t spe_acc;
     uint32_t spe_fscr;
@@ -1045,10 +1047,9 @@ struct CPUPPCState {
 #endif
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
-    target_phys_addr_t vpa;
-    target_phys_addr_t slb_shadow;
-    target_phys_addr_t dispatch_trace_log;
-    uint32_t dtl_size;
+    uint64_t vpa_addr;
+    uint64_t slb_shadow_addr, slb_shadow_size;
+    uint64_t dtl_addr, dtl_size;
 #endif /* TARGET_PPC64 */
 
     int error_code;
@@ -1066,7 +1067,9 @@ struct CPUPPCState {
     target_ulong ivor_mask;
     target_ulong ivpr_mask;
     target_ulong hreset_vector;
-    target_phys_addr_t mpic_cpu_base;
+    hwaddr mpic_iack;
+    /* true when the external proxy facility mode is enabled */
+    bool mpic_proxy;
 #endif
 
     /* Those resources are used only during code translation */
@@ -1079,7 +1082,6 @@ struct CPUPPCState {
     int mmu_idx;         /* precomputed MMU index to speed up mem accesses */
 
     /* Power management */
-    int power_mode;
     int (*check_pow)(CPUPPCState *env);
 
 #if !defined(CONFIG_USER_ONLY)
@@ -1118,10 +1120,10 @@ do {                                            \
 /* Context used internally during MMU translations */
 typedef struct mmu_ctx_t mmu_ctx_t;
 struct mmu_ctx_t {
-    target_phys_addr_t raddr;      /* Real address              */
-    target_phys_addr_t eaddr;      /* Effective address         */
+    hwaddr raddr;      /* Real address              */
+    hwaddr eaddr;      /* Effective address         */
     int prot;                      /* Protection bits           */
-    target_phys_addr_t hash[2];    /* Pagetable hash values     */
+    hwaddr hash[2];    /* Pagetable hash values     */
     target_ulong ptem;             /* Virtual segment ID | API  */
     int key;                       /* Access key                */
     int nx;                        /* Non-execute area          */
@@ -1142,10 +1144,6 @@ int cpu_ppc_signal_handler (int host_signum, void *pinfo,
 int cpu_ppc_handle_mmu_fault (CPUPPCState *env, target_ulong address, int rw,
                               int mmu_idx);
 #define cpu_handle_mmu_fault cpu_ppc_handle_mmu_fault
-#if !defined(CONFIG_USER_ONLY)
-int get_physical_address (CPUPPCState *env, mmu_ctx_t *ctx, target_ulong vaddr,
-                          int rw, int access_type);
-#endif
 void do_interrupt (CPUPPCState *env);
 void ppc_hw_interrupt (CPUPPCState *env);
 
@@ -1160,10 +1158,6 @@ void ppc_store_msr (CPUPPCState *env, target_ulong value);
 
 void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 
-const ppc_def_t *ppc_find_by_pvr(uint32_t pvr);
-const ppc_def_t *cpu_ppc_find_by_name (const char *name);
-int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def);
-
 /* Time-base and decrementer management */
 #ifndef NO_CPU_IO_DEFS
 uint64_t cpu_ppc_load_tbl (CPUPPCState *env);
@@ -1179,7 +1173,6 @@ void cpu_ppc_store_decr (CPUPPCState *env, uint32_t value);
 uint32_t cpu_ppc_load_hdecr (CPUPPCState *env);
 void cpu_ppc_store_hdecr (CPUPPCState *env, uint32_t value);
 uint64_t cpu_ppc_load_purr (CPUPPCState *env);
-void cpu_ppc_store_purr (CPUPPCState *env, uint64_t value);
 uint32_t cpu_ppc601_load_rtcl (CPUPPCState *env);
 uint32_t cpu_ppc601_load_rtcu (CPUPPCState *env);
 #if !defined(CONFIG_USER_ONLY)
@@ -1191,10 +1184,8 @@ void store_40x_dbcr0 (CPUPPCState *env, uint32_t val);
 void store_40x_sler (CPUPPCState *env, uint32_t val);
 void store_booke_tcr (CPUPPCState *env, target_ulong val);
 void store_booke_tsr (CPUPPCState *env, target_ulong val);
-void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot);
-target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env, ppcmas_tlb_t *tlb);
 int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                     target_phys_addr_t *raddrp, target_ulong address,
+                     hwaddr *raddrp, target_ulong address,
                      uint32_t pid);
 void ppc_tlb_invalidate_all (CPUPPCState *env);
 void ppc_tlb_invalidate_one (CPUPPCState *env, target_ulong addr);
@@ -1258,7 +1249,7 @@ static inline void cpu_clone_regs(CPUPPCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /*****************************************************************************/
 /* CRF definitions */
@@ -2222,14 +2213,16 @@ static inline bool msr_is_64bit(CPUPPCState *env, target_ulong msr)
     return msr & (1ULL << MSR_SF);
 }
 
-extern void (*cpu_ppc_hypercall)(CPUPPCState *);
+extern void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
-static inline bool cpu_has_work(CPUPPCState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUPPCState *env = &POWERPC_CPU(cpu)->env;
+
     return msr_ee && (env->interrupt_request & CPU_INTERRUPT_HARD);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUPPCState *env, TranslationBlock *tb)
 {
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index 1a593f6f3fd..0a1ac86a42a 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -33,7 +33,7 @@
 /*****************************************************************************/
 /* PowerPC Hypercall emulation */
 
-void (*cpu_ppc_hypercall)(CPUPPCState *);
+void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
 /*****************************************************************************/
 /* Exception processing */
@@ -63,8 +63,9 @@ static inline void dump_syscall(CPUPPCState *env)
 /* Note that this function should be greatly optimized
  * when called with a constant excp, from ppc_hw_interrupt
  */
-static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
+static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong msr, new_msr, vector;
     int srr0, srr1, asrr0, asrr1;
     int lpes0, lpes1, lev;
@@ -83,7 +84,11 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
                   " => %08x (%02x)\n", env->nip, excp, env->error_code);
 
     /* new srr1 value excluding must-be-zero bits */
-    msr = env->msr & ~0x783f0000ULL;
+    if (excp_model == POWERPC_EXCP_BOOKE) {
+        msr = env->msr;
+    } else {
+        msr = env->msr & ~0x783f0000ULL;
+    }
 
     /* new interrupt handler msr */
     new_msr = env->msr & ((target_ulong)1 << MSR_ME);
@@ -144,6 +149,7 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
             srr1 = SPR_40x_SRR3;
             break;
         case POWERPC_EXCP_BOOKE:
+            /* FIXME: choose one or the other based on CPU type */
             srr0 = SPR_BOOKE_MCSRR0;
             srr1 = SPR_BOOKE_MCSRR1;
             asrr0 = SPR_BOOKE_CSRR0;
@@ -172,6 +178,10 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
         if (lpes0 == 1) {
             new_msr |= (target_ulong)MSR_HVB;
         }
+        if (env->mpic_proxy) {
+            /* IACK the IRQ on delivery */
+            env->spr[SPR_BOOKE_EPR] = ldl_phys(env->mpic_iack);
+        }
         goto store_next;
     case POWERPC_EXCP_ALIGN:     /* Alignment exception                      */
         if (lpes1 == 0) {
@@ -238,7 +248,7 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
         dump_syscall(env);
         lev = env->error_code;
         if ((lev == 1) && cpu_ppc_hypercall) {
-            cpu_ppc_hypercall(env);
+            cpu_ppc_hypercall(cpu);
             return;
         }
         if (lev == 1 || (lpes0 == 0 && lpes1 == 0)) {
@@ -274,6 +284,7 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
     case POWERPC_EXCP_DEBUG:     /* Debug interrupt                          */
         switch (excp_model) {
         case POWERPC_EXCP_BOOKE:
+            /* FIXME: choose one or the other based on CPU type */
             srr0 = SPR_BOOKE_DSRR0;
             srr1 = SPR_BOOKE_DSRR1;
             asrr0 = SPR_BOOKE_CSRR0;
@@ -643,11 +654,14 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
 
 void do_interrupt(CPUPPCState *env)
 {
-    powerpc_excp(env, env->excp_model, env->exception_index);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    powerpc_excp(cpu, env->excp_model, env->exception_index);
 }
 
 void ppc_hw_interrupt(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     int hdice;
 
 #if 0
@@ -658,20 +672,20 @@ void ppc_hw_interrupt(CPUPPCState *env)
     /* External reset */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_RESET)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_RESET);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_RESET);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_RESET);
         return;
     }
     /* Machine check exception */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_MCK)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_MCK);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_MCHECK);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_MCHECK);
         return;
     }
 #if 0 /* TODO */
     /* External debug exception */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_DEBUG)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DEBUG);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_DEBUG);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DEBUG);
         return;
     }
 #endif
@@ -685,7 +699,7 @@ void ppc_hw_interrupt(CPUPPCState *env)
         /* Hypervisor decrementer exception */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_HDECR)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_HDECR);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_HDECR);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_HDECR);
             return;
         }
     }
@@ -698,7 +712,7 @@ void ppc_hw_interrupt(CPUPPCState *env)
 #if 0
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_CEXT);
 #endif
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_CRITICAL);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_CRITICAL);
             return;
         }
     }
@@ -706,30 +720,30 @@ void ppc_hw_interrupt(CPUPPCState *env)
         /* Watchdog timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_WDT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_WDT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_WDT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_WDT);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_CDOORBELL)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_CDOORBELL);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DOORCI);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DOORCI);
             return;
         }
         /* Fixed interval timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_FIT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_FIT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_FIT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_FIT);
             return;
         }
         /* Programmable interval timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_PIT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_PIT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_PIT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_PIT);
             return;
         }
         /* Decrementer exception */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_DECR)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DECR);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DECR);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DECR);
             return;
         }
         /* External interrupt */
@@ -740,23 +754,23 @@ void ppc_hw_interrupt(CPUPPCState *env)
 #if 0
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_EXT);
 #endif
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_EXTERNAL);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_EXTERNAL);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_DOORBELL)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DOORBELL);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DOORI);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DOORI);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_PERFM)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_PERFM);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_PERFM);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_PERFM);
             return;
         }
         /* Thermal interrupt */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_THERM)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_THERM);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_THERM);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_THERM);
             return;
         }
     }
@@ -832,8 +846,13 @@ static inline void do_rfi(CPUPPCState *env, target_ulong nip, target_ulong msr,
 
 void helper_rfi(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
-           ~((target_ulong)0x783F0000), 1);
+    if (env->excp_model == POWERPC_EXCP_BOOKE) {
+        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
+               ~((target_ulong)0), 0);
+    } else {
+        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
+               ~((target_ulong)0x783F0000), 1);
+    }
 }
 
 #if defined(TARGET_PPC64)
@@ -860,20 +879,22 @@ void helper_40x_rfci(CPUPPCState *env)
 
 void helper_rfci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], SPR_BOOKE_CSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], env->spr[SPR_BOOKE_CSRR1],
+           ~((target_ulong)0), 0);
 }
 
 void helper_rfdi(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], SPR_BOOKE_DSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    /* FIXME: choose CSRR1 or DSRR1 based on cpu type */
+    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], env->spr[SPR_BOOKE_DSRR1],
+           ~((target_ulong)0), 0);
 }
 
 void helper_rfmci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], SPR_BOOKE_MCSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    /* FIXME: choose CSRR1 or MCSRR1 based on cpu type */
+    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], env->spr[SPR_BOOKE_MCSRR1],
+           ~((target_ulong)0), 0);
 }
 #endif
 
diff --git a/target-ppc/helper.c b/target-ppc/helper.c
deleted file mode 100644
index 48b19a7e1d1..00000000000
--- a/target-ppc/helper.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- *  PowerPC emulation helpers for QEMU.
- *
- *  Copyright (c) 2003-2007 Jocelyn Mayer
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cpu.h"
-#include "helper_regs.h"
-#include "kvm.h"
-#include "kvm_ppc.h"
-#include "cpus.h"
-
-PowerPCCPU *cpu_ppc_init(const char *cpu_model)
-{
-    PowerPCCPU *cpu;
-    CPUPPCState *env;
-    const ppc_def_t *def;
-
-    def = cpu_ppc_find_by_name(cpu_model);
-    if (!def) {
-        return NULL;
-    }
-
-    cpu = POWERPC_CPU(object_new(TYPE_POWERPC_CPU));
-    env = &cpu->env;
-
-    if (tcg_enabled()) {
-        ppc_translate_init();
-    }
-
-    env->cpu_model_str = cpu_model;
-    cpu_ppc_register_internal(env, def);
-
-    qemu_init_vcpu(env);
-
-    return cpu;
-}
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index fd04c063e23..83139d5225a 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_3(raise_exception_err, void, env, i32, i32)
 DEF_HELPER_2(raise_exception, void, env, i32)
@@ -31,24 +31,24 @@ DEF_HELPER_2(icbi, void, env, tl)
 DEF_HELPER_5(lscbx, tl, env, tl, i32, i32, i32)
 
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_2(mulhd, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(mulhdu, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(mulhd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(mulhdu, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_3(mulldo, i64, env, i64, i64)
 #endif
 
-DEF_HELPER_FLAGS_1(cntlzw, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntb, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(cntlzw, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntb, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_3(sraw, tl, env, tl, tl)
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_1(cntlzd, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntd, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(cntlzd, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntd, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_3(srad, tl, env, tl, tl)
 #endif
 
-DEF_HELPER_FLAGS_1(cntlsw32, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_1(cntlzw32, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_2(brinc, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(cntlsw32, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(cntlzw32, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_2(brinc, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 
 DEF_HELPER_1(float_check_status, void, env)
 DEF_HELPER_1(reset_fpstatus, void, env)
@@ -345,25 +345,25 @@ DEF_HELPER_2(6xx_tlbd, void, env, tl)
 DEF_HELPER_2(6xx_tlbi, void, env, tl)
 DEF_HELPER_2(74xx_tlbd, void, env, tl)
 DEF_HELPER_2(74xx_tlbi, void, env, tl)
-DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_CONST, void, env, tl)
+DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_NO_RWG, void, env, tl)
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_CONST, void, env, tl, tl)
+DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_NO_RWG, void, env, tl, tl)
 DEF_HELPER_2(load_slb_esid, tl, env, tl)
 DEF_HELPER_2(load_slb_vsid, tl, env, tl)
-DEF_HELPER_FLAGS_1(slbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(slbie, TCG_CALL_CONST, void, env, tl)
+DEF_HELPER_FLAGS_1(slbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(slbie, TCG_CALL_NO_RWG, void, env, tl)
 #endif
-DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_CONST, tl, env, tl);
-DEF_HELPER_FLAGS_3(store_sr, TCG_CALL_CONST, void, env, tl, tl)
+DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_NO_RWG, tl, env, tl);
+DEF_HELPER_FLAGS_3(store_sr, TCG_CALL_NO_RWG, void, env, tl, tl)
 
-DEF_HELPER_FLAGS_1(602_mfrom, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(602_mfrom, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_1(msgsnd, void, tl)
 DEF_HELPER_2(msgclr, void, env, tl)
 #endif
 
 DEF_HELPER_4(dlmzb, tl, env, tl, tl, i32)
-DEF_HELPER_FLAGS_2(clcs, TCG_CALL_CONST | TCG_CALL_PURE, tl, env, i32)
+DEF_HELPER_FLAGS_2(clcs, TCG_CALL_NO_RWG_SE, tl, env, i32)
 #if !defined(CONFIG_USER_ONLY)
 DEF_HELPER_2(rac, tl, env, tl)
 #endif
@@ -405,7 +405,6 @@ DEF_HELPER_2(store_40x_dbcr0, void, env, tl)
 DEF_HELPER_2(store_40x_sler, void, env, tl)
 DEF_HELPER_2(store_booke_tcr, void, env, tl)
 DEF_HELPER_2(store_booke_tsr, void, env, tl)
-DEF_HELPER_1(load_epr, tl, env)
 DEF_HELPER_3(store_ibatl, void, env, i32, tl)
 DEF_HELPER_3(store_ibatu, void, env, i32, tl)
 DEF_HELPER_3(store_dbatl, void, env, i32, tl)
@@ -414,4 +413,4 @@ DEF_HELPER_3(store_601_batl, void, env, i32, tl)
 DEF_HELPER_3(store_601_batu, void, env, i32, tl)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index f638b2a07c6..783079d995d 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 #include "helper_regs.h"
@@ -287,23 +287,6 @@ target_ulong helper_602_mfrom(target_ulong arg)
     for (index = ARRAY_SIZE(r->element)-1; index >= 0; index--)
 #endif
 
-/* If X is a NaN, store the corresponding QNaN into RESULT.  Otherwise,
- * execute the following block.  */
-#define DO_HANDLE_NAN(result, x)                        \
-    if (float32_is_any_nan(x)) {                        \
-        CPU_FloatU __f;                                 \
-        __f.f = x;                                      \
-        __f.l = __f.l | (1 << 22);  /* Set QNaN bit. */ \
-        result = __f.f;                                 \
-    } else
-
-#define HANDLE_NAN1(result, x)                  \
-    DO_HANDLE_NAN(result, x)
-#define HANDLE_NAN2(result, x, y)                       \
-    DO_HANDLE_NAN(result, x) DO_HANDLE_NAN(result, y)
-#define HANDLE_NAN3(result, x, y, z)                                    \
-    DO_HANDLE_NAN(result, x) DO_HANDLE_NAN(result, y) DO_HANDLE_NAN(result, z)
-
 /* Saturating arithmetic helpers.  */
 #define SATCVT(from, to, from_type, to_type, min, max)          \
     static inline to_type cvt##from##to(from_type x, int *sat)  \
@@ -409,15 +392,29 @@ VARITH(uwm, u32)
         int i;                                                          \
                                                                         \
         for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
-            HANDLE_NAN2(r->f[i], a->f[i], b->f[i]) {                    \
-                r->f[i] = func(a->f[i], b->f[i], &env->vec_status);     \
-            }                                                           \
+            r->f[i] = func(a->f[i], b->f[i], &env->vec_status);         \
         }                                                               \
     }
 VARITHFP(addfp, float32_add)
 VARITHFP(subfp, float32_sub)
+VARITHFP(minfp, float32_min)
+VARITHFP(maxfp, float32_max)
 #undef VARITHFP
 
+#define VARITHFPFMA(suffix, type)                                       \
+    void helper_v##suffix(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, \
+                           ppc_avr_t *b, ppc_avr_t *c)                  \
+    {                                                                   \
+        int i;                                                          \
+        for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
+            r->f[i] = float32_muladd(a->f[i], c->f[i], b->f[i],         \
+                                     type, &env->vec_status);           \
+        }                                                               \
+    }
+VARITHFPFMA(maddfp, 0);
+VARITHFPFMA(nmsubfp, float_muladd_negate_result | float_muladd_negate_c);
+#undef VARITHFPFMA
+
 #define VARITHSAT_CASE(type, op, cvt, element)                          \
     {                                                                   \
         type result = (type)a->element[i] op (type)b->element[i];       \
@@ -649,27 +646,6 @@ VCT(uxs, cvtsduw, u32)
 VCT(sxs, cvtsdsw, s32)
 #undef VCT
 
-void helper_vmaddfp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b,
-                    ppc_avr_t *c)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN3(r->f[i], a->f[i], b->f[i], c->f[i]) {
-            /* Need to do the computation in higher precision and round
-             * once at the end.  */
-            float64 af, bf, cf, t;
-
-            af = float32_to_float64(a->f[i], &env->vec_status);
-            bf = float32_to_float64(b->f[i], &env->vec_status);
-            cf = float32_to_float64(c->f[i], &env->vec_status);
-            t = float64_mul(af, cf, &env->vec_status);
-            t = float64_add(t, bf, &env->vec_status);
-            r->f[i] = float64_to_float32(t, &env->vec_status);
-        }
-    }
-}
-
 void helper_vmhaddshs(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a,
                       ppc_avr_t *b, ppc_avr_t *c)
 {
@@ -730,27 +706,6 @@ VMINMAX(uw, u32)
 #undef VMINMAX_DO
 #undef VMINMAX
 
-#define VMINMAXFP(suffix, rT, rF)                                       \
-    void helper_v##suffix(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, \
-                          ppc_avr_t *b)                                 \
-    {                                                                   \
-        int i;                                                          \
-                                                                        \
-        for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
-            HANDLE_NAN2(r->f[i], a->f[i], b->f[i]) {                    \
-                if (float32_lt_quiet(a->f[i], b->f[i],                  \
-                                     &env->vec_status)) {               \
-                    r->f[i] = rT->f[i];                                 \
-                } else {                                                \
-                    r->f[i] = rF->f[i];                                 \
-                }                                                       \
-            }                                                           \
-        }                                                               \
-    }
-VMINMAXFP(minfp, a, b)
-VMINMAXFP(maxfp, b, a)
-#undef VMINMAXFP
-
 void helper_vmladduhm(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b, ppc_avr_t *c)
 {
     int i;
@@ -930,28 +885,6 @@ VMUL(uh, u16, u32)
 #undef VMUL_DO
 #undef VMUL
 
-void helper_vnmsubfp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a,
-                     ppc_avr_t *b, ppc_avr_t *c)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN3(r->f[i], a->f[i], b->f[i], c->f[i]) {
-            /* Need to do the computation is higher precision and round
-             * once at the end.  */
-            float64 af, bf, cf, t;
-
-            af = float32_to_float64(a->f[i], &env->vec_status);
-            bf = float32_to_float64(b->f[i], &env->vec_status);
-            cf = float32_to_float64(c->f[i], &env->vec_status);
-            t = float64_mul(af, cf, &env->vec_status);
-            t = float64_sub(t, bf, &env->vec_status);
-            t = float64_chs(t);
-            r->f[i] = float64_to_float32(t, &env->vec_status);
-        }
-    }
-}
-
 void helper_vperm(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b,
                   ppc_avr_t *c)
 {
@@ -1039,9 +972,7 @@ void helper_vrefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_div(float32_one, b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_div(float32_one, b->f[i], &env->vec_status);
     }
 }
 
@@ -1054,9 +985,7 @@ void helper_vrefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
                                                                 \
         set_float_rounding_mode(rounding, &s);                  \
         for (i = 0; i < ARRAY_SIZE(r->f); i++) {                \
-            HANDLE_NAN1(r->f[i], b->f[i]) {                     \
-                r->f[i] = float32_round_to_int (b->f[i], &s);   \
-            }                                                   \
+            r->f[i] = float32_round_to_int (b->f[i], &s);       \
         }                                                       \
     }
 VRFI(n, float_round_nearest_even)
@@ -1089,11 +1018,9 @@ void helper_vrsqrtefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            float32 t = float32_sqrt(b->f[i], &env->vec_status);
+        float32 t = float32_sqrt(b->f[i], &env->vec_status);
 
-            r->f[i] = float32_div(float32_one, t, &env->vec_status);
-        }
+        r->f[i] = float32_div(float32_one, t, &env->vec_status);
     }
 }
 
@@ -1109,9 +1036,7 @@ void helper_vexptefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_exp2(b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_exp2(b->f[i], &env->vec_status);
     }
 }
 
@@ -1120,9 +1045,7 @@ void helper_vlogefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_log2(b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_log2(b->f[i], &env->vec_status);
     }
 }
 
@@ -1473,10 +1396,6 @@ VUPK(lsh, s32, s16, UPKLO)
 #undef UPKHI
 #undef UPKLO
 
-#undef DO_HANDLE_NAN
-#undef HANDLE_NAN1
-#undef HANDLE_NAN2
-#undef HANDLE_NAN3
 #undef VECTOR_FOR_INORDER_I
 #undef HI_IDX
 #undef LO_IDX
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index a31d278a5f4..2c64c634f13 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -23,13 +23,13 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "cpu.h"
-#include "cpus.h"
-#include "device_tree.h"
+#include "sysemu/cpus.h"
+#include "sysemu/device_tree.h"
 #include "hw/sysbus.h"
 #include "hw/spapr.h"
 
@@ -60,6 +60,7 @@ static int cap_booke_sregs;
 static int cap_ppc_smt;
 static int cap_ppc_rma;
 static int cap_spapr_tce;
+static int cap_hior;
 
 /* XXX We have a race condition where we actually have a level triggered
  *     interrupt, but the infrastructure can't expose that yet, so the guest
@@ -72,9 +73,11 @@ static int cap_spapr_tce;
  */
 static QEMUTimer *idle_timer;
 
-static void kvm_kick_env(void *env)
+static void kvm_kick_cpu(void *opaque)
 {
-    qemu_cpu_kick(env);
+    PowerPCCPU *cpu = opaque;
+
+    qemu_cpu_kick(CPU(cpu));
 }
 
 int kvm_arch_init(KVMState *s)
@@ -86,6 +89,7 @@ int kvm_arch_init(KVMState *s)
     cap_ppc_smt = kvm_check_extension(s, KVM_CAP_PPC_SMT);
     cap_ppc_rma = kvm_check_extension(s, KVM_CAP_PPC_RMA);
     cap_spapr_tce = kvm_check_extension(s, KVM_CAP_SPAPR_TCE);
+    cap_hior = kvm_check_extension(s, KVM_CAP_PPC_HIOR);
 
     if (!cap_interrupt_level) {
         fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the "
@@ -95,8 +99,10 @@ int kvm_arch_init(KVMState *s)
     return 0;
 }
 
-static int kvm_arch_sync_sregs(CPUPPCState *cenv)
+static int kvm_arch_sync_sregs(PowerPCCPU *cpu)
 {
+    CPUPPCState *cenv = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_sregs sregs;
     int ret;
 
@@ -113,18 +119,20 @@ static int kvm_arch_sync_sregs(CPUPPCState *cenv)
         }
     }
 
-    ret = kvm_vcpu_ioctl(cenv, KVM_GET_SREGS, &sregs);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
     if (ret) {
         return ret;
     }
 
     sregs.pvr = cenv->spr[SPR_PVR];
-    return kvm_vcpu_ioctl(cenv, KVM_SET_SREGS, &sregs);
+    return kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
 }
 
 /* Set up a shared TLB array with KVM */
-static int kvm_booke206_tlb_init(CPUPPCState *env)
+static int kvm_booke206_tlb_init(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_book3e_206_tlb_params params = {};
     struct kvm_config_tlb cfg = {};
     struct kvm_enable_cap encap = {};
@@ -132,7 +140,7 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
     int ret, i;
 
     if (!kvm_enabled() ||
-        !kvm_check_extension(env->kvm_state, KVM_CAP_SW_TLB)) {
+        !kvm_check_extension(cs->kvm_state, KVM_CAP_SW_TLB)) {
         return 0;
     }
 
@@ -157,7 +165,7 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
     encap.cap = KVM_CAP_SW_TLB;
     encap.args[0] = (uintptr_t)&cfg;
 
-    ret = kvm_vcpu_ioctl(env, KVM_ENABLE_CAP, &encap);
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &encap);
     if (ret < 0) {
         fprintf(stderr, "%s: couldn't enable KVM_CAP_SW_TLB: %s\n",
                 __func__, strerror(-ret));
@@ -170,9 +178,12 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
 
 
 #if defined(TARGET_PPC64)
-static void kvm_get_fallback_smmu_info(CPUPPCState *env,
+static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
                                        struct kvm_ppc_smmu_info *info)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
     memset(info, 0, sizeof(*info));
 
     /* We don't have the new KVM_PPC_GET_SMMU_INFO ioctl, so
@@ -198,7 +209,7 @@ static void kvm_get_fallback_smmu_info(CPUPPCState *env,
      *   implements KVM_CAP_PPC_GET_SMMU_INFO and thus doesn't hit
      *   this fallback.
      */
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
         /* No flags */
         info->flags = 0;
         info->slb_size = 64;
@@ -254,18 +265,19 @@ static void kvm_get_fallback_smmu_info(CPUPPCState *env,
     }
 }
 
-static void kvm_get_smmu_info(CPUPPCState *env, struct kvm_ppc_smmu_info *info)
+static void kvm_get_smmu_info(PowerPCCPU *cpu, struct kvm_ppc_smmu_info *info)
 {
+    CPUState *cs = CPU(cpu);
     int ret;
 
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_SMMU_INFO)) {
-        ret = kvm_vm_ioctl(env->kvm_state, KVM_PPC_GET_SMMU_INFO, info);
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_SMMU_INFO)) {
+        ret = kvm_vm_ioctl(cs->kvm_state, KVM_PPC_GET_SMMU_INFO, info);
         if (ret == 0) {
             return;
         }
     }
 
-    kvm_get_fallback_smmu_info(env, info);
+    kvm_get_fallback_smmu_info(cpu, info);
 }
 
 static long getrampagesize(void)
@@ -308,10 +320,11 @@ static bool kvm_valid_page_size(uint32_t flags, long rampgsize, uint32_t shift)
     return (1ul << shift) <= rampgsize;
 }
 
-static void kvm_fixup_page_sizes(CPUPPCState *env)
+static void kvm_fixup_page_sizes(PowerPCCPU *cpu)
 {
     static struct kvm_ppc_smmu_info smmu_info;
     static bool has_smmu_info;
+    CPUPPCState *env = &cpu->env;
     long rampagesize;
     int iq, ik, jq, jk;
 
@@ -322,7 +335,7 @@ static void kvm_fixup_page_sizes(CPUPPCState *env)
 
     /* Collect MMU info from kernel if not already */
     if (!has_smmu_info) {
-        kvm_get_smmu_info(env, &smmu_info);
+        kvm_get_smmu_info(cpu, &smmu_info);
         has_smmu_info = true;
     }
 
@@ -365,31 +378,38 @@ static void kvm_fixup_page_sizes(CPUPPCState *env)
 }
 #else /* defined (TARGET_PPC64) */
 
-static inline void kvm_fixup_page_sizes(CPUPPCState *env)
+static inline void kvm_fixup_page_sizes(PowerPCCPU *cpu)
 {
 }
 
 #endif /* !defined (TARGET_PPC64) */
 
-int kvm_arch_init_vcpu(CPUPPCState *cenv)
+unsigned long kvm_arch_vcpu_id(CPUState *cpu)
 {
+    return cpu->cpu_index;
+}
+
+int kvm_arch_init_vcpu(CPUState *cs)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *cenv = &cpu->env;
     int ret;
 
     /* Gather server mmu info from KVM and update the CPU state */
-    kvm_fixup_page_sizes(cenv);
+    kvm_fixup_page_sizes(cpu);
 
     /* Synchronize sregs with kvm */
-    ret = kvm_arch_sync_sregs(cenv);
+    ret = kvm_arch_sync_sregs(cpu);
     if (ret) {
         return ret;
     }
 
-    idle_timer = qemu_new_timer_ns(vm_clock, kvm_kick_env, cenv);
+    idle_timer = qemu_new_timer_ns(vm_clock, kvm_kick_cpu, cpu);
 
     /* Some targets support access to KVM's guest TLB. */
     switch (cenv->mmu_model) {
     case POWERPC_MMU_BOOKE206:
-        ret = kvm_booke206_tlb_init(cenv);
+        ret = kvm_booke206_tlb_init(cpu);
         break;
     default:
         break;
@@ -398,12 +418,14 @@ int kvm_arch_init_vcpu(CPUPPCState *cenv)
     return ret;
 }
 
-void kvm_arch_reset_vcpu(CPUPPCState *env)
+void kvm_arch_reset_vcpu(CPUState *cpu)
 {
 }
 
-static void kvm_sw_tlb_put(CPUPPCState *env)
+static void kvm_sw_tlb_put(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_dirty_tlb dirty_tlb;
     unsigned char *bitmap;
     int ret;
@@ -418,7 +440,7 @@ static void kvm_sw_tlb_put(CPUPPCState *env)
     dirty_tlb.bitmap = (uintptr_t)bitmap;
     dirty_tlb.num_dirty = env->nb_tlb;
 
-    ret = kvm_vcpu_ioctl(env, KVM_DIRTY_TLB, &dirty_tlb);
+    ret = kvm_vcpu_ioctl(cs, KVM_DIRTY_TLB, &dirty_tlb);
     if (ret) {
         fprintf(stderr, "%s: KVM_DIRTY_TLB: %s\n",
                 __func__, strerror(-ret));
@@ -427,15 +449,18 @@ static void kvm_sw_tlb_put(CPUPPCState *env)
     g_free(bitmap);
 }
 
-int kvm_arch_put_registers(CPUPPCState *env, int level)
+int kvm_arch_put_registers(CPUState *cs, int level)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
     int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0)
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
+    if (ret < 0) {
         return ret;
+    }
 
     regs.ctr = env->ctr;
     regs.lr  = env->lr;
@@ -460,26 +485,76 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
     for (i = 0;i < 32; i++)
         regs.gpr[i] = env->gpr[i];
 
-    ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_REGS, &regs);
     if (ret < 0)
         return ret;
 
     if (env->tlb_dirty) {
-        kvm_sw_tlb_put(env);
+        kvm_sw_tlb_put(cpu);
         env->tlb_dirty = false;
     }
 
+    if (cap_segstate && (level >= KVM_PUT_RESET_STATE)) {
+        struct kvm_sregs sregs;
+
+        sregs.pvr = env->spr[SPR_PVR];
+
+        sregs.u.s.sdr1 = env->spr[SPR_SDR1];
+
+        /* Sync SLB */
+#ifdef TARGET_PPC64
+        for (i = 0; i < 64; i++) {
+            sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
+            sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
+        }
+#endif
+
+        /* Sync SRs */
+        for (i = 0; i < 16; i++) {
+            sregs.u.s.ppc32.sr[i] = env->sr[i];
+        }
+
+        /* Sync BATs */
+        for (i = 0; i < 8; i++) {
+            /* Beware. We have to swap upper and lower bits here */
+            sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
+                | env->DBAT[1][i];
+            sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
+                | env->IBAT[1][i];
+        }
+
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    if (cap_hior && (level >= KVM_PUT_RESET_STATE)) {
+        uint64_t hior = env->spr[SPR_HIOR];
+        struct kvm_one_reg reg = {
+            .id = KVM_REG_PPC_HIOR,
+            .addr = (uintptr_t) &hior,
+        };
+
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }
+
     return ret;
 }
 
-int kvm_arch_get_registers(CPUPPCState *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
     struct kvm_sregs sregs;
     uint32_t cr;
     int i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
     if (ret < 0)
         return ret;
 
@@ -513,7 +588,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
         env->gpr[i] = regs.gpr[i];
 
     if (cap_booke_sregs) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
@@ -617,7 +692,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
     }
 
     if (cap_segstate) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
@@ -649,7 +724,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
     return 0;
 }
 
-int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
+int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     unsigned virq = level ? KVM_INTERRUPT_SET_LEVEL : KVM_INTERRUPT_UNSET;
 
@@ -661,7 +736,7 @@ int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
         return 0;
     }
 
-    kvm_vcpu_ioctl(env, KVM_INTERRUPT, &virq);
+    kvm_vcpu_ioctl(CPU(cpu), KVM_INTERRUPT, &virq);
 
     return 0;
 }
@@ -674,8 +749,10 @@ int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
 #define PPC_INPUT_INT PPC6xx_INPUT_INT
 #endif
 
-void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     int r;
     unsigned irq;
 
@@ -693,9 +770,10 @@ void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
         irq = KVM_INTERRUPT_SET;
 
         dprintf("injected interrupt %d\n", irq);
-        r = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &irq);
-        if (r < 0)
-            printf("cpu %d fail inject %x\n", env->cpu_index, irq);
+        r = kvm_vcpu_ioctl(cs, KVM_INTERRUPT, &irq);
+        if (r < 0) {
+            printf("cpu %d fail inject %x\n", cs->cpu_index, irq);
+        }
 
         /* Always wake up soon in case the interrupt was level based */
         qemu_mod_timer(idle_timer, qemu_get_clock_ns(vm_clock) +
@@ -707,13 +785,14 @@ void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
      * anyways, so we will get a chance to deliver the rest. */
 }
 
-void kvm_arch_post_run(CPUPPCState *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-int kvm_arch_process_async_events(CPUPPCState *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    return env->halted;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    return cpu->env.halted;
 }
 
 static int kvmppc_handle_halt(CPUPPCState *env)
@@ -743,8 +822,10 @@ static int kvmppc_handle_dcr_write(CPUPPCState *env, uint32_t dcrn, uint32_t dat
     return 0;
 }
 
-int kvm_arch_handle_exit(CPUPPCState *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     int ret;
 
     switch (run->exit_reason) {
@@ -764,11 +845,17 @@ int kvm_arch_handle_exit(CPUPPCState *env, struct kvm_run *run)
 #ifdef CONFIG_PSERIES
     case KVM_EXIT_PAPR_HCALL:
         dprintf("handle PAPR hypercall\n");
-        run->papr_hcall.ret = spapr_hypercall(env, run->papr_hcall.nr,
+        run->papr_hcall.ret = spapr_hypercall(cpu,
+                                              run->papr_hcall.nr,
                                               run->papr_hcall.args);
         ret = 0;
         break;
 #endif
+    case KVM_EXIT_EPR:
+        dprintf("handle epr\n");
+        run->epr.epr = ldl_phys(env->mpic_iack);
+        ret = 0;
+        break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
         ret = -1;
@@ -795,7 +882,7 @@ static int read_cpuinfo(const char *field, char *value, int len)
             break;
         }
         if (!strncmp(line, field, field_len)) {
-            strncpy(value, line, len);
+            pstrcpy(value, len, line);
             ret = 0;
             break;
         }
@@ -913,16 +1000,38 @@ uint32_t kvmppc_get_dfp(void)
     return kvmppc_read_int_cpu_dt("ibm,dfp");
 }
 
+static int kvmppc_get_pvinfo(CPUPPCState *env, struct kvm_ppc_pvinfo *pvinfo)
+ {
+     PowerPCCPU *cpu = ppc_env_get_cpu(env);
+     CPUState *cs = CPU(cpu);
+
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
+        !kvm_vm_ioctl(cs->kvm_state, KVM_PPC_GET_PVINFO, pvinfo)) {
+        return 0;
+    }
+
+    return 1;
+}
+
+int kvmppc_get_hasidle(CPUPPCState *env)
+{
+    struct kvm_ppc_pvinfo pvinfo;
+
+    if (!kvmppc_get_pvinfo(env, &pvinfo) &&
+        (pvinfo.flags & KVM_PPC_PVINFO_FLAGS_EV_IDLE)) {
+        return 1;
+    }
+
+    return 0;
+}
+
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
 {
     uint32_t *hc = (uint32_t*)buf;
-
     struct kvm_ppc_pvinfo pvinfo;
 
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
-        !kvm_vm_ioctl(env->kvm_state, KVM_PPC_GET_PVINFO, &pvinfo)) {
+    if (!kvmppc_get_pvinfo(env, &pvinfo)) {
         memcpy(buf, pvinfo.hcall, buf_len);
-
         return 0;
     }
 
@@ -943,55 +1052,35 @@ int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
     return 0;
 }
 
-void kvmppc_set_papr(CPUPPCState *env)
+void kvmppc_set_papr(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_enable_cap cap = {};
-    struct kvm_one_reg reg = {};
-    struct kvm_sregs sregs = {};
     int ret;
-    uint64_t hior = env->spr[SPR_HIOR];
 
     cap.cap = KVM_CAP_PPC_PAPR;
-    ret = kvm_vcpu_ioctl(env, KVM_ENABLE_CAP, &cap);
-
-    if (ret) {
-        goto fail;
-    }
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &cap);
 
-    /*
-     * XXX We set HIOR here. It really should be a qdev property of
-     *     the CPU node, but we don't have CPUs converted to qdev yet.
-     *
-     *     Once we have qdev CPUs, move HIOR to a qdev property and
-     *     remove this chunk.
-     */
-    reg.id = KVM_REG_PPC_HIOR;
-    reg.addr = (uintptr_t)&hior;
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &reg);
     if (ret) {
-        fprintf(stderr, "Couldn't set HIOR. Maybe you're running an old \n"
-                        "kernel with support for HV KVM but no PAPR PR \n"
-                        "KVM in which case things will work. If they don't \n"
-                        "please update your host kernel!\n");
+        cpu_abort(env, "This KVM version does not support PAPR\n");
     }
+}
 
-    /* Set SDR1 so kernel space finds the HTAB */
-    ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
-    if (ret) {
-        goto fail;
-    }
+void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy)
+{
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_enable_cap cap = {};
+    int ret;
 
-    sregs.u.s.sdr1 = env->spr[SPR_SDR1];
+    cap.cap = KVM_CAP_PPC_EPR;
+    cap.args[0] = mpic_proxy;
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &cap);
 
-    ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
-    if (ret) {
-        goto fail;
+    if (ret && mpic_proxy) {
+        cpu_abort(env, "This KVM version does not support EPR\n");
     }
-
-    return;
-
-fail:
-    cpu_abort(env, "This KVM version does not support PAPR\n");
 }
 
 int kvmppc_smt_threads(void)
@@ -999,6 +1088,7 @@ int kvmppc_smt_threads(void)
     return cap_ppc_smt ? cap_ppc_smt : 1;
 }
 
+#ifdef TARGET_PPC64
 off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem)
 {
     void *rma;
@@ -1042,6 +1132,16 @@ off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem)
     return size;
 }
 
+uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift)
+{
+    if (cap_ppc_rma >= 2) {
+        return current_size;
+    }
+    return MIN(current_size,
+               getrampagesize() << (hash_shift - 7));
+}
+#endif
+
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd)
 {
     struct kvm_create_spapr_tce args = {
@@ -1101,6 +1201,44 @@ int kvmppc_remove_spapr_tce(void *table, int fd, uint32_t window_size)
     return 0;
 }
 
+int kvmppc_reset_htab(int shift_hint)
+{
+    uint32_t shift = shift_hint;
+
+    if (!kvm_enabled()) {
+        /* Full emulation, tell caller to allocate htab itself */
+        return 0;
+    }
+    if (kvm_check_extension(kvm_state, KVM_CAP_PPC_ALLOC_HTAB)) {
+        int ret;
+        ret = kvm_vm_ioctl(kvm_state, KVM_PPC_ALLOCATE_HTAB, &shift);
+        if (ret == -ENOTTY) {
+            /* At least some versions of PR KVM advertise the
+             * capability, but don't implement the ioctl().  Oops.
+             * Return 0 so that we allocate the htab in qemu, as is
+             * correct for PR. */
+            return 0;
+        } else if (ret < 0) {
+            return ret;
+        }
+        return shift;
+    }
+
+    /* We have a kernel that predates the htab reset calls.  For PR
+     * KVM, we need to allocate the htab ourselves, for an HV KVM of
+     * this era, it has allocated a 16MB fixed size hash table
+     * already.  Kernels of this era have the GET_PVINFO capability
+     * only on PR, so we use this hack to determine the right
+     * answer */
+    if (kvm_check_extension(kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
+        /* PR - tell caller to allocate htab */
+        return 0;
+    } else {
+        /* HV - assume 16MB kernel allocated htab */
+        return 24;
+    }
+}
+
 static inline uint32_t mfpvr(void)
 {
     uint32_t pvr;
@@ -1119,18 +1257,37 @@ static void alter_insns(uint64_t *word, uint64_t flags, bool on)
     }
 }
 
-const ppc_def_t *kvmppc_host_cpu_def(void)
+static void kvmppc_host_cpu_initfn(Object *obj)
+{
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(obj);
+
+    assert(kvm_enabled());
+
+    if (pcc->info->pvr != mfpvr()) {
+        fprintf(stderr, "Your host CPU is unsupported.\n"
+                "Please choose a supported model instead, see -cpu ?.\n");
+        exit(1);
+    }
+}
+
+static void kvmppc_host_cpu_class_init(ObjectClass *oc, void *data)
 {
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
     uint32_t host_pvr = mfpvr();
-    const ppc_def_t *base_spec;
+    PowerPCCPUClass *pvr_pcc;
     ppc_def_t *spec;
     uint32_t vmx = kvmppc_get_vmx();
     uint32_t dfp = kvmppc_get_dfp();
 
-    base_spec = ppc_find_by_pvr(host_pvr);
-
     spec = g_malloc0(sizeof(*spec));
-    memcpy(spec, base_spec, sizeof(*spec));
+
+    pvr_pcc = ppc_cpu_class_by_pvr(host_pvr);
+    if (pvr_pcc != NULL) {
+        memcpy(spec, pvr_pcc->info, sizeof(*spec));
+    }
+    pcc->info = spec;
+    /* Override the display name for -cpu ? and QMP */
+    pcc->info->name = "host";
 
     /* Now fix up the spec with information we can query from the host */
 
@@ -1143,29 +1300,28 @@ const ppc_def_t *kvmppc_host_cpu_def(void)
         /* Only override when we know what the host supports */
         alter_insns(&spec->insns_flags2, PPC2_DFP, dfp);
     }
-
-    return spec;
 }
 
-int kvmppc_fixup_cpu(CPUPPCState *env)
+int kvmppc_fixup_cpu(PowerPCCPU *cpu)
 {
+    CPUState *cs = CPU(cpu);
     int smt;
 
     /* Adjust cpu index for SMT */
     smt = kvmppc_smt_threads();
-    env->cpu_index = (env->cpu_index / smp_threads) * smt
-        + (env->cpu_index % smp_threads);
+    cs->cpu_index = (cs->cpu_index / smp_threads) * smt
+        + (cs->cpu_index % smp_threads);
 
     return 0;
 }
 
 
-bool kvm_arch_stop_on_emulation_error(CPUPPCState *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu)
 {
     return true;
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUPPCState *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
@@ -1174,3 +1330,17 @@ int kvm_arch_on_sigbus(int code, void *addr)
 {
     return 1;
 }
+
+static const TypeInfo kvm_host_cpu_type_info = {
+    .name = TYPE_HOST_POWERPC_CPU,
+    .parent = TYPE_POWERPC_CPU,
+    .instance_init = kvmppc_host_cpu_initfn,
+    .class_init = kvmppc_host_cpu_class_init,
+};
+
+static void kvm_ppc_register_types(void)
+{
+    type_register_static(&kvm_host_cpu_type_info);
+}
+
+type_init(kvm_ppc_register_types)
diff --git a/target-ppc/kvm_ppc.c b/target-ppc/kvm_ppc.c
index a2e49cd4235..1b192a80385 100644
--- a/target-ppc/kvm_ppc.c
+++ b/target-ppc/kvm_ppc.c
@@ -12,9 +12,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 #define PROC_DEVTREE_PATH "/proc/device-tree"
 
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index e2f87038536..c30b0066744 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -9,7 +9,9 @@
 #ifndef __KVM_PPC_H__
 #define __KVM_PPC_H__
 
-#include "memory.h"
+#include "exec/memory.h"
+
+#define TYPE_HOST_POWERPC_CPU "host-" TYPE_POWERPC_CPU
 
 void kvmppc_init(void);
 
@@ -19,17 +21,20 @@ uint32_t kvmppc_get_tbfreq(void);
 uint64_t kvmppc_get_clockfreq(void);
 uint32_t kvmppc_get_vmx(void);
 uint32_t kvmppc_get_dfp(void);
+int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
-int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level);
-void kvmppc_set_papr(CPUPPCState *env);
+int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
+void kvmppc_set_papr(PowerPCCPU *cpu);
+void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
 int kvmppc_smt_threads(void);
 #ifndef CONFIG_USER_ONLY
 off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem);
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd);
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
+int kvmppc_reset_htab(int shift_hint);
+uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
 #endif /* !CONFIG_USER_ONLY */
-const ppc_def_t *kvmppc_host_cpu_def(void);
-int kvmppc_fixup_cpu(CPUPPCState *env);
+int kvmppc_fixup_cpu(PowerPCCPU *cpu);
 
 #else
 
@@ -53,6 +58,11 @@ static inline uint32_t kvmppc_get_dfp(void)
     return 0;
 }
 
+static inline int kvmppc_get_hasidle(CPUPPCState *env)
+{
+    return 0;
+}
+
 static inline int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
 {
     return -1;
@@ -63,12 +73,16 @@ static inline int kvmppc_read_segment_page_sizes(uint32_t *prop, int maxcells)
     return -1;
 }
 
-static inline int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
+static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     return -1;
 }
 
-static inline void kvmppc_set_papr(CPUPPCState *env)
+static inline void kvmppc_set_papr(PowerPCCPU *cpu)
+{
+}
+
+static inline void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy)
 {
 }
 
@@ -94,14 +108,26 @@ static inline int kvmppc_remove_spapr_tce(void *table, int pfd,
 {
     return -1;
 }
-#endif /* !CONFIG_USER_ONLY */
 
-static inline const ppc_def_t *kvmppc_host_cpu_def(void)
+static inline int kvmppc_reset_htab(int shift_hint)
 {
-    return NULL;
+    return -1;
+}
+
+static inline uint64_t kvmppc_rma_size(uint64_t current_size,
+                                       unsigned int hash_shift)
+{
+    return ram_size;
+}
+
+static inline int kvmppc_update_sdr1(CPUPPCState *env)
+{
+    return 0;
 }
 
-static inline int kvmppc_fixup_cpu(CPUPPCState *env)
+#endif /* !CONFIG_USER_ONLY */
+
+static inline int kvmppc_fixup_cpu(PowerPCCPU *cpu)
 {
     return -1;
 }
diff --git a/target-ppc/machine.c b/target-ppc/machine.c
index d6c2ee41b39..e014c0c1aff 100644
--- a/target-ppc/machine.c
+++ b/target-ppc/machine.c
@@ -1,11 +1,12 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 void cpu_save(QEMUFile *f, void *opaque)
 {
     CPUPPCState *env = (CPUPPCState *)opaque;
     unsigned int i, j;
+    uint32_t fpscr;
 
     for (i = 0; i < 32; i++)
         qemu_put_betls(f, &env->gpr[i]);
@@ -30,7 +31,8 @@ void cpu_save(QEMUFile *f, void *opaque)
         u.d = env->fpr[i];
         qemu_put_be64(f, u.l);
     }
-    qemu_put_be32s(f, &env->fpscr);
+    fpscr = env->fpscr;
+    qemu_put_be32s(f, &fpscr);
     qemu_put_sbe32s(f, &env->access_type);
 #if defined(TARGET_PPC64)
     qemu_put_betls(f, &env->asr);
@@ -82,7 +84,7 @@ void cpu_save(QEMUFile *f, void *opaque)
     qemu_put_betls(f, &env->hflags);
     qemu_put_betls(f, &env->hflags_nmsr);
     qemu_put_sbe32s(f, &env->mmu_idx);
-    qemu_put_sbe32s(f, &env->power_mode);
+    qemu_put_sbe32(f, 0);
 }
 
 int cpu_load(QEMUFile *f, void *opaque, int version_id)
@@ -90,6 +92,7 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
     CPUPPCState *env = (CPUPPCState *)opaque;
     unsigned int i, j;
     target_ulong sdr1;
+    uint32_t fpscr;
 
     for (i = 0; i < 32; i++)
         qemu_get_betls(f, &env->gpr[i]);
@@ -114,7 +117,8 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
         u.l = qemu_get_be64(f);
         env->fpr[i] = u.d;
     }
-    qemu_get_be32s(f, &env->fpscr);
+    qemu_get_be32s(f, &fpscr);
+    env->fpscr = fpscr;
     qemu_get_sbe32s(f, &env->access_type);
 #if defined(TARGET_PPC64)
     qemu_get_betls(f, &env->asr);
@@ -167,7 +171,7 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
     qemu_get_betls(f, &env->hflags);
     qemu_get_betls(f, &env->hflags_nmsr);
     qemu_get_sbe32s(f, &env->mmu_idx);
-    qemu_get_sbe32s(f, &env->power_mode);
+    qemu_get_sbe32(f); /* Discard unused power_mode */
 
     return 0;
 }
diff --git a/target-ppc/mem_helper.c b/target-ppc/mem_helper.c
index 5b5f1bdd23d..902b1cd8231 100644
--- a/target-ppc/mem_helper.c
+++ b/target-ppc/mem_helper.c
@@ -17,13 +17,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 #include "helper_regs.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 //#define DEBUG_OP
@@ -257,16 +257,16 @@ STVE(stvewx, cpu_stl_data, bswap32, u32)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -275,19 +275,13 @@ STVE(stvewx, cpu_stl_data, bswap32, u32)
 void tlb_fill(CPUPPCState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_ppc_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret != 0)) {
         if (likely(retaddr)) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (likely(tb)) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         helper_raise_exception_err(env, env->exception_index, env->error_code);
     }
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index d2664acef01..0aee7a90633 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -18,7 +18,7 @@
  */
 #include "cpu.h"
 #include "helper.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 //#define DEBUG_MMU
@@ -215,7 +215,7 @@ static inline int pte_check(mmu_ctx_t *ctx, int is_64b, target_ulong pte0,
             pp = pte1 & 0x00000003;
         }
         if (ptem == ctx->ptem) {
-            if (ctx->raddr != (target_phys_addr_t)-1ULL) {
+            if (ctx->raddr != (hwaddr)-1ULL) {
                 /* all matches should have equal RPN, WIMG & PP */
                 if ((ctx->raddr & mmask) != (pte1 & mmask)) {
                     qemu_log("Bad RPN/WIMG/PP\n");
@@ -556,8 +556,8 @@ static inline int get_bat(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-static inline target_phys_addr_t get_pteg_offset(CPUPPCState *env,
-                                                 target_phys_addr_t hash,
+static inline hwaddr get_pteg_offset(CPUPPCState *env,
+                                                 hwaddr hash,
                                                  int pte_size)
 {
     return (hash * pte_size * 8) & env->htab_mask;
@@ -567,7 +567,7 @@ static inline target_phys_addr_t get_pteg_offset(CPUPPCState *env,
 static inline int find_pte2(CPUPPCState *env, mmu_ctx_t *ctx, int is_64b, int h,
                             int rw, int type, int target_page_bits)
 {
-    target_phys_addr_t pteg_off;
+    hwaddr pteg_off;
     target_ulong pte0, pte1;
     int i, good = -1;
     int ret, r;
@@ -817,7 +817,7 @@ static int ppc_load_slb_vsid(CPUPPCState *env, target_ulong rb,
 static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
                               target_ulong eaddr, int rw, int type)
 {
-    target_phys_addr_t hash;
+    hwaddr hash;
     target_ulong vsid;
     int ds, pr, target_page_bits;
     int ret, ret2;
@@ -896,7 +896,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
             ctx->hash[1] = ~hash;
 
             /* Initialize real address with an invalid value */
-            ctx->raddr = (target_phys_addr_t)-1ULL;
+            ctx->raddr = (hwaddr)-1ULL;
             if (unlikely(env->mmu_model == POWERPC_MMU_SOFT_6xx ||
                          env->mmu_model == POWERPC_MMU_SOFT_74xx)) {
                 /* Software TLB search */
@@ -926,7 +926,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
             }
 #if defined(DUMP_PAGE_TABLES)
             if (qemu_log_enabled()) {
-                target_phys_addr_t curaddr;
+                hwaddr curaddr;
                 uint32_t a0, a1, a2, a3;
 
                 qemu_log("Page table: " TARGET_FMT_plx " len " TARGET_FMT_plx
@@ -1009,7 +1009,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
 
 /* Generic TLB check function for embedded PowerPC implementations */
 static int ppcemb_tlb_check(CPUPPCState *env, ppcemb_tlb_t *tlb,
-                            target_phys_addr_t *raddrp,
+                            hwaddr *raddrp,
                             target_ulong address, uint32_t pid, int ext,
                             int i)
 {
@@ -1032,12 +1032,10 @@ static int ppcemb_tlb_check(CPUPPCState *env, ppcemb_tlb_t *tlb,
         return -1;
     }
     *raddrp = (tlb->RPN & mask) | (address & ~mask);
-#if (TARGET_PHYS_ADDR_BITS >= 36)
     if (ext) {
         /* Extend the physical address to 36 bits */
-        *raddrp |= (target_phys_addr_t)(tlb->RPN & 0xF) << 32;
+        *raddrp |= (uint64_t)(tlb->RPN & 0xF) << 32;
     }
-#endif
 
     return 0;
 }
@@ -1047,7 +1045,7 @@ static int ppcemb_tlb_search(CPUPPCState *env, target_ulong address,
                              uint32_t pid)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret;
 
     /* Default return value is no match */
@@ -1081,7 +1079,7 @@ static inline void ppc4xx_tlb_invalidate_virt(CPUPPCState *env,
 {
 #if !defined(FLUSH_ALL_TLBS)
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     target_ulong page, end;
     int i;
 
@@ -1106,11 +1104,11 @@ static int mmu40x_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                        int access_type)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret, zsel, zpr, pr;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
     pr = msr_pr;
     for (i = 0; i < env->nb_tlb; i++) {
         tlb = &env->tlb.tlbe[i];
@@ -1177,7 +1175,7 @@ void store_40x_sler(CPUPPCState *env, uint32_t val)
 }
 
 static inline int mmubooke_check_tlb(CPUPPCState *env, ppcemb_tlb_t *tlb,
-                                     target_phys_addr_t *raddr, int *prot,
+                                     hwaddr *raddr, int *prot,
                                      target_ulong address, int rw,
                                      int access_type, int i)
 {
@@ -1251,11 +1249,11 @@ static int mmubooke_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                          int access_type)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
     for (i = 0; i < env->nb_tlb; i++) {
         tlb = &env->tlb.tlbe[i];
         ret = mmubooke_check_tlb(env, tlb, &raddr, &ctx->prot, address, rw,
@@ -1278,7 +1276,8 @@ static int mmubooke_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot)
+static void booke206_flush_tlb(CPUPPCState *env, int flags,
+                               const int check_iprot)
 {
     int tlb_size;
     int i, j;
@@ -1299,8 +1298,8 @@ void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot)
     tlb_flush(env, 1);
 }
 
-target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env,
-                                             ppcmas_tlb_t *tlb)
+static hwaddr booke206_tlb_to_page_size(CPUPPCState *env,
+                                        ppcmas_tlb_t *tlb)
 {
     int tlbm_size;
 
@@ -1311,7 +1310,7 @@ target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env,
 
 /* TLB check function for MAS based SoftTLBs */
 int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                     target_phys_addr_t *raddrp,
+                     hwaddr *raddrp,
                      target_ulong address, uint32_t pid)
 {
     target_ulong mask;
@@ -1347,7 +1346,7 @@ int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
 }
 
 static int mmubooke206_check_tlb(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                                 target_phys_addr_t *raddr, int *prot,
+                                 hwaddr *raddr, int *prot,
                                  target_ulong address, int rw,
                                  int access_type)
 {
@@ -1437,11 +1436,11 @@ static int mmubooke206_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                             int access_type)
 {
     ppcmas_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, j, ret;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
 
     for (i = 0; i < BOOKE206_MAX_TLBN; i++) {
         int ways = booke206_tlb_ways(env, i);
@@ -1498,7 +1497,7 @@ static void mmubooke_dump_mmu(FILE *f, fprintf_function cpu_fprintf,
 
     entry = &env->tlb.tlbe[0];
     for (i = 0; i < env->nb_tlb; i++, entry++) {
-        target_phys_addr_t ea, pa;
+        hwaddr ea, pa;
         target_ulong mask;
         uint64_t size = (uint64_t)entry->size;
         char size_buf[20];
@@ -1511,10 +1510,8 @@ static void mmubooke_dump_mmu(FILE *f, fprintf_function cpu_fprintf,
         mask = ~(entry->size - 1);
         ea = entry->EPN & mask;
         pa = entry->RPN & mask;
-#if (TARGET_PHYS_ADDR_BITS >= 36)
         /* Extend the physical address to 36 bits */
-        pa |= (target_phys_addr_t)(entry->RPN & 0xF) << 32;
-#endif
+        pa |= (hwaddr)(entry->RPN & 0xF) << 32;
         size /= 1024;
         if (size >= 1024) {
             snprintf(size_buf, sizeof(size_buf), "%3" PRId64 "M", size / 1024);
@@ -1541,7 +1538,7 @@ static void mmubooke206_dump_one_tlb(FILE *f, fprintf_function cpu_fprintf,
 
     entry = &env->tlb.tlbm[offset];
     for (i = 0; i < tlbsize; i++, entry++) {
-        target_phys_addr_t ea, pa, size;
+        hwaddr ea, pa, size;
         int tsize;
 
         if (!(entry->mas1 & MAS1_VALID)) {
@@ -1710,8 +1707,8 @@ static inline int check_physical(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx, target_ulong eaddr,
-                         int rw, int access_type)
+static int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
+                                target_ulong eaddr, int rw, int access_type)
 {
     int ret;
 
@@ -1787,7 +1784,7 @@ int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx, target_ulong eaddr,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
 {
     mmu_ctx_t ctx;
 
@@ -3147,7 +3144,7 @@ void helper_booke206_tlbsx(CPUPPCState *env, target_ulong address)
 {
     ppcmas_tlb_t *tlb = NULL;
     int i, j;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     uint32_t spid, sas;
 
     spid = (env->spr[SPR_BOOKE_MAS6] & MAS6_SPID_MASK) >> MAS6_SPID_SHIFT;
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index ac915ccadec..798b7acfc90 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -19,9 +19,9 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -68,10 +68,10 @@ static TCGv cpu_cfar;
 #endif
 static TCGv cpu_xer;
 static TCGv cpu_reserve;
-static TCGv_i32 cpu_fpscr;
+static TCGv cpu_fpscr;
 static TCGv_i32 cpu_access_type;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void ppc_translate_init(void)
 {
@@ -163,8 +163,8 @@ void ppc_translate_init(void)
                                      offsetof(CPUPPCState, reserve_addr),
                                      "reserve_addr");
 
-    cpu_fpscr = tcg_global_mem_new_i32(TCG_AREG0,
-                                       offsetof(CPUPPCState, fpscr), "fpscr");
+    cpu_fpscr = tcg_global_mem_new(TCG_AREG0,
+                                   offsetof(CPUPPCState, fpscr), "fpscr");
 
     cpu_access_type = tcg_global_mem_new_i32(TCG_AREG0,
                                              offsetof(CPUPPCState, access_type), "access_type");
@@ -2302,6 +2302,7 @@ GEN_FLOAT_B(neg, 0x08, 0x01, 0, PPC_FLOAT);
 /* mcrfs */
 static void gen_mcrfs(DisasContext *ctx)
 {
+    TCGv tmp = tcg_temp_new();
     int bfa;
 
     if (unlikely(!ctx->fpu_enabled)) {
@@ -2309,9 +2310,11 @@ static void gen_mcrfs(DisasContext *ctx)
         return;
     }
     bfa = 4 * (7 - crfS(ctx->opcode));
-    tcg_gen_shri_i32(cpu_crf[crfD(ctx->opcode)], cpu_fpscr, bfa);
+    tcg_gen_shri_tl(tmp, cpu_fpscr, bfa);
+    tcg_gen_trunc_tl_i32(cpu_crf[crfD(ctx->opcode)], tmp);
+    tcg_temp_free(tmp);
     tcg_gen_andi_i32(cpu_crf[crfD(ctx->opcode)], cpu_crf[crfD(ctx->opcode)], 0xf);
-    tcg_gen_andi_i32(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
+    tcg_gen_andi_tl(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
 }
 
 /* mffs */
@@ -2322,7 +2325,7 @@ static void gen_mffs(DisasContext *ctx)
         return;
     }
     gen_reset_fpstatus();
-    tcg_gen_extu_i32_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpscr);
+    tcg_gen_extu_tl_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpscr);
     gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
 }
 
@@ -2346,7 +2349,8 @@ static void gen_mtfsb0(DisasContext *ctx)
         tcg_temp_free_i32(t0);
     }
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
 }
 
@@ -2371,7 +2375,8 @@ static void gen_mtfsb1(DisasContext *ctx)
         tcg_temp_free_i32(t0);
     }
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -2397,7 +2402,8 @@ static void gen_mtfsf(DisasContext *ctx)
     gen_helper_store_fpscr(cpu_env, cpu_fpr[rB(ctx->opcode)], t0);
     tcg_temp_free_i32(t0);
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -2425,7 +2431,8 @@ static void gen_mtfsfi(DisasContext *ctx)
     tcg_temp_free_i64(t0);
     tcg_temp_free_i32(t1);
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -3466,7 +3473,8 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         if (unlikely(ctx->singlestep_enabled)) {
             if ((ctx->singlestep_enabled &
                 (CPU_BRANCH_STEP | CPU_SINGLE_STEP)) &&
-                ctx->exception == POWERPC_EXCP_BRANCH) {
+                (ctx->exception == POWERPC_EXCP_BRANCH ||
+                 ctx->exception == POWERPC_EXCP_TRACE)) {
                 target_ulong tmp = ctx->nip;
                 ctx->nip = dest;
                 gen_exception(ctx, POWERPC_EXCP_TRACE);
@@ -9463,7 +9471,7 @@ void cpu_dump_state (CPUPPCState *env, FILE *f, fprintf_function cpu_fprintf,
         if ((i & (RFPL - 1)) == (RFPL - 1))
             cpu_fprintf(f, "\n");
     }
-    cpu_fprintf(f, "FPSCR %08x\n", env->fpscr);
+    cpu_fprintf(f, "FPSCR " TARGET_FMT_lx "\n", env->fpscr);
 #if !defined(CONFIG_USER_ONLY)
     cpu_fprintf(f, " SRR0 " TARGET_FMT_lx "  SRR1 " TARGET_FMT_lx
                    "    PVR " TARGET_FMT_lx " VRSAVE " TARGET_FMT_lx "\n",
@@ -9617,7 +9625,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.nip = pc_start;
     ctx.tb = tb;
     ctx.exception = POWERPC_EXCP_NONE;
@@ -9657,7 +9665,8 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
 
     gen_icount_start();
     /* Set env in case of segfault during code fetch */
-    while (ctx.exception == POWERPC_EXCP_NONE && gen_opc_ptr < gen_opc_end) {
+    while (ctx.exception == POWERPC_EXCP_NONE
+            && tcg_ctx.gen_opc_ptr < gen_opc_end) {
         if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (bp->pc == ctx.nip) {
@@ -9667,15 +9676,15 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
             }
         }
         if (unlikely(search_pc)) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.nip;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = ctx.nip;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         LOG_DISAS("----------------\n");
         LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
@@ -9690,8 +9699,9 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
                     opc3(ctx.opcode), little_endian ? "little" : "big");
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(ctx.nip);
+        }
         ctx.nip += 4;
         table = env->opcodes;
         num_insns++;
@@ -9766,12 +9776,12 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         tcg_gen_exit_tb(0);
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (unlikely(search_pc)) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.nip - pc_start;
         tb->icount = num_insns;
@@ -9782,7 +9792,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         flags = env->bfd_mach;
         flags |= ctx.le_mode << 16;
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.nip - pc_start, flags);
+        log_target_disas(env, pc_start, ctx.nip - pc_start, flags);
         qemu_log("\n");
     }
 #endif
@@ -9800,5 +9810,5 @@ void gen_intermediate_code_pc (CPUPPCState *env, struct TranslationBlock *tb)
 
 void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->nip = gen_opc_pc[pc_pos];
+    env->nip = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index fba2b424277..e143af532ac 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -23,11 +23,12 @@
  * inside "#if defined(TODO) ... #endif" statements to make tests easier.
  */
 
-#include "dis-asm.h"
-#include "gdbstub.h"
-#include <kvm.h>
+#include "disas/bfd.h"
+#include "exec/gdbstub.h"
+#include <sysemu/kvm.h>
 #include "kvm_ppc.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/cpus.h"
 
 //#define PPC_DUMP_CPU
 //#define PPC_DEBUG_SPR
@@ -1498,7 +1499,7 @@ static void gen_spr_BookE (CPUPPCState *env, uint64_t ivor_mask)
     /* XXX : not implemented */
     spr_register(env, SPR_BOOKE_DBCR0, "DBCR0",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
+                 &spr_read_generic, &spr_write_40x_dbcr0,
                  0x00000000);
     /* XXX : not implemented */
     spr_register(env, SPR_BOOKE_DBCR1, "DBCR1",
@@ -4493,11 +4494,6 @@ static void spr_read_mas73(void *opaque, int gprn, int sprn)
     tcg_temp_free(mas7);
 }
 
-static void spr_load_epr(void *opaque, int gprn, int sprn)
-{
-    gen_helper_load_epr(cpu_gpr[gprn], cpu_env);
-}
-
 #endif
 
 enum fsl_e500_version {
@@ -4656,7 +4652,7 @@ static void init_proc_e500 (CPUPPCState *env, int version)
                  0x00000000);
     spr_register(env, SPR_BOOKE_EPR, "EPR",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_load_epr, SPR_NOACCESS,
+                 &spr_read_generic, SPR_NOACCESS,
                  0x00000000);
     /* XXX better abstract into Emb.xxx features */
     if (version == fsl_e5500) {
@@ -8638,9 +8634,9 @@ static const ppc_def_t ppc_defs[] = {
     POWERPC_DEF("e500v2_v22",    CPU_POWERPC_e500v2_v22,             e500v2),
     /* PowerPC e500v2 v3.0 core                                              */
     POWERPC_DEF("e500v2_v30",    CPU_POWERPC_e500v2_v30,             e500v2),
-    POWERPC_DEF("e500mc",        CPU_POWERPC_e500mc,                 e500mc),
+    POWERPC_DEF_SVR("e500mc", CPU_POWERPC_e500mc, POWERPC_SVR_E500,  e500mc),
 #ifdef TARGET_PPC64
-    POWERPC_DEF("e5500",         CPU_POWERPC_e5500,                  e5500),
+    POWERPC_DEF_SVR("e5500",    CPU_POWERPC_e5500, POWERPC_SVR_E500, e5500),
 #endif
     /* PowerPC e500 microcontrollers                                         */
     /* MPC8533                                                               */
@@ -9797,8 +9793,11 @@ static void fix_opcode_tables (opc_handler_t **ppc_opcodes)
 }
 
 /*****************************************************************************/
-static int create_ppc_opcodes (CPUPPCState *env, const ppc_def_t *def)
+static void create_ppc_opcodes(PowerPCCPU *cpu, Error **errp)
 {
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
+    CPUPPCState *env = &cpu->env;
+    const ppc_def_t *def = pcc->info;
     opcode_t *opc;
 
     fill_new_table(env->opcodes, 0x40);
@@ -9806,18 +9805,16 @@ static int create_ppc_opcodes (CPUPPCState *env, const ppc_def_t *def)
         if (((opc->handler.type & def->insns_flags) != 0) ||
             ((opc->handler.type2 & def->insns_flags2) != 0)) {
             if (register_insn(env->opcodes, opc) < 0) {
-                printf("*** ERROR initializing PowerPC instruction "
-                       "0x%02x 0x%02x 0x%02x\n", opc->opc1, opc->opc2,
-                       opc->opc3);
-                return -1;
+                error_setg(errp, "ERROR initializing PowerPC instruction "
+                           "0x%02x 0x%02x 0x%02x\n", opc->opc1, opc->opc2,
+                           opc->opc3);
+                return;
             }
         }
     }
     fix_opcode_tables(env->opcodes);
     fflush(stdout);
     fflush(stderr);
-
-    return 0;
 }
 
 #if defined(PPC_DUMP_CPU)
@@ -10009,8 +10006,10 @@ static int gdb_set_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
     return 0;
 }
 
-static int ppc_fixup_cpu(CPUPPCState *env)
+static int ppc_fixup_cpu(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+
     /* TCG doesn't (yet) emulate some groups of instructions that
      * are implemented on some otherwise supported CPUs (e.g. VSX
      * and decimal floating point instructions on POWER7).  We
@@ -10031,53 +10030,42 @@ static int ppc_fixup_cpu(CPUPPCState *env)
     return 0;
 }
 
-int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
+static void ppc_cpu_realize(Object *obj, Error **errp)
 {
-    env->msr_mask = def->msr_mask;
-    env->mmu_model = def->mmu_model;
-    env->excp_model = def->excp_model;
-    env->bus_model = def->bus_model;
-    env->insns_flags = def->insns_flags;
-    env->insns_flags2 = def->insns_flags2;
-    env->flags = def->flags;
-    env->bfd_mach = def->bfd_mach;
-    env->check_pow = def->check_pow;
+    PowerPCCPU *cpu = POWERPC_CPU(obj);
+    CPUPPCState *env = &cpu->env;
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
+    ppc_def_t *def = pcc->info;
+    Error *local_err = NULL;
+#if !defined(CONFIG_USER_ONLY)
+    int max_smt = kvm_enabled() ? kvmppc_smt_threads() : 1;
+#endif
 
-#if defined(TARGET_PPC64)
-    if (def->sps)
-        env->sps = *def->sps;
-    else if (env->mmu_model & POWERPC_MMU_64) {
-        /* Use default sets of page sizes */
-        static const struct ppc_segment_page_sizes defsps = {
-            .sps = {
-                { .page_shift = 12, /* 4K */
-                  .slb_enc = 0,
-                  .enc = { { .page_shift = 12, .pte_enc = 0 } }
-                },
-                { .page_shift = 24, /* 16M */
-                  .slb_enc = 0x100,
-                  .enc = { { .page_shift = 24, .pte_enc = 0 } }
-                },
-            },
-        };
-        env->sps = defsps;
+#if !defined(CONFIG_USER_ONLY)
+    if (smp_threads > max_smt) {
+        fprintf(stderr, "Cannot support more than %d threads on PPC with %s\n",
+                max_smt, kvm_enabled() ? "KVM" : "TCG");
+        exit(1);
     }
-#endif /* defined(TARGET_PPC64) */
+#endif
 
     if (kvm_enabled()) {
-        if (kvmppc_fixup_cpu(env) != 0) {
-            fprintf(stderr, "Unable to virtualize selected CPU with KVM\n");
-            exit(1);
+        if (kvmppc_fixup_cpu(cpu) != 0) {
+            error_setg(errp, "Unable to virtualize selected CPU with KVM");
+            return;
         }
     } else {
-        if (ppc_fixup_cpu(env) != 0) {
-            fprintf(stderr, "Unable to emulate selected CPU with TCG\n");
-            exit(1);
+        if (ppc_fixup_cpu(cpu) != 0) {
+            error_setg(errp, "Unable to emulate selected CPU with TCG");
+            return;
         }
     }
 
-    if (create_ppc_opcodes(env, def) < 0)
-        return -1;
+    create_ppc_opcodes(cpu, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
     init_ppc_proc(env, def);
 
     if (def->insns_flags & PPC_FLOAT) {
@@ -10093,6 +10081,8 @@ int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
                                  34, "power-spe.xml", 0);
     }
 
+    qemu_init_vcpu(env);
+
 #if defined(PPC_DUMP_CPU)
     {
         const char *mmu_model, *excp_model, *bus_model;
@@ -10254,50 +10244,65 @@ int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
     dump_ppc_sprs(env);
     fflush(stdout);
 #endif
-
-    return 0;
 }
 
-static bool ppc_cpu_usable(const ppc_def_t *def)
+static gint ppc_cpu_compare_class_pvr(gconstpointer a, gconstpointer b)
 {
-#if defined(TARGET_PPCEMB)
-    /* When using the ppcemb target, we only support 440 style cores */
-    if (def->mmu_model != POWERPC_MMU_BOOKE) {
-        return false;
+    ObjectClass *oc = (ObjectClass *)a;
+    uint32_t pvr = *(uint32_t *)b;
+    PowerPCCPUClass *pcc = (PowerPCCPUClass *)a;
+
+    /* -cpu host does a PVR lookup during construction */
+    if (unlikely(strcmp(object_class_get_name(oc),
+                        TYPE_HOST_POWERPC_CPU) == 0)) {
+        return -1;
     }
-#endif
 
-    return true;
+    return pcc->info->pvr == pvr ? 0 : -1;
 }
 
-const ppc_def_t *ppc_find_by_pvr(uint32_t pvr)
+PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr)
 {
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    GSList *list, *item;
+    PowerPCCPUClass *pcc = NULL;
 
-        /* If we have an exact match, we're done */
-        if (pvr == ppc_defs[i].pvr) {
-            return &ppc_defs[i];
-        }
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    item = g_slist_find_custom(list, &pvr, ppc_cpu_compare_class_pvr);
+    if (item != NULL) {
+        pcc = POWERPC_CPU_CLASS(item->data);
     }
+    g_slist_free(list);
+
+    return pcc;
+}
 
-    return NULL;
+static gint ppc_cpu_compare_class_name(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *oc = (ObjectClass *)a;
+    const char *name = b;
+
+    if (strncasecmp(name, object_class_get_name(oc), strlen(name)) == 0 &&
+        strcmp(object_class_get_name(oc) + strlen(name),
+               "-" TYPE_POWERPC_CPU) == 0) {
+        return 0;
+    }
+    return -1;
 }
 
 #include <ctype.h>
 
-const ppc_def_t *cpu_ppc_find_by_name (const char *name)
+static ObjectClass *ppc_cpu_class_by_name(const char *name)
 {
-    const ppc_def_t *ret;
+    GSList *list, *item;
+    ObjectClass *ret = NULL;
     const char *p;
-    int i, max, len;
+    int i, len;
 
-    if (kvm_enabled() && (strcasecmp(name, "host") == 0)) {
-        return kvmppc_host_cpu_def();
+    if (strcasecmp(name, "host") == 0) {
+        if (kvm_enabled()) {
+            ret = object_class_by_name(TYPE_HOST_POWERPC_CPU);
+        }
+        return ret;
     }
 
     /* Check if the given name is a PVR */
@@ -10312,63 +10317,152 @@ const ppc_def_t *cpu_ppc_find_by_name (const char *name)
             if (!qemu_isxdigit(*p++))
                 break;
         }
-        if (i == 8)
-            return ppc_find_by_pvr(strtoul(name, NULL, 16));
-    }
-    ret = NULL;
-    max = ARRAY_SIZE(ppc_defs);
-    for (i = 0; i < max; i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
+        if (i == 8) {
+            ret = OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
+            return ret;
         }
+    }
 
-        if (strcasecmp(name, ppc_defs[i].name) == 0) {
-            ret = &ppc_defs[i];
-            break;
-        }
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    item = g_slist_find_custom(list, name, ppc_cpu_compare_class_name);
+    if (item != NULL) {
+        ret = OBJECT_CLASS(item->data);
     }
+    g_slist_free(list);
 
     return ret;
 }
 
-void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf)
+PowerPCCPU *cpu_ppc_init(const char *cpu_model)
 {
-    int i, max;
+    PowerPCCPU *cpu;
+    CPUPPCState *env;
+    ObjectClass *oc;
+    Error *err = NULL;
 
-    max = ARRAY_SIZE(ppc_defs);
-    for (i = 0; i < max; i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    oc = ppc_cpu_class_by_name(cpu_model);
+    if (oc == NULL) {
+        return NULL;
+    }
+
+    cpu = POWERPC_CPU(object_new(object_class_get_name(oc)));
+    env = &cpu->env;
 
-        (*cpu_fprintf)(f, "PowerPC %-16s PVR %08x\n",
-                       ppc_defs[i].name, ppc_defs[i].pvr);
+    if (tcg_enabled()) {
+        ppc_translate_init();
     }
+
+    env->cpu_model_str = cpu_model;
+
+    ppc_cpu_realize(OBJECT(cpu), &err);
+    if (err != NULL) {
+        fprintf(stderr, "%s\n", error_get_pretty(err));
+        error_free(err);
+        object_delete(OBJECT(cpu));
+        return NULL;
+    }
+
+    return cpu;
+}
+
+/* Sort by PVR, ordering special case "host" last. */
+static gint ppc_cpu_list_compare(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *oc_a = (ObjectClass *)a;
+    ObjectClass *oc_b = (ObjectClass *)b;
+    PowerPCCPUClass *pcc_a = POWERPC_CPU_CLASS(oc_a);
+    PowerPCCPUClass *pcc_b = POWERPC_CPU_CLASS(oc_b);
+    const char *name_a = object_class_get_name(oc_a);
+    const char *name_b = object_class_get_name(oc_b);
+
+    if (strcmp(name_a, TYPE_HOST_POWERPC_CPU) == 0) {
+        return 1;
+    } else if (strcmp(name_b, TYPE_HOST_POWERPC_CPU) == 0) {
+        return -1;
+    } else {
+        /* Avoid an integer overflow during subtraction */
+        if (pcc_a->info->pvr < pcc_b->info->pvr) {
+            return -1;
+        } else if (pcc_a->info->pvr > pcc_b->info->pvr) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static void ppc_cpu_list_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CPUListState *s = user_data;
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+
+    (*s->cpu_fprintf)(s->file, "PowerPC %-16s PVR %08x\n",
+                      pcc->info->name, pcc->info->pvr);
+}
+
+void ppc_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+    CPUListState s = {
+        .file = f,
+        .cpu_fprintf = cpu_fprintf,
+    };
+    GSList *list;
+
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    list = g_slist_sort(list, ppc_cpu_list_compare);
+    g_slist_foreach(list, ppc_cpu_list_entry, &s);
+    g_slist_free(list);
+}
+
+static void ppc_cpu_defs_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CpuDefinitionInfoList **first = user_data;
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    CpuDefinitionInfoList *entry;
+    CpuDefinitionInfo *info;
+
+    info = g_malloc0(sizeof(*info));
+    info->name = g_strdup(pcc->info->name);
+
+    entry = g_malloc0(sizeof(*entry));
+    entry->value = info;
+    entry->next = *first;
+    *first = entry;
 }
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
     CpuDefinitionInfoList *cpu_list = NULL;
-    int i;
+    GSList *list;
 
-    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
-        CpuDefinitionInfoList *entry;
-        CpuDefinitionInfo *info;
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    g_slist_foreach(list, ppc_cpu_defs_entry, &cpu_list);
+    g_slist_free(list);
 
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    return cpu_list;
+}
 
-        info = g_malloc0(sizeof(*info));
-        info->name = g_strdup(ppc_defs[i].name);
+static void ppc_cpu_def_class_init(ObjectClass *oc, void *data)
+{
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    ppc_def_t *info = data;
 
-        entry = g_malloc0(sizeof(*entry));
-        entry->value = info;
-        entry->next = cpu_list;
-        cpu_list = entry;
-    }
+    pcc->info = info;
+}
 
-    return cpu_list;
+static void ppc_cpu_register_model(const ppc_def_t *def)
+{
+    TypeInfo type_info = {
+        .parent = TYPE_POWERPC_CPU,
+        .class_init = ppc_cpu_def_class_init,
+        .class_data = (void *)def,
+    };
+
+    type_info.name = g_strdup_printf("%s-" TYPE_POWERPC_CPU, def->name),
+    type_register(&type_info);
+    g_free((gpointer)type_info.name);
 }
 
 /* CPUClass::reset() */
@@ -10380,7 +10474,7 @@ static void ppc_cpu_reset(CPUState *s)
     target_ulong msr;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -10423,6 +10517,15 @@ static void ppc_cpu_reset(CPUState *s)
     env->pending_interrupts = 0;
     env->exception_index = POWERPC_EXCP_NONE;
     env->error_code = 0;
+
+#if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
+    env->vpa_addr = 0;
+    env->slb_shadow_addr = 0;
+    env->slb_shadow_size = 0;
+    env->dtl_addr = 0;
+    env->dtl_size = 0;
+#endif /* TARGET_PPC64 */
+
     /* Flush all TLBs */
     tlb_flush(env, 1);
 }
@@ -10430,9 +10533,42 @@ static void ppc_cpu_reset(CPUState *s)
 static void ppc_cpu_initfn(Object *obj)
 {
     PowerPCCPU *cpu = POWERPC_CPU(obj);
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
     CPUPPCState *env = &cpu->env;
+    ppc_def_t *def = pcc->info;
 
     cpu_exec_init(env);
+
+    env->msr_mask = def->msr_mask;
+    env->mmu_model = def->mmu_model;
+    env->excp_model = def->excp_model;
+    env->bus_model = def->bus_model;
+    env->insns_flags = def->insns_flags;
+    env->insns_flags2 = def->insns_flags2;
+    env->flags = def->flags;
+    env->bfd_mach = def->bfd_mach;
+    env->check_pow = def->check_pow;
+
+#if defined(TARGET_PPC64)
+    if (def->sps) {
+        env->sps = *def->sps;
+    } else if (env->mmu_model & POWERPC_MMU_64) {
+        /* Use default sets of page sizes */
+        static const struct ppc_segment_page_sizes defsps = {
+            .sps = {
+                { .page_shift = 12, /* 4K */
+                  .slb_enc = 0,
+                  .enc = { { .page_shift = 12, .pte_enc = 0 } }
+                },
+                { .page_shift = 24, /* 16M */
+                  .slb_enc = 0x100,
+                  .enc = { { .page_shift = 24, .pte_enc = 0 } }
+                },
+            },
+        };
+        env->sps = defsps;
+    }
+#endif /* defined(TARGET_PPC64) */
 }
 
 static void ppc_cpu_class_init(ObjectClass *oc, void *data)
@@ -10442,6 +10578,8 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
 
     pcc->parent_reset = cc->reset;
     cc->reset = ppc_cpu_reset;
+
+    cc->class_by_name = ppc_cpu_class_by_name;
 }
 
 static const TypeInfo ppc_cpu_type_info = {
@@ -10449,14 +10587,27 @@ static const TypeInfo ppc_cpu_type_info = {
     .parent = TYPE_CPU,
     .instance_size = sizeof(PowerPCCPU),
     .instance_init = ppc_cpu_initfn,
-    .abstract = false,
+    .abstract = true,
     .class_size = sizeof(PowerPCCPUClass),
     .class_init = ppc_cpu_class_init,
 };
 
 static void ppc_cpu_register_types(void)
 {
+    int i;
+
     type_register_static(&ppc_cpu_type_info);
+
+    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
+        const ppc_def_t *def = &ppc_defs[i];
+#if defined(TARGET_PPCEMB)
+        /* When using the ppcemb target, we only support 440 style cores */
+        if (def->mmu_model != POWERPC_MMU_BOOKE) {
+            continue;
+        }
+#endif
+        ppc_cpu_register_model(def);
+    }
 }
 
 type_init(ppc_cpu_register_types)
diff --git a/target-s390x/Makefile.objs b/target-s390x/Makefile.objs
index 80be3bbdb27..e728abf7598 100644
--- a/target-s390x/Makefile.objs
+++ b/target-s390x/Makefile.objs
@@ -1,5 +1,4 @@
-obj-y += translate.o op_helper.o helper.o cpu.o interrupt.o
+obj-y += translate.o helper.o cpu.o interrupt.o
+obj-y += int_helper.o fpu_helper.o cc_helper.o mem_helper.o misc_helper.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-s390x/cc_helper.c b/target-s390x/cc_helper.c
new file mode 100644
index 00000000000..a6d60bf8851
--- /dev/null
+++ b/target-s390x/cc_helper.c
@@ -0,0 +1,577 @@
+/*
+ *  S/390 condition code helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+#include "qemu/host-utils.h"
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+static uint32_t cc_calc_ltgt_32(int32_t src, int32_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static uint32_t cc_calc_ltgt0_32(int32_t dst)
+{
+    return cc_calc_ltgt_32(dst, 0);
+}
+
+static uint32_t cc_calc_ltgt_64(int64_t src, int64_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static uint32_t cc_calc_ltgt0_64(int64_t dst)
+{
+    return cc_calc_ltgt_64(dst, 0);
+}
+
+static uint32_t cc_calc_ltugtu_32(uint32_t src, uint32_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static uint32_t cc_calc_ltugtu_64(uint64_t src, uint64_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static uint32_t cc_calc_tm_32(uint32_t val, uint32_t mask)
+{
+    uint32_t r = val & mask;
+
+    if (r == 0) {
+        return 0;
+    } else if (r == mask) {
+        return 3;
+    } else {
+        return 1;
+    }
+}
+
+static uint32_t cc_calc_tm_64(uint64_t val, uint64_t mask)
+{
+    uint64_t r = val & mask;
+
+    if (r == 0) {
+        return 0;
+    } else if (r == mask) {
+        return 3;
+    } else {
+        int top = clz64(mask);
+        if ((int64_t)(val << top) < 0) {
+            return 2;
+        } else {
+            return 1;
+        }
+    }
+}
+
+static uint32_t cc_calc_nz(uint64_t dst)
+{
+    return !!dst;
+}
+
+static uint32_t cc_calc_add_64(int64_t a1, int64_t a2, int64_t ar)
+{
+    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static uint32_t cc_calc_addu_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    return (ar != 0) + 2 * (ar < a1);
+}
+
+static uint32_t cc_calc_addc_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    /* Recover a2 + carry_in.  */
+    uint64_t a2c = ar - a1;
+    /* Check for a2+carry_in overflow, then a1+a2c overflow.  */
+    int carry_out = (a2c < a2) || (ar < a1);
+
+    return (ar != 0) + 2 * carry_out;
+}
+
+static uint32_t cc_calc_sub_64(int64_t a1, int64_t a2, int64_t ar)
+{
+    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static uint32_t cc_calc_subu_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    if (ar == 0) {
+        return 2;
+    } else {
+        if (a2 > a1) {
+            return 1;
+        } else {
+            return 3;
+        }
+    }
+}
+
+static uint32_t cc_calc_subb_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    /* We had borrow-in if normal subtraction isn't equal.  */
+    int borrow_in = ar - (a1 - a2);
+    int borrow_out;
+
+    /* If a2 was ULONG_MAX, and borrow_in, then a2 is logically 65 bits,
+       and we must have had borrow out.  */
+    if (borrow_in && a2 == (uint64_t)-1) {
+        borrow_out = 1;
+    } else {
+        a2 += borrow_in;
+        borrow_out = (a2 > a1);
+    }
+
+    return (ar != 0) + 2 * !borrow_out;
+}
+
+static uint32_t cc_calc_abs_64(int64_t dst)
+{
+    if ((uint64_t)dst == 0x8000000000000000ULL) {
+        return 3;
+    } else if (dst) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static uint32_t cc_calc_nabs_64(int64_t dst)
+{
+    return !!dst;
+}
+
+static uint32_t cc_calc_comp_64(int64_t dst)
+{
+    if ((uint64_t)dst == 0x8000000000000000ULL) {
+        return 3;
+    } else if (dst < 0) {
+        return 1;
+    } else if (dst > 0) {
+        return 2;
+    } else {
+        return 0;
+    }
+}
+
+
+static uint32_t cc_calc_add_32(int32_t a1, int32_t a2, int32_t ar)
+{
+    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static uint32_t cc_calc_addu_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    return (ar != 0) + 2 * (ar < a1);
+}
+
+static uint32_t cc_calc_addc_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    /* Recover a2 + carry_in.  */
+    uint32_t a2c = ar - a1;
+    /* Check for a2+carry_in overflow, then a1+a2c overflow.  */
+    int carry_out = (a2c < a2) || (ar < a1);
+
+    return (ar != 0) + 2 * carry_out;
+}
+
+static uint32_t cc_calc_sub_32(int32_t a1, int32_t a2, int32_t ar)
+{
+    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static uint32_t cc_calc_subu_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    if (ar == 0) {
+        return 2;
+    } else {
+        if (a2 > a1) {
+            return 1;
+        } else {
+            return 3;
+        }
+    }
+}
+
+static uint32_t cc_calc_subb_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    /* We had borrow-in if normal subtraction isn't equal.  */
+    int borrow_in = ar - (a1 - a2);
+    int borrow_out;
+
+    /* If a2 was UINT_MAX, and borrow_in, then a2 is logically 65 bits,
+       and we must have had borrow out.  */
+    if (borrow_in && a2 == (uint32_t)-1) {
+        borrow_out = 1;
+    } else {
+        a2 += borrow_in;
+        borrow_out = (a2 > a1);
+    }
+
+    return (ar != 0) + 2 * !borrow_out;
+}
+
+static uint32_t cc_calc_abs_32(int32_t dst)
+{
+    if ((uint32_t)dst == 0x80000000UL) {
+        return 3;
+    } else if (dst) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static uint32_t cc_calc_nabs_32(int32_t dst)
+{
+    return !!dst;
+}
+
+static uint32_t cc_calc_comp_32(int32_t dst)
+{
+    if ((uint32_t)dst == 0x80000000UL) {
+        return 3;
+    } else if (dst < 0) {
+        return 1;
+    } else if (dst > 0) {
+        return 2;
+    } else {
+        return 0;
+    }
+}
+
+/* calculate condition code for insert character under mask insn */
+static uint32_t cc_calc_icm(uint64_t mask, uint64_t val)
+{
+    if ((val & mask) == 0) {
+        return 0;
+    } else {
+        int top = clz64(mask);
+        if ((int64_t)(val << top) < 0) {
+            return 1;
+        } else {
+            return 2;
+        }
+    }
+}
+
+static uint32_t cc_calc_sla_32(uint32_t src, int shift)
+{
+    uint32_t mask = ((1U << shift) - 1U) << (32 - shift);
+    uint32_t sign = 1U << 31;
+    uint32_t match;
+    int32_t r;
+
+    /* Check if the sign bit stays the same.  */
+    if (src & sign) {
+        match = mask;
+    } else {
+        match = 0;
+    }
+    if ((src & mask) != match) {
+        /* Overflow.  */
+        return 3;
+    }
+
+    r = ((src << shift) & ~sign) | (src & sign);
+    if (r == 0) {
+        return 0;
+    } else if (r < 0) {
+        return 1;
+    }
+    return 2;
+}
+
+static uint32_t cc_calc_sla_64(uint64_t src, int shift)
+{
+    uint64_t mask = ((1ULL << shift) - 1ULL) << (64 - shift);
+    uint64_t sign = 1ULL << 63;
+    uint64_t match;
+    int64_t r;
+
+    /* Check if the sign bit stays the same.  */
+    if (src & sign) {
+        match = mask;
+    } else {
+        match = 0;
+    }
+    if ((src & mask) != match) {
+        /* Overflow.  */
+        return 3;
+    }
+
+    r = ((src << shift) & ~sign) | (src & sign);
+    if (r == 0) {
+        return 0;
+    } else if (r < 0) {
+        return 1;
+    }
+    return 2;
+}
+
+static uint32_t cc_calc_flogr(uint64_t dst)
+{
+    return dst ? 2 : 0;
+}
+
+static uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
+                                  uint64_t src, uint64_t dst, uint64_t vr)
+{
+    uint32_t r = 0;
+
+    switch (cc_op) {
+    case CC_OP_CONST0:
+    case CC_OP_CONST1:
+    case CC_OP_CONST2:
+    case CC_OP_CONST3:
+        /* cc_op value _is_ cc */
+        r = cc_op;
+        break;
+    case CC_OP_LTGT0_32:
+        r = cc_calc_ltgt0_32(dst);
+        break;
+    case CC_OP_LTGT0_64:
+        r =  cc_calc_ltgt0_64(dst);
+        break;
+    case CC_OP_LTGT_32:
+        r =  cc_calc_ltgt_32(src, dst);
+        break;
+    case CC_OP_LTGT_64:
+        r =  cc_calc_ltgt_64(src, dst);
+        break;
+    case CC_OP_LTUGTU_32:
+        r =  cc_calc_ltugtu_32(src, dst);
+        break;
+    case CC_OP_LTUGTU_64:
+        r =  cc_calc_ltugtu_64(src, dst);
+        break;
+    case CC_OP_TM_32:
+        r =  cc_calc_tm_32(src, dst);
+        break;
+    case CC_OP_TM_64:
+        r =  cc_calc_tm_64(src, dst);
+        break;
+    case CC_OP_NZ:
+        r =  cc_calc_nz(dst);
+        break;
+    case CC_OP_ADD_64:
+        r =  cc_calc_add_64(src, dst, vr);
+        break;
+    case CC_OP_ADDU_64:
+        r =  cc_calc_addu_64(src, dst, vr);
+        break;
+    case CC_OP_ADDC_64:
+        r =  cc_calc_addc_64(src, dst, vr);
+        break;
+    case CC_OP_SUB_64:
+        r =  cc_calc_sub_64(src, dst, vr);
+        break;
+    case CC_OP_SUBU_64:
+        r =  cc_calc_subu_64(src, dst, vr);
+        break;
+    case CC_OP_SUBB_64:
+        r =  cc_calc_subb_64(src, dst, vr);
+        break;
+    case CC_OP_ABS_64:
+        r =  cc_calc_abs_64(dst);
+        break;
+    case CC_OP_NABS_64:
+        r =  cc_calc_nabs_64(dst);
+        break;
+    case CC_OP_COMP_64:
+        r =  cc_calc_comp_64(dst);
+        break;
+
+    case CC_OP_ADD_32:
+        r =  cc_calc_add_32(src, dst, vr);
+        break;
+    case CC_OP_ADDU_32:
+        r =  cc_calc_addu_32(src, dst, vr);
+        break;
+    case CC_OP_ADDC_32:
+        r =  cc_calc_addc_32(src, dst, vr);
+        break;
+    case CC_OP_SUB_32:
+        r =  cc_calc_sub_32(src, dst, vr);
+        break;
+    case CC_OP_SUBU_32:
+        r =  cc_calc_subu_32(src, dst, vr);
+        break;
+    case CC_OP_SUBB_32:
+        r =  cc_calc_subb_32(src, dst, vr);
+        break;
+    case CC_OP_ABS_32:
+        r =  cc_calc_abs_32(dst);
+        break;
+    case CC_OP_NABS_32:
+        r =  cc_calc_nabs_32(dst);
+        break;
+    case CC_OP_COMP_32:
+        r =  cc_calc_comp_32(dst);
+        break;
+
+    case CC_OP_ICM:
+        r =  cc_calc_icm(src, dst);
+        break;
+    case CC_OP_SLA_32:
+        r =  cc_calc_sla_32(src, dst);
+        break;
+    case CC_OP_SLA_64:
+        r =  cc_calc_sla_64(src, dst);
+        break;
+    case CC_OP_FLOGR:
+        r = cc_calc_flogr(dst);
+        break;
+
+    case CC_OP_NZ_F32:
+        r = set_cc_nz_f32(dst);
+        break;
+    case CC_OP_NZ_F64:
+        r = set_cc_nz_f64(dst);
+        break;
+    case CC_OP_NZ_F128:
+        r = set_cc_nz_f128(make_float128(src, dst));
+        break;
+
+    default:
+        cpu_abort(env, "Unknown CC operation: %s\n", cc_name(cc_op));
+    }
+
+    HELPER_LOG("%s: %15s 0x%016lx 0x%016lx 0x%016lx = %d\n", __func__,
+               cc_name(cc_op), src, dst, vr, r);
+    return r;
+}
+
+uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
+                 uint64_t vr)
+{
+    return do_calc_cc(env, cc_op, src, dst, vr);
+}
+
+uint32_t HELPER(calc_cc)(CPUS390XState *env, uint32_t cc_op, uint64_t src,
+                         uint64_t dst, uint64_t vr)
+{
+    return do_calc_cc(env, cc_op, src, dst, vr);
+}
+
+#ifndef CONFIG_USER_ONLY
+void HELPER(load_psw)(CPUS390XState *env, uint64_t mask, uint64_t addr)
+{
+    load_psw(env, mask, addr);
+    cpu_loop_exit(env);
+}
+
+void HELPER(sacf)(CPUS390XState *env, uint64_t a1)
+{
+    HELPER_LOG("%s: %16" PRIx64 "\n", __func__, a1);
+
+    switch (a1 & 0xf00) {
+    case 0x000:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_PRIMARY;
+        break;
+    case 0x100:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_SECONDARY;
+        break;
+    case 0x300:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_HOME;
+        break;
+    default:
+        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        break;
+    }
+}
+#endif
diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index 6fa55a80a59..d54e4a2ee2a 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_S390_CPU_QOM_H
 #define QEMU_S390_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_S390_CPU "s390-cpu"
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index 619b202b92c..0b68db83050 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2009 Ulrich Hecht
  * Copyright (c) 2011 Alexander Graf
  * Copyright (c) 2012 SUSE LINUX Products GmbH
+ * Copyright (c) 2012 IBM Corp.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -18,12 +19,44 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ * Contributions after 2012-12-11 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
  */
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
+#ifndef CONFIG_USER_ONLY
+#include "hw/hw.h"
+#include "sysemu/arch_init.h"
+#endif
+
+#define CR0_RESET       0xE0UL
+#define CR14_RESET      0xC2000000UL;
+
+/* generate CPU information for cpu -? */
+void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+#ifdef CONFIG_KVM
+    (*cpu_fprintf)(f, "s390 %16s\n", "host");
+#endif
+}
+
+#ifndef CONFIG_USER_ONLY
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
+{
+    CpuDefinitionInfoList *entry;
+    CpuDefinitionInfo *info;
 
+    info = g_malloc0(sizeof(*info));
+    info->name = g_strdup("host");
+
+    entry = g_malloc0(sizeof(*entry));
+    entry->value = info;
+
+    return entry;
+}
+#endif
 
 /* CPUClass::reset() */
 static void s390_cpu_reset(CPUState *s)
@@ -33,18 +66,37 @@ static void s390_cpu_reset(CPUState *s)
     CPUS390XState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
+    s390_del_running_cpu(env);
+
     scc->parent_reset(s);
 
     memset(env, 0, offsetof(CPUS390XState, breakpoints));
-    /* FIXME: reset vector? */
+
+    /* architectured initial values for CR 0 and 14 */
+    env->cregs[0] = CR0_RESET;
+    env->cregs[14] = CR14_RESET;
+    /* set halted to 1 to make sure we can add the cpu in
+     * s390_ipl_cpu code, where env->halted is set back to 0
+     * after incrementing the cpu counter */
+#if !defined(CONFIG_USER_ONLY)
+    env->halted = 1;
+#endif
     tlb_flush(env, 1);
-    s390_add_running_cpu(env);
 }
 
+#if !defined(CONFIG_USER_ONLY)
+static void s390_cpu_machine_reset_cb(void *opaque)
+{
+    S390CPU *cpu = opaque;
+
+    cpu_reset(CPU(cpu));
+}
+#endif
+
 static void s390_cpu_initfn(Object *obj)
 {
     S390CPU *cpu = S390_CPU(obj);
@@ -56,12 +108,17 @@ static void s390_cpu_initfn(Object *obj)
 
     cpu_exec_init(env);
 #if !defined(CONFIG_USER_ONLY)
+    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
     qemu_get_timedate(&tm, 0);
     env->tod_offset = TOD_UNIX_EPOCH +
                       (time2tod(mktimegm(&tm)) * 1000000000ULL);
     env->tod_basetime = 0;
     env->tod_timer = qemu_new_timer_ns(vm_clock, s390x_tod_timer, cpu);
     env->cpu_timer = qemu_new_timer_ns(vm_clock, s390x_cpu_timer, cpu);
+    /* set env->halted state to 1 to avoid decrementing the running
+     * cpu counter in s390_cpu_reset to a negative number at
+     * initial ipl */
+    env->halted = 1;
 #endif
     env->cpu_num = cpu_num++;
     env->ext_index = -1;
@@ -69,6 +126,15 @@ static void s390_cpu_initfn(Object *obj)
     cpu_reset(CPU(cpu));
 }
 
+static void s390_cpu_finalize(Object *obj)
+{
+#if !defined(CONFIG_USER_ONLY)
+    S390CPU *cpu = S390_CPU(obj);
+
+    qemu_unregister_reset(s390_cpu_machine_reset_cb, cpu);
+#endif
+}
+
 static void s390_cpu_class_init(ObjectClass *oc, void *data)
 {
     S390CPUClass *scc = S390_CPU_CLASS(oc);
@@ -83,6 +149,7 @@ static const TypeInfo s390_cpu_type_info = {
     .parent = TYPE_CPU,
     .instance_size = sizeof(S390CPU),
     .instance_init = s390_cpu_initfn,
+    .instance_finalize = s390_cpu_finalize,
     .abstract = false,
     .class_size = sizeof(S390CPUClass),
     .class_init = s390_cpu_class_init,
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 18ac6e3938e..1f2d94218a4 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -13,7 +13,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef CPU_S390X_H
@@ -28,15 +31,15 @@
 
 #define CPUArchState struct CPUS390XState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 #define TARGET_PAGE_BITS 12
 
 #define TARGET_PHYS_ADDR_SPACE_BITS 64
 #define TARGET_VIRT_ADDR_SPACE_BITS 64
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define NB_MMU_MODES 3
 
@@ -60,17 +63,20 @@ typedef struct ExtQueue {
 } ExtQueue;
 
 typedef struct CPUS390XState {
-    uint64_t regs[16];	/* GP registers */
+    uint64_t regs[16];     /* GP registers */
+    CPU_DoubleU fregs[16]; /* FP registers */
+    uint32_t aregs[16];    /* access registers */
 
-    uint32_t aregs[16];	/* access registers */
+    uint32_t fpc;          /* floating-point control register */
+    uint32_t cc_op;
 
-    uint32_t fpc;	/* floating-point control register */
-    CPU_DoubleU fregs[16]; /* FP registers */
     float_status fpu_status; /* passed to softfloat lib */
 
+    /* The low part of a 128-bit return, or remainder of a divide.  */
+    uint64_t retxl;
+
     PSW psw;
 
-    uint32_t cc_op;
     uint64_t cc_src;
     uint64_t cc_dst;
     uint64_t cc_vr;
@@ -79,15 +85,15 @@ typedef struct CPUS390XState {
     uint64_t psa;
 
     uint32_t int_pgm_code;
-    uint32_t int_pgm_ilc;
+    uint32_t int_pgm_ilen;
 
     uint32_t int_svc_code;
-    uint32_t int_svc_ilc;
+    uint32_t int_svc_ilen;
 
     uint64_t cregs[16]; /* control registers */
 
-    int pending_int;
     ExtQueue ext_queue[MAX_EXT_QUEUE];
+    int pending_int;
 
     int ext_index;
 
@@ -113,7 +119,7 @@ static inline void cpu_clone_regs(CPUS390XState *env, target_ulong newsp)
     if (newsp) {
         env->regs[15] = newsp;
     }
-    env->regs[0] = 0;
+    env->regs[2] = 0;
 }
 #endif
 
@@ -253,25 +259,31 @@ static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc,
              ((env->psw.mask & PSW_MASK_32) ? FLAG_MASK_32 : 0);
 }
 
-static inline int get_ilc(uint8_t opc)
+/* While the PoO talks about ILC (a number between 1-3) what is actually
+   stored in LowCore is shifted left one bit (an even between 2-6).  As
+   this is the actual length of the insn and therefore more useful, that
+   is what we want to pass around and manipulate.  To make sure that we
+   have applied this distinction universally, rename the "ILC" to "ILEN".  */
+static inline int get_ilen(uint8_t opc)
 {
     switch (opc >> 6) {
     case 0:
-        return 1;
+        return 2;
     case 1:
     case 2:
-        return 2;
-    case 3:
-        return 3;
+        return 4;
+    default:
+        return 6;
     }
-
-    return 0;
 }
 
-#define ILC_LATER       0x20
-#define ILC_LATER_INC   0x21
-#define ILC_LATER_INC_2 0x22
-
+#ifndef CONFIG_USER_ONLY
+/* In several cases of runtime exceptions, we havn't recorded the true
+   instruction length.  Use these codes when raising exceptions in order
+   to re-compute the length by examining the insn in memory.  */
+#define ILEN_LATER       0x20
+#define ILEN_LATER_INC   0x21
+#endif
 
 S390CPU *cpu_s390x_init(const char *cpu_model);
 void s390x_translate_init(void);
@@ -293,24 +305,24 @@ int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong address, int rw
 void s390x_tod_timer(void *opaque);
 void s390x_cpu_timer(void *opaque);
 
-int s390_virtio_hypercall(CPUS390XState *env, uint64_t mem, uint64_t hypercall);
+int s390_virtio_hypercall(CPUS390XState *env);
 
 #ifdef CONFIG_KVM
-void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code);
-void kvm_s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token);
-void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
+void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code);
+void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token);
+void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
                                  uint64_t parm64, int vm);
 #else
-static inline void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code)
+static inline void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
 {
 }
 
-static inline void kvm_s390_virtio_irq(CPUS390XState *env, int config_change,
+static inline void kvm_s390_virtio_irq(S390CPU *cpu, int config_change,
                                        uint64_t token)
 {
 }
 
-static inline void kvm_s390_interrupt_internal(CPUS390XState *env, int type,
+static inline void kvm_s390_interrupt_internal(S390CPU *cpu, int type,
                                                uint32_t parm, uint64_t parm64,
                                                int vm)
 {
@@ -324,7 +336,7 @@ unsigned s390_del_running_cpu(CPUS390XState *env);
 void s390_sclp_extint(uint32_t parm);
 
 /* from s390-virtio-bus */
-extern const target_phys_addr_t virtio_size;
+extern const hwaddr virtio_size;
 
 #else
 static inline void s390_add_running_cpu(CPUS390XState *env)
@@ -350,23 +362,15 @@ static inline void cpu_set_tls(CPUS390XState *env, target_ulong newtls)
 #define cpu_gen_code cpu_s390x_gen_code
 #define cpu_signal_handler cpu_s390x_signal_handler
 
-#include "exec-all.h"
-
-#ifdef CONFIG_USER_ONLY
-
-#define EXCP_OPEX 1 /* operation exception (sigill) */
-#define EXCP_SVC 2 /* supervisor call (syscall) */
-#define EXCP_ADDR 5 /* addressing exception */
-#define EXCP_SPEC 6 /* specification exception */
+void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
+#define cpu_list s390_cpu_list
 
-#else
+#include "exec/exec-all.h"
 
 #define EXCP_EXT 1 /* external interrupt */
 #define EXCP_SVC 2 /* supervisor call (syscall) */
 #define EXCP_PGM 3 /* program interruption */
 
-#endif /* CONFIG_USER_ONLY */
-
 #define INTERRUPT_EXT        (1 << 0)
 #define INTERRUPT_TOD        (1 << 1)
 #define INTERRUPT_CPUTIMER   (1 << 2)
@@ -430,79 +434,6 @@ static inline void cpu_set_tls(CPUS390XState *env, target_ulong newtls)
 /* Total.  */
 #define S390_NUM_REGS 51
 
-/* Pseudo registers -- PC and condition code.  */
-#define S390_PC_REGNUM S390_NUM_REGS
-#define S390_CC_REGNUM (S390_NUM_REGS+1)
-#define S390_NUM_PSEUDO_REGS 2
-#define S390_NUM_TOTAL_REGS (S390_NUM_REGS+2)
-
-
-
-/* Program Status Word.  */
-#define S390_PSWM_REGNUM 0
-#define S390_PSWA_REGNUM 1
-/* General Purpose Registers.  */
-#define S390_R0_REGNUM 2
-#define S390_R1_REGNUM 3
-#define S390_R2_REGNUM 4
-#define S390_R3_REGNUM 5
-#define S390_R4_REGNUM 6
-#define S390_R5_REGNUM 7
-#define S390_R6_REGNUM 8
-#define S390_R7_REGNUM 9
-#define S390_R8_REGNUM 10
-#define S390_R9_REGNUM 11
-#define S390_R10_REGNUM 12
-#define S390_R11_REGNUM 13
-#define S390_R12_REGNUM 14
-#define S390_R13_REGNUM 15
-#define S390_R14_REGNUM 16
-#define S390_R15_REGNUM 17
-/* Access Registers.  */
-#define S390_A0_REGNUM 18
-#define S390_A1_REGNUM 19
-#define S390_A2_REGNUM 20
-#define S390_A3_REGNUM 21
-#define S390_A4_REGNUM 22
-#define S390_A5_REGNUM 23
-#define S390_A6_REGNUM 24
-#define S390_A7_REGNUM 25
-#define S390_A8_REGNUM 26
-#define S390_A9_REGNUM 27
-#define S390_A10_REGNUM 28
-#define S390_A11_REGNUM 29
-#define S390_A12_REGNUM 30
-#define S390_A13_REGNUM 31
-#define S390_A14_REGNUM 32
-#define S390_A15_REGNUM 33
-/* Floating Point Control Word.  */
-#define S390_FPC_REGNUM 34
-/* Floating Point Registers.  */
-#define S390_F0_REGNUM 35
-#define S390_F1_REGNUM 36
-#define S390_F2_REGNUM 37
-#define S390_F3_REGNUM 38
-#define S390_F4_REGNUM 39
-#define S390_F5_REGNUM 40
-#define S390_F6_REGNUM 41
-#define S390_F7_REGNUM 42
-#define S390_F8_REGNUM 43
-#define S390_F9_REGNUM 44
-#define S390_F10_REGNUM 45
-#define S390_F11_REGNUM 46
-#define S390_F12_REGNUM 47
-#define S390_F13_REGNUM 48
-#define S390_F14_REGNUM 49
-#define S390_F15_REGNUM 50
-/* Total.  */
-#define S390_NUM_REGS 51
-
-/* Pseudo registers -- PC and condition code.  */
-#define S390_PC_REGNUM S390_NUM_REGS
-#define S390_CC_REGNUM (S390_NUM_REGS+1)
-#define S390_NUM_PSEUDO_REGS 2
-#define S390_NUM_TOTAL_REGS (S390_NUM_REGS+2)
-
 /* CC optimization */
 
 enum cc_op {
@@ -524,15 +455,19 @@ enum cc_op {
 
     CC_OP_ADD_64,               /* overflow on add (64bit) */
     CC_OP_ADDU_64,              /* overflow on unsigned add (64bit) */
+    CC_OP_ADDC_64,              /* overflow on unsigned add-carry (64bit) */
     CC_OP_SUB_64,               /* overflow on subtraction (64bit) */
     CC_OP_SUBU_64,              /* overflow on unsigned subtraction (64bit) */
+    CC_OP_SUBB_64,              /* overflow on unsigned sub-borrow (64bit) */
     CC_OP_ABS_64,               /* sign eval on abs (64bit) */
     CC_OP_NABS_64,              /* sign eval on nabs (64bit) */
 
     CC_OP_ADD_32,               /* overflow on add (32bit) */
     CC_OP_ADDU_32,              /* overflow on unsigned add (32bit) */
+    CC_OP_ADDC_32,              /* overflow on unsigned add-carry (32bit) */
     CC_OP_SUB_32,               /* overflow on subtraction (32bit) */
     CC_OP_SUBU_32,              /* overflow on unsigned subtraction (32bit) */
+    CC_OP_SUBB_32,              /* overflow on unsigned sub-borrow (32bit) */
     CC_OP_ABS_32,               /* sign eval on abs (64bit) */
     CC_OP_NABS_32,              /* sign eval on nabs (64bit) */
 
@@ -542,14 +477,14 @@ enum cc_op {
     CC_OP_TM_32,                /* test under mask (32bit) */
     CC_OP_TM_64,                /* test under mask (64bit) */
 
-    CC_OP_LTGT_F32,             /* FP compare (32bit) */
-    CC_OP_LTGT_F64,             /* FP compare (64bit) */
-
     CC_OP_NZ_F32,               /* FP dst != 0 (32bit) */
     CC_OP_NZ_F64,               /* FP dst != 0 (64bit) */
+    CC_OP_NZ_F128,              /* FP dst != 0 (128bit) */
 
     CC_OP_ICM,                  /* insert characters under mask */
-    CC_OP_SLAG,                 /* Calculate shift left signed */
+    CC_OP_SLA_32,               /* Calculate shift left signed (32bit) */
+    CC_OP_SLA_64,               /* Calculate shift left signed (64bit) */
+    CC_OP_FLOGR,                /* find leftmost one */
     CC_OP_MAX
 };
 
@@ -569,26 +504,31 @@ static const char *cc_names[] = {
     [CC_OP_LTGT0_64]  = "CC_OP_LTGT0_64",
     [CC_OP_ADD_64]    = "CC_OP_ADD_64",
     [CC_OP_ADDU_64]   = "CC_OP_ADDU_64",
+    [CC_OP_ADDC_64]   = "CC_OP_ADDC_64",
     [CC_OP_SUB_64]    = "CC_OP_SUB_64",
     [CC_OP_SUBU_64]   = "CC_OP_SUBU_64",
+    [CC_OP_SUBB_64]   = "CC_OP_SUBB_64",
     [CC_OP_ABS_64]    = "CC_OP_ABS_64",
     [CC_OP_NABS_64]   = "CC_OP_NABS_64",
     [CC_OP_ADD_32]    = "CC_OP_ADD_32",
     [CC_OP_ADDU_32]   = "CC_OP_ADDU_32",
+    [CC_OP_ADDC_32]   = "CC_OP_ADDC_32",
     [CC_OP_SUB_32]    = "CC_OP_SUB_32",
     [CC_OP_SUBU_32]   = "CC_OP_SUBU_32",
+    [CC_OP_SUBB_32]   = "CC_OP_SUBB_32",
     [CC_OP_ABS_32]    = "CC_OP_ABS_32",
     [CC_OP_NABS_32]   = "CC_OP_NABS_32",
     [CC_OP_COMP_32]   = "CC_OP_COMP_32",
     [CC_OP_COMP_64]   = "CC_OP_COMP_64",
     [CC_OP_TM_32]     = "CC_OP_TM_32",
     [CC_OP_TM_64]     = "CC_OP_TM_64",
-    [CC_OP_LTGT_F32]  = "CC_OP_LTGT_F32",
-    [CC_OP_LTGT_F64]  = "CC_OP_LTGT_F64",
     [CC_OP_NZ_F32]    = "CC_OP_NZ_F32",
     [CC_OP_NZ_F64]    = "CC_OP_NZ_F64",
+    [CC_OP_NZ_F128]   = "CC_OP_NZ_F128",
     [CC_OP_ICM]       = "CC_OP_ICM",
-    [CC_OP_SLAG]      = "CC_OP_SLAG",
+    [CC_OP_SLA_32]    = "CC_OP_SLA_32",
+    [CC_OP_SLA_64]    = "CC_OP_SLA_64",
+    [CC_OP_FLOGR]     = "CC_OP_FLOGR",
 };
 
 static inline const char *cc_name(int cc_op)
@@ -596,17 +536,6 @@ static inline const char *cc_name(int cc_op)
     return cc_names[cc_op];
 }
 
-/* SCLP PV interface defines */
-#define SCLP_CMDW_READ_SCP_INFO         0x00020001
-#define SCLP_CMDW_READ_SCP_INFO_FORCED  0x00120001
-
-#define SCP_LENGTH                      0x00
-#define SCP_FUNCTION_CODE               0x02
-#define SCP_CONTROL_MASK                0x03
-#define SCP_RESPONSE_CODE               0x06
-#define SCP_MEM_CODE                    0x08
-#define SCP_INCREMENT                   0x0a
-
 typedef struct LowCore
 {
     /* prefix area: defined by architecture */
@@ -616,9 +545,9 @@ typedef struct LowCore
     uint32_t        ext_params;               /* 0x080 */
     uint16_t        cpu_addr;                 /* 0x084 */
     uint16_t        ext_int_code;             /* 0x086 */
-    uint16_t        svc_ilc;                  /* 0x088 */
+    uint16_t        svc_ilen;                 /* 0x088 */
     uint16_t        svc_code;                 /* 0x08a */
-    uint16_t        pgm_ilc;                  /* 0x08c */
+    uint16_t        pgm_ilen;                 /* 0x08c */
     uint16_t        pgm_code;                 /* 0x08e */
     uint32_t        data_exc_code;            /* 0x090 */
     uint16_t        mon_class_num;            /* 0x094 */
@@ -846,87 +775,6 @@ struct sysib_322 {
 #define SK_F                    (0x1 << 3)
 #define SK_ACC_MASK             (0xf << 4)
 
-
-/* EBCDIC handling */
-static const uint8_t ebcdic2ascii[] = {
-    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
-    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
-    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
-    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
-    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
-    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
-    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
-    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
-    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
-    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
-    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
-    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
-    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
-    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
-    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
-    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
-    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
-    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
-    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
-    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
-    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
-    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
-    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
-    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
-    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
-    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
-    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
-    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
-};
-
-static const uint8_t ascii2ebcdic [] = {
-    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
-    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
-    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
-    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
-    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
-    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
-    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
-    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
-    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
-    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
-    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
-    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
-};
-
-static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
-{
-    int i;
-
-    for (i = 0; i < len; i++) {
-        p[i] = ascii2ebcdic[(int)ascii[i]];
-    }
-}
-
 #define SIGP_SENSE             0x01
 #define SIGP_EXTERNAL_CALL     0x02
 #define SIGP_EMERGENCY         0x03
@@ -955,7 +803,7 @@ static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr);
 int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
                   target_ulong *raddr, int *flags);
-int sclp_service_call(CPUS390XState *env, uint32_t sccb, uint64_t code);
+int sclp_service_call(uint32_t sccb, uint64_t code);
 uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
                  uint64_t vr);
 
@@ -988,8 +836,10 @@ static inline void cpu_inject_ext(CPUS390XState *env, uint32_t code, uint32_t pa
     cpu_interrupt(env, CPU_INTERRUPT_HARD);
 }
 
-static inline bool cpu_has_work(CPUS390XState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUS390XState *env = &S390_CPU(cpu)->env;
+
     return (env->interrupt_request & CPU_INTERRUPT_HARD) &&
         (env->psw.mask & PSW_MASK_EXT);
 }
@@ -999,4 +849,14 @@ static inline void cpu_pc_from_tb(CPUS390XState *env, TranslationBlock* tb)
     env->psw.addr = tb->pc;
 }
 
+/* fpu_helper.c */
+uint32_t set_cc_nz_f32(float32 v);
+uint32_t set_cc_nz_f64(float64 v);
+uint32_t set_cc_nz_f128(float128 v);
+
+/* misc_helper.c */
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilen);
+void QEMU_NORETURN runtime_exception(CPUS390XState *env, int excp,
+                                     uintptr_t retaddr);
+
 #endif
diff --git a/target-s390x/fpu_helper.c b/target-s390x/fpu_helper.c
new file mode 100644
index 00000000000..94375b6a63e
--- /dev/null
+++ b/target-s390x/fpu_helper.c
@@ -0,0 +1,713 @@
+/*
+ *  S/390 FPU helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+#if !defined(CONFIG_USER_ONLY)
+#include "exec/softmmu_exec.h"
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+#define RET128(F) (env->retxl = F.low, F.high)
+
+#define convert_bit(mask, from, to) \
+    (to < from                      \
+     ? (mask / (from / to)) & to    \
+     : (mask & from) * (to / from))
+
+static void ieee_exception(CPUS390XState *env, uint32_t dxc, uintptr_t retaddr)
+{
+    /* Install the DXC code.  */
+    env->fpc = (env->fpc & ~0xff00) | (dxc << 8);
+    /* Trap.  */
+    runtime_exception(env, PGM_DATA, retaddr);
+}
+
+/* Should be called after any operation that may raise IEEE exceptions.  */
+static void handle_exceptions(CPUS390XState *env, uintptr_t retaddr)
+{
+    unsigned s390_exc, qemu_exc;
+
+    /* Get the exceptions raised by the current operation.  Reset the
+       fpu_status contents so that the next operation has a clean slate.  */
+    qemu_exc = env->fpu_status.float_exception_flags;
+    if (qemu_exc == 0) {
+        return;
+    }
+    env->fpu_status.float_exception_flags = 0;
+
+    /* Convert softfloat exception bits to s390 exception bits.  */
+    s390_exc = 0;
+    s390_exc |= convert_bit(qemu_exc, float_flag_invalid, 0x80);
+    s390_exc |= convert_bit(qemu_exc, float_flag_divbyzero, 0x40);
+    s390_exc |= convert_bit(qemu_exc, float_flag_overflow, 0x20);
+    s390_exc |= convert_bit(qemu_exc, float_flag_underflow, 0x10);
+    s390_exc |= convert_bit(qemu_exc, float_flag_inexact, 0x08);
+
+    /* Install the exceptions that we raised.  */
+    env->fpc |= s390_exc << 16;
+
+    /* Send signals for enabled exceptions.  */
+    s390_exc &= env->fpc >> 24;
+    if (s390_exc) {
+        ieee_exception(env, s390_exc, retaddr);
+    }
+}
+
+static inline int float_comp_to_cc(CPUS390XState *env, int float_compare)
+{
+    switch (float_compare) {
+    case float_relation_equal:
+        return 0;
+    case float_relation_less:
+        return 1;
+    case float_relation_greater:
+        return 2;
+    case float_relation_unordered:
+        return 3;
+    default:
+        cpu_abort(env, "unknown return value for float compare\n");
+    }
+}
+
+/* condition codes for unary FP ops */
+uint32_t set_cc_nz_f32(float32 v)
+{
+    if (float32_is_any_nan(v)) {
+        return 3;
+    } else if (float32_is_zero(v)) {
+        return 0;
+    } else if (float32_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+uint32_t set_cc_nz_f64(float64 v)
+{
+    if (float64_is_any_nan(v)) {
+        return 3;
+    } else if (float64_is_zero(v)) {
+        return 0;
+    } else if (float64_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+uint32_t set_cc_nz_f128(float128 v)
+{
+    if (float128_is_any_nan(v)) {
+        return 3;
+    } else if (float128_is_zero(v)) {
+        return 0;
+    } else if (float128_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+/* 32-bit FP addition */
+uint64_t HELPER(aeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float32 ret = float32_add(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP addition */
+uint64_t HELPER(adb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float64 ret = float64_add(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 128-bit FP addition */
+uint64_t HELPER(axb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
+{
+    float128 ret = float128_add(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* 32-bit FP subtraction */
+uint64_t HELPER(seb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float32 ret = float32_sub(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP subtraction */
+uint64_t HELPER(sdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float64 ret = float64_sub(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 128-bit FP subtraction */
+uint64_t HELPER(sxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
+{
+    float128 ret = float128_sub(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* 32-bit FP division */
+uint64_t HELPER(deb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float32 ret = float32_div(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP division */
+uint64_t HELPER(ddb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float64 ret = float64_div(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 128-bit FP division */
+uint64_t HELPER(dxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
+{
+    float128 ret = float128_div(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* 32-bit FP multiplication */
+uint64_t HELPER(meeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float32 ret = float32_mul(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP multiplication */
+uint64_t HELPER(mdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float64 ret = float64_mul(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64/32-bit FP multiplication */
+uint64_t HELPER(mdeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    float64 ret = float32_to_float64(f2, &env->fpu_status);
+    ret = float64_mul(f1, ret, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 128-bit FP multiplication */
+uint64_t HELPER(mxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
+{
+    float128 ret = float128_mul(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* 128/64-bit FP multiplication */
+uint64_t HELPER(mxdb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                      uint64_t f2)
+{
+    float128 ret = float64_to_float128(f2, &env->fpu_status);
+    ret = float128_mul(make_float128(ah, al), ret, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* convert 32-bit float to 64-bit float */
+uint64_t HELPER(ldeb)(CPUS390XState *env, uint64_t f2)
+{
+    float64 ret = float32_to_float64(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 64-bit float */
+uint64_t HELPER(ldxb)(CPUS390XState *env, uint64_t ah, uint64_t al)
+{
+    float64 ret = float128_to_float64(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit float to 128-bit float */
+uint64_t HELPER(lxdb)(CPUS390XState *env, uint64_t f2)
+{
+    float128 ret = float64_to_float128(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* convert 32-bit float to 128-bit float */
+uint64_t HELPER(lxeb)(CPUS390XState *env, uint64_t f2)
+{
+    float128 ret = float32_to_float128(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* convert 64-bit float to 32-bit float */
+uint64_t HELPER(ledb)(CPUS390XState *env, uint64_t f2)
+{
+    float32 ret = float64_to_float32(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 32-bit float */
+uint64_t HELPER(lexb)(CPUS390XState *env, uint64_t ah, uint64_t al)
+{
+    float32 ret = float128_to_float32(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 32-bit FP compare */
+uint32_t HELPER(ceb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    int cmp = float32_compare_quiet(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
+}
+
+/* 64-bit FP compare */
+uint32_t HELPER(cdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
+{
+    int cmp = float64_compare_quiet(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
+}
+
+/* 128-bit FP compare */
+uint32_t HELPER(cxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
+{
+    int cmp = float128_compare_quiet(make_float128(ah, al),
+                                     make_float128(bh, bl),
+                                     &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
+}
+
+static int swap_round_mode(CPUS390XState *env, int m3)
+{
+    int ret = env->fpu_status.float_rounding_mode;
+    switch (m3) {
+    case 0:
+        /* current mode */
+        break;
+    case 1:
+        /* biased round no nearest */
+    case 4:
+        /* round to nearest */
+        set_float_rounding_mode(float_round_nearest_even, &env->fpu_status);
+        break;
+    case 5:
+        /* round to zero */
+        set_float_rounding_mode(float_round_to_zero, &env->fpu_status);
+        break;
+    case 6:
+        /* round to +inf */
+        set_float_rounding_mode(float_round_up, &env->fpu_status);
+        break;
+    case 7:
+        /* round to -inf */
+        set_float_rounding_mode(float_round_down, &env->fpu_status);
+        break;
+    }
+    return ret;
+}
+
+/* convert 64-bit int to 32-bit float */
+uint64_t HELPER(cegb)(CPUS390XState *env, int64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float32 ret = int64_to_float32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit int to 64-bit float */
+uint64_t HELPER(cdgb)(CPUS390XState *env, int64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float64 ret = int64_to_float64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit int to 128-bit float */
+uint64_t HELPER(cxgb)(CPUS390XState *env, int64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 ret = int64_to_float128(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* convert 64-bit uint to 32-bit float */
+uint64_t HELPER(celgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float32 ret = uint64_to_float32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit uint to 64-bit float */
+uint64_t HELPER(cdlgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float64 ret = uint64_to_float64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit uint to 128-bit float */
+uint64_t HELPER(cxlgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 ret = uint64_to_float128(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+/* convert 32-bit float to 64-bit int */
+uint64_t HELPER(cgeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    int64_t ret = float32_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit float to 64-bit int */
+uint64_t HELPER(cgdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    int64_t ret = float64_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 64-bit int */
+uint64_t HELPER(cgxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    int64_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 32-bit float to 32-bit int */
+uint64_t HELPER(cfeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    int32_t ret = float32_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit float to 32-bit int */
+uint64_t HELPER(cfdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    int32_t ret = float64_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 32-bit int */
+uint64_t HELPER(cfxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    int32_t ret = float128_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 32-bit float to 64-bit uint */
+uint64_t HELPER(clgeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    uint64_t ret;
+    v2 = float32_to_float64(v2, &env->fpu_status);
+    ret = float64_to_uint64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit float to 64-bit uint */
+uint64_t HELPER(clgdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    uint64_t ret = float64_to_uint64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 64-bit uint */
+uint64_t HELPER(clgxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    /* ??? Not 100% correct.  */
+    uint64_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 32-bit float to 32-bit uint */
+uint64_t HELPER(clfeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    uint32_t ret = float32_to_uint32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 64-bit float to 32-bit uint */
+uint64_t HELPER(clfdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    uint32_t ret = float64_to_uint32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* convert 128-bit float to 32-bit uint */
+uint64_t HELPER(clfxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    /* Not 100% correct.  */
+    uint32_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 32-bit FP multiply and add */
+uint64_t HELPER(maeb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float32 ret = float32_muladd(f2, f3, f1, 0, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP multiply and add */
+uint64_t HELPER(madb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float64 ret = float64_muladd(f2, f3, f1, 0, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 32-bit FP multiply and subtract */
+uint64_t HELPER(mseb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float32 ret = float32_muladd(f2, f3, f1, float_muladd_negate_c,
+                                 &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* 64-bit FP multiply and subtract */
+uint64_t HELPER(msdb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float64 ret = float64_muladd(f2, f3, f1, float_muladd_negate_c,
+                                 &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* test data class 32-bit */
+uint32_t HELPER(tceb)(uint64_t f1, uint64_t m2)
+{
+    float32 v1 = f1;
+    int neg = float32_is_neg(v1);
+    uint32_t cc = 0;
+
+    if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float32_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* test data class 64-bit */
+uint32_t HELPER(tcdb)(uint64_t v1, uint64_t m2)
+{
+    int neg = float64_is_neg(v1);
+    uint32_t cc = 0;
+
+    if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float64_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* test data class 128-bit */
+uint32_t HELPER(tcxb)(uint64_t ah, uint64_t al, uint64_t m2)
+{
+    float128 v1 = make_float128(ah, al);
+    int neg = float128_is_neg(v1);
+    uint32_t cc = 0;
+
+    if ((float128_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float128_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float128_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float128_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* square root 32-bit */
+uint64_t HELPER(sqeb)(CPUS390XState *env, uint64_t f2)
+{
+    float32 ret = float32_sqrt(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* square root 64-bit */
+uint64_t HELPER(sqdb)(CPUS390XState *env, uint64_t f2)
+{
+    float64 ret = float64_sqrt(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* square root 128-bit */
+uint64_t HELPER(sqxb)(CPUS390XState *env, uint64_t ah, uint64_t al)
+{
+    float128 ret = float128_sqrt(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+static const int fpc_to_rnd[4] = {
+    float_round_nearest_even,
+    float_round_to_zero,
+    float_round_up,
+    float_round_down
+};
+
+/* set fpc */
+void HELPER(sfpc)(CPUS390XState *env, uint64_t fpc)
+{
+    /* Install everything in the main FPC.  */
+    env->fpc = fpc;
+
+    /* Install the rounding mode in the shadow fpu_status.  */
+    set_float_rounding_mode(fpc_to_rnd[fpc & 3], &env->fpu_status);
+}
+
+/* set fpc and signal */
+void HELPER(sfas)(CPUS390XState *env, uint64_t val)
+{
+    uint32_t signalling = env->fpc;
+    uint32_t source = val;
+    uint32_t s390_exc;
+
+    /* The contents of the source operand are placed in the FPC register;
+       then the flags in the FPC register are set to the logical OR of the
+       signalling flags and the source flags.  */
+    env->fpc = source | (signalling & 0x00ff0000);
+    set_float_rounding_mode(fpc_to_rnd[source & 3], &env->fpu_status);
+
+    /* If any signalling flag is 1 and the corresponding source mask
+       is also 1, a simulated-iee-exception trap occurs.  */
+    s390_exc = (signalling >> 16) & (source >> 24);
+    if (s390_exc) {
+        ieee_exception(env, s390_exc | 3, GETPC());
+    }
+}
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index d0a1180a834..9a132e6d2c4 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -19,10 +19,10 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
-#include "qemu-timer.h"
+#include "exec/gdbstub.h"
+#include "qemu/timer.h"
 #ifndef CONFIG_USER_ONLY
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #endif
 
 //#define DEBUG_S390
@@ -74,7 +74,7 @@ S390CPU *cpu_s390x_init(const char *cpu_model)
 {
     S390CPU *cpu;
     CPUS390XState *env;
-    static int inited = 0;
+    static int inited;
 
     cpu = S390_CPU(object_new(TYPE_S390_CPU));
     env = &cpu->env;
@@ -91,29 +91,31 @@ S390CPU *cpu_s390x_init(const char *cpu_model)
 
 #if defined(CONFIG_USER_ONLY)
 
-void do_interrupt (CPUS390XState *env)
+void do_interrupt(CPUS390XState *env)
 {
     env->exception_index = -1;
 }
 
-int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong address, int rw,
-                                int mmu_idx)
+int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong address,
+                               int rw, int mmu_idx)
 {
-    /* fprintf(stderr,"%s: address 0x%lx rw %d mmu_idx %d\n",
-            __FUNCTION__, address, rw, mmu_idx); */
-    env->exception_index = EXCP_ADDR;
-    env->__excp_addr = address; /* FIXME: find out how this works on a real machine */
+    env->exception_index = EXCP_PGM;
+    env->int_pgm_code = PGM_ADDRESSING;
+    /* On real machines this value is dropped into LowMem.  Since this
+       is userland, simply put this someplace that cpu_loop can find it.  */
+    env->__excp_addr = address;
     return 1;
 }
 
 #else /* !CONFIG_USER_ONLY */
 
 /* Ensure to exit the TB after this call! */
-static void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilc)
+static void trigger_pgm_exception(CPUS390XState *env, uint32_t code,
+                                  uint32_t ilen)
 {
     env->exception_index = EXCP_PGM;
     env->int_pgm_code = code;
-    env->int_pgm_ilc = ilc;
+    env->int_pgm_ilen = ilen;
 }
 
 static int trans_bits(CPUS390XState *env, uint64_t mode)
@@ -138,48 +140,49 @@ static int trans_bits(CPUS390XState *env, uint64_t mode)
     return bits;
 }
 
-static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr, uint64_t mode)
+static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint64_t mode)
 {
-    int ilc = ILC_LATER_INC_2;
+    int ilen = ILEN_LATER_INC;
     int bits = trans_bits(env, mode) | 4;
 
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __FUNCTION__, vaddr, bits);
+    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, PGM_PROTECTION, ilc);
+    trigger_pgm_exception(env, PGM_PROTECTION, ilen);
 }
 
-static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr, uint32_t type,
-                               uint64_t asc, int rw)
+static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint32_t type, uint64_t asc, int rw)
 {
-    int ilc = ILC_LATER;
+    int ilen = ILEN_LATER;
     int bits = trans_bits(env, asc);
 
+    /* Code accesses have an undefined ilc.  */
     if (rw == 2) {
-        /* code has is undefined ilc */
-        ilc = 2;
+        ilen = 2;
     }
 
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __FUNCTION__, vaddr, bits);
+    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, type, ilc);
+    trigger_pgm_exception(env, type, ilen);
 }
 
-static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t asc,
-                              uint64_t asce, int level, target_ulong *raddr,
-                              int *flags, int rw)
+static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
+                              uint64_t asc, uint64_t asce, int level,
+                              target_ulong *raddr, int *flags, int rw)
 {
     uint64_t offs = 0;
     uint64_t origin;
     uint64_t new_asce;
 
-    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __FUNCTION__, asce);
+    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __func__, asce);
 
     if (((level != _ASCE_TYPE_SEGMENT) && (asce & _REGION_ENTRY_INV)) ||
         ((level == _ASCE_TYPE_SEGMENT) && (asce & _SEGMENT_ENTRY_INV))) {
         /* XXX different regions have different faults */
-        DPRINTF("%s: invalid region\n", __FUNCTION__);
+        DPRINTF("%s: invalid region\n", __func__);
         trigger_page_fault(env, vaddr, PGM_SEGMENT_TRANS, asc, rw);
         return -1;
     }
@@ -222,7 +225,7 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     new_asce = ldq_phys(origin + offs);
     PTE_DPRINTF("%s: 0x%" PRIx64 " + 0x%" PRIx64 " => 0x%016" PRIx64 "\n",
-                __FUNCTION__, origin, offs, new_asce);
+                __func__, origin, offs, new_asce);
 
     if (level != _ASCE_TYPE_SEGMENT) {
         /* yet another region */
@@ -232,7 +235,7 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     /* PTE */
     if (new_asce & _PAGE_INVALID) {
-        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __FUNCTION__, new_asce);
+        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __func__, new_asce);
         trigger_page_fault(env, vaddr, PGM_PAGE_TRANS, asc, rw);
         return -1;
     }
@@ -243,13 +246,14 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     *raddr = new_asce & _ASCE_ORIGIN;
 
-    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __FUNCTION__, new_asce);
+    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __func__, new_asce);
 
     return 0;
 }
 
-static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t asc,
-                             target_ulong *raddr, int *flags, int rw)
+static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr,
+                             uint64_t asc, target_ulong *raddr, int *flags,
+                             int rw)
 {
     uint64_t asce = 0;
     int level, new_level;
@@ -257,15 +261,15 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
 
     switch (asc) {
     case PSW_ASC_PRIMARY:
-        PTE_DPRINTF("%s: asc=primary\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=primary\n", __func__);
         asce = env->cregs[1];
         break;
     case PSW_ASC_SECONDARY:
-        PTE_DPRINTF("%s: asc=secondary\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=secondary\n", __func__);
         asce = env->cregs[7];
         break;
     case PSW_ASC_HOME:
-        PTE_DPRINTF("%s: asc=home\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=home\n", __func__);
         asce = env->cregs[13];
         break;
     }
@@ -276,8 +280,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_REGION2:
         if (vaddr & 0xffe0000000000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xffe0000000000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xffe0000000000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -285,8 +288,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_REGION3:
         if (vaddr & 0xfffffc0000000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xfffffc0000000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xfffffc0000000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -294,8 +296,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_SEGMENT:
         if (vaddr & 0xffffffff80000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xffffffff80000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xffffffff80000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -358,7 +359,7 @@ int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
         break;
     }
 
-out:
+ out:
     /* Convert real address -> absolute address */
     if (*raddr < 0x2000) {
         *raddr = *raddr + env->psa;
@@ -378,18 +379,18 @@ out:
     return r;
 }
 
-int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong _vaddr, int rw,
-                                int mmu_idx)
+int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong orig_vaddr,
+                               int rw, int mmu_idx)
 {
     uint64_t asc = env->psw.mask & PSW_MASK_ASC;
     target_ulong vaddr, raddr;
     int prot;
 
     DPRINTF("%s: address 0x%" PRIx64 " rw %d mmu_idx %d\n",
-            __FUNCTION__, _vaddr, rw, mmu_idx);
+            __func__, _vaddr, rw, mmu_idx);
 
-    _vaddr &= TARGET_PAGE_MASK;
-    vaddr = _vaddr;
+    orig_vaddr &= TARGET_PAGE_MASK;
+    vaddr = orig_vaddr;
 
     /* 31-Bit mode */
     if (!(env->psw.mask & PSW_MASK_64)) {
@@ -403,22 +404,23 @@ int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong _vaddr, int rw,
 
     /* check out of RAM access */
     if (raddr > (ram_size + virtio_size)) {
-        DPRINTF("%s: aaddr %" PRIx64 " > ram_size %" PRIx64 "\n", __FUNCTION__,
+        DPRINTF("%s: aaddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
                 (uint64_t)aaddr, (uint64_t)ram_size);
-        trigger_pgm_exception(env, PGM_ADDRESSING, ILC_LATER);
+        trigger_pgm_exception(env, PGM_ADDRESSING, ILEN_LATER);
         return 1;
     }
 
-    DPRINTF("%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n", __FUNCTION__,
+    DPRINTF("%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n", __func__,
             (uint64_t)vaddr, (uint64_t)raddr, prot);
 
-    tlb_set_page(env, _vaddr, raddr, prot,
+    tlb_set_page(env, orig_vaddr, raddr, prot,
                  mmu_idx, TARGET_PAGE_SIZE);
 
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUS390XState *env, target_ulong vaddr)
+hwaddr cpu_get_phys_page_debug(CPUS390XState *env,
+                                           target_ulong vaddr)
 {
     target_ulong raddr;
     int prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
@@ -452,18 +454,19 @@ void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 
     env->psw.addr = addr;
     env->psw.mask = mask;
-    env->cc_op = (mask >> 13) & 3;
+    env->cc_op = (mask >> 44) & 3;
 }
 
 static uint64_t get_psw_mask(CPUS390XState *env)
 {
-    uint64_t r = env->psw.mask;
+    uint64_t r;
 
     env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
 
-    r &= ~(3ULL << 13);
+    r = env->psw.mask;
+    r &= ~PSW_MASK_CC;
     assert(!(env->cc_op & ~3));
-    r |= env->cc_op << 13;
+    r |= (uint64_t)env->cc_op << 44;
 
     return r;
 }
@@ -472,14 +475,14 @@ static void do_svc_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
+    hwaddr len = TARGET_PAGE_SIZE;
 
     lowcore = cpu_physical_memory_map(env->psa, &len, 1);
 
     lowcore->svc_code = cpu_to_be16(env->int_svc_code);
-    lowcore->svc_ilc = cpu_to_be16(env->int_svc_ilc);
+    lowcore->svc_ilen = cpu_to_be16(env->int_svc_ilen);
     lowcore->svc_old_psw.mask = cpu_to_be64(get_psw_mask(env));
-    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + (env->int_svc_ilc));
+    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + env->int_svc_ilen);
     mask = be64_to_cpu(lowcore->svc_new_psw.mask);
     addr = be64_to_cpu(lowcore->svc_new_psw.addr);
 
@@ -492,28 +495,27 @@ static void do_program_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
-    int ilc = env->int_pgm_ilc;
+    hwaddr len = TARGET_PAGE_SIZE;
+    int ilen = env->int_pgm_ilen;
 
-    switch (ilc) {
-    case ILC_LATER:
-        ilc = get_ilc(ldub_code(env->psw.addr));
-        break;
-    case ILC_LATER_INC:
-        ilc = get_ilc(ldub_code(env->psw.addr));
-        env->psw.addr += ilc * 2;
+    switch (ilen) {
+    case ILEN_LATER:
+        ilen = get_ilen(cpu_ldub_code(env, env->psw.addr));
         break;
-    case ILC_LATER_INC_2:
-        ilc = get_ilc(ldub_code(env->psw.addr)) * 2;
-        env->psw.addr += ilc;
+    case ILEN_LATER_INC:
+        ilen = get_ilen(cpu_ldub_code(env, env->psw.addr));
+        env->psw.addr += ilen;
         break;
+    default:
+        assert(ilen == 2 || ilen == 4 || ilen == 6);
     }
 
-    qemu_log("%s: code=0x%x ilc=%d\n", __FUNCTION__, env->int_pgm_code, ilc);
+    qemu_log_mask(CPU_LOG_INT, "%s: code=0x%x ilen=%d\n",
+                  __func__, env->int_pgm_code, ilen);
 
     lowcore = cpu_physical_memory_map(env->psa, &len, 1);
 
-    lowcore->pgm_ilc = cpu_to_be16(ilc);
+    lowcore->pgm_ilen = cpu_to_be16(ilen);
     lowcore->pgm_code = cpu_to_be16(env->int_pgm_code);
     lowcore->program_old_psw.mask = cpu_to_be64(get_psw_mask(env));
     lowcore->program_old_psw.addr = cpu_to_be64(env->psw.addr);
@@ -522,8 +524,8 @@ static void do_program_interrupt(CPUS390XState *env)
 
     cpu_physical_memory_unmap(lowcore, len, 1, len);
 
-    DPRINTF("%s: %x %x %" PRIx64 " %" PRIx64 "\n", __FUNCTION__,
-            env->int_pgm_code, ilc, env->psw.mask,
+    DPRINTF("%s: %x %x %" PRIx64 " %" PRIx64 "\n", __func__,
+            env->int_pgm_code, ilen, env->psw.mask,
             env->psw.addr);
 
     load_psw(env, mask, addr);
@@ -535,7 +537,7 @@ static void do_ext_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
+    hwaddr len = TARGET_PAGE_SIZE;
     ExtQueue *q;
 
     if (!(env->psw.mask & PSW_MASK_EXT)) {
@@ -565,16 +567,16 @@ static void do_ext_interrupt(CPUS390XState *env)
         env->pending_int &= ~INTERRUPT_EXT;
     }
 
-    DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __FUNCTION__,
+    DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
             env->psw.mask, env->psw.addr);
 
     load_psw(env, mask, addr);
 }
 
-void do_interrupt (CPUS390XState *env)
+void do_interrupt(CPUS390XState *env)
 {
-    qemu_log("%s: %d at pc=%" PRIx64 "\n", __FUNCTION__, env->exception_index,
-             env->psw.addr);
+    qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n",
+                  __func__, env->exception_index, env->psw.addr);
 
     s390_add_running_cpu(env);
     /* handle external interrupts */
diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index 01c8d0ea84f..dd90d93bee4 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -1,152 +1,120 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_1(exception, void, i32)
-DEF_HELPER_3(nc, i32, i32, i64, i64)
-DEF_HELPER_3(oc, i32, i32, i64, i64)
-DEF_HELPER_3(xc, i32, i32, i64, i64)
-DEF_HELPER_3(mvc, void, i32, i64, i64)
-DEF_HELPER_3(clc, i32, i32, i64, i64)
-DEF_HELPER_2(mvcl, i32, i32, i32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64)
-DEF_HELPER_FLAGS_2(set_cc_icm, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32)
-DEF_HELPER_3(clm, i32, i32, i32, i64)
-DEF_HELPER_3(stcm, void, i32, i32, i64)
-DEF_HELPER_2(mlg, void, i32, i64)
-DEF_HELPER_2(dlg, void, i32, i64)
-DEF_HELPER_FLAGS_3(set_cc_add64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_addu64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_add32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_addu32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_sub64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_subu64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_sub32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_subu32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_3(srst, i32, i32, i32, i32)
-DEF_HELPER_3(clst, i32, i32, i32, i32)
-DEF_HELPER_3(mvpg, void, i64, i64, i64)
-DEF_HELPER_3(mvst, void, i32, i32, i32)
-DEF_HELPER_3(csg, i32, i32, i64, i32)
-DEF_HELPER_3(cdsg, i32, i32, i64, i32)
-DEF_HELPER_3(cs, i32, i32, i64, i32)
-DEF_HELPER_4(ex, i32, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32)
-DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_PURE|TCG_CALL_CONST, s32, s32)
-DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_PURE|TCG_CALL_CONST, i64, s64)
-DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_PURE|TCG_CALL_CONST, s64, s64)
-DEF_HELPER_3(stcmh, void, i32, i64, i32)
-DEF_HELPER_3(icmh, i32, i32, i64, i32)
-DEF_HELPER_2(ipm, void, i32, i32)
-DEF_HELPER_FLAGS_3(addc_u32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_addc_u64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_3(stam, void, i32, i64, i32)
-DEF_HELPER_3(lam, void, i32, i64, i32)
-DEF_HELPER_3(mvcle, i32, i32, i64, i32)
-DEF_HELPER_3(clcle, i32, i32, i64, i32)
-DEF_HELPER_3(slb, i32, i32, i32, i32)
-DEF_HELPER_4(slbg, i32, i32, i32, i64, i64)
-DEF_HELPER_2(cefbr, void, i32, s32)
-DEF_HELPER_2(cdfbr, void, i32, s32)
-DEF_HELPER_2(cxfbr, void, i32, s32)
-DEF_HELPER_2(cegbr, void, i32, s64)
-DEF_HELPER_2(cdgbr, void, i32, s64)
-DEF_HELPER_2(cxgbr, void, i32, s64)
-DEF_HELPER_2(adbr, i32, i32, i32)
-DEF_HELPER_2(aebr, i32, i32, i32)
-DEF_HELPER_2(sebr, i32, i32, i32)
-DEF_HELPER_2(sdbr, i32, i32, i32)
-DEF_HELPER_2(debr, void, i32, i32)
-DEF_HELPER_2(dxbr, void, i32, i32)
-DEF_HELPER_2(mdbr, void, i32, i32)
-DEF_HELPER_2(mxbr, void, i32, i32)
-DEF_HELPER_2(ldebr, void, i32, i32)
-DEF_HELPER_2(ldxbr, void, i32, i32)
-DEF_HELPER_2(lxdbr, void, i32, i32)
-DEF_HELPER_2(ledbr, void, i32, i32)
-DEF_HELPER_2(lexbr, void, i32, i32)
-DEF_HELPER_2(lpebr, i32, i32, i32)
-DEF_HELPER_2(lpdbr, i32, i32, i32)
-DEF_HELPER_2(lpxbr, i32, i32, i32)
-DEF_HELPER_2(ltebr, i32, i32, i32)
-DEF_HELPER_2(ltdbr, i32, i32, i32)
-DEF_HELPER_2(ltxbr, i32, i32, i32)
-DEF_HELPER_2(lcebr, i32, i32, i32)
-DEF_HELPER_2(lcdbr, i32, i32, i32)
-DEF_HELPER_2(lcxbr, i32, i32, i32)
-DEF_HELPER_2(aeb, void, i32, i32)
-DEF_HELPER_2(deb, void, i32, i32)
-DEF_HELPER_2(meeb, void, i32, i32)
-DEF_HELPER_2(cdb, i32, i32, i64)
-DEF_HELPER_2(adb, i32, i32, i64)
-DEF_HELPER_2(seb, void, i32, i32)
-DEF_HELPER_2(sdb, i32, i32, i64)
-DEF_HELPER_2(mdb, void, i32, i64)
-DEF_HELPER_2(ddb, void, i32, i64)
-DEF_HELPER_FLAGS_2(cebr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(cdbr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(cxbr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_3(cgebr, i32, i32, i32, i32)
-DEF_HELPER_3(cgdbr, i32, i32, i32, i32)
-DEF_HELPER_3(cgxbr, i32, i32, i32, i32)
-DEF_HELPER_1(lzer, void, i32)
-DEF_HELPER_1(lzdr, void, i32)
-DEF_HELPER_1(lzxr, void, i32)
-DEF_HELPER_3(cfebr, i32, i32, i32, i32)
-DEF_HELPER_3(cfdbr, i32, i32, i32, i32)
-DEF_HELPER_3(cfxbr, i32, i32, i32, i32)
-DEF_HELPER_2(axbr, i32, i32, i32)
-DEF_HELPER_2(sxbr, i32, i32, i32)
-DEF_HELPER_2(meebr, void, i32, i32)
-DEF_HELPER_2(ddbr, void, i32, i32)
-DEF_HELPER_3(madb, void, i32, i64, i32)
-DEF_HELPER_3(maebr, void, i32, i32, i32)
-DEF_HELPER_3(madbr, void, i32, i32, i32)
-DEF_HELPER_3(msdbr, void, i32, i32, i32)
-DEF_HELPER_2(ldeb, void, i32, i64)
-DEF_HELPER_2(lxdb, void, i32, i64)
-DEF_HELPER_FLAGS_2(tceb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_FLAGS_2(tcdb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_FLAGS_2(tcxb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_2(flogr, i32, i32, i64)
-DEF_HELPER_2(sqdbr, void, i32, i32)
-DEF_HELPER_FLAGS_1(cvd, TCG_CALL_PURE|TCG_CALL_CONST, i64, s32)
-DEF_HELPER_3(unpk, void, i32, i64, i64)
-DEF_HELPER_3(tr, void, i32, i64, i64)
+DEF_HELPER_2(exception, noreturn, env, i32)
+DEF_HELPER_FLAGS_4(nc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(oc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(xc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(mvc, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(clc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_3(mvcl, i32, env, i32, i32)
+DEF_HELPER_FLAGS_4(clm, TCG_CALL_NO_WG, i32, env, i32, i32, i64)
+DEF_HELPER_FLAGS_3(mul128, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs32, TCG_CALL_NO_WG, s64, env, s64, s64)
+DEF_HELPER_FLAGS_3(divu32, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs64, TCG_CALL_NO_WG, s64, env, s64, s64)
+DEF_HELPER_FLAGS_4(divu64, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_4(srst, i64, env, i64, i64, i64)
+DEF_HELPER_4(clst, i64, env, i64, i64, i64)
+DEF_HELPER_4(mvpg, void, env, i64, i64, i64)
+DEF_HELPER_4(mvst, i64, env, i64, i64, i64)
+DEF_HELPER_5(ex, i32, env, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_NO_RWG_SE, i32, s32)
+DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_NO_RWG_SE, s32, s32)
+DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_NO_RWG_SE, i64, s64)
+DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_NO_RWG_SE, s64, s64)
+DEF_HELPER_FLAGS_4(stam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(lam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_4(mvcle, i32, env, i32, i64, i32)
+DEF_HELPER_4(clcle, i32, env, i32, i64, i32)
+DEF_HELPER_3(cegb, i64, env, s64, i32)
+DEF_HELPER_3(cdgb, i64, env, s64, i32)
+DEF_HELPER_3(cxgb, i64, env, s64, i32)
+DEF_HELPER_3(celgb, i64, env, i64, i32)
+DEF_HELPER_3(cdlgb, i64, env, i64, i32)
+DEF_HELPER_3(cxlgb, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(aeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(adb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(axb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(seb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(sdb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(sxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(deb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(ddb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(dxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(meeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mdeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mdb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(mxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_4(mxdb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_2(ldeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(ldxb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(lxdb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(lxeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(ledb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(lexb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(ceb, TCG_CALL_NO_WG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_3(cdb, TCG_CALL_NO_WG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_5(cxb, TCG_CALL_NO_WG_SE, i32, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cgeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(cgdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(cgxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(cfeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(cfdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(cfxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(clgeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(clgdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(clgxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(clfeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(clfdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(clfxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_4(maeb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(madb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(mseb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(msdb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_2(tceb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_2(tcdb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(tcxb, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_2(sqeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(sqdb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(sqxb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_1(cvd, TCG_CALL_NO_RWG_SE, i64, s32)
+DEF_HELPER_FLAGS_4(unpk, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(tr, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_4(cksm, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_5(calc_cc, TCG_CALL_NO_RWG_SE, i32, env, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_2(sfpc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(sfas, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_1(popcnt, TCG_CALL_NO_RWG_SE, i64, i64)
 
-DEF_HELPER_2(servc, i32, i32, i64)
-DEF_HELPER_3(diag, i64, i32, i64, i64)
-DEF_HELPER_2(load_psw, void, i64, i64)
-DEF_HELPER_1(program_interrupt, void, i32)
-DEF_HELPER_FLAGS_1(stidp, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(spx, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(sck, TCG_CALL_CONST, i32, i64)
-DEF_HELPER_1(stck, i32, i64)
-DEF_HELPER_1(stcke, i32, i64)
-DEF_HELPER_FLAGS_1(sckc, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(stckc, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(spt, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(stpt, TCG_CALL_CONST, void, i64)
-DEF_HELPER_3(stsi, i32, i64, i32, i32)
-DEF_HELPER_3(lctl, void, i32, i64, i32)
-DEF_HELPER_3(lctlg, void, i32, i64, i32)
-DEF_HELPER_3(stctl, void, i32, i64, i32)
-DEF_HELPER_3(stctg, void, i32, i64, i32)
-DEF_HELPER_FLAGS_2(tprot, TCG_CALL_CONST, i32, i64, i64)
-DEF_HELPER_FLAGS_1(iske, TCG_CALL_PURE|TCG_CALL_CONST, i64, i64)
-DEF_HELPER_FLAGS_2(sske, TCG_CALL_CONST, void, i32, i64)
-DEF_HELPER_FLAGS_2(rrbe, TCG_CALL_CONST, i32, i32, i64)
-DEF_HELPER_2(csp, i32, i32, i32)
-DEF_HELPER_3(mvcs, i32, i64, i64, i64)
-DEF_HELPER_3(mvcp, i32, i64, i64, i64)
-DEF_HELPER_3(sigp, i32, i64, i32, i64)
-DEF_HELPER_1(sacf, void, i64)
-DEF_HELPER_FLAGS_2(ipte, TCG_CALL_CONST, void, i64, i64)
-DEF_HELPER_FLAGS_0(ptlb, TCG_CALL_CONST, void)
-DEF_HELPER_2(lra, i32, i64, i32)
-DEF_HELPER_2(stura, void, i64, i32)
-DEF_HELPER_2(cksm, void, i32, i32)
+#ifndef CONFIG_USER_ONLY
+DEF_HELPER_3(servc, i32, env, i64, i64)
+DEF_HELPER_4(diag, i64, env, i32, i64, i64)
+DEF_HELPER_3(load_psw, noreturn, env, i64, i64)
+DEF_HELPER_FLAGS_2(spx, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_1(stck, TCG_CALL_NO_RWG_SE, i64, env)
+DEF_HELPER_FLAGS_2(sckc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_1(stckc, TCG_CALL_NO_RWG, i64, env)
+DEF_HELPER_FLAGS_2(spt, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_1(stpt, TCG_CALL_NO_RWG, i64, env)
+DEF_HELPER_4(stsi, i32, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(lctl, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(lctlg, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(stctl, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(stctg, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_2(tprot, TCG_CALL_NO_RWG, i32, i64, i64)
+DEF_HELPER_FLAGS_2(iske, TCG_CALL_NO_RWG_SE, i64, env, i64)
+DEF_HELPER_FLAGS_3(sske, TCG_CALL_NO_RWG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(rrbe, TCG_CALL_NO_RWG, i32, env, i64)
+DEF_HELPER_3(csp, i32, env, i32, i64)
+DEF_HELPER_4(mvcs, i32, env, i64, i64, i64)
+DEF_HELPER_4(mvcp, i32, env, i64, i64, i64)
+DEF_HELPER_4(sigp, i32, env, i64, i32, i64)
+DEF_HELPER_FLAGS_2(sacf, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_3(ipte, TCG_CALL_NO_RWG, void, env, i64, i64)
+DEF_HELPER_FLAGS_1(ptlb, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_2(lra, i64, env, i64)
+DEF_HELPER_FLAGS_3(stura, TCG_CALL_NO_WG, void, env, i64, i64)
+#endif
 
-DEF_HELPER_FLAGS_4(calc_cc, TCG_CALL_PURE|TCG_CALL_CONST,
-                   i32, i32, i64, i64, i64)
-
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-s390x/insn-data.def b/target-s390x/insn-data.def
new file mode 100644
index 00000000000..b42ebb6a1a3
--- /dev/null
+++ b/target-s390x/insn-data.def
@@ -0,0 +1,813 @@
+/* ADD */
+    C(0x1a00, AR,      RR_a,  Z,   r1, r2, new, r1_32, add, adds32)
+    C(0xb9f8, ARK,     RRF_a, DO,  r2, r3, new, r1_32, add, adds32)
+    C(0x5a00, A,       RX_a,  Z,   r1, m2_32s, new, r1_32, add, adds32)
+    C(0xe35a, AY,      RXY_a, LD,  r1, m2_32s, new, r1_32, add, adds32)
+    C(0xb908, AGR,     RRE,   Z,   r1, r2, r1, 0, add, adds64)
+    C(0xb918, AGFR,    RRE,   Z,   r1, r2_32s, r1, 0, add, adds64)
+    C(0xb9e8, AGRK,    RRF_a, DO,  r2, r3, r1, 0, add, adds64)
+    C(0xe308, AG,      RXY_a, Z,   r1, m2_64, r1, 0, add, adds64)
+    C(0xe318, AGF,     RXY_a, Z,   r1, m2_32s, r1, 0, add, adds64)
+    C(0xb30a, AEBR,    RRE,   Z,   e1, e2, new, e1, aeb, f32)
+    C(0xb31a, ADBR,    RRE,   Z,   f1_o, f2_o, f1, 0, adb, f64)
+    C(0xb34a, AXBR,    RRE,   Z,   0, x2_o, x1, 0, axb, f128)
+    C(0xed0a, AEB,     RXE,   Z,   e1, m2_32u, new, e1, aeb, f32)
+    C(0xed1a, ADB,     RXE,   Z,   f1_o, m2_64, f1, 0, adb, f64)
+/* ADD IMMEDIATE */
+    C(0xc209, AFI,     RIL_a, EI,  r1, i2, new, r1_32, add, adds32)
+    C(0xeb6a, ASI,     SIY,   GIE, m1_32s, i2, new, m1_32, add, adds32)
+    C(0xecd8, AHIK,    RIE_d, DO,  r3, i2, new, r1_32, add, adds32)
+    C(0xc208, AGFI,    RIL_a, EI,  r1, i2, r1, 0, add, adds64)
+    C(0xeb7a, AGSI,    SIY,   GIE, m1_64, i2, new, m1_64, add, adds64)
+    C(0xecd9, AGHIK,   RIE_d, DO,  r3, i2, r1, 0, add, adds64)
+/* ADD HALFWORD */
+    C(0x4a00, AH,      RX_a,  Z,   r1, m2_16s, new, r1_32, add, adds32)
+    C(0xe37a, AHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, add, adds32)
+/* ADD HALFWORD IMMEDIATE */
+    C(0xa70a, AHI,     RI_a,  Z,   r1, i2, new, r1_32, add, adds32)
+    C(0xa70b, AGHI,    RI_a,  Z,   r1, i2, r1, 0, add, adds64)
+
+/* ADD LOGICAL */
+    C(0x1e00, ALR,     RR_a,  Z,   r1, r2, new, r1_32, add, addu32)
+    C(0xb9fa, ALRK,    RRF_a, DO,  r2, r3, new, r1_32, add, addu32)
+    C(0x5e00, AL,      RX_a,  Z,   r1, m2_32u, new, r1_32, add, addu32)
+    C(0xe35e, ALY,     RXY_a, LD,  r1, m2_32u, new, r1_32, add, addu32)
+    C(0xb90a, ALGR,    RRE,   Z,   r1, r2, r1, 0, add, addu64)
+    C(0xb91a, ALGFR,   RRE,   Z,   r1, r2_32u, r1, 0, add, addu64)
+    C(0xb9ea, ALGRK,   RRF_a, DO,  r2, r3, r1, 0, add, addu64)
+    C(0xe30a, ALG,     RXY_a, Z,   r1, m2_64, r1, 0, add, addu64)
+    C(0xe31a, ALGF,    RXY_a, Z,   r1, m2_32u, r1, 0, add, addu64)
+/* ADD LOGICAL IMMEDIATE */
+    C(0xc20b, ALFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, add, addu32)
+    C(0xc20a, ALGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, add, addu64)
+/* ADD LOGICAL WITH SIGNED IMMEDIATE */
+    C(0xeb6e, ALSI,    SIY,   GIE, m1_32u, i2, new, m1_32, add, addu32)
+    C(0xecda, ALHSIK,  RIE_d, DO,  r3, i2, new, r1_32, add, addu32)
+    C(0xeb7e, ALGSI,   SIY,   GIE, m1_64, i2, new, m1_64, add, addu64)
+    C(0xecdb, ALGHSIK, RIE_d, DO,  r3, i2, r1, 0, add, addu64)
+/* ADD LOGICAL WITH CARRY */
+    C(0xb998, ALCR,    RRE,   Z,   r1, r2, new, r1_32, addc, addc32)
+    C(0xb988, ALCGR,   RRE,   Z,   r1, r2, r1, 0, addc, addc64)
+    C(0xe398, ALC,     RXY_a, Z,   r1, m2_32u, new, r1_32, addc, addc32)
+    C(0xe388, ALCG,    RXY_a, Z,   r1, m2_64, r1, 0, addc, addc64)
+
+/* AND */
+    C(0x1400, NR,      RR_a,  Z,   r1, r2, new, r1_32, and, nz32)
+    C(0xb9f4, NRK,     RRF_a, DO,  r2, r3, new, r1_32, and, nz32)
+    C(0x5400, N,       RX_a,  Z,   r1, m2_32s, new, r1_32, and, nz32)
+    C(0xe354, NY,      RXY_a, LD,  r1, m2_32s, new, r1_32, and, nz32)
+    C(0xb980, NGR,     RRE,   Z,   r1, r2, r1, 0, and, nz64)
+    C(0xb9e4, NGRK,    RRF_a, DO,  r2, r3, r1, 0, and, nz64)
+    C(0xe380, NG,      RXY_a, Z,   r1, m2_64, r1, 0, and, nz64)
+    C(0xd400, NC,      SS_a,  Z,   la1, a2, 0, 0, nc, 0)
+/* AND IMMEDIATE */
+    D(0xc00a, NIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, andi, 0, 0x2020)
+    D(0xc00b, NILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, andi, 0, 0x2000)
+    D(0xa504, NIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1030)
+    D(0xa505, NIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1020)
+    D(0xa506, NILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1010)
+    D(0xa507, NILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1000)
+    C(0x9400, NI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, and, nz64)
+    C(0xeb54, NIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, and, nz64)
+
+/* BRANCH AND SAVE */
+    C(0x0d00, BASR,    RR_a,  Z,   0, r2_nz, r1, 0, bas, 0)
+    C(0x4d00, BAS,     RX_a,  Z,   0, a2, r1, 0, bas, 0)
+/* BRANCH RELATIVE AND SAVE */
+    C(0xa705, BRAS,    RI_b,  Z,   0, 0, r1, 0, basi, 0)
+    C(0xc005, BRASL,   RIL_b, Z,   0, 0, r1, 0, basi, 0)
+/* BRANCH ON CONDITION */
+    C(0x0700, BCR,     RR_b,  Z,   0, r2_nz, 0, 0, bc, 0)
+    C(0x4700, BC,      RX_b,  Z,   0, a2, 0, 0, bc, 0)
+/* BRANCH RELATIVE ON CONDITION */
+    C(0xa704, BRC,     RI_c,  Z,   0, 0, 0, 0, bc, 0)
+    C(0xc004, BRCL,    RIL_c, Z,   0, 0, 0, 0, bc, 0)
+/* BRANCH ON COUNT */
+    C(0x0600, BCTR,    RR_a,  Z,   0, r2_nz, 0, 0, bct32, 0)
+    C(0xb946, BCTGR,   RRE,   Z,   0, r2_nz, 0, 0, bct64, 0)
+    C(0x4600, BCT,     RX_a,  Z,   0, a2, 0, 0, bct32, 0)
+    C(0xe346, BCTG,    RXY_a, Z,   0, a2, 0, 0, bct64, 0)
+/* BRANCH RELATIVE ON COUNT */
+    C(0xa706, BRCT,    RI_b,  Z,   0, 0, 0, 0, bct32, 0)
+    C(0xa707, BRCTG,   RI_b,  Z,   0, 0, 0, 0, bct64, 0)
+/* BRANCH ON INDEX */
+    D(0x8600, BXH,     RS_a,  Z,   0, a2, 0, 0, bx32, 0, 0)
+    D(0x8700, BXLE,    RS_a,  Z,   0, a2, 0, 0, bx32, 0, 1)
+    D(0xeb44, BXHG,    RSY_a, Z,   0, a2, 0, 0, bx64, 0, 0)
+    D(0xeb45, BXLEG,   RSY_a, Z,   0, a2, 0, 0, bx64, 0, 1)
+/* BRANCH RELATIVE ON INDEX */
+    D(0x8400, BRXH,    RSI,   Z,   0, 0, 0, 0, bx32, 0, 0)
+    D(0x8500, BRXLE,   RSI,   Z,   0, 0, 0, 0, bx32, 0, 1)
+    D(0xec44, BRXHG,   RIE_e, Z,   0, 0, 0, 0, bx64, 0, 0)
+    D(0xec45, BRXHLE,  RIE_e, Z,   0, 0, 0, 0, bx64, 0, 1)
+
+/* CHECKSUM */
+    C(0xb241, CKSM,    RRE,   Z,   r1_o, ra2, new, r1_32, cksm, 0)
+
+/* COPY SIGN */
+    C(0xb372, CPSDR,   RRF_b, FPSSH, f3_o, f2_o, f1, 0, cps, 0)
+
+/* COMPARE */
+    C(0x1900, CR,      RR_a,  Z,   r1_o, r2_o, 0, 0, 0, cmps32)
+    C(0x5900, C,       RX_a,  Z,   r1_o, m2_32s, 0, 0, 0, cmps32)
+    C(0xe359, CY,      RXY_a, LD,  r1_o, m2_32s, 0, 0, 0, cmps32)
+    C(0xb920, CGR,     RRE,   Z,   r1_o, r2_o, 0, 0, 0, cmps64)
+    C(0xb930, CGFR,    RRE,   Z,   r1_o, r2_32s, 0, 0, 0, cmps64)
+    C(0xe320, CG,      RXY_a, Z,   r1_o, m2_64, 0, 0, 0, cmps64)
+    C(0xe330, CGF,     RXY_a, Z,   r1_o, m2_32s, 0, 0, 0, cmps64)
+    C(0xb309, CEBR,    RRE,   Z,   e1, e2, 0, 0, ceb, 0)
+    C(0xb319, CDBR,    RRE,   Z,   f1_o, f2_o, 0, 0, cdb, 0)
+    C(0xb349, CXBR,    RRE,   Z,   x1_o, x2_o, 0, 0, cxb, 0)
+    C(0xed09, CEB,     RXE,   Z,   e1, m2_32u, 0, 0, ceb, 0)
+    C(0xed19, CDB,     RXE,   Z,   f1_o, m2_64, 0, 0, cdb, 0)
+/* COMPARE IMMEDIATE */
+    C(0xc20d, CFI,     RIL_a, EI,  r1, i2, 0, 0, 0, cmps32)
+    C(0xc20c, CGFI,    RIL_a, EI,  r1, i2, 0, 0, 0, cmps64)
+/* COMPARE RELATIVE LONG */
+    C(0xc60d, CRL,     RIL_b, GIE, r1, mri2_32s, 0, 0, 0, cmps32)
+    C(0xc608, CGRL,    RIL_b, GIE, r1, mri2_64, 0, 0, 0, cmps64)
+    C(0xc60c, CGFRL,   RIL_b, GIE, r1, mri2_32s, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD */
+    C(0x4900, CH,      RX_a,  Z,   r1_o, m2_16s, 0, 0, 0, cmps32)
+    C(0xe379, CHY,     RXY_a, LD,  r1_o, m2_16s, 0, 0, 0, cmps32)
+    C(0xe334, CGH,     RXY_a, GIE, r1_o, m2_16s, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD IMMEDIATE */
+    C(0xa70e, CHI,     RI_a,  Z,   r1_o, i2, 0, 0, 0, cmps32)
+    C(0xa70f, CGHI,    RI_a,  Z,   r1_o, i2, 0, 0, 0, cmps64)
+    C(0xe554, CHHSI,   SIL,   GIE, m1_16s, i2, 0, 0, 0, cmps64)
+    C(0xe55c, CHSI,    SIL,   GIE, m1_32s, i2, 0, 0, 0, cmps64)
+    C(0xe558, CGHSI,   SIL,   GIE, m1_64, i2, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD RELATIVE LONG */
+    C(0xc605, CHRL,    RIL_a, GIE, r1_o, mri2_32s, 0, 0, 0, cmps32)
+    C(0xc604, CGHRL,   RIL_a, GIE, r1_o, mri2_64, 0, 0, 0, cmps64)
+
+/* COMPARE LOGICAL */
+    C(0x1500, CLR,     RR_a,  Z,   r1, r2, 0, 0, 0, cmpu32)
+    C(0x5500, CL,      RX_a,  Z,   r1, m2_32s, 0, 0, 0, cmpu32)
+    C(0xe355, CLY,     RXY_a, LD,  r1, m2_32s, 0, 0, 0, cmpu32)
+    C(0xb921, CLGR,    RRE,   Z,   r1, r2, 0, 0, 0, cmpu64)
+    C(0xb931, CLGFR,   RRE,   Z,   r1, r2_32u, 0, 0, 0, cmpu64)
+    C(0xe321, CLG,     RXY_a, Z,   r1, m2_64, 0, 0, 0, cmpu64)
+    C(0xe331, CLGF,    RXY_a, Z,   r1, m2_32u, 0, 0, 0, cmpu64)
+    C(0xd500, CLC,     SS_a,  Z,   la1, a2, 0, 0, clc, 0)
+/* COMPARE LOGICAL IMMEDIATE */
+    C(0xc20f, CLFI,    RIL_a, EI,  r1, i2, 0, 0, 0, cmpu32)
+    C(0xc20e, CLGFI,   RIL_a, EI,  r1, i2_32u, 0, 0, 0, cmpu64)
+    C(0x9500, CLI,     SI,    Z,   m1_8u, i2_8u, 0, 0, 0, cmpu64)
+    C(0xeb55, CLIY,    SIY,   LD,  m1_8u, i2_8u, 0, 0, 0, cmpu64)
+    C(0xe555, CLHHSI,  SIL,   GIE, m1_16u, i2_16u, 0, 0, 0, cmpu64)
+    C(0xe55d, CLFHSI,  SIL,   GIE, m1_32u, i2_16u, 0, 0, 0, cmpu64)
+    C(0xe559, CLGHSI,  SIL,   GIE, m1_64, i2_16u, 0, 0, 0, cmpu64)
+/* COMPARE LOGICAL RELATIVE LONG */
+    C(0xc60f, CLRL,    RIL_b, GIE, r1_o, mri2_32u, 0, 0, 0, cmpu32)
+    C(0xc60a, CLGRL,   RIL_b, GIE, r1_o, mri2_64, 0, 0, 0, cmpu64)
+    C(0xc60e, CLGFRL,  RIL_b, GIE, r1_o, mri2_32u, 0, 0, 0, cmpu64)
+    C(0xc607, CLHRL,   RIL_b, GIE, r1_o, mri2_16u, 0, 0, 0, cmpu32)
+    C(0xc606, CLGHRL,  RIL_b, GIE, r1_o, mri2_16u, 0, 0, 0, cmpu64)
+/* COMPARE LOGICAL LONG EXTENDED */
+    C(0xa900, CLCLE,   RS_a,  Z,   0, a2, 0, 0, clcle, 0)
+/* COMPARE LOGICAL CHARACTERS UNDER MASK */
+    C(0xbd00, CLM,     RS_b,  Z,   r1_o, a2, 0, 0, clm, 0)
+    C(0xeb21, CLMY,    RSY_b, LD,  r1_o, a2, 0, 0, clm, 0)
+    C(0xeb20, CLMH,    RSY_b, Z,   r1_sr32, a2, 0, 0, clm, 0)
+/* COMPARE LOGICAL STRING */
+    C(0xb25d, CLST,    RRE,   Z,   r1_o, r2_o, 0, 0, clst, 0)
+
+/* COMPARE AND BRANCH */
+    D(0xecf6, CRB,     RRS,   GIE, r1_32s, r2_32s, 0, 0, cj, 0, 0)
+    D(0xece4, CGRB,    RRS,   GIE, r1_o, r2_o, 0, 0, cj, 0, 0)
+    D(0xec76, CRJ,     RIE_b, GIE, r1_32s, r2_32s, 0, 0, cj, 0, 0)
+    D(0xec64, CGRJ,    RIE_b, GIE, r1_o, r2_o, 0, 0, cj, 0, 0)
+    D(0xecfe, CIB,     RIS,   GIE, r1_32s, i2, 0, 0, cj, 0, 0)
+    D(0xecfc, CGIB,    RIS,   GIE, r1_o, i2, 0, 0, cj, 0, 0)
+    D(0xec7e, CIJ,     RIE_c, GIE, r1_32s, i2, 0, 0, cj, 0, 0)
+    D(0xec7c, CGIJ,    RIE_c, GIE, r1_o, i2, 0, 0, cj, 0, 0)
+/* COMPARE LOGICAL AND BRANCH */
+    D(0xecf7, CLRB,    RRS,   GIE, r1_32u, r2_32u, 0, 0, cj, 0, 1)
+    D(0xece5, CLGRB,   RRS,   GIE, r1_o, r2_o, 0, 0, cj, 0, 1)
+    D(0xec77, CLRJ,    RIE_b, GIE, r1_32u, r2_32u, 0, 0, cj, 0, 1)
+    D(0xec65, CLGRJ,   RIE_b, GIE, r1_o, r2_o, 0, 0, cj, 0, 1)
+    D(0xecff, CLIB,    RIS,   GIE, r1_32u, i2_8u, 0, 0, cj, 0, 1)
+    D(0xecfd, CLGIB,   RIS,   GIE, r1_o, i2_8u, 0, 0, cj, 0, 1)
+    D(0xec7f, CLIJ,    RIE_c, GIE, r1_32u, i2_8u, 0, 0, cj, 0, 1)
+    D(0xec7d, CLGIJ,   RIE_c, GIE, r1_o, i2_8u, 0, 0, cj, 0, 1)
+
+/* COMPARE AND SWAP */
+    D(0xba00, CS,      RS_a,  Z,   r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb14, CSY,     RSY_a, LD,  r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb30, CSG,     RSY_a, Z,   r3_o, r1_o, new, r1, cs, 0, 1)
+/* COMPARE DOUBLE AND SWAP */
+    D(0xbb00, CDS,     RS_a,  Z,   r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    D(0xeb31, CDSY,    RSY_a, LD,  r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    C(0xeb3e, CDSG,    RSY_a, Z,   0, 0, 0, 0, cdsg, 0)
+
+/* COMPARE AND TRAP */
+    D(0xb972, CRT,     RRF_c, GIE, r1_32s, r2_32s, 0, 0, ct, 0, 0)
+    D(0xb960, CGRT,    RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 0)
+    D(0xec72, CIT,     RIE_a, GIE, r1_32s, i2, 0, 0, ct, 0, 0)
+    D(0xec70, CGIT,    RIE_a, GIE, r1_o, i2, 0, 0, ct, 0, 0)
+/* COMPARE LOGICAL AND TRAP */
+    D(0xb973, CLRT,    RRF_c, GIE, r1_32u, r2_32u, 0, 0, ct, 0, 1)
+    D(0xb961, CLGRT,   RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 1)
+    D(0xec73, CLFIT,   RIE_a, GIE, r1_32u, i2_32u, 0, 0, ct, 0, 1)
+    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_32u, 0, 0, ct, 0, 0)
+
+/* CONVERT TO DECIMAL */
+    C(0x4e00, CVD,     RX_a,  Z,   r1_o, a2, 0, 0, cvd, 0)
+    C(0xe326, CVDY,    RXY_a, LD,  r1_o, a2, 0, 0, cvd, 0)
+/* CONVERT TO FIXED */
+    C(0xb398, CFEBR,   RRF_e, Z,   0, e2, new, r1_32, cfeb, 0)
+    C(0xb399, CFDBR,   RRF_e, Z,   0, f2_o, new, r1_32, cfdb, 0)
+    C(0xb39a, CFXBR,   RRF_e, Z,   0, x2_o, new, r1_32, cfxb, 0)
+    C(0xb3a8, CGEBR,   RRF_e, Z,   0, e2, r1, 0, cgeb, 0)
+    C(0xb3a9, CGDBR,   RRF_e, Z,   0, f2_o, r1, 0, cgdb, 0)
+    C(0xb3aa, CGXBR,   RRF_e, Z,   0, x2_o, r1, 0, cgxb, 0)
+/* CONVERT FROM FIXED */
+    C(0xb394, CEFBR,   RRF_e, Z,   0, r2_32s, new, e1, cegb, 0)
+    C(0xb395, CDFBR,   RRF_e, Z,   0, r2_32s, f1, 0, cdgb, 0)
+    C(0xb396, CXFBR,   RRF_e, Z,   0, r2_32s, x1, 0, cxgb, 0)
+    C(0xb3a4, CEGBR,   RRF_e, Z,   0, r2_o, new, e1, cegb, 0)
+    C(0xb3a5, CDGBR,   RRF_e, Z,   0, r2_o, f1, 0, cdgb, 0)
+    C(0xb3a6, CXGBR,   RRF_e, Z,   0, r2_o, x1, 0, cxgb, 0)
+/* CONVERT TO LOGICAL */
+    C(0xb39c, CLFEBR,  RRF_e, FPE, 0, e2, new, r1_32, clfeb, 0)
+    C(0xb39d, CLFDBR,  RRF_e, FPE, 0, f2_o, new, r1_32, clfdb, 0)
+    C(0xb39e, CLFXBR,  RRF_e, FPE, 0, x2_o, new, r1_32, clfxb, 0)
+    C(0xb3ac, CLGEBR,  RRF_e, FPE, 0, e2, r1, 0, clgeb, 0)
+    C(0xb3ad, CLGDBR,  RRF_e, FPE, 0, f2_o, r1, 0, clgdb, 0)
+    C(0xb3ae, CLGXBR,  RRF_e, FPE, 0, x2_o, r1, 0, clgxb, 0)
+/* CONVERT FROM LOGICAL */
+    C(0xb390, CELFBR,  RRF_e, FPE, 0, r2_32u, new, e1, celgb, 0)
+    C(0xb391, CDLFBR,  RRF_e, FPE, 0, r2_32u, f1, 0, cdlgb, 0)
+    C(0xb392, CXLFBR,  RRF_e, FPE, 0, r2_32u, x1, 0, cxlgb, 0)
+    C(0xb3a0, CELGBR,  RRF_e, FPE, 0, r2_o, new, e1, celgb, 0)
+    C(0xb3a1, CDLGBR,  RRF_e, FPE, 0, r2_o, f1, 0, cdlgb, 0)
+    C(0xb3a2, CXLGBR,  RRF_e, FPE, 0, r2_o, x1, 0, cxlgb, 0)
+
+/* DIVIDE */
+    C(0x1d00, DR,      RR_a,  Z,   r1_D32, r2_32s, new_P, r1_P32, divs32, 0)
+    C(0x5d00, D,       RX_a,  Z,   r1_D32, m2_32s, new_P, r1_P32, divs32, 0)
+    C(0xb30d, DEBR,    RRE,   Z,   e1, e2, new, e1, deb, 0)
+    C(0xb31d, DDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, ddb, 0)
+    C(0xb34d, DXBR,    RRE,   Z,   0, x2_o, x1, 0, dxb, 0)
+    C(0xed0d, DEB,     RXE,   Z,   e1, m2_32u, new, e1, deb, 0)
+    C(0xed1d, DDB,     RXE,   Z,   f1_o, m2_64, f1, 0, ddb, 0)
+/* DIVIDE LOGICAL */
+    C(0xb997, DLR,     RRE,   Z,   r1_D32, r2_32u, new_P, r1_P32, divu32, 0)
+    C(0xe397, DL,      RXY_a, Z,   r1_D32, m2_32u, new_P, r1_P32, divu32, 0)
+    C(0xb987, DLGR,    RRE,   Z,   0, r2_o, r1_P, 0, divu64, 0)
+    C(0xe387, DLG,     RXY_a, Z,   0, m2_64, r1_P, 0, divu64, 0)
+/* DIVIDE SINGLE */
+    C(0xb90d, DSGR,    RRE,   Z,   r1p1, r2, r1_P, 0, divs64, 0)
+    C(0xb91d, DSGFR,   RRE,   Z,   r1p1, r2_32s, r1_P, 0, divs64, 0)
+    C(0xe30d, DSG,     RXY_a, Z,   r1p1, m2_64, r1_P, 0, divs64, 0)
+    C(0xe31d, DSGF,    RXY_a, Z,   r1p1, m2_32s, r1_P, 0, divs64, 0)
+
+/* EXCLUSIVE OR */
+    C(0x1700, XR,      RR_a,  Z,   r1, r2, new, r1_32, xor, nz32)
+    C(0xb9f7, XRK,     RRF_a, DO,  r2, r3, new, r1_32, xor, nz32)
+    C(0x5700, X,       RX_a,  Z,   r1, m2_32s, new, r1_32, xor, nz32)
+    C(0xe357, XY,      RXY_a, LD,  r1, m2_32s, new, r1_32, xor, nz32)
+    C(0xb982, XGR,     RRE,   Z,   r1, r2, r1, 0, xor, nz64)
+    C(0xb9e7, XGRK,    RRF_a, DO,  r2, r3, r1, 0, xor, nz64)
+    C(0xe382, XG,      RXY_a, Z,   r1, m2_64, r1, 0, xor, nz64)
+    C(0xd700, XC,      SS_a,  Z,   0, 0, 0, 0, xc, 0)
+/* EXCLUSIVE OR IMMEDIATE */
+    D(0xc006, XIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, xori, 0, 0x2020)
+    D(0xc007, XILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, xori, 0, 0x2000)
+    C(0x9700, XI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, xor, nz64)
+    C(0xeb57, XIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, xor, nz64)
+
+/* EXECUTE */
+    C(0x4400, EX,      RX_a,  Z,   r1_o, a2, 0, 0, ex, 0)
+/* EXECUTE RELATIVE LONG */
+    C(0xc600, EXRL,    RIL_b, EE,  r1_o, ri2, 0, 0, ex, 0)
+
+/* EXTRACT ACCESS */
+    C(0xb24f, EAR,     RRE,   Z,   0, 0, new, r1_32, ear, 0)
+/* EXTRACT FPC */
+    C(0xb38c, EFPC,    RRE,   Z,   0, 0, new, r1_32, efpc, 0)
+
+/* FIND LEFTMOST ONE */
+    C(0xb983, FLOGR,   RRE,   EI,  0, r2_o, r1_P, 0, flogr, 0)
+
+/* INSERT CHARACTER */
+    C(0x4300, IC,      RX_a,  Z,   0, m2_8u, 0, r1_8, mov2, 0)
+    C(0xe373, ICY,     RXY_a, LD,  0, m2_8u, 0, r1_8, mov2, 0)
+/* INSERT CHARACTERS UNDER MASK */
+    D(0xbf00, ICM,     RS_b,  Z,   0, a2, r1, 0, icm, 0, 0)
+    D(0xeb81, ICMY,    RSY_b, LD,  0, a2, r1, 0, icm, 0, 0)
+    D(0xeb80, ICMH,    RSY_b, Z,   0, a2, r1, 0, icm, 0, 32)
+/* INSERT IMMEDIATE */
+    D(0xc008, IIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, insi, 0, 0x2020)
+    D(0xc009, IILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, insi, 0, 0x2000)
+    D(0xa500, IIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1030)
+    D(0xa501, IIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1020)
+    D(0xa502, IILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1010)
+    D(0xa503, IILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1000)
+/* INSERT PROGRAM MASK */
+    C(0xb222, IPM,     RRE,   Z,   0, 0, r1, 0, ipm, 0)
+
+/* LOAD */
+    C(0x1800, LR,      RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, 0)
+    C(0x5800, L,       RX_a,  Z,   0, a2, new, r1_32, ld32s, 0)
+    C(0xe358, LY,      RXY_a, Z,   0, a2, new, r1_32, ld32s, 0)
+    C(0xb904, LGR,     RRE,   Z,   0, r2_o, 0, r1, mov2, 0)
+    C(0xb914, LGFR,    RRE,   Z,   0, r2_32s, 0, r1, mov2, 0)
+    C(0xe304, LG,      RXY_a, Z,   0, a2, r1, 0, ld64, 0)
+    C(0xe314, LGF,     RXY_a, Z,   0, a2, r1, 0, ld32s, 0)
+    C(0x2800, LDR,     RR_a,  Z,   0, f2_o, 0, f1, mov2, 0)
+    C(0x6800, LD,      RX_a,  Z,   0, m2_64, 0, f1, mov2, 0)
+    C(0xed65, LDY,     RXY_a, LD,  0, m2_64, 0, f1, mov2, 0)
+    C(0x3800, LER,     RR_a,  Z,   0, e2, 0, cond_e1e2, mov2, 0)
+    C(0x7800, LE,      RX_a,  Z,   0, m2_32u, 0, e1, mov2, 0)
+    C(0xed64, LEY,     RXY_a, LD,  0, m2_32u, 0, e1, mov2, 0)
+    C(0xb365, LXR,     RRE,   Z,   0, x2_o, 0, x1, movx, 0)
+/* LOAD IMMEDIATE */
+    C(0xc001, LGFI,    RIL_a, EI,  0, i2, 0, r1, mov2, 0)
+/* LOAD RELATIVE LONG */
+    C(0xc40d, LRL,     RIL_b, GIE, 0, ri2, new, r1_32, ld32s, 0)
+    C(0xc408, LGRL,    RIL_b, GIE, 0, ri2, r1, 0, ld64, 0)
+    C(0xc40c, LGFRL,   RIL_b, GIE, 0, ri2, r1, 0, ld32s, 0)
+/* LOAD ADDRESS */
+    C(0x4100, LA,      RX_a,  Z,   0, a2, 0, r1, mov2, 0)
+    C(0xe371, LAY,     RXY_a, LD,  0, a2, 0, r1, mov2, 0)
+/* LOAD ADDRESS RELATIVE LONG */
+    C(0xc000, LARL,    RIL_b, Z,   0, ri2, 0, r1, mov2, 0)
+/* LOAD AND TEST */
+    C(0x1200, LTR,     RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, s32)
+    C(0xb902, LTGR,    RRE,   Z,   0, r2_o, 0, r1, mov2, s64)
+    C(0xb912, LTGFR,   RRE,   Z,   0, r2_32s, 0, r1, mov2, s64)
+    C(0xe312, LT,      RXY_a, EI,  0, a2, new, r1_32, ld32s, s64)
+    C(0xe302, LTG,     RXY_a, EI,  0, a2, r1, 0, ld64, s64)
+    C(0xe332, LTGF,    RXY_a, GIE, 0, a2, r1, 0, ld32s, s64)
+    C(0xb302, LTEBR,   RRE,   Z,   0, e2, 0, cond_e1e2, mov2, f32)
+    C(0xb312, LTDBR,   RRE,   Z,   0, f2_o, 0, f1, mov2, f64)
+    C(0xb342, LTXBR,   RRE,   Z,   0, x2_o, 0, x1, movx, f128)
+/* LOAD BYTE */
+    C(0xb926, LBR,     RRE,   EI,  0, r2_8s, 0, r1_32, mov2, 0)
+    C(0xb906, LGBR,    RRE,   EI,  0, r2_8s, 0, r1, mov2, 0)
+    C(0xe376, LB,      RXY_a, LD,  0, a2, new, r1_32, ld8s, 0)
+    C(0xe377, LGB,     RXY_a, LD,  0, a2, r1, 0, ld8s, 0)
+/* LOAD COMPLEMENT */
+    C(0x1300, LCR,     RR_a,  Z,   0, r2, new, r1_32, neg, neg32)
+    C(0xb903, LCGR,    RRE,   Z,   0, r2, r1, 0, neg, neg64)
+    C(0xb913, LCGFR,   RRE,   Z,   0, r2_32s, r1, 0, neg, neg64)
+    C(0xb303, LCEBR,   RRE,   Z,   0, e2, new, e1, negf32, f32)
+    C(0xb313, LCDBR,   RRE,   Z,   0, f2_o, f1, 0, negf64, f64)
+    C(0xb343, LCXBR,   RRE,   Z,   0, x2_o, x1, 0, negf128, f128)
+    C(0xb373, LCDFR,   RRE,   FPSSH, 0, f2_o, f1, 0, negf64, 0)
+/* LOAD HALFWORD */
+    C(0xb927, LHR,     RRE,   EI,  0, r2_16s, 0, r1_32, mov2, 0)
+    C(0xb907, LGHR,    RRE,   EI,  0, r2_16s, 0, r1, mov2, 0)
+    C(0x4800, LH,      RX_a,  Z,   0, a2, new, r1_32, ld16s, 0)
+    C(0xe378, LHY,     RXY_a, LD,  0, a2, new, r1_32, ld16s, 0)
+    C(0xe315, LGH,     RXY_a, Z,   0, a2, r1, 0, ld16s, 0)
+/* LOAD HALFWORD IMMEDIATE */
+    C(0xa708, LHI,     RI_a,  Z,   0, i2, 0, r1_32, mov2, 0)
+    C(0xa709, LGHI,    RI_a,  Z,   0, i2, 0, r1, mov2, 0)
+/* LOAD HALFWORD RELATIVE LONG */
+    C(0xc405, LHRL,    RIL_b, GIE, 0, ri2, new, r1_32, ld16s, 0)
+    C(0xc404, LGHRL,   RIL_b, GIE, 0, ri2, r1, 0, ld16s, 0)
+/* LOAD LOGICAL */
+    C(0xb916, LLGFR,   RRE,   Z,   0, r2_32u, 0, r1, mov2, 0)
+    C(0xe316, LLGF,    RXY_a, Z,   0, a2, r1, 0, ld32u, 0)
+/* LOAD LOGICAL RELATIVE LONG */
+    C(0xc40e, LLGFRL,  RIL_b, GIE, 0, ri2, r1, 0, ld32u, 0)
+/* LOAD LOGICAL CHARACTER */
+    C(0xb994, LLCR,    RRE,   EI,  0, r2_8u, 0, r1_32, mov2, 0)
+    C(0xb984, LLGCR,   RRE,   EI,  0, r2_8u, 0, r1, mov2, 0)
+    C(0xe394, LLC,     RXY_a, EI,  0, a2, new, r1_32, ld8u, 0)
+    C(0xe390, LLGC,    RXY_a, Z,   0, a2, r1, 0, ld8u, 0)
+/* LOAD LOGICAL HALFWORD */
+    C(0xb995, LLHR,    RRE,   EI,  0, r2_16u, 0, r1_32, mov2, 0)
+    C(0xb985, LLGHR,   RRE,   EI,  0, r2_16u, 0, r1, mov2, 0)
+    C(0xe395, LLH,     RXY_a, EI,  0, a2, new, r1_32, ld16u, 0)
+    C(0xe391, LLGH,    RXY_a, Z,   0, a2, r1, 0, ld16u, 0)
+/* LOAD LOGICAL HALFWORD RELATIVE LONG */
+    C(0xc402, LLHRL,   RIL_b, GIE, 0, ri2, new, r1_32, ld16u, 0)
+    C(0xc406, LLGHRL,  RIL_b, GIE, 0, ri2, r1, 0, ld16u, 0)
+/* LOAD LOGICAL IMMEDATE */
+    D(0xc00e, LLIHF,   RIL_a, EI, 0, i2_32u_shl, 0, r1, mov2, 0, 32)
+    D(0xc00f, LLILF,   RIL_a, EI, 0, i2_32u_shl, 0, r1, mov2, 0, 0)
+    D(0xa50c, LLIHH,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 48)
+    D(0xa50d, LLIHL,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 32)
+    D(0xa50e, LLILH,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 16)
+    D(0xa50f, LLILL,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 0)
+/* LOAD LOGICAL THIRTY ONE BITS */
+    C(0xb917, LLGTR,   RRE,   Z,  0, r2_o, r1, 0, llgt, 0)
+    C(0xe317, LLGT,    RXY_a, Z,  0, m2_32u, r1, 0, llgt, 0)
+/* LOAD FPR FROM GR */
+    C(0xb3c1, LDGR,    RRE,   FPRGR, 0, r2_o, 0, f1, mov2, 0)
+/* LOAD GR FROM FPR */
+    C(0xb3cd, LGDR,    RRE,   FPRGR, 0, f2_o, 0, r1, mov2, 0)
+/* LOAD NEGATIVE */
+    C(0x1100, LNR,     RR_a,  Z,   0, r2_32s, new, r1_32, nabs, nabs32)
+    C(0xb901, LNGR,    RRE,   Z,   0, r2, r1, 0, nabs, nabs64)
+    C(0xb911, LNGFR,   RRE,   Z,   0, r2_32s, r1, 0, nabs, nabs64)
+    C(0xb301, LNEBR,   RRE,   Z,   0, e2, new, e1, nabsf32, f32)
+    C(0xb311, LNDBR,   RRE,   Z,   0, f2_o, f1, 0, nabsf64, f64)
+    C(0xb341, LNXBR,   RRE,   Z,   0, x2_o, x1, 0, nabsf128, f128)
+/* LOAD ON CONDITION */
+    C(0xb9f2, LOCR,    RRF_c, LOC, r1, r2, new, r1_32, loc, 0)
+    C(0xb9e2, LOCGR,   RRF_c, LOC, r1, r2, r1, 0, loc, 0)
+    C(0xebf2, LOC,     RSY_b, LOC, r1, m2_32u, new, r1_32, loc, 0)
+    C(0xebe2, LOCG,    RSY_b, LOC, r1, m2_64, r1, 0, loc, 0)
+/* LOAD POSITIVE */
+    C(0x1000, LPR,     RR_a,  Z,   0, r2_32s, new, r1_32, abs, abs32)
+    C(0xb900, LPGR,    RRE,   Z,   0, r2, r1, 0, abs, abs64)
+    C(0xb910, LPGFR,   RRE,   Z,   0, r2_32s, r1, 0, abs, abs64)
+    C(0xb300, LPEBR,   RRE,   Z,   0, e2, new, e1, absf32, f32)
+    C(0xb310, LPDBR,   RRE,   Z,   0, f2_o, f1, 0, absf64, f64)
+    C(0xb340, LPXBR,   RRE,   Z,   0, x2_o, x1, 0, absf128, f128)
+/* LOAD REVERSED */
+    C(0xb91f, LRVR,    RRE,   Z,   0, r2_32u, new, r1_32, rev32, 0)
+    C(0xb90f, LRVGR,   RRE,   Z,   0, r2_o, r1, 0, rev64, 0)
+    C(0xe31f, LRVH,    RXY_a, Z,   0, m2_16u, new, r1_16, rev16, 0)
+    C(0xe31e, LRV,     RXY_a, Z,   0, m2_32u, new, r1_32, rev32, 0)
+    C(0xe30f, LRVG,    RXY_a, Z,   0, m2_64, r1, 0, rev64, 0)
+/* LOAD ZERO */
+    C(0xb374, LZER,    RRE,   Z,   0, 0, 0, e1, zero, 0)
+    C(0xb375, LZDR,    RRE,   Z,   0, 0, 0, f1, zero, 0)
+    C(0xb376, LZXR,    RRE,   Z,   0, 0, 0, x1, zero2, 0)
+
+/* LOAD FPC */
+    C(0xb29d, LFPC,    S,     Z,   0, m2_32u, 0, 0, sfpc, 0)
+/* LOAD FPC AND SIGNAL */
+    C(0xb2bd, LFAS,    S,     IEEEE_SIM, 0, m2_32u, 0, 0, sfas, 0)
+
+/* LOAD LENGTHENED */
+    C(0xb304, LDEBR,   RRE,   Z,   0, e2, f1, 0, ldeb, 0)
+    C(0xb305, LXDBR,   RRE,   Z,   0, f2_o, x1, 0, lxdb, 0)
+    C(0xb306, LXEBR,   RRE,   Z,   0, e2, x1, 0, lxeb, 0)
+    C(0xed04, LDEB,    RXE,   Z,   0, m2_32u, f1, 0, ldeb, 0)
+    C(0xed05, LXDB,    RXE,   Z,   0, m2_64, x1, 0, lxdb, 0)
+    C(0xed06, LXEB,    RXE,   Z,   0, m2_32u, x1, 0, lxeb, 0)
+/* LOAD ROUNDED */
+    C(0xb344, LEDBR,   RRE,   Z,   0, f2_o, new, e1, ledb, 0)
+    C(0xb345, LDXBR,   RRE,   Z,   0, x2_o, f1, 0, ldxb, 0)
+    C(0xb346, LEXBR,   RRE,   Z,   0, x2_o, new, e1, lexb, 0)
+
+/* LOAD MULTIPLE */
+    C(0x9800, LM,      RS_a,  Z,   0, a2, 0, 0, lm32, 0)
+    C(0xeb98, LMY,     RSY_a, LD,  0, a2, 0, 0, lm32, 0)
+    C(0xeb04, LMG,     RSY_a, Z,   0, a2, 0, 0, lm64, 0)
+/* LOAD MULTIPLE HIGH */
+    C(0xeb96, LMH,     RSY_a, Z,   0, a2, 0, 0, lmh, 0)
+/* LOAD ACCESS MULTIPLE */
+    C(0x9a00, LAM,     RS_a,  Z,   0, a2, 0, 0, lam, 0)
+    C(0xeb9a, LAMY,    RSY_a, LD,  0, a2, 0, 0, lam, 0)
+
+/* MOVE */
+    C(0xd200, MVC,     SS_a,  Z,   la1, a2, 0, 0, mvc, 0)
+    C(0xe544, MVHHI,   SIL,   GIE, la1, i2, 0, m1_16, mov2, 0)
+    C(0xe54c, MVHI,    SIL,   GIE, la1, i2, 0, m1_32, mov2, 0)
+    C(0xe548, MVGHI,   SIL,   GIE, la1, i2, 0, m1_64, mov2, 0)
+    C(0x9200, MVI,     SI,    Z,   la1, i2, 0, m1_8, mov2, 0)
+    C(0xeb52, MVIY,    SIY,   LD,  la1, i2, 0, m1_8, mov2, 0)
+/* MOVE LONG */
+    C(0x0e00, MVCL,    RR_a,  Z,   0, 0, 0, 0, mvcl, 0)
+/* MOVE LONG EXTENDED */
+    C(0xa800, MVCLE,   RS_a,  Z,   0, a2, 0, 0, mvcle, 0)
+/* MOVE PAGE */
+    C(0xb254, MVPG,    RRE,   Z,   r1_o, r2_o, 0, 0, mvpg, 0)
+/* MOVE STRING */
+    C(0xb255, MVST,    RRE,   Z,   r1_o, r2_o, 0, 0, mvst, 0)
+
+/* MULTIPLY */
+    C(0x1c00, MR,      RR_a,  Z,   r1p1_32s, r2_32s, new, r1_D32, mul, 0)
+    C(0x5c00, M,       RX_a,  Z,   r1p1_32s, m2_32s, new, r1_D32, mul, 0)
+    C(0xe35c, MFY,     RXY_a, GIE, r1p1_32s, m2_32s, new, r1_D32, mul, 0)
+    C(0xb317, MEEBR,   RRE,   Z,   e1, e2, new, e1, meeb, 0)
+    C(0xb31c, MDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, mdb, 0)
+    C(0xb34c, MXBR,    RRE,   Z,   0, x2_o, x1, 0, mxb, 0)
+    C(0xb30c, MDEBR,   RRE,   Z,   f1_o, e2, f1, 0, mdeb, 0)
+    C(0xb307, MXDBR,   RRE,   Z,   0, f2_o, x1, 0, mxdb, 0)
+    C(0xed17, MEEB,    RXE,   Z,   e1, m2_32u, new, e1, meeb, 0)
+    C(0xed1c, MDB,     RXE,   Z,   f1_o, m2_64, f1, 0, mdb, 0)
+    C(0xed0c, MDEB,    RXE,   Z,   f1_o, m2_32u, f1, 0, mdeb, 0)
+    C(0xed07, MXDB,    RXE,   Z,   0, m2_64, x1, 0, mxdb, 0)
+/* MULTIPLY HALFWORD */
+    C(0x4c00, MH,      RX_a,  Z,   r1_o, m2_16s, new, r1_32, mul, 0)
+    C(0xe37c, MHY,     RXY_a, GIE, r1_o, m2_16s, new, r1_32, mul, 0)
+/* MULTIPLY HALFWORD IMMEDIATE */
+    C(0xa70c, MHI,     RI_a,  Z,   r1_o, i2, new, r1_32, mul, 0)
+    C(0xa70d, MGHI,    RI_a,  Z,   r1_o, i2, r1, 0, mul, 0)
+/* MULTIPLY LOGICAL */
+    C(0xb996, MLR,     RRE,   Z,   r1p1_32u, r2_32u, new, r1_D32, mul, 0)
+    C(0xe396, ML,      RXY_a, Z,   r1p1_32u, m2_32u, new, r1_D32, mul, 0)
+    C(0xb986, MLGR,    RRE,   Z,   r1p1, r2_o, r1_P, 0, mul128, 0)
+    C(0xe386, MLG,     RXY_a, Z,   r1p1, m2_64, r1_P, 0, mul128, 0)
+/* MULTIPLY SINGLE */
+    C(0xb252, MSR,     RRE,   Z,   r1_o, r2_o, new, r1_32, mul, 0)
+    C(0x7100, MS,      RX_a,  Z,   r1_o, m2_32s, new, r1_32, mul, 0)
+    C(0xe351, MSY,     RXY_a, LD,  r1_o, m2_32s, new, r1_32, mul, 0)
+    C(0xb90c, MSGR,    RRE,   Z,   r1_o, r2_o, r1, 0, mul, 0)
+    C(0xb91c, MSGFR,   RRE,   Z,   r1_o, r2_32s, r1, 0, mul, 0)
+    C(0xe30c, MSG,     RXY_a, Z,   r1_o, m2_64, r1, 0, mul, 0)
+    C(0xe31c, MSGF,    RXY_a, Z,   r1_o, m2_32s, r1, 0, mul, 0)
+/* MULTIPLY SINGLE IMMEDIATE */
+    C(0xc201, MSFI,    RIL_a, GIE, r1_o, i2, new, r1_32, mul, 0)
+    C(0xc200, MSGFI,   RIL_a, GIE, r1_o, i2, r1, 0, mul, 0)
+
+/* MULTIPLY AND ADD */
+    C(0xb30e, MAEBR,   RRD,   Z,   e1, e2, new, e1, maeb, 0)
+    C(0xb31e, MADBR,   RRD,   Z,   f1_o, f2_o, f1, 0, madb, 0)
+    C(0xed0e, MAEB,    RXF,   Z,   e1, m2_32u, new, e1, maeb, 0)
+    C(0xed1e, MADB,    RXF,   Z,   f1_o, m2_64, f1, 0, madb, 0)
+/* MULTIPLY AND SUBTRACT */
+    C(0xb30f, MSEBR,   RRD,   Z,   e1, e2, new, e1, mseb, 0)
+    C(0xb31f, MSDBR,   RRD,   Z,   f1_o, f2_o, f1, 0, msdb, 0)
+    C(0xed0f, MSEB,    RXF,   Z,   e1, m2_32u, new, e1, mseb, 0)
+    C(0xed1f, MSDB,    RXF,   Z,   f1_o, m2_64, f1, 0, msdb, 0)
+
+/* OR */
+    C(0x1600, OR,      RR_a,  Z,   r1, r2, new, r1_32, or, nz32)
+    C(0xb9f6, ORK,     RRF_a, DO,  r2, r3, new, r1_32, or, nz32)
+    C(0x5600, O,       RX_a,  Z,   r1, m2_32s, new, r1_32, or, nz32)
+    C(0xe356, OY,      RXY_a, LD,  r1, m2_32s, new, r1_32, or, nz32)
+    C(0xb981, OGR,     RRE,   Z,   r1, r2, r1, 0, or, nz64)
+    C(0xb9e6, OGRK,    RRF_a, DO,  r2, r3, r1, 0, or, nz64)
+    C(0xe381, OG,      RXY_a, Z,   r1, m2_64, r1, 0, or, nz64)
+    C(0xd600, OC,      SS_a,  Z,   la1, a2, 0, 0, oc, 0)
+/* OR IMMEDIATE */
+    D(0xc00c, OIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, ori, 0, 0x2020)
+    D(0xc00d, OILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, ori, 0, 0x2000)
+    D(0xa508, OIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1030)
+    D(0xa509, OIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1020)
+    D(0xa50a, OILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1010)
+    D(0xa50b, OILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1000)
+    C(0x9600, OI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, or, nz64)
+    C(0xeb56, OIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, or, nz64)
+
+/* PREFETCH */
+    /* Implemented as nops of course.  */
+    C(0xe336, PFD,     RXY_b, GIE, 0, 0, 0, 0, 0, 0)
+    C(0xc602, PFDRL,   RIL_c, GIE, 0, 0, 0, 0, 0, 0)
+
+/* POPULATION COUNT */
+    C(0xb9e1, POPCNT,  RRE,   PC,  0, r2_o, r1, 0, popcnt, nz64)
+
+/* ROTATE LEFT SINGLE LOGICAL */
+    C(0xeb1d, RLL,     RSY_a, Z,   r3_o, sh32, new, r1_32, rll32, 0)
+    C(0xeb1c, RLLG,    RSY_a, Z,   r3_o, sh64, r1, 0, rll64, 0)
+
+/* ROTATE THEN INSERT SELECTED BITS */
+    C(0xec55, RISBG,   RIE_f, GIE, 0, r2, r1, 0, risbg, s64)
+    C(0xec5d, RISBHG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
+    C(0xec51, RISBLG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
+/* ROTATE_THEN <OP> SELECTED BITS */
+    C(0xec54, RNSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+    C(0xec56, ROSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+    C(0xec57, RXSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+
+/* SEARCH STRING */
+    C(0xb25e, SRST,    RRE,   Z,   r1_o, r2_o, 0, 0, srst, 0)
+
+/* SET ACCESS */
+    C(0xb24e, SAR,     RRE,   Z,   0, r2_o, 0, 0, sar, 0)
+/* SET FPC */
+    C(0xb384, SFPC,    RRE,   Z,   0, r1_o, 0, 0, sfpc, 0)
+/* SET FPC AND SIGNAL */
+    C(0xb385, SFASR,   RRE,   IEEEE_SIM, 0, r1_o, 0, 0, sfas, 0)
+/* SET BFP ROUNDING MODE */
+    C(0xb299, SRNM,    S,     Z,   0, 0, 0, 0, srnm, 0)
+    C(0xb2b8, SRNMB,   S,     FPE, 0, 0, 0, 0, srnm, 0)
+/* SET DFP ROUNDING MODE */
+    C(0xb2b9, SRNMT,   S,     DFP, 0, 0, 0, 0, srnm, 0)
+
+/* SHIFT LEFT SINGLE */
+    D(0x8b00, SLA,     RS_a,  Z,   r1, sh32, new, r1_32, sla, 0, 31)
+    D(0xebdd, SLAK,    RSY_a, DO,  r3, sh32, new, r1_32, sla, 0, 31)
+    D(0xeb0b, SLAG,    RSY_a, Z,   r3, sh64, r1, 0, sla, 0, 63)
+/* SHIFT LEFT SINGLE LOGICAL */
+    C(0x8900, SLL,     RS_a,  Z,   r1_o, sh32, new, r1_32, sll, 0)
+    C(0xebdf, SLLK,    RSY_a, DO,  r3_o, sh32, new, r1_32, sll, 0)
+    C(0xeb0d, SLLG,    RSY_a, Z,   r3_o, sh64, r1, 0, sll, 0)
+/* SHIFT RIGHT SINGLE */
+    C(0x8a00, SRA,     RS_a,  Z,   r1_32s, sh32, new, r1_32, sra, s32)
+    C(0xebdc, SRAK,    RSY_a, DO,  r3_32s, sh32, new, r1_32, sra, s32)
+    C(0xeb0a, SRAG,    RSY_a, Z,   r3_o, sh64, r1, 0, sra, s64)
+/* SHIFT RIGHT SINGLE LOGICAL */
+    C(0x8800, SRL,     RS_a,  Z,   r1_32u, sh32, new, r1_32, srl, 0)
+    C(0xebde, SRLK,    RSY_a, DO,  r3_32u, sh32, new, r1_32, srl, 0)
+    C(0xeb0c, SRLG,    RSY_a, Z,   r3_o, sh64, r1, 0, srl, 0)
+/* SHIFT LEFT DOUBLE */
+    D(0x8f00, SLDA,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sla, 0, 31)
+/* SHIFT LEFT DOUBLE LOGICAL */
+    C(0x8d00, SLDL,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sll, 0)
+/* SHIFT RIGHT DOUBLE */
+    C(0x8e00, SRDA,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sra, s64)
+/* SHIFT RIGHT DOUBLE LOGICAL */
+    C(0x8c00, SRDL,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, srl, 0)
+
+/* SQUARE ROOT */
+    C(0xb314, SQEBR,   RRE,   Z,   0, e2, new, e1, sqeb, 0)
+    C(0xb315, SQDBR,   RRE,   Z,   0, f2_o, f1, 0, sqdb, 0)
+    C(0xb316, SQXBR,   RRE,   Z,   0, x2_o, x1, 0, sqxb, 0)
+    C(0xed14, SQEB,    RXE,   Z,   0, m2_32u, new, e1, sqeb, 0)
+    C(0xed15, SQDB,    RXE,   Z,   0, m2_64, f1, 0, sqdb, 0)
+
+/* STORE */
+    C(0x5000, ST,      RX_a,  Z,   r1_o, a2, 0, 0, st32, 0)
+    C(0xe350, STY,     RXY_a, LD,  r1_o, a2, 0, 0, st32, 0)
+    C(0xe324, STG,     RXY_a, Z,   r1_o, a2, 0, 0, st64, 0)
+    C(0x6000, STD,     RX_a,  Z,   f1_o, a2, 0, 0, st64, 0)
+    C(0xed67, STDY,    RXY_a, LD,  f1_o, a2, 0, 0, st64, 0)
+    C(0x7000, STE,     RX_a,  Z,   e1, a2, 0, 0, st32, 0)
+    C(0xed66, STEY,    RXY_a, LD,  e1, a2, 0, 0, st32, 0)
+/* STORE RELATIVE LONG */
+    C(0xc40f, STRL,    RIL_b, GIE, r1_o, ri2, 0, 0, st32, 0)
+    C(0xc40b, STGRL,   RIL_b, GIE, r1_o, ri2, 0, 0, st64, 0)
+/* STORE CHARACTER */
+    C(0x4200, STC,     RX_a,  Z,   r1_o, a2, 0, 0, st8, 0)
+    C(0xe372, STCY,    RXY_a, LD,  r1_o, a2, 0, 0, st8, 0)
+/* STORE CHARACTERS UNDER MASK */
+    D(0xbe00, STCM,    RS_b,  Z,   r1_o, a2, 0, 0, stcm, 0, 0)
+    D(0xeb2d, STCMY,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 0)
+    D(0xeb2c, STCMH,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 32)
+/* STORE HALFWORD */
+    C(0x4000, STH,     RX_a,  Z,   r1_o, a2, 0, 0, st16, 0)
+    C(0xe370, STHY,    RXY_a, LD,  r1_o, a2, 0, 0, st16, 0)
+/* STORE HALFWORD RELATIVE LONG */
+    C(0xc407, STHRL,   RIL_b, GIE, r1_o, ri2, 0, 0, st16, 0)
+/* STORE ON CONDITION */
+    D(0xebf3, STOC,    RSY_b, LOC, 0, 0, 0, 0, soc, 0, 0)
+    D(0xebe3, STOCG,   RSY_b, LOC, 0, 0, 0, 0, soc, 0, 1)
+/* STORE REVERSED */
+    C(0xe33f, STRVH,   RXY_a, Z,   la2, r1_16u, new, m1_16, rev16, 0)
+    C(0xe33e, STRV,    RXY_a, Z,   la2, r1_32u, new, m1_32, rev32, 0)
+    C(0xe32f, STRVG,   RXY_a, Z,   la2, r1_o, new, m1_64, rev64, 0)
+
+/* STORE FPC */
+    C(0xb29c, STFPC,   S,     Z,   0, a2, new, m2_32, efpc, 0)
+
+/* STORE MULTIPLE */
+    D(0x9000, STM,     RS_a,  Z,   0, a2, 0, 0, stm, 0, 4)
+    D(0xeb90, STMY,    RSY_a, LD,  0, a2, 0, 0, stm, 0, 4)
+    D(0xeb24, STMG,    RSY_a, Z,   0, a2, 0, 0, stm, 0, 8)
+/* STORE MULTIPLE HIGH */
+    C(0xeb26, STMH,    RSY_a, Z,   0, a2, 0, 0, stmh, 0)
+/* STORE ACCESS MULTIPLE */
+    C(0x9b00, STAM,    RS_a,  Z,   0, a2, 0, 0, stam, 0)
+    C(0xeb9b, STAMY,   RSY_a, LD,  0, a2, 0, 0, stam, 0)
+
+/* SUBTRACT */
+    C(0x1b00, SR,      RR_a,  Z,   r1, r2, new, r1_32, sub, subs32)
+    C(0xb9f9, SRK,     RRF_a, DO,  r2, r3, new, r1_32, sub, subs32)
+    C(0x5b00, S,       RX_a,  Z,   r1, m2_32s, new, r1_32, sub, subs32)
+    C(0xe35b, SY,      RXY_a, LD,  r1, m2_32s, new, r1_32, sub, subs32)
+    C(0xb909, SGR,     RRE,   Z,   r1, r2, r1, 0, sub, subs64)
+    C(0xb919, SGFR,    RRE,   Z,   r1, r2_32s, r1, 0, sub, subs64)
+    C(0xb9e9, SGRK,    RRF_a, DO,  r2, r3, r1, 0, sub, subs64)
+    C(0xe309, SG,      RXY_a, Z,   r1, m2_64, r1, 0, sub, subs64)
+    C(0xe319, SGF,     RXY_a, Z,   r1, m2_32s, r1, 0, sub, subs64)
+    C(0xb30b, SEBR,    RRE,   Z,   e1, e2, new, e1, seb, f32)
+    C(0xb31b, SDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, sdb, f64)
+    C(0xb34b, SXBR,    RRE,   Z,   0, x2_o, x1, 0, sxb, f128)
+    C(0xed0b, SEB,     RXE,   Z,   e1, m2_32u, new, e1, seb, f32)
+    C(0xed1b, SDB,     RXE,   Z,   f1_o, m2_64, f1, 0, sdb, f64)
+/* SUBTRACT HALFWORD */
+    C(0x4b00, SH,      RX_a,  Z,   r1, m2_16s, new, r1_32, sub, subs32)
+    C(0xe37b, SHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, sub, subs32)
+/* SUBTRACT LOGICAL */
+    C(0x1f00, SLR,     RR_a,  Z,   r1, r2, new, r1_32, sub, subu32)
+    C(0xb9fb, SLRK,    RRF_a, DO,  r2, r3, new, r1_32, sub, subu32)
+    C(0x5f00, SL,      RX_a,  Z,   r1, m2_32u, new, r1_32, sub, subu32)
+    C(0xe35f, SLY,     RXY_a, LD,  r1, m2_32u, new, r1_32, sub, subu32)
+    C(0xb90b, SLGR,    RRE,   Z,   r1, r2, r1, 0, sub, subu64)
+    C(0xb91b, SLGFR,   RRE,   Z,   r1, r2_32u, r1, 0, sub, subu64)
+    C(0xb9eb, SLGRK,   RRF_a, DO,  r2, r3, r1, 0, sub, subu64)
+    C(0xe30b, SLG,     RXY_a, Z,   r1, m2_64, r1, 0, sub, subu64)
+    C(0xe31b, SLGF,    RXY_a, Z,   r1, m2_32u, r1, 0, sub, subu64)
+/* SUBTRACT LOGICAL IMMEDIATE */
+    C(0xc205, SLFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, sub, subu32)
+    C(0xc204, SLGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, sub, subu64)
+/* SUBTRACT LOGICAL WITH BORROW */
+    C(0xb999, SLBR,    RRE,   Z,   r1, r2, new, r1_32, subb, subb32)
+    C(0xb989, SLBGR,   RRE,   Z,   r1, r2, r1, 0, subb, subb64)
+    C(0xe399, SLB,     RXY_a, Z,   r1, m2_32u, new, r1_32, subb, subb32)
+    C(0xe389, SLBG,    RXY_a, Z,   r1, m2_64, r1, 0, subb, subb64)
+
+/* SUPERVISOR CALL */
+    C(0x0a00, SVC,     I,     Z,   0, 0, 0, 0, svc, 0)
+
+/* TEST DATA CLASS */
+    C(0xed10, TCEB,    RXE,   Z,   e1, a2, 0, 0, tceb, 0)
+    C(0xed11, TCDB,    RXE,   Z,   f1_o, a2, 0, 0, tcdb, 0)
+    C(0xed12, TCXB,    RXE,   Z,   x1_o, a2, 0, 0, tcxb, 0)
+
+/* TEST UNDER MASK */
+    C(0x9100, TM,      SI,    Z,   m1_8u, i2_8u, 0, 0, 0, tm32)
+    C(0xeb51, TMY,     SIY,   LD,  m1_8u, i2_8u, 0, 0, 0, tm32)
+    D(0xa702, TMHH,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 48)
+    D(0xa703, TMHL,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 32)
+    D(0xa700, TMLH,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 16)
+    D(0xa701, TMLL,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 0)
+
+/* TRANSLATE */
+    C(0xdc00, TR,      SS_a,  Z,   la1, a2, 0, 0, tr, 0)
+
+/* UNPACK */
+    /* Really format SS_b, but we pack both lengths into one argument
+       for the helper call, so we might as well leave one 8-bit field.  */
+    C(0xf300, UNPK,    SS_a,  Z,   la1, a2, 0, 0, unpk, 0)
+
+#ifndef CONFIG_USER_ONLY
+/* COMPARE AND SWAP AND PURGE */
+    C(0xb250, CSP,     RRE,   Z,   0, ra2, 0, 0, csp, 0)
+/* DIAGNOSE (KVM hypercall) */
+    C(0x8300, DIAG,    RX_a,  Z,   0, 0, 0, 0, diag, 0)
+/* INSERT STORAGE KEY EXTENDED */
+    C(0xb229, ISKE,    RRE,   Z,   0, r2_o, new, r1_8, iske, 0)
+/* INVALIDATE PAGE TABLE ENTRY */
+    C(0xb221, IPTE,    RRF_a, Z,   r1_o, r2_o, 0, 0, ipte, 0)
+/* LOAD CONTROL */
+    C(0xb700, LCTL,    RS_a,  Z,   0, a2, 0, 0, lctl, 0)
+    C(0xeb2f, LCTLG,   RSY_a, Z,   0, a2, 0, 0, lctlg, 0)
+/* LOAD PSW */
+    C(0x8200, LPSW,    S,     Z,   0, a2, 0, 0, lpsw, 0)
+/* LOAD PSW EXTENDED */
+    C(0xb2b2, LPSWE,   S,     Z,   0, a2, 0, 0, lpswe, 0)
+/* LOAD REAL ADDRESS */
+    C(0xb100, LRA,     RX_a,  Z,   0, a2, r1, 0, lra, 0)
+    C(0xe313, LRAY,    RXY_a, LD,  0, a2, r1, 0, lra, 0)
+    C(0xe303, LRAG,    RXY_a, Z,   0, a2, r1, 0, lra, 0)
+/* MOVE TO PRIMARY */
+    C(0xda00, MVCP,    SS_d,  Z,   la1, a2, 0, 0, mvcp, 0)
+/* MOVE TO SECONDARY */
+    C(0xdb00, MVCS,    SS_d,  Z,   la1, a2, 0, 0, mvcs, 0)
+/* PURGE TLB */
+    C(0xb20d, PTLB,    S,     Z,   0, 0, 0, 0, ptlb, 0)
+/* RESET REFERENCE BIT EXTENDED */
+    C(0xb22a, RRBE,    RRE,   Z,   0, r2_o, 0, 0, rrbe, 0)
+/* SERVICE CALL LOGICAL PROCESSOR (PV hypercall) */
+    C(0xb220, SERVC,   RRE,   Z,   r1_o, r2_o, 0, 0, servc, 0)
+/* SET ADDRESSING MODE */
+    /* We only do 64-bit, so accept this as a no-op.
+       Let SAM24 and SAM31 signal illegal instruction.  */
+    C(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, 0, 0)
+/* SET ADDRESS SPACE CONTROL FAST */
+    C(0xb279, SACF,    S,     Z,   0, a2, 0, 0, sacf, 0)
+/* SET CLOCK */
+    /* ??? Not implemented - is it necessary? */
+    C(0xb204, SCK,     S,     Z,   0, 0, 0, 0, 0, 0)
+/* SET CLOCK COMPARATOR */
+    C(0xb206, SCKC,    S,     Z,   0, m2_64, 0, 0, sckc, 0)
+/* SET CPU TIMER */
+    C(0xb208, SPT,     S,     Z,   0, m2_64, 0, 0, spt, 0)
+/* SET PREFIX */
+    C(0xb210, SPX,     S,     Z,   0, m2_32u, 0, 0, spx, 0)
+/* SET PSW KEY FROM ADDRESS */
+    C(0xb20a, SPKA,    S,     Z,   0, a2, 0, 0, spka, 0)
+/* SET STORAGE KEY EXTENDED */
+    C(0xb22b, SSKE,    RRF_c, Z,   r1_o, r2_o, 0, 0, sske, 0)
+/* SET SYSTEM MASK */
+    C(0x8000, SSM,     S,     Z,   0, m2_8u, 0, 0, ssm, 0)
+/* SIGNAL PROCESSOR */
+    C(0xae00, SIGP,    RS_a,  Z,   r3_o, a2, 0, 0, sigp, 0)
+/* STORE CLOCK */
+    C(0xb205, STCK,    S,     Z,   la2, 0, new, m1_64, stck, 0)
+    C(0xb27c, STCKF,   S,     Z,   la2, 0, new, m1_64, stck, 0)
+/* STORE CLOCK EXTENDED */
+    C(0xb278, STCKE,   S,     Z,   0, a2, 0, 0, stcke, 0)
+/* STORE CLOCK COMPARATOR */
+    C(0xb207, STCKC,   S,     Z,   la2, 0, new, m1_64, stckc, 0)
+/* STORE CONTROL */
+    C(0xb600, STCTL,   RS_a,  Z,   0, a2, 0, 0, stctl, 0)
+    C(0xeb25, STCTG,   RSY_a, Z,   0, a2, 0, 0, stctg, 0)
+/* STORE CPU ADDRESS */
+    C(0xb212, STAP,    S,     Z,   la2, 0, new, m1_16, stap, 0)
+/* STORE CPU ID */
+    C(0xb202, STIDP,   S,     Z,   la2, 0, new, m1_64, stidp, 0)
+/* STORE CPU TIMER */
+    C(0xb209, STPT,    S,     Z,   la2, 0, new, m1_64, stpt, 0)
+/* STORE FACILITY LIST */
+    C(0xb2b1, STFL,    S,     Z,   0, 0, 0, 0, stfl, 0)
+/* STORE PREFIX */
+    C(0xb211, STPX,    S,     Z,   la2, 0, new, m1_32, stpx, 0)
+/* STORE SYSTEM INFORMATION */
+    C(0xb27d, STSI,    S,     Z,   0, a2, 0, 0, stsi, 0)
+/* STORE THEN AND SYSTEM MASK */
+    C(0xac00, STNSM,   SI,    Z,   la1, 0, 0, 0, stnosm, 0)
+/* STORE THEN OR SYSTEM MASK */
+    C(0xad00, STOSM,   SI,    Z,   la1, 0, 0, 0, stnosm, 0)
+/* STORE USING REAL ADDRESS */
+    C(0xb246, STURA,   RRE,   Z,   r1_o, r2_o, 0, 0, stura, 0)
+/* TEST PROTECTION */
+    C(0xe501, TPROT,   SSE,   Z,   la1, a2, 0, 0, tprot, 0)
+
+/* I/O Instructions.  For each we simply indicate non-operation.  */
+    C(0xb276, XSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb230, CSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb231, HSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb232, MSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb23b, RCHP,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb238, RSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb233, SSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb234, STSCH,   S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb235, TSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    /* ??? Not listed in PoO ninth edition, but there's a linux driver that
+       uses it: "A CHSC subchannel is usually present on LPAR only."  */
+    C(0xb25f, CHSC,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+#endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/insn-format.def b/target-s390x/insn-format.def
new file mode 100644
index 00000000000..0e898b90bd6
--- /dev/null
+++ b/target-s390x/insn-format.def
@@ -0,0 +1,55 @@
+/* Description of s390 insn formats.  */
+/* NAME   F1,          F2... */
+F0(E)
+F1(I,     I(1, 8, 8))
+F2(RI_a,  R(1, 8),     I(2,16,16))
+F2(RI_b,  R(1, 8),     I(2,16,16))
+F2(RI_c,  M(1, 8),     I(2,16,16))
+F3(RIE_a, R(1, 8),     I(2,16,16),  M(3,32))
+F4(RIE_b, R(1, 8),     R(2,12),     M(3,32),   I(4,16,16))
+F4(RIE_c, R(1, 8),     I(2,32, 8),  M(3,12),   I(4,16,16))
+F3(RIE_d, R(1, 8),     I(2,16,16),  R(3,12))
+F3(RIE_e, R(1, 8),     I(2,16,16),  R(3,12))
+F5(RIE_f, R(1, 8),     R(2,12),     I(3,16,8), I(4,24,8),  I(5,32,8))
+F2(RIL_a, R(1, 8),     I(2,16,32))
+F2(RIL_b, R(1, 8),     I(2,16,32))
+F2(RIL_c, M(1, 8),     I(2,16,32))
+F4(RIS,   R(1, 8),     I(2,32, 8),  M(3,12),   BD(4,16,20))
+/* ??? The PoO does not call out subtypes _a and _b for RR, as it does
+   for e.g. RX.  Our checking requires this for e.g. BCR.  */
+F2(RR_a,  R(1, 8),     R(2,12))
+F2(RR_b,  M(1, 8),     R(2,12))
+F2(RRE,   R(1,24),     R(2,28))
+F3(RRD,   R(1,16),     R(2,28),     R(3,24))
+F4(RRF_a, R(1,24),     R(2,28),     R(3,16),   M(4,20))
+F4(RRF_b, R(1,24),     R(2,28),     R(3,16),   M(4,20))
+F4(RRF_c, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRF_d, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRF_e, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRS,   R(1, 8),     R(2,12),     M(3,32),   BD(4,16,20))
+F3(RS_a,  R(1, 8),     BD(2,16,20), R(3,12))
+F3(RS_b,  R(1, 8),     BD(2,16,20), M(3,12))
+F3(RSI,   R(1, 8),     I(2,16,16),  R(3,12))
+F2(RSL,   L(1, 8, 4),  BD(1,16,20))
+F3(RSY_a, R(1, 8),     BDL(2),      R(3,12))
+F3(RSY_b, R(1, 8),     BDL(2),      M(3,12))
+F2(RX_a,  R(1, 8),     BXD(2))
+F2(RX_b,  M(1, 8),     BXD(2))
+F2(RXE,   R(1, 8),     BXD(2))
+F3(RXF,   R(1,32),     BXD(2),      R(3, 8))
+F2(RXY_a, R(1, 8),     BXDL(2))
+F2(RXY_b, M(1, 8),     BXDL(2))
+F1(S,     BD(2,16,20))
+F2(SI,    BD(1,16,20), I(2,8,8))
+F2(SIL,   BD(1,16,20), I(2,32,16))
+F2(SIY,   BDL(1),      I(2, 8, 8))
+F3(SS_a,  L(1, 8, 8),  BD(1,16,20), BD(2,32,36))
+F4(SS_b,  L(1, 8, 4),  BD(1,16,20), L(2,12,4),   BD(2,32,36))
+F4(SS_c,  L(1, 8, 4),  BD(1,16,20), BD(2,32,36), I(3,12, 4))
+/* ??? Odd man out.  The L1 field here is really a register, but the
+   easy way to compress the fields has R1 and B1 overlap.  */
+F4(SS_d,  L(1, 8, 4),  BD(1,16,20), BD(2,32,36), R(3,12))
+F4(SS_e,  R(1, 8),     BD(2,16,20), R(3,12),     BD(4,32,36))
+F3(SS_f,  BD(1,16,20), L(2,8,8),    BD(2,32,36))
+F2(SSE,   BD(1,16,20), BD(2,32,36))
+F3(SSF,   BD(1,16,20), BD(2,32,36), R(3,8))
diff --git a/target-s390x/int_helper.c b/target-s390x/int_helper.c
new file mode 100644
index 00000000000..685830124f6
--- /dev/null
+++ b/target-s390x/int_helper.c
@@ -0,0 +1,205 @@
+/*
+ *  S/390 integer helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "qemu/host-utils.h"
+#include "helper.h"
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+/* 64/64 -> 128 unsigned multiplication */
+uint64_t HELPER(mul128)(CPUS390XState *env, uint64_t v1, uint64_t v2)
+{
+    uint64_t reth;
+    mulu64(&env->retxl, &reth, v1, v2);
+    return reth;
+}
+
+/* 64/32 -> 32 signed division */
+int64_t HELPER(divs32)(CPUS390XState *env, int64_t a, int64_t b64)
+{
+    int32_t ret, b = b64;
+    int64_t q;
+
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    ret = q = a / b;
+    env->retxl = a % b;
+
+    /* Catch non-representable quotient.  */
+    if (ret != q) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    return ret;
+}
+
+/* 64/32 -> 32 unsigned division */
+uint64_t HELPER(divu32)(CPUS390XState *env, uint64_t a, uint64_t b64)
+{
+    uint32_t ret, b = b64;
+    uint64_t q;
+
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    ret = q = a / b;
+    env->retxl = a % b;
+
+    /* Catch non-representable quotient.  */
+    if (ret != q) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    return ret;
+}
+
+/* 64/64 -> 64 signed division */
+int64_t HELPER(divs64)(CPUS390XState *env, int64_t a, int64_t b)
+{
+    /* Catch divide by zero, and non-representable quotient (MIN / -1).  */
+    if (b == 0 || (b == -1 && a == (1ll << 63))) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+    env->retxl = a % b;
+    return a / b;
+}
+
+/* 128 -> 64/64 unsigned division */
+uint64_t HELPER(divu64)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                        uint64_t b)
+{
+    uint64_t ret;
+    /* Signal divide by zero.  */
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+    if (ah == 0) {
+        /* 64 -> 64/64 case */
+        env->retxl = al % b;
+        ret = al / b;
+    } else {
+        /* ??? Move i386 idivq helper to host-utils.  */
+#if HOST_LONG_BITS == 64 && defined(__GNUC__)
+        /* assuming 64-bit hosts have __uint128_t */
+        __uint128_t a = ((__uint128_t)ah << 64) | al;
+        __uint128_t q = a / b;
+        env->retxl = a % b;
+        ret = q;
+        if (ret != q) {
+            runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+        }
+#else
+        /* 32-bit hosts would need special wrapper functionality - just abort if
+           we encounter such a case; it's very unlikely anyways. */
+        cpu_abort(env, "128 -> 64/64 division not implemented\n");
+#endif
+    }
+    return ret;
+}
+
+/* absolute value 32-bit */
+uint32_t HELPER(abs_i32)(int32_t val)
+{
+    if (val < 0) {
+        return -val;
+    } else {
+        return val;
+    }
+}
+
+/* negative absolute value 32-bit */
+int32_t HELPER(nabs_i32)(int32_t val)
+{
+    if (val < 0) {
+        return val;
+    } else {
+        return -val;
+    }
+}
+
+/* absolute value 64-bit */
+uint64_t HELPER(abs_i64)(int64_t val)
+{
+    HELPER_LOG("%s: val 0x%" PRIx64 "\n", __func__, val);
+
+    if (val < 0) {
+        return -val;
+    } else {
+        return val;
+    }
+}
+
+/* negative absolute value 64-bit */
+int64_t HELPER(nabs_i64)(int64_t val)
+{
+    if (val < 0) {
+        return val;
+    } else {
+        return -val;
+    }
+}
+
+/* count leading zeros, for find leftmost one */
+uint64_t HELPER(clz)(uint64_t v)
+{
+    return clz64(v);
+}
+
+uint64_t HELPER(cvd)(int32_t bin)
+{
+    /* positive 0 */
+    uint64_t dec = 0x0c;
+    int shift = 4;
+
+    if (bin < 0) {
+        bin = -bin;
+        dec = 0x0d;
+    }
+
+    for (shift = 4; (shift < 64) && bin; shift += 4) {
+        int current_number = bin % 10;
+
+        dec |= (current_number) << shift;
+        bin /= 10;
+    }
+
+    return dec;
+}
+
+uint64_t HELPER(popcnt)(uint64_t r2)
+{
+    uint64_t ret = 0;
+    int i;
+
+    for (i = 0; i < 64; i += 8) {
+        uint64_t t = ctpop32((r2 >> i) & 0xff);
+        ret |= t << i;
+    }
+    return ret;
+}
diff --git a/target-s390x/interrupt.c b/target-s390x/interrupt.c
index c1b034f7752..e51519dbd7b 100644
--- a/target-s390x/interrupt.c
+++ b/target-s390x/interrupt.c
@@ -8,7 +8,7 @@
  */
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #if !defined(CONFIG_USER_ONLY)
 /* service interrupts are floating therefore we must not pass an cpustate */
@@ -19,7 +19,8 @@ void s390_sclp_extint(uint32_t parm)
 
     if (kvm_enabled()) {
 #ifdef CONFIG_KVM
-        kvm_s390_interrupt_internal(env, KVM_S390_INT_SERVICE, parm, 0, 1);
+        kvm_s390_interrupt_internal(dummy_cpu, KVM_S390_INT_SERVICE, parm,
+                                    0, 1);
 #endif
     } else {
         env->psw.addr += 4;
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 07edf936901..99deddf96d5 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -2,6 +2,7 @@
  * QEMU S390x KVM implementation
  *
  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ * Copyright IBM Corp. 2012
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -13,7 +14,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
@@ -25,11 +29,11 @@
 #include <asm/ptrace.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "cpu.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 /* #define DEBUG_KVM */
 
@@ -60,77 +64,162 @@
 #define SIGP_STORE_STATUS_ADDR          0x0e
 #define SIGP_SET_ARCH                   0x12
 
-#define SCLP_CMDW_READ_SCP_INFO         0x00020001
-#define SCLP_CMDW_READ_SCP_INFO_FORCED  0x00120001
-
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
 
+static int cap_sync_regs;
+
 int kvm_arch_init(KVMState *s)
 {
+    cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
     return 0;
 }
 
-int kvm_arch_init_vcpu(CPUS390XState *env)
+unsigned long kvm_arch_vcpu_id(CPUState *cpu)
+{
+    return cpu->cpu_index;
+}
+
+int kvm_arch_init_vcpu(CPUState *cpu)
 {
     int ret = 0;
 
-    if (kvm_vcpu_ioctl(env, KVM_S390_INITIAL_RESET, NULL) < 0) {
+    if (kvm_vcpu_ioctl(cpu, KVM_S390_INITIAL_RESET, NULL) < 0) {
         perror("cannot init reset vcpu");
     }
 
     return ret;
 }
 
-void kvm_arch_reset_vcpu(CPUS390XState *env)
+void kvm_arch_reset_vcpu(CPUState *cpu)
 {
-    /* FIXME: add code to reset vcpu. */
+    /* The initial reset call is needed here to reset in-kernel
+     * vcpu data that we can't access directly from QEMU
+     * (i.e. with older kernels which don't support sync_regs/ONE_REG).
+     * Before this ioctl cpu_synchronize_state() is called in common kvm
+     * code (kvm-all) */
+    if (kvm_vcpu_ioctl(cpu, KVM_S390_INITIAL_RESET, NULL)) {
+        perror("Can't reset vcpu\n");
+    }
 }
 
-int kvm_arch_put_registers(CPUS390XState *env, int level)
+int kvm_arch_put_registers(CPUState *cs, int level)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
+    struct kvm_sregs sregs;
     struct kvm_regs regs;
     int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
-    }
+    /* always save the PSW  and the GPRS*/
+    cs->kvm_run->psw_addr = env->psw.addr;
+    cs->kvm_run->psw_mask = env->psw.mask;
 
-    for (i = 0; i < 16; i++) {
-        regs.gprs[i] = env->regs[i];
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+        for (i = 0; i < 16; i++) {
+            cs->kvm_run->s.regs.gprs[i] = env->regs[i];
+            cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
+        }
+    } else {
+        for (i = 0; i < 16; i++) {
+            regs.gprs[i] = env->regs[i];
+        }
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_REGS, &regs);
+        if (ret < 0) {
+            return ret;
+        }
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
+    /* Do we need to save more than that? */
+    if (level == KVM_PUT_RUNTIME_STATE) {
+        return 0;
     }
 
-    env->kvm_run->psw_addr = env->psw.addr;
-    env->kvm_run->psw_mask = env->psw.mask;
+    if (cap_sync_regs &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        for (i = 0; i < 16; i++) {
+            cs->kvm_run->s.regs.acrs[i] = env->aregs[i];
+            cs->kvm_run->s.regs.crs[i] = env->cregs[i];
+        }
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_CRS;
+    } else {
+        for (i = 0; i < 16; i++) {
+            sregs.acrs[i] = env->aregs[i];
+            sregs.crs[i] = env->cregs[i];
+        }
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
+        if (ret < 0) {
+            return ret;
+        }
+    }
 
-    return ret;
+    /* Finally the prefix */
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        cs->kvm_run->s.regs.prefix = env->psa;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
+    } else {
+        /* prefix is only supported via sync regs */
+    }
+    return 0;
 }
 
-int kvm_arch_get_registers(CPUS390XState *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
-    int ret;
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
+    struct kvm_sregs sregs;
     struct kvm_regs regs;
+    int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
+    /* get the PSW */
+    env->psw.addr = cs->kvm_run->psw_addr;
+    env->psw.mask = cs->kvm_run->psw_mask;
+
+    /* the GPRS */
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+        for (i = 0; i < 16; i++) {
+            env->regs[i] = cs->kvm_run->s.regs.gprs[i];
+        }
+    } else {
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
+        if (ret < 0) {
+            return ret;
+        }
+         for (i = 0; i < 16; i++) {
+            env->regs[i] = regs.gprs[i];
+        }
     }
 
-    for (i = 0; i < 16; i++) {
-        env->regs[i] = regs.gprs[i];
+    /* The ACRS and CRS */
+    if (cap_sync_regs &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        for (i = 0; i < 16; i++) {
+            env->aregs[i] = cs->kvm_run->s.regs.acrs[i];
+            env->cregs[i] = cs->kvm_run->s.regs.crs[i];
+        }
+    } else {
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
+        if (ret < 0) {
+            return ret;
+        }
+         for (i = 0; i < 16; i++) {
+            env->aregs[i] = sregs.acrs[i];
+            env->cregs[i] = sregs.crs[i];
+        }
     }
 
-    env->psw.addr = env->kvm_run->psw_addr;
-    env->psw.mask = env->kvm_run->psw_mask;
+    /* Finally the prefix */
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        env->psa = cs->kvm_run->s.regs.prefix;
+    } else {
+        /* no prefix without sync regs */
+    }
 
     return 0;
 }
@@ -170,8 +259,10 @@ void *kvm_arch_vmalloc(ram_addr_t size)
     }
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
 
     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
@@ -181,8 +272,10 @@ int kvm_arch_insert_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *
     return 0;
 }
 
-int kvm_arch_remove_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     uint8_t t[4];
     static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
 
@@ -197,26 +290,28 @@ int kvm_arch_remove_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *
     return 0;
 }
 
-void kvm_arch_pre_run(CPUS390XState *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-void kvm_arch_post_run(CPUS390XState *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-int kvm_arch_process_async_events(CPUS390XState *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    return env->halted;
+    S390CPU *cpu = S390_CPU(cs);
+    return cpu->env.halted;
 }
 
-void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
+void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
                                  uint64_t parm64, int vm)
 {
+    CPUState *cs = CPU(cpu);
     struct kvm_s390_interrupt kvmint;
     int r;
 
-    if (!env->kvm_state) {
+    if (!cs->kvm_state) {
         return;
     }
 
@@ -225,9 +320,9 @@ void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
     kvmint.parm64 = parm64;
 
     if (vm) {
-        r = kvm_vm_ioctl(env->kvm_state, KVM_S390_INTERRUPT, &kvmint);
+        r = kvm_vm_ioctl(cs->kvm_state, KVM_S390_INTERRUPT, &kvmint);
     } else {
-        r = kvm_vcpu_ioctl(env, KVM_S390_INTERRUPT, &kvmint);
+        r = kvm_vcpu_ioctl(cs, KVM_S390_INTERRUPT, &kvmint);
     }
 
     if (r < 0) {
@@ -236,34 +331,38 @@ void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
     }
 }
 
-void kvm_s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token)
+void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
 {
-    kvm_s390_interrupt_internal(env, KVM_S390_INT_VIRTIO, config_change,
+    kvm_s390_interrupt_internal(cpu, KVM_S390_INT_VIRTIO, config_change,
                                 token, 1);
 }
 
-void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code)
+void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
 {
-    kvm_s390_interrupt_internal(env, type, code, 0, 0);
+    kvm_s390_interrupt_internal(cpu, type, code, 0, 0);
 }
 
-static void enter_pgmcheck(CPUS390XState *env, uint16_t code)
+static void enter_pgmcheck(S390CPU *cpu, uint16_t code)
 {
-    kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
+    kvm_s390_interrupt(cpu, KVM_S390_PROGRAM_INT, code);
 }
 
-static inline void setcc(CPUS390XState *env, uint64_t cc)
+static inline void setcc(S390CPU *cpu, uint64_t cc)
 {
-    env->kvm_run->psw_mask &= ~(3ull << 44);
-    env->kvm_run->psw_mask |= (cc & 3) << 44;
+    CPUS390XState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
+    cs->kvm_run->psw_mask &= ~(3ull << 44);
+    cs->kvm_run->psw_mask |= (cc & 3) << 44;
 
     env->psw.mask &= ~(3ul << 44);
     env->psw.mask |= (cc & 3) << 44;
 }
 
-static int kvm_sclp_service_call(CPUS390XState *env, struct kvm_run *run,
+static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
                                  uint16_t ipbh0)
 {
+    CPUS390XState *env = &cpu->env;
     uint32_t sccb;
     uint64_t code;
     int r = 0;
@@ -272,16 +371,16 @@ static int kvm_sclp_service_call(CPUS390XState *env, struct kvm_run *run,
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
-    r = sclp_service_call(env, sccb, code);
+    r = sclp_service_call(sccb, code);
     if (r < 0) {
-        enter_pgmcheck(env, -r);
+        enter_pgmcheck(cpu, -r);
     }
-    setcc(env, r);
+    setcc(cpu, r);
 
     return 0;
 }
 
-static int handle_priv(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
+static int handle_priv(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 {
     int r = 0;
     uint16_t ipbh0 = (run->s390_sieic.ipb & 0xffff0000) >> 16;
@@ -289,7 +388,7 @@ static int handle_priv(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
     dprintf("KVM: PRIV: %d\n", ipa1);
     switch (ipa1) {
         case PRIV_SCLP_CALL:
-            r = kvm_sclp_service_call(env, run, ipbh0);
+            r = kvm_sclp_service_call(cpu, run, ipbh0);
             break;
         default:
             dprintf("KVM: unknown PRIV: 0x%x\n", ipa1);
@@ -303,7 +402,7 @@ static int handle_priv(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
 static int handle_hypercall(CPUS390XState *env, struct kvm_run *run)
 {
     cpu_synchronize_state(env);
-    env->regs[2] = s390_virtio_hypercall(env, env->regs[2], env->regs[1]);
+    env->regs[2] = s390_virtio_hypercall(env);
 
     return 0;
 }
@@ -332,9 +431,9 @@ static int s390_cpu_restart(S390CPU *cpu)
 {
     CPUS390XState *env = &cpu->env;
 
-    kvm_s390_interrupt(env, KVM_S390_RESTART, 0);
+    kvm_s390_interrupt(cpu, KVM_S390_RESTART, 0);
     s390_add_running_cpu(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
     dprintf("DONE: SIGP cpu restart: %p\n", env);
     return 0;
 }
@@ -346,12 +445,13 @@ static int s390_store_status(CPUS390XState *env, uint32_t parameter)
     return -1;
 }
 
-static int s390_cpu_initial_reset(CPUS390XState *env)
+static int s390_cpu_initial_reset(S390CPU *cpu)
 {
+    CPUS390XState *env = &cpu->env;
     int i;
 
     s390_del_running_cpu(env);
-    if (kvm_vcpu_ioctl(env, KVM_S390_INITIAL_RESET, NULL) < 0) {
+    if (kvm_vcpu_ioctl(CPU(cpu), KVM_S390_INITIAL_RESET, NULL) < 0) {
         perror("cannot init reset vcpu");
     }
 
@@ -365,8 +465,9 @@ static int s390_cpu_initial_reset(CPUS390XState *env)
     return 0;
 }
 
-static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
+static int handle_sigp(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 {
+    CPUS390XState *env = &cpu->env;
     uint8_t order_code;
     uint32_t parameter;
     uint16_t cpu_addr;
@@ -410,7 +511,7 @@ static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
             /* make the caller panic */
             return -1;
         case SIGP_INITIAL_CPU_RESET:
-            r = s390_cpu_initial_reset(target_env);
+            r = s390_cpu_initial_reset(target_cpu);
             break;
         default:
             fprintf(stderr, "KVM: unknown SIGP: 0x%x\n", order_code);
@@ -418,12 +519,13 @@ static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
     }
 
 out:
-    setcc(env, r ? 3 : 0);
+    setcc(cpu, r ? 3 : 0);
     return 0;
 }
 
-static int handle_instruction(CPUS390XState *env, struct kvm_run *run)
+static int handle_instruction(S390CPU *cpu, struct kvm_run *run)
 {
+    CPUS390XState *env = &cpu->env;
     unsigned int ipa0 = (run->s390_sieic.ipa & 0xff00);
     uint8_t ipa1 = run->s390_sieic.ipa & 0x00ff;
     int ipb_code = (run->s390_sieic.ipb & 0x0fff0000) >> 16;
@@ -432,43 +534,45 @@ static int handle_instruction(CPUS390XState *env, struct kvm_run *run)
     dprintf("handle_instruction 0x%x 0x%x\n", run->s390_sieic.ipa, run->s390_sieic.ipb);
     switch (ipa0) {
         case IPA0_PRIV:
-            r = handle_priv(env, run, ipa1);
+            r = handle_priv(cpu, run, ipa1);
             break;
         case IPA0_DIAG:
             r = handle_diag(env, run, ipb_code);
             break;
         case IPA0_SIGP:
-            r = handle_sigp(env, run, ipa1);
+            r = handle_sigp(cpu, run, ipa1);
             break;
     }
 
     if (r < 0) {
-        enter_pgmcheck(env, 0x0001);
+        enter_pgmcheck(cpu, 0x0001);
     }
     return 0;
 }
 
-static bool is_special_wait_psw(CPUS390XState *env)
+static bool is_special_wait_psw(CPUState *cs)
 {
     /* signal quiesce */
-    return env->kvm_run->psw_addr == 0xfffUL;
+    return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
-static int handle_intercept(CPUS390XState *env)
+static int handle_intercept(S390CPU *cpu)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUS390XState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
     int icpt_code = run->s390_sieic.icptcode;
     int r = 0;
 
     dprintf("intercept: 0x%x (at 0x%lx)\n", icpt_code,
-            (long)env->kvm_run->psw_addr);
+            (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
-            r = handle_instruction(env, run);
+            r = handle_instruction(cpu, run);
             break;
         case ICPT_WAITPSW:
             if (s390_del_running_cpu(env) == 0 &&
-                is_special_wait_psw(env)) {
+                is_special_wait_psw(cs)) {
                 qemu_system_shutdown_request();
             }
             r = EXCP_HALTED;
@@ -496,13 +600,14 @@ static int handle_intercept(CPUS390XState *env)
     return r;
 }
 
-int kvm_arch_handle_exit(CPUS390XState *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
+    S390CPU *cpu = S390_CPU(cs);
     int ret = 0;
 
     switch (run->exit_reason) {
         case KVM_EXIT_S390_SIEIC:
-            ret = handle_intercept(env);
+            ret = handle_intercept(cpu);
             break;
         case KVM_EXIT_S390_RESET:
             qemu_system_reset_request();
@@ -518,12 +623,12 @@ int kvm_arch_handle_exit(CPUS390XState *env, struct kvm_run *run)
     return ret;
 }
 
-bool kvm_arch_stop_on_emulation_error(CPUS390XState *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu)
 {
     return true;
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUS390XState *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
new file mode 100644
index 00000000000..372334b3c8f
--- /dev/null
+++ b/target-s390x/mem_helper.c
@@ -0,0 +1,1076 @@
+/*
+ *  S/390 memory access helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/*****************************************************************************/
+/* Softmmu support */
+#if !defined(CONFIG_USER_ONLY)
+#include "exec/softmmu_exec.h"
+
+#define MMUSUFFIX _mmu
+
+#define SHIFT 0
+#include "exec/softmmu_template.h"
+
+#define SHIFT 1
+#include "exec/softmmu_template.h"
+
+#define SHIFT 2
+#include "exec/softmmu_template.h"
+
+#define SHIFT 3
+#include "exec/softmmu_template.h"
+
+/* try to fill the TLB and return an exception if error. If retaddr is
+   NULL, it means that the function was called in C code (i.e. not
+   from generated code or from helper.c) */
+/* XXX: fix it to restore all registers */
+void tlb_fill(CPUS390XState *env, target_ulong addr, int is_write, int mmu_idx,
+              uintptr_t retaddr)
+{
+    int ret;
+
+    ret = cpu_s390x_handle_mmu_fault(env, addr, is_write, mmu_idx);
+    if (unlikely(ret != 0)) {
+        if (likely(retaddr)) {
+            /* now we have a real cpu fault */
+            cpu_restore_state(env, retaddr);
+        }
+        cpu_loop_exit(env);
+    }
+}
+
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+#ifndef CONFIG_USER_ONLY
+static void mvc_fast_memset(CPUS390XState *env, uint32_t l, uint64_t dest,
+                            uint8_t byte)
+{
+    hwaddr dest_phys;
+    hwaddr len = l;
+    void *dest_p;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    int flags;
+
+    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
+        cpu_stb_data(env, dest, byte);
+        cpu_abort(env, "should never reach here");
+    }
+    dest_phys |= dest & ~TARGET_PAGE_MASK;
+
+    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
+
+    memset(dest_p, byte, len);
+
+    cpu_physical_memory_unmap(dest_p, 1, len, len);
+}
+
+static void mvc_fast_memmove(CPUS390XState *env, uint32_t l, uint64_t dest,
+                             uint64_t src)
+{
+    hwaddr dest_phys;
+    hwaddr src_phys;
+    hwaddr len = l;
+    void *dest_p;
+    void *src_p;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    int flags;
+
+    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
+        cpu_stb_data(env, dest, 0);
+        cpu_abort(env, "should never reach here");
+    }
+    dest_phys |= dest & ~TARGET_PAGE_MASK;
+
+    if (mmu_translate(env, src, 0, asc, &src_phys, &flags)) {
+        cpu_ldub_data(env, src);
+        cpu_abort(env, "should never reach here");
+    }
+    src_phys |= src & ~TARGET_PAGE_MASK;
+
+    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
+    src_p = cpu_physical_memory_map(src_phys, &len, 0);
+
+    memmove(dest_p, src_p, len);
+
+    cpu_physical_memory_unmap(dest_p, 1, len, len);
+    cpu_physical_memory_unmap(src_p, 0, len, len);
+}
+#endif
+
+/* and on array */
+uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) & cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* xor on array */
+uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+
+#ifndef CONFIG_USER_ONLY
+    /* xor with itself is the same as memset(0) */
+    if ((l > 32) && (src == dest) &&
+        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK)) {
+        mvc_fast_memset(env, l + 1, dest, 0);
+        return 0;
+    }
+#else
+    if (src == dest) {
+        memset(g2h(dest), 0, l + 1);
+        return 0;
+    }
+#endif
+
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) ^ cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* or on array */
+uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) | cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* memmove */
+void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
+{
+    int i = 0;
+    int x = 0;
+    uint32_t l_64 = (l + 1) / 8;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+
+#ifndef CONFIG_USER_ONLY
+    if ((l > 32) &&
+        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK) &&
+        (dest & TARGET_PAGE_MASK) == ((dest + l) & TARGET_PAGE_MASK)) {
+        if (dest == (src + 1)) {
+            mvc_fast_memset(env, l + 1, dest, cpu_ldub_data(env, src));
+            return;
+        } else if ((src & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
+            mvc_fast_memmove(env, l + 1, dest, src);
+            return;
+        }
+    }
+#else
+    if (dest == (src + 1)) {
+        memset(g2h(dest), cpu_ldub_data(env, src), l + 1);
+        return;
+    } else {
+        memmove(g2h(dest), g2h(src), l + 1);
+        return;
+    }
+#endif
+
+    /* handle the parts that fit into 8-byte loads/stores */
+    if (dest != (src + 1)) {
+        for (i = 0; i < l_64; i++) {
+            cpu_stq_data(env, dest + x, cpu_ldq_data(env, src + x));
+            x += 8;
+        }
+    }
+
+    /* slow version crossing pages with byte accesses */
+    for (i = x; i <= l; i++) {
+        cpu_stb_data(env, dest + i, cpu_ldub_data(env, src + i));
+    }
+}
+
+/* compare unsigned byte arrays */
+uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
+{
+    int i;
+    unsigned char x, y;
+    uint32_t cc;
+
+    HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
+               __func__, l, s1, s2);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, s1 + i);
+        y = cpu_ldub_data(env, s2 + i);
+        HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
+        if (x < y) {
+            cc = 1;
+            goto done;
+        } else if (x > y) {
+            cc = 2;
+            goto done;
+        }
+    }
+    cc = 0;
+ done:
+    HELPER_LOG("\n");
+    return cc;
+}
+
+/* compare logical under mask */
+uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
+                     uint64_t addr)
+{
+    uint8_t r, d;
+    uint32_t cc;
+
+    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
+               mask, addr);
+    cc = 0;
+    while (mask) {
+        if (mask & 8) {
+            d = cpu_ldub_data(env, addr);
+            r = (r1 & 0xff000000UL) >> 24;
+            HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
+                       addr);
+            if (r < d) {
+                cc = 1;
+                break;
+            } else if (r > d) {
+                cc = 2;
+                break;
+            }
+            addr++;
+        }
+        mask = (mask << 1) & 0xf;
+        r1 <<= 8;
+    }
+    HELPER_LOG("\n");
+    return cc;
+}
+
+static inline uint64_t fix_address(CPUS390XState *env, uint64_t a)
+{
+    /* 31-Bit mode */
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        a &= 0x7fffffff;
+    }
+    return a;
+}
+
+static inline uint64_t get_address(CPUS390XState *env, int x2, int b2, int d2)
+{
+    uint64_t r = d2;
+    if (x2) {
+        r += env->regs[x2];
+    }
+    if (b2) {
+        r += env->regs[b2];
+    }
+    return fix_address(env, r);
+}
+
+static inline uint64_t get_address_31fix(CPUS390XState *env, int reg)
+{
+    return fix_address(env, env->regs[reg]);
+}
+
+/* search string (c is byte to search, r2 is string, r1 end of string) */
+uint64_t HELPER(srst)(CPUS390XState *env, uint64_t r0, uint64_t end,
+                      uint64_t str)
+{
+    uint32_t len;
+    uint8_t v, c = r0;
+
+    str = fix_address(env, str);
+    end = fix_address(env, end);
+
+    /* Assume for now that R2 is unmodified.  */
+    env->retxl = str;
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        if (str + len == end) {
+            /* Character not found.  R1 & R2 are unmodified.  */
+            env->cc_op = 2;
+            return end;
+        }
+        v = cpu_ldub_data(env, str + len);
+        if (v == c) {
+            /* Character found.  Set R1 to the location; R2 is unmodified.  */
+            env->cc_op = 1;
+            return str + len;
+        }
+    }
+
+    /* CPU-determined bytes processed.  Advance R2 to next byte to process.  */
+    env->retxl = str + len;
+    env->cc_op = 3;
+    return end;
+}
+
+/* unsigned string compare (c is string terminator) */
+uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
+{
+    uint32_t len;
+
+    c = c & 0xff;
+    s1 = fix_address(env, s1);
+    s2 = fix_address(env, s2);
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        uint8_t v1 = cpu_ldub_data(env, s1 + len);
+        uint8_t v2 = cpu_ldub_data(env, s2 + len);
+        if (v1 == v2) {
+            if (v1 == c) {
+                /* Equal.  CC=0, and don't advance the registers.  */
+                env->cc_op = 0;
+                env->retxl = s2;
+                return s1;
+            }
+        } else {
+            /* Unequal.  CC={1,2}, and advance the registers.  Note that
+               the terminator need not be zero, but the string that contains
+               the terminator is by definition "low".  */
+            env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
+            env->retxl = s2 + len;
+            return s1 + len;
+        }
+    }
+
+    /* CPU-determined bytes equal; advance the registers.  */
+    env->cc_op = 3;
+    env->retxl = s2 + len;
+    return s1 + len;
+}
+
+/* move page */
+void HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
+{
+    /* XXX missing r0 handling */
+    env->cc_op = 0;
+#ifdef CONFIG_USER_ONLY
+    memmove(g2h(r1), g2h(r2), TARGET_PAGE_SIZE);
+#else
+    mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
+#endif
+}
+
+/* string copy (c is string terminator) */
+uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
+{
+    uint32_t len;
+
+    c = c & 0xff;
+    d = fix_address(env, d);
+    s = fix_address(env, s);
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        uint8_t v = cpu_ldub_data(env, s + len);
+        cpu_stb_data(env, d + len, v);
+        if (v == c) {
+            /* Complete.  Set CC=1 and advance R1.  */
+            env->cc_op = 1;
+            env->retxl = s;
+            return d + len;
+        }
+    }
+
+    /* Incomplete.  Set CC=3 and signal to advance R1 and R2.  */
+    env->cc_op = 3;
+    env->retxl = s + len;
+    return d + len;
+}
+
+static uint32_t helper_icm(CPUS390XState *env, uint32_t r1, uint64_t address,
+                           uint32_t mask)
+{
+    int pos = 24; /* top of the lower half of r1 */
+    uint64_t rmask = 0xff000000ULL;
+    uint8_t val = 0;
+    int ccd = 0;
+    uint32_t cc = 0;
+
+    while (mask) {
+        if (mask & 8) {
+            env->regs[r1] &= ~rmask;
+            val = cpu_ldub_data(env, address);
+            if ((val & 0x80) && !ccd) {
+                cc = 1;
+            }
+            ccd = 1;
+            if (val && cc == 0) {
+                cc = 2;
+            }
+            env->regs[r1] |= (uint64_t)val << pos;
+            address++;
+        }
+        mask = (mask << 1) & 0xf;
+        pos -= 8;
+        rmask >>= 8;
+    }
+
+    return cc;
+}
+
+/* execute instruction
+   this instruction executes an insn modified with the contents of r1
+   it does not change the executed instruction in memory
+   it does not change the program counter
+   in other words: tricky...
+   currently implemented by interpreting the cases it is most commonly used in
+*/
+uint32_t HELPER(ex)(CPUS390XState *env, uint32_t cc, uint64_t v1,
+                    uint64_t addr, uint64_t ret)
+{
+    uint16_t insn = cpu_lduw_code(env, addr);
+
+    HELPER_LOG("%s: v1 0x%lx addr 0x%lx insn 0x%x\n", __func__, v1, addr,
+               insn);
+    if ((insn & 0xf0ff) == 0xd000) {
+        uint32_t l, insn2, b1, b2, d1, d2;
+
+        l = v1 & 0xff;
+        insn2 = cpu_ldl_code(env, addr + 2);
+        b1 = (insn2 >> 28) & 0xf;
+        b2 = (insn2 >> 12) & 0xf;
+        d1 = (insn2 >> 16) & 0xfff;
+        d2 = insn2 & 0xfff;
+        switch (insn & 0xf00) {
+        case 0x200:
+            helper_mvc(env, l, get_address(env, 0, b1, d1),
+                       get_address(env, 0, b2, d2));
+            break;
+        case 0x500:
+            cc = helper_clc(env, l, get_address(env, 0, b1, d1),
+                            get_address(env, 0, b2, d2));
+            break;
+        case 0x700:
+            cc = helper_xc(env, l, get_address(env, 0, b1, d1),
+                           get_address(env, 0, b2, d2));
+            break;
+        case 0xc00:
+            helper_tr(env, l, get_address(env, 0, b1, d1),
+                      get_address(env, 0, b2, d2));
+            break;
+        default:
+            goto abort;
+            break;
+        }
+    } else if ((insn & 0xff00) == 0x0a00) {
+        /* supervisor call */
+        HELPER_LOG("%s: svc %ld via execute\n", __func__, (insn | v1) & 0xff);
+        env->psw.addr = ret - 4;
+        env->int_svc_code = (insn | v1) & 0xff;
+        env->int_svc_ilen = 4;
+        helper_exception(env, EXCP_SVC);
+    } else if ((insn & 0xff00) == 0xbf00) {
+        uint32_t insn2, r1, r3, b2, d2;
+
+        insn2 = cpu_ldl_code(env, addr + 2);
+        r1 = (insn2 >> 20) & 0xf;
+        r3 = (insn2 >> 16) & 0xf;
+        b2 = (insn2 >> 12) & 0xf;
+        d2 = insn2 & 0xfff;
+        cc = helper_icm(env, r1, get_address(env, 0, b2, d2), r3);
+    } else {
+    abort:
+        cpu_abort(env, "EXECUTE on instruction prefix 0x%x not implemented\n",
+                  insn);
+    }
+    return cc;
+}
+
+/* load access registers r1 to r3 from memory at a2 */
+void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->aregs[i] = cpu_ldl_data(env, a2);
+        a2 += 4;
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+/* store access registers r1 to r3 in memory at a2 */
+void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stl_data(env, a2, env->aregs[i]);
+        a2 += 4;
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+/* move long */
+uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
+    uint64_t dest = get_address_31fix(env, r1);
+    uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
+    uint64_t src = get_address_31fix(env, r2);
+    uint8_t pad = src >> 24;
+    uint8_t v;
+    uint32_t cc;
+
+    if (destlen == srclen) {
+        cc = 0;
+    } else if (destlen < srclen) {
+        cc = 1;
+    } else {
+        cc = 2;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
+        v = cpu_ldub_data(env, src);
+        cpu_stb_data(env, dest, v);
+    }
+
+    for (; destlen; dest++, destlen--) {
+        cpu_stb_data(env, dest, pad);
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    env->regs[r2 + 1] -= src - env->regs[r2];
+    env->regs[r1] = dest;
+    env->regs[r2] = src;
+
+    return cc;
+}
+
+/* move long extended another memcopy insn with more bells and whistles */
+uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
+                       uint32_t r3)
+{
+    uint64_t destlen = env->regs[r1 + 1];
+    uint64_t dest = env->regs[r1];
+    uint64_t srclen = env->regs[r3 + 1];
+    uint64_t src = env->regs[r3];
+    uint8_t pad = a2 & 0xff;
+    uint8_t v;
+    uint32_t cc;
+
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        destlen = (uint32_t)destlen;
+        srclen = (uint32_t)srclen;
+        dest &= 0x7fffffff;
+        src &= 0x7fffffff;
+    }
+
+    if (destlen == srclen) {
+        cc = 0;
+    } else if (destlen < srclen) {
+        cc = 1;
+    } else {
+        cc = 2;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
+        v = cpu_ldub_data(env, src);
+        cpu_stb_data(env, dest, v);
+    }
+
+    for (; destlen; dest++, destlen--) {
+        cpu_stb_data(env, dest, pad);
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    /* FIXME: 31-bit mode! */
+    env->regs[r3 + 1] -= src - env->regs[r3];
+    env->regs[r1] = dest;
+    env->regs[r3] = src;
+
+    return cc;
+}
+
+/* compare logical long extended memcompare insn with padding */
+uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
+                       uint32_t r3)
+{
+    uint64_t destlen = env->regs[r1 + 1];
+    uint64_t dest = get_address_31fix(env, r1);
+    uint64_t srclen = env->regs[r3 + 1];
+    uint64_t src = get_address_31fix(env, r3);
+    uint8_t pad = a2 & 0xff;
+    uint8_t v1 = 0, v2 = 0;
+    uint32_t cc = 0;
+
+    if (!(destlen || srclen)) {
+        return cc;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen || srclen; src++, dest++, destlen--, srclen--) {
+        v1 = srclen ? cpu_ldub_data(env, src) : pad;
+        v2 = destlen ? cpu_ldub_data(env, dest) : pad;
+        if (v1 != v2) {
+            cc = (v1 < v2) ? 1 : 2;
+            break;
+        }
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    env->regs[r3 + 1] -= src - env->regs[r3];
+    env->regs[r1] = dest;
+    env->regs[r3] = src;
+
+    return cc;
+}
+
+/* checksum */
+uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
+                      uint64_t src, uint64_t src_len)
+{
+    uint64_t max_len, len;
+    uint64_t cksm = (uint32_t)r1;
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    max_len = (src_len > 0x2000 ? 0x2000 : src_len);
+
+    /* Process full words as available.  */
+    for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
+        cksm += (uint32_t)cpu_ldl_data(env, src);
+    }
+
+    switch (max_len - len) {
+    case 1:
+        cksm += cpu_ldub_data(env, src) << 24;
+        len += 1;
+        break;
+    case 2:
+        cksm += cpu_lduw_data(env, src) << 16;
+        len += 2;
+        break;
+    case 3:
+        cksm += cpu_lduw_data(env, src) << 16;
+        cksm += cpu_ldub_data(env, src + 2) << 8;
+        len += 3;
+        break;
+    }
+
+    /* Fold the carry from the checksum.  Note that we can see carry-out
+       during folding more than once (but probably not more than twice).  */
+    while (cksm > 0xffffffffull) {
+        cksm = (uint32_t)cksm + (cksm >> 32);
+    }
+
+    /* Indicate whether or not we've processed everything.  */
+    env->cc_op = (len == src_len ? 0 : 3);
+
+    /* Return both cksm and processed length.  */
+    env->retxl = cksm;
+    return len;
+}
+
+void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
+                  uint64_t src)
+{
+    int len_dest = len >> 4;
+    int len_src = len & 0xf;
+    uint8_t b;
+    int second_nibble = 0;
+
+    dest += len_dest;
+    src += len_src;
+
+    /* last byte is special, it only flips the nibbles */
+    b = cpu_ldub_data(env, src);
+    cpu_stb_data(env, dest, (b << 4) | (b >> 4));
+    src--;
+    len_src--;
+
+    /* now pad every nibble with 0xf0 */
+
+    while (len_dest > 0) {
+        uint8_t cur_byte = 0;
+
+        if (len_src > 0) {
+            cur_byte = cpu_ldub_data(env, src);
+        }
+
+        len_dest--;
+        dest--;
+
+        /* only advance one nibble at a time */
+        if (second_nibble) {
+            cur_byte >>= 4;
+            len_src--;
+            src--;
+        }
+        second_nibble = !second_nibble;
+
+        /* digit */
+        cur_byte = (cur_byte & 0xf);
+        /* zone bits */
+        cur_byte |= 0xf0;
+
+        cpu_stb_data(env, dest, cur_byte);
+    }
+}
+
+void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
+                uint64_t trans)
+{
+    int i;
+
+    for (i = 0; i <= len; i++) {
+        uint8_t byte = cpu_ldub_data(env, array + i);
+        uint8_t new_byte = cpu_ldub_data(env, trans + byte);
+
+        cpu_stb_data(env, array + i, new_byte);
+    }
+}
+
+#if !defined(CONFIG_USER_ONLY)
+void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t src = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->cregs[i] = cpu_ldq_data(env, src);
+        HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
+                   i, src, env->cregs[i]);
+        src += sizeof(uint64_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+
+    tlb_flush(env, 1);
+}
+
+void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t src = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) |
+            cpu_ldl_data(env, src);
+        src += sizeof(uint32_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+
+    tlb_flush(env, 1);
+}
+
+void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t dest = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stq_data(env, dest, env->cregs[i]);
+        dest += sizeof(uint64_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t dest = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stl_data(env, dest, env->cregs[i]);
+        dest += sizeof(uint32_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
+{
+    /* XXX implement */
+
+    return 0;
+}
+
+/* insert storage key extended */
+uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
+{
+    uint64_t addr = get_address(env, 0, 0, r2);
+
+    if (addr > ram_size) {
+        return 0;
+    }
+
+    return env->storage_keys[addr / TARGET_PAGE_SIZE];
+}
+
+/* set storage key extended */
+void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
+{
+    uint64_t addr = get_address(env, 0, 0, r2);
+
+    if (addr > ram_size) {
+        return;
+    }
+
+    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
+}
+
+/* reset reference bit extended */
+uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
+{
+    uint8_t re;
+    uint8_t key;
+
+    if (r2 > ram_size) {
+        return 0;
+    }
+
+    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
+    re = key & (SK_R | SK_C);
+    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
+
+    /*
+     * cc
+     *
+     * 0  Reference bit zero; change bit zero
+     * 1  Reference bit zero; change bit one
+     * 2  Reference bit one; change bit zero
+     * 3  Reference bit one; change bit one
+     */
+
+    return re >> 1;
+}
+
+/* compare and swap and purge */
+uint32_t HELPER(csp)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+{
+    uint32_t cc;
+    uint32_t o1 = env->regs[r1];
+    uint64_t a2 = r2 & ~3ULL;
+    uint32_t o2 = cpu_ldl_data(env, a2);
+
+    if (o1 == o2) {
+        cpu_stl_data(env, a2, env->regs[(r1 + 1) & 15]);
+        if (r2 & 0x3) {
+            /* flush TLB / ALB */
+            tlb_flush(env, 1);
+        }
+        cc = 0;
+    } else {
+        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | o2;
+        cc = 1;
+    }
+
+    return cc;
+}
+
+static uint32_t mvc_asc(CPUS390XState *env, int64_t l, uint64_t a1,
+                        uint64_t mode1, uint64_t a2, uint64_t mode2)
+{
+    target_ulong src, dest;
+    int flags, cc = 0, i;
+
+    if (!l) {
+        return 0;
+    } else if (l > 256) {
+        /* max 256 */
+        l = 256;
+        cc = 3;
+    }
+
+    if (mmu_translate(env, a1 & TARGET_PAGE_MASK, 1, mode1, &dest, &flags)) {
+        cpu_loop_exit(env);
+    }
+    dest |= a1 & ~TARGET_PAGE_MASK;
+
+    if (mmu_translate(env, a2 & TARGET_PAGE_MASK, 0, mode2, &src, &flags)) {
+        cpu_loop_exit(env);
+    }
+    src |= a2 & ~TARGET_PAGE_MASK;
+
+    /* XXX replace w/ memcpy */
+    for (i = 0; i < l; i++) {
+        /* XXX be more clever */
+        if ((((dest + i) & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) ||
+            (((src + i) & TARGET_PAGE_MASK) != (src & TARGET_PAGE_MASK))) {
+            mvc_asc(env, l - i, a1 + i, mode1, a2 + i, mode2);
+            break;
+        }
+        stb_phys(dest + i, ldub_phys(src + i));
+    }
+
+    return cc;
+}
+
+uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
+{
+    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
+               __func__, l, a1, a2);
+
+    return mvc_asc(env, l, a1, PSW_ASC_SECONDARY, a2, PSW_ASC_PRIMARY);
+}
+
+uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
+{
+    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
+               __func__, l, a1, a2);
+
+    return mvc_asc(env, l, a1, PSW_ASC_PRIMARY, a2, PSW_ASC_SECONDARY);
+}
+
+/* invalidate pte */
+void HELPER(ipte)(CPUS390XState *env, uint64_t pte_addr, uint64_t vaddr)
+{
+    uint64_t page = vaddr & TARGET_PAGE_MASK;
+    uint64_t pte = 0;
+
+    /* XXX broadcast to other CPUs */
+
+    /* XXX Linux is nice enough to give us the exact pte address.
+       According to spec we'd have to find it out ourselves */
+    /* XXX Linux is fine with overwriting the pte, the spec requires
+       us to only set the invalid bit */
+    stq_phys(pte_addr, pte | _PAGE_INVALID);
+
+    /* XXX we exploit the fact that Linux passes the exact virtual
+       address here - it's not obliged to! */
+    tlb_flush_page(env, page);
+
+    /* XXX 31-bit hack */
+    if (page & 0x80000000) {
+        tlb_flush_page(env, page & ~0x80000000);
+    } else {
+        tlb_flush_page(env, page | 0x80000000);
+    }
+}
+
+/* flush local tlb */
+void HELPER(ptlb)(CPUS390XState *env)
+{
+    tlb_flush(env, 1);
+}
+
+/* store using real address */
+void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
+{
+    stw_phys(get_address(env, 0, 0, addr), (uint32_t)v1);
+}
+
+/* load real address */
+uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
+{
+    uint32_t cc = 0;
+    int old_exc = env->exception_index;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    uint64_t ret;
+    int flags;
+
+    /* XXX incomplete - has more corner cases */
+    if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
+        program_interrupt(env, PGM_SPECIAL_OP, 2);
+    }
+
+    env->exception_index = old_exc;
+    if (mmu_translate(env, addr, 0, asc, &ret, &flags)) {
+        cc = 3;
+    }
+    if (env->exception_index == EXCP_PGM) {
+        ret = env->int_pgm_code | 0x80000000;
+    } else {
+        ret |= addr & ~TARGET_PAGE_MASK;
+    }
+    env->exception_index = old_exc;
+
+    env->cc_op = cc;
+    return ret;
+}
+#endif
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
new file mode 100644
index 00000000000..09301d0a6f0
--- /dev/null
+++ b/target-s390x/misc_helper.c
@@ -0,0 +1,450 @@
+/*
+ *  S/390 misc helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "exec/memory.h"
+#include "qemu/host-utils.h"
+#include "helper.h"
+#include <string.h>
+#include "sysemu/kvm.h"
+#include "qemu/timer.h"
+#ifdef CONFIG_KVM
+#include <linux/kvm.h>
+#endif
+
+#if !defined(CONFIG_USER_ONLY)
+#include "exec/softmmu_exec.h"
+#include "sysemu/sysemu.h"
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+/* Raise an exception dynamically from a helper function.  */
+void QEMU_NORETURN runtime_exception(CPUS390XState *env, int excp,
+                                     uintptr_t retaddr)
+{
+    int t;
+
+    env->exception_index = EXCP_PGM;
+    env->int_pgm_code = excp;
+
+    /* Use the (ultimate) callers address to find the insn that trapped.  */
+    cpu_restore_state(env, retaddr);
+
+    /* Advance past the insn.  */
+    t = cpu_ldub_code(env, env->psw.addr);
+    env->int_pgm_ilen = t = get_ilen(t);
+    env->psw.addr += 2 * t;
+
+    cpu_loop_exit(env);
+}
+
+/* Raise an exception statically from a TB.  */
+void HELPER(exception)(CPUS390XState *env, uint32_t excp)
+{
+    HELPER_LOG("%s: exception %d\n", __func__, excp);
+    env->exception_index = excp;
+    cpu_loop_exit(env);
+}
+
+#ifndef CONFIG_USER_ONLY
+
+/* EBCDIC handling */
+static const uint8_t ebcdic2ascii[] = {
+    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
+    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
+    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
+    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
+    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
+    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
+    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
+    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
+    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
+    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
+    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
+    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
+    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
+    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
+    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
+    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
+    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
+    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
+    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
+    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
+    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
+};
+
+static const uint8_t ascii2ebcdic[] = {
+    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
+    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
+    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
+    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
+    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
+    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
+    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
+    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
+};
+
+static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
+{
+    int i;
+
+    for (i = 0; i < len; i++) {
+        p[i] = ascii2ebcdic[(uint8_t)ascii[i]];
+    }
+}
+
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilen)
+{
+    qemu_log_mask(CPU_LOG_INT, "program interrupt at %#" PRIx64 "\n",
+                  env->psw.addr);
+
+    if (kvm_enabled()) {
+#ifdef CONFIG_KVM
+        kvm_s390_interrupt(s390_env_get_cpu(env), KVM_S390_PROGRAM_INT, code);
+#endif
+    } else {
+        env->int_pgm_code = code;
+        env->int_pgm_ilen = ilen;
+        env->exception_index = EXCP_PGM;
+        cpu_loop_exit(env);
+    }
+}
+
+/* SCLP service call */
+uint32_t HELPER(servc)(CPUS390XState *env, uint64_t r1, uint64_t r2)
+{
+    int r = sclp_service_call(r1, r2);
+    if (r < 0) {
+        program_interrupt(env, -r, 4);
+        return 0;
+    }
+    return r;
+}
+
+/* DIAG */
+uint64_t HELPER(diag)(CPUS390XState *env, uint32_t num, uint64_t mem,
+                      uint64_t code)
+{
+    uint64_t r;
+
+    switch (num) {
+    case 0x500:
+        /* KVM hypercall */
+        r = s390_virtio_hypercall(env);
+        break;
+    case 0x44:
+        /* yield */
+        r = 0;
+        break;
+    case 0x308:
+        /* ipl */
+        r = 0;
+        break;
+    default:
+        r = -1;
+        break;
+    }
+
+    if (r) {
+        program_interrupt(env, PGM_OPERATION, ILEN_LATER_INC);
+    }
+
+    return r;
+}
+
+/* Set Prefix */
+void HELPER(spx)(CPUS390XState *env, uint64_t a1)
+{
+    uint32_t prefix = a1 & 0x7fffe000;
+    env->psa = prefix;
+    qemu_log("prefix: %#x\n", prefix);
+    tlb_flush_page(env, 0);
+    tlb_flush_page(env, TARGET_PAGE_SIZE);
+}
+
+static inline uint64_t clock_value(CPUS390XState *env)
+{
+    uint64_t time;
+
+    time = env->tod_offset +
+        time2tod(qemu_get_clock_ns(vm_clock) - env->tod_basetime);
+
+    return time;
+}
+
+/* Store Clock */
+uint64_t HELPER(stck)(CPUS390XState *env)
+{
+    return clock_value(env);
+}
+
+/* Set Clock Comparator */
+void HELPER(sckc)(CPUS390XState *env, uint64_t time)
+{
+    if (time == -1ULL) {
+        return;
+    }
+
+    /* difference between now and then */
+    time -= clock_value(env);
+    /* nanoseconds */
+    time = (time * 125) >> 9;
+
+    qemu_mod_timer(env->tod_timer, qemu_get_clock_ns(vm_clock) + time);
+}
+
+/* Store Clock Comparator */
+uint64_t HELPER(stckc)(CPUS390XState *env)
+{
+    /* XXX implement */
+    return 0;
+}
+
+/* Set CPU Timer */
+void HELPER(spt)(CPUS390XState *env, uint64_t time)
+{
+    if (time == -1ULL) {
+        return;
+    }
+
+    /* nanoseconds */
+    time = (time * 125) >> 9;
+
+    qemu_mod_timer(env->cpu_timer, qemu_get_clock_ns(vm_clock) + time);
+}
+
+/* Store CPU Timer */
+uint64_t HELPER(stpt)(CPUS390XState *env)
+{
+    /* XXX implement */
+    return 0;
+}
+
+/* Store System Information */
+uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0,
+                      uint64_t r0, uint64_t r1)
+{
+    int cc = 0;
+    int sel1, sel2;
+
+    if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 &&
+        ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) {
+        /* valid function code, invalid reserved bits */
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+    }
+
+    sel1 = r0 & STSI_R0_SEL1_MASK;
+    sel2 = r1 & STSI_R1_SEL2_MASK;
+
+    /* XXX: spec exception if sysib is not 4k-aligned */
+
+    switch (r0 & STSI_LEVEL_MASK) {
+    case STSI_LEVEL_1:
+        if ((sel1 == 1) && (sel2 == 1)) {
+            /* Basic Machine Configuration */
+            struct sysib_111 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            ebcdic_put(sysib.manuf, "QEMU            ", 16);
+            /* same as machine type number in STORE CPU ID */
+            ebcdic_put(sysib.type, "QEMU", 4);
+            /* same as model number in STORE CPU ID */
+            ebcdic_put(sysib.model, "QEMU            ", 16);
+            ebcdic_put(sysib.sequence, "QEMU            ", 16);
+            ebcdic_put(sysib.plant, "QEMU", 4);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else if ((sel1 == 2) && (sel2 == 1)) {
+            /* Basic Machine CPU */
+            struct sysib_121 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            /* XXX make different for different CPUs? */
+            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
+            ebcdic_put(sysib.plant, "QEMU", 4);
+            stw_p(&sysib.cpu_addr, env->cpu_num);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else if ((sel1 == 2) && (sel2 == 2)) {
+            /* Basic Machine CPUs */
+            struct sysib_122 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            stl_p(&sysib.capability, 0x443afc29);
+            /* XXX change when SMP comes */
+            stw_p(&sysib.total_cpus, 1);
+            stw_p(&sysib.active_cpus, 1);
+            stw_p(&sysib.standby_cpus, 0);
+            stw_p(&sysib.reserved_cpus, 0);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else {
+            cc = 3;
+        }
+        break;
+    case STSI_LEVEL_2:
+        {
+            if ((sel1 == 2) && (sel2 == 1)) {
+                /* LPAR CPU */
+                struct sysib_221 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                /* XXX make different for different CPUs? */
+                ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
+                ebcdic_put(sysib.plant, "QEMU", 4);
+                stw_p(&sysib.cpu_addr, env->cpu_num);
+                stw_p(&sysib.cpu_id, 0);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else if ((sel1 == 2) && (sel2 == 2)) {
+                /* LPAR CPUs */
+                struct sysib_222 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                stw_p(&sysib.lpar_num, 0);
+                sysib.lcpuc = 0;
+                /* XXX change when SMP comes */
+                stw_p(&sysib.total_cpus, 1);
+                stw_p(&sysib.conf_cpus, 1);
+                stw_p(&sysib.standby_cpus, 0);
+                stw_p(&sysib.reserved_cpus, 0);
+                ebcdic_put(sysib.name, "QEMU    ", 8);
+                stl_p(&sysib.caf, 1000);
+                stw_p(&sysib.dedicated_cpus, 0);
+                stw_p(&sysib.shared_cpus, 0);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else {
+                cc = 3;
+            }
+            break;
+        }
+    case STSI_LEVEL_3:
+        {
+            if ((sel1 == 2) && (sel2 == 2)) {
+                /* VM CPUs */
+                struct sysib_322 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                sysib.count = 1;
+                /* XXX change when SMP comes */
+                stw_p(&sysib.vm[0].total_cpus, 1);
+                stw_p(&sysib.vm[0].conf_cpus, 1);
+                stw_p(&sysib.vm[0].standby_cpus, 0);
+                stw_p(&sysib.vm[0].reserved_cpus, 0);
+                ebcdic_put(sysib.vm[0].name, "KVMguest", 8);
+                stl_p(&sysib.vm[0].caf, 1000);
+                ebcdic_put(sysib.vm[0].cpi, "KVM/Linux       ", 16);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else {
+                cc = 3;
+            }
+            break;
+        }
+    case STSI_LEVEL_CURRENT:
+        env->regs[0] = STSI_LEVEL_3;
+        break;
+    default:
+        cc = 3;
+        break;
+    }
+
+    return cc;
+}
+
+uint32_t HELPER(sigp)(CPUS390XState *env, uint64_t order_code, uint32_t r1,
+                      uint64_t cpu_addr)
+{
+    int cc = 0;
+
+    HELPER_LOG("%s: %016" PRIx64 " %08x %016" PRIx64 "\n",
+               __func__, order_code, r1, cpu_addr);
+
+    /* Remember: Use "R1 or R1 + 1, whichever is the odd-numbered register"
+       as parameter (input). Status (output) is always R1. */
+
+    switch (order_code) {
+    case SIGP_SET_ARCH:
+        /* switch arch */
+        break;
+    case SIGP_SENSE:
+        /* enumerate CPU status */
+        if (cpu_addr) {
+            /* XXX implement when SMP comes */
+            return 3;
+        }
+        env->regs[r1] &= 0xffffffff00000000ULL;
+        cc = 1;
+        break;
+#if !defined(CONFIG_USER_ONLY)
+    case SIGP_RESTART:
+        qemu_system_reset_request();
+        cpu_loop_exit(env);
+        break;
+    case SIGP_STOP:
+        qemu_system_shutdown_request();
+        cpu_loop_exit(env);
+        break;
+#endif
+    default:
+        /* unknown sigp */
+        fprintf(stderr, "XXX unknown sigp: 0x%" PRIx64 "\n", order_code);
+        cc = 3;
+    }
+
+    return cc;
+}
+#endif
diff --git a/target-s390x/op_helper.c b/target-s390x/op_helper.c
deleted file mode 100644
index abc35ddd7b3..00000000000
--- a/target-s390x/op_helper.c
+++ /dev/null
@@ -1,3019 +0,0 @@
-/*
- *  S/390 helper routines
- *
- *  Copyright (c) 2009 Ulrich Hecht
- *  Copyright (c) 2009 Alexander Graf
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cpu.h"
-#include "memory.h"
-#include "cputlb.h"
-#include "dyngen-exec.h"
-#include "host-utils.h"
-#include "helper.h"
-#include <string.h>
-#include "kvm.h"
-#include "qemu-timer.h"
-#ifdef CONFIG_KVM
-#include <linux/kvm.h>
-#endif
-
-#if !defined (CONFIG_USER_ONLY)
-#include "sysemu.h"
-#endif
-
-/*****************************************************************************/
-/* Softmmu support */
-#if !defined (CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "softmmu_template.h"
-
-#define SHIFT 1
-#include "softmmu_template.h"
-
-#define SHIFT 2
-#include "softmmu_template.h"
-
-#define SHIFT 3
-#include "softmmu_template.h"
-
-/* try to fill the TLB and return an exception if error. If retaddr is
-   NULL, it means that the function was called in C code (i.e. not
-   from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUS390XState *env1, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-    CPUS390XState *saved_env;
-    int ret;
-
-    saved_env = env;
-    env = env1;
-    ret = cpu_s390x_handle_mmu_fault(env, addr, is_write, mmu_idx);
-    if (unlikely(ret != 0)) {
-        if (likely(retaddr)) {
-            /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (likely(tb)) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
-        }
-        cpu_loop_exit(env);
-    }
-    env = saved_env;
-}
-
-#endif
-
-/* #define DEBUG_HELPER */
-#ifdef DEBUG_HELPER
-#define HELPER_LOG(x...) qemu_log(x)
-#else
-#define HELPER_LOG(x...)
-#endif
-
-/* raise an exception */
-void HELPER(exception)(uint32_t excp)
-{
-    HELPER_LOG("%s: exception %d\n", __FUNCTION__, excp);
-    env->exception_index = excp;
-    cpu_loop_exit(env);
-}
-
-#ifndef CONFIG_USER_ONLY
-static void mvc_fast_memset(CPUS390XState *env, uint32_t l, uint64_t dest,
-                            uint8_t byte)
-{
-    target_phys_addr_t dest_phys;
-    target_phys_addr_t len = l;
-    void *dest_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        stb(dest, byte);
-        cpu_abort(env, "should never reach here");
-    }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-
-    memset(dest_p, byte, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-}
-
-static void mvc_fast_memmove(CPUS390XState *env, uint32_t l, uint64_t dest,
-                             uint64_t src)
-{
-    target_phys_addr_t dest_phys;
-    target_phys_addr_t src_phys;
-    target_phys_addr_t len = l;
-    void *dest_p;
-    void *src_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        stb(dest, 0);
-        cpu_abort(env, "should never reach here");
-    }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, src, 0, asc, &src_phys, &flags)) {
-        ldub(src);
-        cpu_abort(env, "should never reach here");
-    }
-    src_phys |= src & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-    src_p = cpu_physical_memory_map(src_phys, &len, 0);
-
-    memmove(dest_p, src_p, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-    cpu_physical_memory_unmap(src_p, 0, len, len);
-}
-#endif
-
-/* and on array */
-uint32_t HELPER(nc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) & ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* xor on array */
-uint32_t HELPER(xc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-
-#ifndef CONFIG_USER_ONLY
-    /* xor with itself is the same as memset(0) */
-    if ((l > 32) && (src == dest) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK)) {
-        mvc_fast_memset(env, l + 1, dest, 0);
-        return 0;
-    }
-#else
-    if (src == dest) {
-        memset(g2h(dest), 0, l + 1);
-        return 0;
-    }
-#endif
-
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) ^ ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* or on array */
-uint32_t HELPER(oc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) | ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* memmove */
-void HELPER(mvc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i = 0;
-    int x = 0;
-    uint32_t l_64 = (l + 1) / 8;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-
-#ifndef CONFIG_USER_ONLY
-    if ((l > 32) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK) &&
-        (dest & TARGET_PAGE_MASK) == ((dest + l) & TARGET_PAGE_MASK)) {
-        if (dest == (src + 1)) {
-            mvc_fast_memset(env, l + 1, dest, ldub(src));
-            return;
-        } else if ((src & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
-            mvc_fast_memmove(env, l + 1, dest, src);
-            return;
-        }
-    }
-#else
-    if (dest == (src + 1)) {
-        memset(g2h(dest), ldub(src), l + 1);
-        return;
-    } else {
-        memmove(g2h(dest), g2h(src), l + 1);
-        return;
-    }
-#endif
-
-    /* handle the parts that fit into 8-byte loads/stores */
-    if (dest != (src + 1)) {
-        for (i = 0; i < l_64; i++) {
-            stq(dest + x, ldq(src + x));
-            x += 8;
-        }
-    }
-
-    /* slow version crossing pages with byte accesses */
-    for (i = x; i <= l; i++) {
-        stb(dest + i, ldub(src + i));
-    }
-}
-
-/* compare unsigned byte arrays */
-uint32_t HELPER(clc)(uint32_t l, uint64_t s1, uint64_t s2)
-{
-    int i;
-    unsigned char x,y;
-    uint32_t cc;
-    HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
-               __FUNCTION__, l, s1, s2);
-    for (i = 0; i <= l; i++) {
-        x = ldub(s1 + i);
-        y = ldub(s2 + i);
-        HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
-        if (x < y) {
-            cc = 1;
-            goto done;
-        } else if (x > y) {
-            cc = 2;
-            goto done;
-        }
-    }
-    cc = 0;
-done:
-    HELPER_LOG("\n");
-    return cc;
-}
-
-/* compare logical under mask */
-uint32_t HELPER(clm)(uint32_t r1, uint32_t mask, uint64_t addr)
-{
-    uint8_t r,d;
-    uint32_t cc;
-    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __FUNCTION__, r1,
-               mask, addr);
-    cc = 0;
-    while (mask) {
-        if (mask & 8) {
-            d = ldub(addr);
-            r = (r1 & 0xff000000UL) >> 24;
-            HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
-                        addr);
-            if (r < d) {
-                cc = 1;
-                break;
-            } else if (r > d) {
-                cc = 2;
-                break;
-            }
-            addr++;
-        }
-        mask = (mask << 1) & 0xf;
-        r1 <<= 8;
-    }
-    HELPER_LOG("\n");
-    return cc;
-}
-
-/* store character under mask */
-void HELPER(stcm)(uint32_t r1, uint32_t mask, uint64_t addr)
-{
-    uint8_t r;
-    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%lx\n", __FUNCTION__, r1, mask,
-               addr);
-    while (mask) {
-        if (mask & 8) {
-            r = (r1 & 0xff000000UL) >> 24;
-            stb(addr, r);
-            HELPER_LOG("mask 0x%x %02x (0x%lx) ", mask, r, addr);
-            addr++;
-        }
-        mask = (mask << 1) & 0xf;
-        r1 <<= 8;
-    }
-    HELPER_LOG("\n");
-}
-
-/* 64/64 -> 128 unsigned multiplication */
-void HELPER(mlg)(uint32_t r1, uint64_t v2)
-{
-#if HOST_LONG_BITS == 64 && defined(__GNUC__)
-    /* assuming 64-bit hosts have __uint128_t */
-    __uint128_t res = (__uint128_t)env->regs[r1 + 1];
-    res *= (__uint128_t)v2;
-    env->regs[r1] = (uint64_t)(res >> 64);
-    env->regs[r1 + 1] = (uint64_t)res;
-#else
-    mulu64(&env->regs[r1 + 1], &env->regs[r1], env->regs[r1 + 1], v2);
-#endif
-}
-
-/* 128 -> 64/64 unsigned division */
-void HELPER(dlg)(uint32_t r1, uint64_t v2)
-{
-    uint64_t divisor = v2;
-
-    if (!env->regs[r1]) {
-        /* 64 -> 64/64 case */
-        env->regs[r1] = env->regs[r1+1] % divisor;
-        env->regs[r1+1] = env->regs[r1+1] / divisor;
-        return;
-    } else {
-
-#if HOST_LONG_BITS == 64 && defined(__GNUC__)
-        /* assuming 64-bit hosts have __uint128_t */
-        __uint128_t dividend = (((__uint128_t)env->regs[r1]) << 64) |
-                               (env->regs[r1+1]);
-        __uint128_t quotient = dividend / divisor;
-        env->regs[r1+1] = quotient;
-        __uint128_t remainder = dividend % divisor;
-        env->regs[r1] = remainder;
-#else
-        /* 32-bit hosts would need special wrapper functionality - just abort if
-           we encounter such a case; it's very unlikely anyways. */
-        cpu_abort(env, "128 -> 64/64 division not implemented\n");
-#endif
-    }
-}
-
-static inline uint64_t get_address(int x2, int b2, int d2)
-{
-    uint64_t r = d2;
-
-    if (x2) {
-        r += env->regs[x2];
-    }
-
-    if (b2) {
-        r += env->regs[b2];
-    }
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
-}
-
-static inline uint64_t get_address_31fix(int reg)
-{
-    uint64_t r = env->regs[reg];
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
-}
-
-/* search string (c is byte to search, r2 is string, r1 end of string) */
-uint32_t HELPER(srst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t i;
-    uint32_t cc = 2;
-    uint64_t str = get_address_31fix(r2);
-    uint64_t end = get_address_31fix(r1);
-
-    HELPER_LOG("%s: c %d *r1 0x%" PRIx64 " *r2 0x%" PRIx64 "\n", __FUNCTION__,
-               c, env->regs[r1], env->regs[r2]);
-
-    for (i = str; i != end; i++) {
-        if (ldub(i) == c) {
-            env->regs[r1] = i;
-            cc = 1;
-            break;
-        }
-    }
-
-    return cc;
-}
-
-/* unsigned string compare (c is string terminator) */
-uint32_t HELPER(clst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t s1 = get_address_31fix(r1);
-    uint64_t s2 = get_address_31fix(r2);
-    uint8_t v1, v2;
-    uint32_t cc;
-    c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: comparing '%s' and '%s'\n",
-                   __FUNCTION__, (char*)g2h(s1), (char*)g2h(s2));
-    }
-#endif
-    for (;;) {
-        v1 = ldub(s1);
-        v2 = ldub(s2);
-        if ((v1 == c || v2 == c) || (v1 != v2)) {
-            break;
-        }
-        s1++;
-        s2++;
-    }
-
-    if (v1 == v2) {
-        cc = 0;
-    } else {
-        cc = (v1 < v2) ? 1 : 2;
-        /* FIXME: 31-bit mode! */
-        env->regs[r1] = s1;
-        env->regs[r2] = s2;
-    }
-    return cc;
-}
-
-/* move page */
-void HELPER(mvpg)(uint64_t r0, uint64_t r1, uint64_t r2)
-{
-    /* XXX missing r0 handling */
-#ifdef CONFIG_USER_ONLY
-    int i;
-
-    for (i = 0; i < TARGET_PAGE_SIZE; i++) {
-        stb(r1 + i, ldub(r2 + i));
-    }
-#else
-    mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
-#endif
-}
-
-/* string copy (c is string terminator) */
-void HELPER(mvst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t src = get_address_31fix(r2);
-    uint8_t v;
-    c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: copy '%s' to 0x%lx\n", __FUNCTION__, (char*)g2h(src),
-                   dest);
-    }
-#endif
-    for (;;) {
-        v = ldub(src);
-        stb(dest, v);
-        if (v == c) {
-            break;
-        }
-        src++;
-        dest++;
-    }
-    env->regs[r1] = dest; /* FIXME: 31-bit mode! */
-}
-
-/* compare and swap 64-bit */
-uint32_t HELPER(csg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2 = ldq(a2);
-    if (env->regs[r1] == v2) {
-        cc = 0;
-        stq(a2, env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2;
-    }
-    return cc;
-}
-
-/* compare double and swap 64-bit */
-uint32_t HELPER(cdsg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2_hi = ldq(a2);
-    uint64_t v2_lo = ldq(a2 + 8);
-    uint64_t v1_hi = env->regs[r1];
-    uint64_t v1_lo = env->regs[r1 + 1];
-
-    if ((v1_hi == v2_hi) && (v1_lo == v2_lo)) {
-        cc = 0;
-        stq(a2, env->regs[r3]);
-        stq(a2 + 8, env->regs[r3 + 1]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2_hi;
-        env->regs[r1 + 1] = v2_lo;
-    }
-
-    return cc;
-}
-
-/* compare and swap 32-bit */
-uint32_t HELPER(cs)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    HELPER_LOG("%s: r1 %d a2 0x%lx r3 %d\n", __FUNCTION__, r1, a2, r3);
-    uint32_t v2 = ldl(a2);
-    if (((uint32_t)env->regs[r1]) == v2) {
-        cc = 0;
-        stl(a2, (uint32_t)env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | v2;
-    }
-    return cc;
-}
-
-static uint32_t helper_icm(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 24; /* top of the lower half of r1 */
-    uint64_t rmask = 0xff000000ULL;
-    uint8_t val = 0;
-    int ccd = 0;
-    uint32_t cc = 0;
-
-    while (mask) {
-        if (mask & 8) {
-            env->regs[r1] &= ~rmask;
-            val = ldub(address);
-            if ((val & 0x80) && !ccd) {
-                cc = 1;
-            }
-            ccd = 1;
-            if (val && cc == 0) {
-                cc = 2;
-            }
-            env->regs[r1] |= (uint64_t)val << pos;
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-        rmask >>= 8;
-    }
-
-    return cc;
-}
-
-/* execute instruction
-   this instruction executes an insn modified with the contents of r1
-   it does not change the executed instruction in memory
-   it does not change the program counter
-   in other words: tricky...
-   currently implemented by interpreting the cases it is most commonly used in
- */
-uint32_t HELPER(ex)(uint32_t cc, uint64_t v1, uint64_t addr, uint64_t ret)
-{
-    uint16_t insn = lduw_code(addr);
-    HELPER_LOG("%s: v1 0x%lx addr 0x%lx insn 0x%x\n", __FUNCTION__, v1, addr,
-             insn);
-    if ((insn & 0xf0ff) == 0xd000) {
-        uint32_t l, insn2, b1, b2, d1, d2;
-        l = v1 & 0xff;
-        insn2 = ldl_code(addr + 2);
-        b1 = (insn2 >> 28) & 0xf;
-        b2 = (insn2 >> 12) & 0xf;
-        d1 = (insn2 >> 16) & 0xfff;
-        d2 = insn2 & 0xfff;
-        switch (insn & 0xf00) {
-        case 0x200:
-            helper_mvc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0x500:
-            cc = helper_clc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0x700:
-            cc = helper_xc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0xc00:
-            helper_tr(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        default:
-            goto abort;
-            break;
-        }
-    } else if ((insn & 0xff00) == 0x0a00) {
-        /* supervisor call */
-        HELPER_LOG("%s: svc %ld via execute\n", __FUNCTION__, (insn|v1) & 0xff);
-        env->psw.addr = ret - 4;
-        env->int_svc_code = (insn|v1) & 0xff;
-        env->int_svc_ilc = 4;
-        helper_exception(EXCP_SVC);
-    } else if ((insn & 0xff00) == 0xbf00) {
-        uint32_t insn2, r1, r3, b2, d2;
-        insn2 = ldl_code(addr + 2);
-        r1 = (insn2 >> 20) & 0xf;
-        r3 = (insn2 >> 16) & 0xf;
-        b2 = (insn2 >> 12) & 0xf;
-        d2 = insn2 & 0xfff;
-        cc = helper_icm(r1, get_address(0, b2, d2), r3);
-    } else {
-abort:
-        cpu_abort(env, "EXECUTE on instruction prefix 0x%x not implemented\n",
-                  insn);
-    }
-    return cc;
-}
-
-/* absolute value 32-bit */
-uint32_t HELPER(abs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 32-bit */
-int32_t HELPER(nabs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
-/* absolute value 64-bit */
-uint64_t HELPER(abs_i64)(int64_t val)
-{
-    HELPER_LOG("%s: val 0x%" PRIx64 "\n", __FUNCTION__, val);
-
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 64-bit */
-int64_t HELPER(nabs_i64)(int64_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
-/* add with carry 32-bit unsigned */
-uint32_t HELPER(addc_u32)(uint32_t cc, uint32_t v1, uint32_t v2)
-{
-    uint32_t res;
-
-    res = v1 + v2;
-    if (cc & 2) {
-        res++;
-    }
-
-    return res;
-}
-
-/* store character under mask high operates on the upper half of r1 */
-void HELPER(stcmh)(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-
-    while (mask) {
-        if (mask & 8) {
-            stb(address, (env->regs[r1] >> pos) & 0xff);
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-    }
-}
-
-/* insert character under mask high; same as icm, but operates on the
-   upper half of r1 */
-uint32_t HELPER(icmh)(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-    uint64_t rmask = 0xff00000000000000ULL;
-    uint8_t val = 0;
-    int ccd = 0;
-    uint32_t cc = 0;
-
-    while (mask) {
-        if (mask & 8) {
-            env->regs[r1] &= ~rmask;
-            val = ldub(address);
-            if ((val & 0x80) && !ccd) {
-                cc = 1;
-            }
-            ccd = 1;
-            if (val && cc == 0) {
-                cc = 2;
-            }
-            env->regs[r1] |= (uint64_t)val << pos;
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-        rmask >>= 8;
-    }
-
-    return cc;
-}
-
-/* insert psw mask and condition code into r1 */
-void HELPER(ipm)(uint32_t cc, uint32_t r1)
-{
-    uint64_t r = env->regs[r1];
-
-    r &= 0xffffffff00ffffffULL;
-    r |= (cc << 28) | ( (env->psw.mask >> 40) & 0xf );
-    env->regs[r1] = r;
-    HELPER_LOG("%s: cc %d psw.mask 0x%lx r1 0x%lx\n", __FUNCTION__,
-               cc, env->psw.mask, r);
-}
-
-/* load access registers r1 to r3 from memory at a2 */
-void HELPER(lam)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->aregs[i] = ldl(a2);
-        a2 += 4;
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-/* store access registers r1 to r3 in memory at a2 */
-void HELPER(stam)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stl(a2, env->aregs[i]);
-        a2 += 4;
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-/* move long */
-uint32_t HELPER(mvcl)(uint32_t r1, uint32_t r2)
-{
-    uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
-    uint64_t src = get_address_31fix(r2);
-    uint8_t pad = src >> 24;
-    uint8_t v;
-    uint32_t cc;
-
-    if (destlen == srclen) {
-        cc = 0;
-    } else if (destlen < srclen) {
-        cc = 1;
-    } else {
-        cc = 2;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
-        v = ldub(src);
-        stb(dest, v);
-    }
-
-    for (; destlen; dest++, destlen--) {
-        stb(dest, pad);
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    env->regs[r2 + 1] -= src - env->regs[r2];
-    env->regs[r1] = dest;
-    env->regs[r2] = src;
-
-    return cc;
-}
-
-/* move long extended another memcopy insn with more bells and whistles */
-uint32_t HELPER(mvcle)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    uint64_t destlen = env->regs[r1 + 1];
-    uint64_t dest = env->regs[r1];
-    uint64_t srclen = env->regs[r3 + 1];
-    uint64_t src = env->regs[r3];
-    uint8_t pad = a2 & 0xff;
-    uint8_t v;
-    uint32_t cc;
-
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        destlen = (uint32_t)destlen;
-        srclen = (uint32_t)srclen;
-        dest &= 0x7fffffff;
-        src &= 0x7fffffff;
-    }
-
-    if (destlen == srclen) {
-        cc = 0;
-    } else if (destlen < srclen) {
-        cc = 1;
-    } else {
-        cc = 2;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
-        v = ldub(src);
-        stb(dest, v);
-    }
-
-    for (; destlen; dest++, destlen--) {
-        stb(dest, pad);
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    /* FIXME: 31-bit mode! */
-    env->regs[r3 + 1] -= src - env->regs[r3];
-    env->regs[r1] = dest;
-    env->regs[r3] = src;
-
-    return cc;
-}
-
-/* compare logical long extended memcompare insn with padding */
-uint32_t HELPER(clcle)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    uint64_t destlen = env->regs[r1 + 1];
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t srclen = env->regs[r3 + 1];
-    uint64_t src = get_address_31fix(r3);
-    uint8_t pad = a2 & 0xff;
-    uint8_t v1 = 0,v2 = 0;
-    uint32_t cc = 0;
-
-    if (!(destlen || srclen)) {
-        return cc;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen || srclen; src++, dest++, destlen--, srclen--) {
-        v1 = srclen ? ldub(src) : pad;
-        v2 = destlen ? ldub(dest) : pad;
-        if (v1 != v2) {
-            cc = (v1 < v2) ? 1 : 2;
-            break;
-        }
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    env->regs[r3 + 1] -= src - env->regs[r3];
-    env->regs[r1] = dest;
-    env->regs[r3] = src;
-
-    return cc;
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slb)(uint32_t cc, uint32_t r1, uint32_t v2)
-{
-    uint32_t v1 = env->regs[r1];
-    uint32_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slbg)(uint32_t cc, uint32_t r1, uint64_t v1, uint64_t v2)
-{
-    uint64_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-static inline int float_comp_to_cc(int float_compare)
-{
-    switch (float_compare) {
-    case float_relation_equal:
-        return 0;
-    case float_relation_less:
-        return 1;
-    case float_relation_greater:
-        return 2;
-    case float_relation_unordered:
-        return 3;
-    default:
-        cpu_abort(env, "unknown return value for float compare\n");
-    }
-}
-
-/* condition codes for binary FP ops */
-static uint32_t set_cc_f32(float32 v1, float32 v2)
-{
-    return float_comp_to_cc(float32_compare_quiet(v1, v2, &env->fpu_status));
-}
-
-static uint32_t set_cc_f64(float64 v1, float64 v2)
-{
-    return float_comp_to_cc(float64_compare_quiet(v1, v2, &env->fpu_status));
-}
-
-/* condition codes for unary FP ops */
-static uint32_t set_cc_nz_f32(float32 v)
-{
-    if (float32_is_any_nan(v)) {
-        return 3;
-    } else if (float32_is_zero(v)) {
-        return 0;
-    } else if (float32_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static uint32_t set_cc_nz_f64(float64 v)
-{
-    if (float64_is_any_nan(v)) {
-        return 3;
-    } else if (float64_is_zero(v)) {
-        return 0;
-    } else if (float64_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static uint32_t set_cc_nz_f128(float128 v)
-{
-    if (float128_is_any_nan(v)) {
-        return 3;
-    } else if (float128_is_zero(v)) {
-        return 0;
-    } else if (float128_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-/* convert 32-bit int to 64-bit float */
-void HELPER(cdfbr)(uint32_t f1, int32_t v2)
-{
-    HELPER_LOG("%s: converting %d to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].d = int32_to_float64(v2, &env->fpu_status);
-}
-
-/* convert 32-bit int to 128-bit float */
-void HELPER(cxfbr)(uint32_t f1, int32_t v2)
-{
-    CPU_QuadU v1;
-    v1.q = int32_to_float128(v2, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-}
-
-/* convert 64-bit int to 32-bit float */
-void HELPER(cegbr)(uint32_t f1, int64_t v2)
-{
-    HELPER_LOG("%s: converting %ld to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].l.upper = int64_to_float32(v2, &env->fpu_status);
-}
-
-/* convert 64-bit int to 64-bit float */
-void HELPER(cdgbr)(uint32_t f1, int64_t v2)
-{
-    HELPER_LOG("%s: converting %ld to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].d = int64_to_float64(v2, &env->fpu_status);
-}
-
-/* convert 64-bit int to 128-bit float */
-void HELPER(cxgbr)(uint32_t f1, int64_t v2)
-{
-    CPU_QuadU x1;
-    x1.q = int64_to_float128(v2, &env->fpu_status);
-    HELPER_LOG("%s: converted %ld to 0x%lx and 0x%lx\n", __FUNCTION__, v2,
-               x1.ll.upper, x1.ll.lower);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
-}
-
-/* convert 32-bit int to 32-bit float */
-void HELPER(cefbr)(uint32_t f1, int32_t v2)
-{
-    env->fregs[f1].l.upper = int32_to_float32(v2, &env->fpu_status);
-    HELPER_LOG("%s: converting %d to 0x%d in f%d\n", __FUNCTION__, v2,
-               env->fregs[f1].l.upper, f1);
-}
-
-/* 32-bit FP addition RR */
-uint32_t HELPER(aebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __FUNCTION__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* 64-bit FP addition RR */
-uint32_t HELPER(adbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_add(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%ld resulting in 0x%ld in f%d\n", __FUNCTION__,
-               env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* 32-bit FP subtraction RR */
-uint32_t HELPER(sebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_sub(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __FUNCTION__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* 64-bit FP subtraction RR */
-uint32_t HELPER(sdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_sub(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: subtracting 0x%ld resulting in 0x%ld in f%d\n",
-               __FUNCTION__, env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* 32-bit FP division RR */
-void HELPER(debr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_div(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-}
-
-/* 128-bit FP division RR */
-void HELPER(dxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_div(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* 64-bit FP multiplication RR */
-void HELPER(mdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_mul(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-}
-
-/* 128-bit FP multiplication RR */
-void HELPER(mxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_mul(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* convert 32-bit float to 64-bit float */
-void HELPER(ldebr)(uint32_t r1, uint32_t r2)
-{
-    env->fregs[r1].d = float32_to_float64(env->fregs[r2].l.upper,
-                                          &env->fpu_status);
-}
-
-/* convert 128-bit float to 64-bit float */
-void HELPER(ldxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].d = float128_to_float64(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%ld\n", __FUNCTION__, env->fregs[f1].d);
-}
-
-/* convert 64-bit float to 128-bit float */
-void HELPER(lxdbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU res;
-    res.q = float64_to_float128(env->fregs[f2].d, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* convert 64-bit float to 32-bit float */
-void HELPER(ledbr)(uint32_t f1, uint32_t f2)
-{
-    float64 d2 = env->fregs[f2].d;
-    env->fregs[f1].l.upper = float64_to_float32(d2, &env->fpu_status);
-}
-
-/* convert 128-bit float to 32-bit float */
-void HELPER(lexbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].l.upper = float128_to_float32(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%d\n", __FUNCTION__, env->fregs[f1].l.upper);
-}
-
-/* absolute value of 32-bit float */
-uint32_t HELPER(lpebr)(uint32_t f1, uint32_t f2)
-{
-    float32 v1;
-    float32 v2 = env->fregs[f2].d;
-    v1 = float32_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f32(v1);
-}
-
-/* absolute value of 64-bit float */
-uint32_t HELPER(lpdbr)(uint32_t f1, uint32_t f2)
-{
-    float64 v1;
-    float64 v2 = env->fregs[f2].d;
-    v1 = float64_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f64(v1);
-}
-
-/* absolute value of 128-bit float */
-uint32_t HELPER(lpxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    v1.q = float128_abs(v2.q);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-    return set_cc_nz_f128(v1.q);
-}
-
-/* load and test 64-bit float */
-uint32_t HELPER(ltdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = env->fregs[f2].d;
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load and test 32-bit float */
-uint32_t HELPER(ltebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = env->fregs[f2].l.upper;
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load and test 128-bit float */
-uint32_t HELPER(ltxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x;
-    x.ll.upper = env->fregs[f2].ll;
-    x.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 2].ll = x.ll.lower;
-    return set_cc_nz_f128(x.q);
-}
-
-/* load complement of 32-bit float */
-uint32_t HELPER(lcebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_chs(env->fregs[f2].l.upper);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load complement of 64-bit float */
-uint32_t HELPER(lcdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_chs(env->fregs[f2].d);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load complement of 128-bit float */
-uint32_t HELPER(lcxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x1, x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    x1.q = float128_chs(x2.q);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
-    return set_cc_nz_f128(x1.q);
-}
-
-/* 32-bit FP addition RM */
-void HELPER(aeb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: adding 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_add(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP division RM */
-void HELPER(deb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: dividing 0x%d from f%d by 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_div(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP multiplication RM */
-void HELPER(meeb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: multiplying 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_mul(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP compare RR */
-uint32_t HELPER(cebr)(uint32_t f1, uint32_t f2)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    float32 v2 = env->fregs[f2].l.upper;
-    HELPER_LOG("%s: comparing 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2);
-    return set_cc_f32(v1, v2);
-}
-
-/* 64-bit FP compare RR */
-uint32_t HELPER(cdbr)(uint32_t f1, uint32_t f2)
-{
-    float64 v1 = env->fregs[f1].d;
-    float64 v2 = env->fregs[f2].d;
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%ld\n", __FUNCTION__,
-               v1, f1, v2);
-    return set_cc_f64(v1, v2);
-}
-
-/* 128-bit FP compare RR */
-uint32_t HELPER(cxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-
-    return float_comp_to_cc(float128_compare_quiet(v1.q, v2.q,
-                            &env->fpu_status));
-}
-
-/* 64-bit FP compare RM */
-uint32_t HELPER(cdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%lx\n", __FUNCTION__, v1,
-               f1, v2.d);
-    return set_cc_f64(v1, v2.d);
-}
-
-/* 64-bit FP addition RM */
-uint32_t HELPER(adb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: adding 0x%lx from f%d and 0x%lx\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = v1 = float64_add(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
-}
-
-/* 32-bit FP subtraction RM */
-void HELPER(seb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    env->fregs[f1].l.upper = float32_sub(v1, v2.f, &env->fpu_status);
-}
-
-/* 64-bit FP subtraction RM */
-uint32_t HELPER(sdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    env->fregs[f1].d = v1 = float64_sub(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
-}
-
-/* 64-bit FP multiplication RM */
-void HELPER(mdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: multiplying 0x%lx from f%d and 0x%ld\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_mul(v1, v2.d, &env->fpu_status);
-}
-
-/* 64-bit FP division RM */
-void HELPER(ddb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: dividing 0x%lx from f%d by 0x%ld\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_div(v1, v2.d, &env->fpu_status);
-}
-
-static void set_round_mode(int m3)
-{
-    switch (m3) {
-    case 0:
-        /* current mode */
-        break;
-    case 1:
-        /* biased round no nearest */
-    case 4:
-        /* round to nearest */
-        set_float_rounding_mode(float_round_nearest_even, &env->fpu_status);
-        break;
-    case 5:
-        /* round to zero */
-        set_float_rounding_mode(float_round_to_zero, &env->fpu_status);
-        break;
-    case 6:
-        /* round to +inf */
-        set_float_rounding_mode(float_round_up, &env->fpu_status);
-        break;
-    case 7:
-        /* round to -inf */
-        set_float_rounding_mode(float_round_down, &env->fpu_status);
-        break;
-    }
-}
-
-/* convert 32-bit float to 64-bit int */
-uint32_t HELPER(cgebr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float32 v2 = env->fregs[f2].l.upper;
-    set_round_mode(m3);
-    env->regs[r1] = float32_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
-}
-
-/* convert 64-bit float to 64-bit int */
-uint32_t HELPER(cgdbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float64 v2 = env->fregs[f2].d;
-    set_round_mode(m3);
-    env->regs[r1] = float64_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
-}
-
-/* convert 128-bit float to 64-bit int */
-uint32_t HELPER(cgxbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    set_round_mode(m3);
-    env->regs[r1] = float128_to_int64(v2.q, &env->fpu_status);
-    if (float128_is_any_nan(v2.q)) {
-        return 3;
-    } else if (float128_is_zero(v2.q)) {
-        return 0;
-    } else if (float128_is_neg(v2.q)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-/* convert 32-bit float to 32-bit int */
-uint32_t HELPER(cfebr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float32 v2 = env->fregs[f2].l.upper;
-    set_round_mode(m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float32_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
-}
-
-/* convert 64-bit float to 32-bit int */
-uint32_t HELPER(cfdbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float64 v2 = env->fregs[f2].d;
-    set_round_mode(m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float64_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
-}
-
-/* convert 128-bit float to 32-bit int */
-uint32_t HELPER(cfxbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float128_to_int32(v2.q, &env->fpu_status);
-    return set_cc_nz_f128(v2.q);
-}
-
-/* load 32-bit FP zero */
-void HELPER(lzer)(uint32_t f1)
-{
-    env->fregs[f1].l.upper = float32_zero;
-}
-
-/* load 64-bit FP zero */
-void HELPER(lzdr)(uint32_t f1)
-{
-    env->fregs[f1].d = float64_zero;
-}
-
-/* load 128-bit FP zero */
-void HELPER(lzxr)(uint32_t f1)
-{
-    CPU_QuadU x;
-    x.q = float64_to_float128(float64_zero, &env->fpu_status);
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 1].ll = x.ll.lower;
-}
-
-/* 128-bit FP subtraction RR */
-uint32_t HELPER(sxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_sub(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
-}
-
-/* 128-bit FP addition RR */
-uint32_t HELPER(axbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_add(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
-}
-
-/* 32-bit FP multiplication RR */
-void HELPER(meebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_mul(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-}
-
-/* 64-bit FP division RR */
-void HELPER(ddbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_div(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-}
-
-/* 64-bit FP multiply and add RM */
-void HELPER(madb)(uint32_t f1, uint64_t a2, uint32_t f3)
-{
-    HELPER_LOG("%s: f1 %d a2 0x%lx f3 %d\n", __FUNCTION__, f1, a2, f3);
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    env->fregs[f1].d = float64_add(env->fregs[f1].d,
-                                   float64_mul(v2.d, env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   &env->fpu_status);
-}
-
-/* 64-bit FP multiply and add RR */
-void HELPER(madbr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __FUNCTION__, f1, f2, f3);
-    env->fregs[f1].d = float64_add(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
-}
-
-/* 64-bit FP multiply and subtract RR */
-void HELPER(msdbr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __FUNCTION__, f1, f2, f3);
-    env->fregs[f1].d = float64_sub(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
-}
-
-/* 32-bit FP multiply and add RR */
-void HELPER(maebr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         float32_mul(env->fregs[f2].l.upper,
-                                                     env->fregs[f3].l.upper,
-                                                     &env->fpu_status),
-                                         &env->fpu_status);
-}
-
-/* convert 32-bit float to 64-bit float */
-void HELPER(ldeb)(uint32_t f1, uint64_t a2)
-{
-    uint32_t v2;
-    v2 = ldl(a2);
-    env->fregs[f1].d = float32_to_float64(v2,
-                                          &env->fpu_status);
-}
-
-/* convert 64-bit float to 128-bit float */
-void HELPER(lxdb)(uint32_t f1, uint64_t a2)
-{
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    CPU_QuadU v1;
-    v1.q = float64_to_float128(v2.d, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-}
-
-/* test data class 32-bit */
-uint32_t HELPER(tceb)(uint32_t f1, uint64_t m2)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    int neg = float32_is_neg(v1);
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __FUNCTION__, (long)v1, m2, neg);
-    if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
-        (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
-        (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float32_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* test data class 64-bit */
-uint32_t HELPER(tcdb)(uint32_t f1, uint64_t m2)
-{
-    float64 v1 = env->fregs[f1].d;
-    int neg = float64_is_neg(v1);
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __FUNCTION__, v1, m2, neg);
-    if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
-        (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
-        (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float64_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* test data class 128-bit */
-uint32_t HELPER(tcxb)(uint32_t f1, uint64_t m2)
-{
-    CPU_QuadU v1;
-    uint32_t cc = 0;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-
-    int neg = float128_is_neg(v1.q);
-    if ((float128_is_zero(v1.q) && (m2 & (1 << (11-neg)))) ||
-        (float128_is_infinity(v1.q) && (m2 & (1 << (5-neg)))) ||
-        (float128_is_any_nan(v1.q) && (m2 & (1 << (3-neg)))) ||
-        (float128_is_signaling_nan(v1.q) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* find leftmost one */
-uint32_t HELPER(flogr)(uint32_t r1, uint64_t v2)
-{
-    uint64_t res = 0;
-    uint64_t ov2 = v2;
-
-    while (!(v2 & 0x8000000000000000ULL) && v2) {
-        v2 <<= 1;
-        res++;
-    }
-
-    if (!v2) {
-        env->regs[r1] = 64;
-        env->regs[r1 + 1] = 0;
-        return 0;
-    } else {
-        env->regs[r1] = res;
-        env->regs[r1 + 1] = ov2 & ~(0x8000000000000000ULL >> res);
-        return 2;
-    }
-}
-
-/* square root 64-bit RR */
-void HELPER(sqdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_sqrt(env->fregs[f2].d, &env->fpu_status);
-}
-
-/* checksum */
-void HELPER(cksm)(uint32_t r1, uint32_t r2)
-{
-    uint64_t src = get_address_31fix(r2);
-    uint64_t src_len = env->regs[(r2 + 1) & 15];
-    uint64_t cksm = (uint32_t)env->regs[r1];
-
-    while (src_len >= 4) {
-        cksm += ldl(src);
-
-        /* move to next word */
-        src_len -= 4;
-        src += 4;
-    }
-
-    switch (src_len) {
-    case 0:
-        break;
-    case 1:
-        cksm += ldub(src) << 24;
-        break;
-    case 2:
-        cksm += lduw(src) << 16;
-        break;
-    case 3:
-        cksm += lduw(src) << 16;
-        cksm += ldub(src + 2) << 8;
-        break;
-    }
-
-    /* indicate we've processed everything */
-    env->regs[r2] = src + src_len;
-    env->regs[(r2 + 1) & 15] = 0;
-
-    /* store result */
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                    ((uint32_t)cksm + (cksm >> 32));
-}
-
-static inline uint32_t cc_calc_ltgt_32(CPUS390XState *env, int32_t src,
-                                       int32_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltgt0_32(CPUS390XState *env, int32_t dst)
-{
-    return cc_calc_ltgt_32(env, dst, 0);
-}
-
-static inline uint32_t cc_calc_ltgt_64(CPUS390XState *env, int64_t src,
-                                       int64_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltgt0_64(CPUS390XState *env, int64_t dst)
-{
-    return cc_calc_ltgt_64(env, dst, 0);
-}
-
-static inline uint32_t cc_calc_ltugtu_32(CPUS390XState *env, uint32_t src,
-                                         uint32_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltugtu_64(CPUS390XState *env, uint64_t src,
-                                         uint64_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_tm_32(CPUS390XState *env, uint32_t val, uint32_t mask)
-{
-    HELPER_LOG("%s: val 0x%x mask 0x%x\n", __FUNCTION__, val, mask);
-    uint16_t r = val & mask;
-    if (r == 0 || mask == 0) {
-        return 0;
-    } else if (r == mask) {
-        return 3;
-    } else {
-        return 1;
-    }
-}
-
-/* set condition code for test under mask */
-static inline uint32_t cc_calc_tm_64(CPUS390XState *env, uint64_t val, uint32_t mask)
-{
-    uint16_t r = val & mask;
-    HELPER_LOG("%s: val 0x%lx mask 0x%x r 0x%x\n", __FUNCTION__, val, mask, r);
-    if (r == 0 || mask == 0) {
-        return 0;
-    } else if (r == mask) {
-        return 3;
-    } else {
-        while (!(mask & 0x8000)) {
-            mask <<= 1;
-            val <<= 1;
-        }
-        if (val & 0x8000) {
-            return 2;
-        } else {
-            return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_nz(CPUS390XState *env, uint64_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_add_64(CPUS390XState *env, int64_t a1, int64_t a2,
-                                      int64_t ar)
-{
-    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_addu_64(CPUS390XState *env, uint64_t a1, uint64_t a2,
-                                       uint64_t ar)
-{
-    if (ar == 0) {
-        if (a1) {
-            return 2;
-        } else {
-            return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-          return 3;
-        } else {
-          return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_sub_64(CPUS390XState *env, int64_t a1, int64_t a2,
-                                      int64_t ar)
-{
-    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_subu_64(CPUS390XState *env, uint64_t a1, uint64_t a2,
-                                       uint64_t ar)
-{
-    if (ar == 0) {
-        return 2;
-    } else {
-        if (a2 > a1) {
-            return 1;
-        } else {
-            return 3;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_abs_64(CPUS390XState *env, int64_t dst)
-{
-    if ((uint64_t)dst == 0x8000000000000000ULL) {
-        return 3;
-    } else if (dst) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-static inline uint32_t cc_calc_nabs_64(CPUS390XState *env, int64_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_comp_64(CPUS390XState *env, int64_t dst)
-{
-    if ((uint64_t)dst == 0x8000000000000000ULL) {
-        return 3;
-    } else if (dst < 0) {
-        return 1;
-    } else if (dst > 0) {
-        return 2;
-    } else {
-        return 0;
-    }
-}
-
-
-static inline uint32_t cc_calc_add_32(CPUS390XState *env, int32_t a1, int32_t a2,
-                                      int32_t ar)
-{
-    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_addu_32(CPUS390XState *env, uint32_t a1, uint32_t a2,
-                                       uint32_t ar)
-{
-    if (ar == 0) {
-        if (a1) {
-          return 2;
-        } else {
-          return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-          return 3;
-        } else {
-          return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_sub_32(CPUS390XState *env, int32_t a1, int32_t a2,
-                                      int32_t ar)
-{
-    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_subu_32(CPUS390XState *env, uint32_t a1, uint32_t a2,
-                                       uint32_t ar)
-{
-    if (ar == 0) {
-        return 2;
-    } else {
-        if (a2 > a1) {
-            return 1;
-        } else {
-            return 3;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_abs_32(CPUS390XState *env, int32_t dst)
-{
-    if ((uint32_t)dst == 0x80000000UL) {
-        return 3;
-    } else if (dst) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-static inline uint32_t cc_calc_nabs_32(CPUS390XState *env, int32_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_comp_32(CPUS390XState *env, int32_t dst)
-{
-    if ((uint32_t)dst == 0x80000000UL) {
-        return 3;
-    } else if (dst < 0) {
-        return 1;
-    } else if (dst > 0) {
-        return 2;
-    } else {
-        return 0;
-    }
-}
-
-/* calculate condition code for insert character under mask insn */
-static inline uint32_t cc_calc_icm_32(CPUS390XState *env, uint32_t mask, uint32_t val)
-{
-    HELPER_LOG("%s: mask 0x%x val %d\n", __FUNCTION__, mask, val);
-    uint32_t cc;
-
-    if (mask == 0xf) {
-        if (!val) {
-            return 0;
-        } else if (val & 0x80000000) {
-            return 1;
-        } else {
-            return 2;
-        }
-    }
-
-    if (!val || !mask) {
-        cc = 0;
-    } else {
-        while (mask != 1) {
-            mask >>= 1;
-            val >>= 8;
-        }
-        if (val & 0x80) {
-            cc = 1;
-        } else {
-            cc = 2;
-        }
-    }
-    return cc;
-}
-
-static inline uint32_t cc_calc_slag(CPUS390XState *env, uint64_t src, uint64_t shift)
-{
-    uint64_t mask = ((1ULL << shift) - 1ULL) << (64 - shift);
-    uint64_t match, r;
-
-    /* check if the sign bit stays the same */
-    if (src & (1ULL << 63)) {
-        match = mask;
-    } else {
-        match = 0;
-    }
-
-    if ((src & mask) != match) {
-        /* overflow */
-        return 3;
-    }
-
-    r = ((src << shift) & ((1ULL << 63) - 1)) | (src & (1ULL << 63));
-
-    if ((int64_t)r == 0) {
-        return 0;
-    } else if ((int64_t)r < 0) {
-        return 1;
-    }
-
-    return 2;
-}
-
-
-static inline uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src,
-                                  uint64_t dst, uint64_t vr)
-{
-    uint32_t r = 0;
-
-    switch (cc_op) {
-    case CC_OP_CONST0:
-    case CC_OP_CONST1:
-    case CC_OP_CONST2:
-    case CC_OP_CONST3:
-        /* cc_op value _is_ cc */
-        r = cc_op;
-        break;
-    case CC_OP_LTGT0_32:
-        r = cc_calc_ltgt0_32(env, dst);
-        break;
-    case CC_OP_LTGT0_64:
-        r =  cc_calc_ltgt0_64(env, dst);
-        break;
-    case CC_OP_LTGT_32:
-        r =  cc_calc_ltgt_32(env, src, dst);
-        break;
-    case CC_OP_LTGT_64:
-        r =  cc_calc_ltgt_64(env, src, dst);
-        break;
-    case CC_OP_LTUGTU_32:
-        r =  cc_calc_ltugtu_32(env, src, dst);
-        break;
-    case CC_OP_LTUGTU_64:
-        r =  cc_calc_ltugtu_64(env, src, dst);
-        break;
-    case CC_OP_TM_32:
-        r =  cc_calc_tm_32(env, src, dst);
-        break;
-    case CC_OP_TM_64:
-        r =  cc_calc_tm_64(env, src, dst);
-        break;
-    case CC_OP_NZ:
-        r =  cc_calc_nz(env, dst);
-        break;
-    case CC_OP_ADD_64:
-        r =  cc_calc_add_64(env, src, dst, vr);
-        break;
-    case CC_OP_ADDU_64:
-        r =  cc_calc_addu_64(env, src, dst, vr);
-        break;
-    case CC_OP_SUB_64:
-        r =  cc_calc_sub_64(env, src, dst, vr);
-        break;
-    case CC_OP_SUBU_64:
-        r =  cc_calc_subu_64(env, src, dst, vr);
-        break;
-    case CC_OP_ABS_64:
-        r =  cc_calc_abs_64(env, dst);
-        break;
-    case CC_OP_NABS_64:
-        r =  cc_calc_nabs_64(env, dst);
-        break;
-    case CC_OP_COMP_64:
-        r =  cc_calc_comp_64(env, dst);
-        break;
-
-    case CC_OP_ADD_32:
-        r =  cc_calc_add_32(env, src, dst, vr);
-        break;
-    case CC_OP_ADDU_32:
-        r =  cc_calc_addu_32(env, src, dst, vr);
-        break;
-    case CC_OP_SUB_32:
-        r =  cc_calc_sub_32(env, src, dst, vr);
-        break;
-    case CC_OP_SUBU_32:
-        r =  cc_calc_subu_32(env, src, dst, vr);
-        break;
-    case CC_OP_ABS_32:
-        r =  cc_calc_abs_64(env, dst);
-        break;
-    case CC_OP_NABS_32:
-        r =  cc_calc_nabs_64(env, dst);
-        break;
-    case CC_OP_COMP_32:
-        r =  cc_calc_comp_32(env, dst);
-        break;
-
-    case CC_OP_ICM:
-        r =  cc_calc_icm_32(env, src, dst);
-        break;
-    case CC_OP_SLAG:
-        r =  cc_calc_slag(env, src, dst);
-        break;
-
-    case CC_OP_LTGT_F32:
-        r = set_cc_f32(src, dst);
-        break;
-    case CC_OP_LTGT_F64:
-        r = set_cc_f64(src, dst);
-        break;
-    case CC_OP_NZ_F32:
-        r = set_cc_nz_f32(dst);
-        break;
-    case CC_OP_NZ_F64:
-        r = set_cc_nz_f64(dst);
-        break;
-
-    default:
-        cpu_abort(env, "Unknown CC operation: %s\n", cc_name(cc_op));
-    }
-
-    HELPER_LOG("%s: %15s 0x%016lx 0x%016lx 0x%016lx = %d\n", __FUNCTION__,
-               cc_name(cc_op), src, dst, vr, r);
-    return r;
-}
-
-uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
-                 uint64_t vr)
-{
-    return do_calc_cc(env, cc_op, src, dst, vr);
-}
-
-uint32_t HELPER(calc_cc)(uint32_t cc_op, uint64_t src, uint64_t dst,
-                         uint64_t vr)
-{
-    return do_calc_cc(env, cc_op, src, dst, vr);
-}
-
-uint64_t HELPER(cvd)(int32_t bin)
-{
-    /* positive 0 */
-    uint64_t dec = 0x0c;
-    int shift = 4;
-
-    if (bin < 0) {
-        bin = -bin;
-        dec = 0x0d;
-    }
-
-    for (shift = 4; (shift < 64) && bin; shift += 4) {
-        int current_number = bin % 10;
-
-        dec |= (current_number) << shift;
-        bin /= 10;
-    }
-
-    return dec;
-}
-
-void HELPER(unpk)(uint32_t len, uint64_t dest, uint64_t src)
-{
-    int len_dest = len >> 4;
-    int len_src = len & 0xf;
-    uint8_t b;
-    int second_nibble = 0;
-
-    dest += len_dest;
-    src += len_src;
-
-    /* last byte is special, it only flips the nibbles */
-    b = ldub(src);
-    stb(dest, (b << 4) | (b >> 4));
-    src--;
-    len_src--;
-
-    /* now pad every nibble with 0xf0 */
-
-    while (len_dest > 0) {
-        uint8_t cur_byte = 0;
-
-        if (len_src > 0) {
-            cur_byte = ldub(src);
-        }
-
-        len_dest--;
-        dest--;
-
-        /* only advance one nibble at a time */
-        if (second_nibble) {
-            cur_byte >>= 4;
-            len_src--;
-            src--;
-        }
-        second_nibble = !second_nibble;
-
-        /* digit */
-        cur_byte = (cur_byte & 0xf);
-        /* zone bits */
-        cur_byte |= 0xf0;
-
-        stb(dest, cur_byte);
-    }
-}
-
-void HELPER(tr)(uint32_t len, uint64_t array, uint64_t trans)
-{
-    int i;
-
-    for (i = 0; i <= len; i++) {
-        uint8_t byte = ldub(array + i);
-        uint8_t new_byte = ldub(trans + byte);
-        stb(array + i, new_byte);
-    }
-}
-
-#ifndef CONFIG_USER_ONLY
-
-void HELPER(load_psw)(uint64_t mask, uint64_t addr)
-{
-    load_psw(env, mask, addr);
-    cpu_loop_exit(env);
-}
-
-static void program_interrupt(CPUS390XState *env, uint32_t code, int ilc)
-{
-    qemu_log("program interrupt at %#" PRIx64 "\n", env->psw.addr);
-
-    if (kvm_enabled()) {
-#ifdef CONFIG_KVM
-        kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
-#endif
-    } else {
-        env->int_pgm_code = code;
-        env->int_pgm_ilc = ilc;
-        env->exception_index = EXCP_PGM;
-        cpu_loop_exit(env);
-    }
-}
-
-/*
- * ret < 0 indicates program check, ret = 0,1,2,3 -> cc
- */
-int sclp_service_call(CPUS390XState *env, uint32_t sccb, uint64_t code)
-{
-    int r = 0;
-    int shift = 0;
-
-#ifdef DEBUG_HELPER
-    printf("sclp(0x%x, 0x%" PRIx64 ")\n", sccb, code);
-#endif
-
-    /* basic checks */
-    if (!memory_region_is_ram(phys_page_find(sccb >> TARGET_PAGE_BITS)->mr)) {
-        return -PGM_ADDRESSING;
-    }
-    if (sccb & ~0x7ffffff8ul) {
-        return -PGM_SPECIFICATION;
-    }
-
-    switch(code) {
-        case SCLP_CMDW_READ_SCP_INFO:
-        case SCLP_CMDW_READ_SCP_INFO_FORCED:
-            while ((ram_size >> (20 + shift)) > 65535) {
-                shift++;
-            }
-            stw_phys(sccb + SCP_MEM_CODE, ram_size >> (20 + shift));
-            stb_phys(sccb + SCP_INCREMENT, 1 << shift);
-            stw_phys(sccb + SCP_RESPONSE_CODE, 0x10);
-
-            s390_sclp_extint(sccb & ~3);
-            break;
-        default:
-#ifdef DEBUG_HELPER
-            printf("KVM: invalid sclp call 0x%x / 0x%" PRIx64 "x\n", sccb, code);
-#endif
-            r = 3;
-            break;
-    }
-
-    return r;
-}
-
-/* SCLP service call */
-uint32_t HELPER(servc)(uint32_t r1, uint64_t r2)
-{
-    int r;
-
-    r = sclp_service_call(env, r1, r2);
-    if (r < 0) {
-        program_interrupt(env, -r, 4);
-        return 0;
-    }
-    return r;
-}
-
-/* DIAG */
-uint64_t HELPER(diag)(uint32_t num, uint64_t mem, uint64_t code)
-{
-    uint64_t r;
-
-    switch (num) {
-    case 0x500:
-        /* KVM hypercall */
-        r = s390_virtio_hypercall(env, mem, code);
-        break;
-    case 0x44:
-        /* yield */
-        r = 0;
-        break;
-    case 0x308:
-        /* ipl */
-        r = 0;
-        break;
-    default:
-        r = -1;
-        break;
-    }
-
-    if (r) {
-        program_interrupt(env, PGM_OPERATION, ILC_LATER_INC);
-    }
-
-    return r;
-}
-
-/* Store CPU ID */
-void HELPER(stidp)(uint64_t a1)
-{
-    stq(a1, env->cpu_num);
-}
-
-/* Set Prefix */
-void HELPER(spx)(uint64_t a1)
-{
-    uint32_t prefix;
-
-    prefix = ldl(a1);
-    env->psa = prefix & 0xfffff000;
-    qemu_log("prefix: %#x\n", prefix);
-    tlb_flush_page(env, 0);
-    tlb_flush_page(env, TARGET_PAGE_SIZE);
-}
-
-/* Set Clock */
-uint32_t HELPER(sck)(uint64_t a1)
-{
-    /* XXX not implemented - is it necessary? */
-
-    return 0;
-}
-
-static inline uint64_t clock_value(CPUS390XState *env)
-{
-    uint64_t time;
-
-    time = env->tod_offset +
-           time2tod(qemu_get_clock_ns(vm_clock) - env->tod_basetime);
-
-    return time;
-}
-
-/* Store Clock */
-uint32_t HELPER(stck)(uint64_t a1)
-{
-    stq(a1, clock_value(env));
-
-    return 0;
-}
-
-/* Store Clock Extended */
-uint32_t HELPER(stcke)(uint64_t a1)
-{
-    stb(a1, 0);
-    /* basically the same value as stck */
-    stq(a1 + 1, clock_value(env) | env->cpu_num);
-    /* more fine grained than stck */
-    stq(a1 + 9, 0);
-    /* XXX programmable fields */
-    stw(a1 + 17, 0);
-
-
-    return 0;
-}
-
-/* Set Clock Comparator */
-void HELPER(sckc)(uint64_t a1)
-{
-    uint64_t time = ldq(a1);
-
-    if (time == -1ULL) {
-        return;
-    }
-
-    /* difference between now and then */
-    time -= clock_value(env);
-    /* nanoseconds */
-    time = (time * 125) >> 9;
-
-    qemu_mod_timer(env->tod_timer, qemu_get_clock_ns(vm_clock) + time);
-}
-
-/* Store Clock Comparator */
-void HELPER(stckc)(uint64_t a1)
-{
-    /* XXX implement */
-    stq(a1, 0);
-}
-
-/* Set CPU Timer */
-void HELPER(spt)(uint64_t a1)
-{
-    uint64_t time = ldq(a1);
-
-    if (time == -1ULL) {
-        return;
-    }
-
-    /* nanoseconds */
-    time = (time * 125) >> 9;
-
-    qemu_mod_timer(env->cpu_timer, qemu_get_clock_ns(vm_clock) + time);
-}
-
-/* Store CPU Timer */
-void HELPER(stpt)(uint64_t a1)
-{
-    /* XXX implement */
-    stq(a1, 0);
-}
-
-/* Store System Information */
-uint32_t HELPER(stsi)(uint64_t a0, uint32_t r0, uint32_t r1)
-{
-    int cc = 0;
-    int sel1, sel2;
-
-    if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 &&
-        ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) {
-        /* valid function code, invalid reserved bits */
-        program_interrupt(env, PGM_SPECIFICATION, 2);
-    }
-
-    sel1 = r0 & STSI_R0_SEL1_MASK;
-    sel2 = r1 & STSI_R1_SEL2_MASK;
-
-    /* XXX: spec exception if sysib is not 4k-aligned */
-
-    switch (r0 & STSI_LEVEL_MASK) {
-    case STSI_LEVEL_1:
-        if ((sel1 == 1) && (sel2 == 1)) {
-            /* Basic Machine Configuration */
-            struct sysib_111 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            ebcdic_put(sysib.manuf, "QEMU            ", 16);
-            /* same as machine type number in STORE CPU ID */
-            ebcdic_put(sysib.type, "QEMU", 4);
-            /* same as model number in STORE CPU ID */
-            ebcdic_put(sysib.model, "QEMU            ", 16);
-            ebcdic_put(sysib.sequence, "QEMU            ", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 1)) {
-            /* Basic Machine CPU */
-            struct sysib_121 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            /* XXX make different for different CPUs? */
-            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            stw_p(&sysib.cpu_addr, env->cpu_num);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 2)) {
-            /* Basic Machine CPUs */
-            struct sysib_122 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            stl_p(&sysib.capability, 0x443afc29);
-            /* XXX change when SMP comes */
-            stw_p(&sysib.total_cpus, 1);
-            stw_p(&sysib.active_cpus, 1);
-            stw_p(&sysib.standby_cpus, 0);
-            stw_p(&sysib.reserved_cpus, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    case STSI_LEVEL_2:
-    {
-        if ((sel1 == 2) && (sel2 == 1)) {
-            /* LPAR CPU */
-            struct sysib_221 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            /* XXX make different for different CPUs? */
-            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            stw_p(&sysib.cpu_addr, env->cpu_num);
-            stw_p(&sysib.cpu_id, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 2)) {
-            /* LPAR CPUs */
-            struct sysib_222 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            stw_p(&sysib.lpar_num, 0);
-            sysib.lcpuc = 0;
-            /* XXX change when SMP comes */
-            stw_p(&sysib.total_cpus, 1);
-            stw_p(&sysib.conf_cpus, 1);
-            stw_p(&sysib.standby_cpus, 0);
-            stw_p(&sysib.reserved_cpus, 0);
-            ebcdic_put(sysib.name, "QEMU    ", 8);
-            stl_p(&sysib.caf, 1000);
-            stw_p(&sysib.dedicated_cpus, 0);
-            stw_p(&sysib.shared_cpus, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    }
-    case STSI_LEVEL_3:
-    {
-        if ((sel1 == 2) && (sel2 == 2)) {
-            /* VM CPUs */
-            struct sysib_322 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            sysib.count = 1;
-            /* XXX change when SMP comes */
-            stw_p(&sysib.vm[0].total_cpus, 1);
-            stw_p(&sysib.vm[0].conf_cpus, 1);
-            stw_p(&sysib.vm[0].standby_cpus, 0);
-            stw_p(&sysib.vm[0].reserved_cpus, 0);
-            ebcdic_put(sysib.vm[0].name, "KVMguest", 8);
-            stl_p(&sysib.vm[0].caf, 1000);
-            ebcdic_put(sysib.vm[0].cpi, "KVM/Linux       ", 16);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    }
-    case STSI_LEVEL_CURRENT:
-        env->regs[0] = STSI_LEVEL_3;
-        break;
-    default:
-        cc = 3;
-        break;
-    }
-
-    return cc;
-}
-
-void HELPER(lctlg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t src = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = ldq(src);
-        HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
-                   i, src, env->cregs[i]);
-        src += sizeof(uint64_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-
-    tlb_flush(env, 1);
-}
-
-void HELPER(lctl)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t src = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) | ldl(src);
-        src += sizeof(uint32_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-
-    tlb_flush(env, 1);
-}
-
-void HELPER(stctg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t dest = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stq(dest, env->cregs[i]);
-        dest += sizeof(uint64_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-void HELPER(stctl)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t dest = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stl(dest, env->cregs[i]);
-        dest += sizeof(uint32_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
-{
-    /* XXX implement */
-
-    return 0;
-}
-
-/* insert storage key extended */
-uint64_t HELPER(iske)(uint64_t r2)
-{
-    uint64_t addr = get_address(0, 0, r2);
-
-    if (addr > ram_size) {
-        return 0;
-    }
-
-    return env->storage_keys[addr / TARGET_PAGE_SIZE];
-}
-
-/* set storage key extended */
-void HELPER(sske)(uint32_t r1, uint64_t r2)
-{
-    uint64_t addr = get_address(0, 0, r2);
-
-    if (addr > ram_size) {
-        return;
-    }
-
-    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
-}
-
-/* reset reference bit extended */
-uint32_t HELPER(rrbe)(uint32_t r1, uint64_t r2)
-{
-    uint8_t re;
-    uint8_t key;
-    if (r2 > ram_size) {
-        return 0;
-    }
-
-    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
-    re = key & (SK_R | SK_C);
-    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
-
-    /*
-     * cc
-     *
-     * 0  Reference bit zero; change bit zero
-     * 1  Reference bit zero; change bit one
-     * 2  Reference bit one; change bit zero
-     * 3  Reference bit one; change bit one
-     */
-
-    return re >> 1;
-}
-
-/* compare and swap and purge */
-uint32_t HELPER(csp)(uint32_t r1, uint32_t r2)
-{
-    uint32_t cc;
-    uint32_t o1 = env->regs[r1];
-    uint64_t a2 = get_address_31fix(r2) & ~3ULL;
-    uint32_t o2 = ldl(a2);
-
-    if (o1 == o2) {
-        stl(a2, env->regs[(r1 + 1) & 15]);
-        if (env->regs[r2] & 0x3) {
-            /* flush TLB / ALB */
-            tlb_flush(env, 1);
-        }
-        cc = 0;
-    } else {
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | o2;
-        cc = 1;
-    }
-
-    return cc;
-}
-
-static uint32_t mvc_asc(int64_t l, uint64_t a1, uint64_t mode1, uint64_t a2,
-                        uint64_t mode2)
-{
-    target_ulong src, dest;
-    int flags, cc = 0, i;
-
-    if (!l) {
-        return 0;
-    } else if (l > 256) {
-        /* max 256 */
-        l = 256;
-        cc = 3;
-    }
-
-    if (mmu_translate(env, a1 & TARGET_PAGE_MASK, 1, mode1, &dest, &flags)) {
-        cpu_loop_exit(env);
-    }
-    dest |= a1 & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, a2 & TARGET_PAGE_MASK, 0, mode2, &src, &flags)) {
-        cpu_loop_exit(env);
-    }
-    src |= a2 & ~TARGET_PAGE_MASK;
-
-    /* XXX replace w/ memcpy */
-    for (i = 0; i < l; i++) {
-        /* XXX be more clever */
-        if ((((dest + i) & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) ||
-            (((src + i) & TARGET_PAGE_MASK) != (src & TARGET_PAGE_MASK))) {
-            mvc_asc(l - i, a1 + i, mode1, a2 + i, mode2);
-            break;
-        }
-        stb_phys(dest + i, ldub_phys(src + i));
-    }
-
-    return cc;
-}
-
-uint32_t HELPER(mvcs)(uint64_t l, uint64_t a1, uint64_t a2)
-{
-    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
-               __FUNCTION__, l, a1, a2);
-
-    return mvc_asc(l, a1, PSW_ASC_SECONDARY, a2, PSW_ASC_PRIMARY);
-}
-
-uint32_t HELPER(mvcp)(uint64_t l, uint64_t a1, uint64_t a2)
-{
-    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
-               __FUNCTION__, l, a1, a2);
-
-    return mvc_asc(l, a1, PSW_ASC_PRIMARY, a2, PSW_ASC_SECONDARY);
-}
-
-uint32_t HELPER(sigp)(uint64_t order_code, uint32_t r1, uint64_t cpu_addr)
-{
-    int cc = 0;
-
-    HELPER_LOG("%s: %016" PRIx64 " %08x %016" PRIx64 "\n",
-               __FUNCTION__, order_code, r1, cpu_addr);
-
-    /* Remember: Use "R1 or R1+1, whichever is the odd-numbered register"
-       as parameter (input). Status (output) is always R1. */
-
-    switch (order_code) {
-    case SIGP_SET_ARCH:
-        /* switch arch */
-        break;
-    case SIGP_SENSE:
-        /* enumerate CPU status */
-        if (cpu_addr) {
-            /* XXX implement when SMP comes */
-            return 3;
-        }
-        env->regs[r1] &= 0xffffffff00000000ULL;
-        cc = 1;
-        break;
-#if !defined (CONFIG_USER_ONLY)
-    case SIGP_RESTART:
-        qemu_system_reset_request();
-        cpu_loop_exit(env);
-        break;
-    case SIGP_STOP:
-        qemu_system_shutdown_request();
-        cpu_loop_exit(env);
-        break;
-#endif
-    default:
-        /* unknown sigp */
-        fprintf(stderr, "XXX unknown sigp: 0x%" PRIx64 "\n", order_code);
-        cc = 3;
-    }
-
-    return cc;
-}
-
-void HELPER(sacf)(uint64_t a1)
-{
-    HELPER_LOG("%s: %16" PRIx64 "\n", __FUNCTION__, a1);
-
-    switch (a1 & 0xf00) {
-    case 0x000:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_PRIMARY;
-        break;
-    case 0x100:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_SECONDARY;
-        break;
-    case 0x300:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_HOME;
-        break;
-    default:
-        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
-        program_interrupt(env, PGM_SPECIFICATION, 2);
-        break;
-    }
-}
-
-/* invalidate pte */
-void HELPER(ipte)(uint64_t pte_addr, uint64_t vaddr)
-{
-    uint64_t page = vaddr & TARGET_PAGE_MASK;
-    uint64_t pte = 0;
-
-    /* XXX broadcast to other CPUs */
-
-    /* XXX Linux is nice enough to give us the exact pte address.
-           According to spec we'd have to find it out ourselves */
-    /* XXX Linux is fine with overwriting the pte, the spec requires
-           us to only set the invalid bit */
-    stq_phys(pte_addr, pte | _PAGE_INVALID);
-
-    /* XXX we exploit the fact that Linux passes the exact virtual
-           address here - it's not obliged to! */
-    tlb_flush_page(env, page);
-
-    /* XXX 31-bit hack */
-    if (page & 0x80000000) {
-        tlb_flush_page(env, page & ~0x80000000);
-    } else {
-        tlb_flush_page(env, page | 0x80000000);
-    }
-}
-
-/* flush local tlb */
-void HELPER(ptlb)(void)
-{
-    tlb_flush(env, 1);
-}
-
-/* store using real address */
-void HELPER(stura)(uint64_t addr, uint32_t v1)
-{
-    stw_phys(get_address(0, 0, addr), v1);
-}
-
-/* load real address */
-uint32_t HELPER(lra)(uint64_t addr, uint32_t r1)
-{
-    uint32_t cc = 0;
-    int old_exc = env->exception_index;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    uint64_t ret;
-    int flags;
-
-    /* XXX incomplete - has more corner cases */
-    if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
-        program_interrupt(env, PGM_SPECIAL_OP, 2);
-    }
-
-    env->exception_index = old_exc;
-    if (mmu_translate(env, addr, 0, asc, &ret, &flags)) {
-        cc = 3;
-    }
-    if (env->exception_index == EXCP_PGM) {
-        ret = env->int_pgm_code | 0x80000000;
-    } else {
-        ret |= addr & ~TARGET_PAGE_MASK;
-    }
-    env->exception_index = old_exc;
-
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | (ret & 0xffffffffULL);
-    } else {
-        env->regs[r1] = ret;
-    }
-
-    return cc;
-}
-
-#endif
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 1c1baf53425..a57296c64ff 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -18,7 +18,6 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-/* #define DEBUG_ILLEGAL_INSTRUCTIONS */
 /* #define DEBUG_INLINE_BRANCHES */
 #define S390X_DEBUG_DISAS
 /* #define S390X_DEBUG_DISAS_VERBOSE */
@@ -30,41 +29,54 @@
 #endif
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
+#include "qemu/host-utils.h"
 
 /* global register indexes */
 static TCGv_ptr cpu_env;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 #include "helper.h"
 #define GEN_HELPER 1
 #include "helper.h"
 
+
+/* Information that (most) every instruction needs to manipulate.  */
 typedef struct DisasContext DisasContext;
+typedef struct DisasInsn DisasInsn;
+typedef struct DisasFields DisasFields;
+
 struct DisasContext {
-    uint64_t pc;
-    int is_jmp;
-    enum cc_op cc_op;
     struct TranslationBlock *tb;
+    const DisasInsn *insn;
+    DisasFields *fields;
+    uint64_t pc, next_pc;
+    enum cc_op cc_op;
+    bool singlestep_enabled;
 };
 
-#define DISAS_EXCP 4
+/* Information carried about a condition to be evaluated.  */
+typedef struct {
+    TCGCond cond:8;
+    bool is_64;
+    bool g1;
+    bool g2;
+    union {
+        struct { TCGv_i64 a, b; } s64;
+        struct { TCGv_i32 a, b; } s32;
+    } u;
+} DisasCompare;
 
-static void gen_op_calc_cc(DisasContext *s);
+#define DISAS_EXCP 4
 
 #ifdef DEBUG_INLINE_BRANCHES
 static uint64_t inline_branch_hit[CC_OP_MAX];
 static uint64_t inline_branch_miss[CC_OP_MAX];
 #endif
 
-static inline void debug_insn(uint64_t insn)
-{
-    LOG_DISAS("insn: 0x%" PRIx64 "\n", insn);
-}
-
-static inline uint64_t pc_to_link_info(DisasContext *s, uint64_t pc)
+static uint64_t pc_to_link_info(DisasContext *s, uint64_t pc)
 {
     if (!(s->tb->flags & FLAG_MASK_64)) {
         if (s->tb->flags & FLAG_MASK_32) {
@@ -79,6 +91,14 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
 {
     int i;
 
+    if (env->cc_op > 3) {
+        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %15s\n",
+                    env->psw.mask, env->psw.addr, cc_name(env->cc_op));
+    } else {
+        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %02x\n",
+                    env->psw.mask, env->psw.addr, env->cc_op);
+    }
+
     for (i = 0; i < 16; i++) {
         cpu_fprintf(f, "R%02d=%016" PRIx64, i, env->regs[i]);
         if ((i % 4) == 3) {
@@ -89,7 +109,7 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
     }
 
     for (i = 0; i < 16; i++) {
-        cpu_fprintf(f, "F%02d=%016" PRIx64, i, *(uint64_t *)&env->fregs[i]);
+        cpu_fprintf(f, "F%02d=%016" PRIx64, i, env->fregs[i].ll);
         if ((i % 4) == 3) {
             cpu_fprintf(f, "\n");
         } else {
@@ -97,8 +117,6 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
         }
     }
 
-    cpu_fprintf(f, "\n");
-
 #ifndef CONFIG_USER_ONLY
     for (i = 0; i < 16; i++) {
         cpu_fprintf(f, "C%02d=%016" PRIx64, i, env->cregs[i]);
@@ -110,22 +128,14 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
     }
 #endif
 
-    cpu_fprintf(f, "\n");
-
-    if (env->cc_op > 3) {
-        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %15s\n",
-                    env->psw.mask, env->psw.addr, cc_name(env->cc_op));
-    } else {
-        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %02x\n",
-                    env->psw.mask, env->psw.addr, env->cc_op);
-    }
-
 #ifdef DEBUG_INLINE_BRANCHES
     for (i = 0; i < CC_OP_MAX; i++) {
         cpu_fprintf(f, "  %15s = %10ld\t%10ld\n", cc_name(i),
                     inline_branch_miss[i], inline_branch_hit[i]);
     }
 #endif
+
+    cpu_fprintf(f, "\n");
 }
 
 static TCGv_i64 psw_addr;
@@ -136,21 +146,22 @@ static TCGv_i64 cc_src;
 static TCGv_i64 cc_dst;
 static TCGv_i64 cc_vr;
 
-static char cpu_reg_names[10*3 + 6*4];
+static char cpu_reg_names[32][4];
 static TCGv_i64 regs[16];
+static TCGv_i64 fregs[16];
 
 static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
 
 void s390x_translate_init(void)
 {
     int i;
-    size_t cpu_reg_names_size = sizeof(cpu_reg_names);
-    char *p;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    psw_addr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, psw.addr),
+    psw_addr = tcg_global_mem_new_i64(TCG_AREG0,
+                                      offsetof(CPUS390XState, psw.addr),
                                       "psw_addr");
-    psw_mask = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, psw.mask),
+    psw_mask = tcg_global_mem_new_i64(TCG_AREG0,
+                                      offsetof(CPUS390XState, psw.mask),
                                       "psw_mask");
 
     cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUS390XState, cc_op),
@@ -162,137 +173,105 @@ void s390x_translate_init(void)
     cc_vr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_vr),
                                    "cc_vr");
 
-    p = cpu_reg_names;
     for (i = 0; i < 16; i++) {
-        snprintf(p, cpu_reg_names_size, "r%d", i);
+        snprintf(cpu_reg_names[i], sizeof(cpu_reg_names[0]), "r%d", i);
         regs[i] = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUS390XState, regs[i]), p);
-        p += (i < 10) ? 3 : 4;
-        cpu_reg_names_size -= (i < 10) ? 3 : 4;
+                                     offsetof(CPUS390XState, regs[i]),
+                                     cpu_reg_names[i]);
     }
-}
-
-static inline TCGv_i64 load_reg(int reg)
-{
-    TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_mov_i64(r, regs[reg]);
-    return r;
-}
 
-static inline TCGv_i64 load_freg(int reg)
-{
-    TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_ld_i64(r, cpu_env, offsetof(CPUS390XState, fregs[reg].d));
-    return r;
-}
+    for (i = 0; i < 16; i++) {
+        snprintf(cpu_reg_names[i + 16], sizeof(cpu_reg_names[0]), "f%d", i);
+        fregs[i] = tcg_global_mem_new(TCG_AREG0,
+                                      offsetof(CPUS390XState, fregs[i].d),
+                                      cpu_reg_names[i + 16]);
+    }
 
-static inline TCGv_i32 load_freg32(int reg)
-{
-    TCGv_i32 r = tcg_temp_new_i32();
-    tcg_gen_ld_i32(r, cpu_env, offsetof(CPUS390XState, fregs[reg].l.upper));
-    return r;
+    /* register helpers */
+#define GEN_HELPER 2
+#include "helper.h"
 }
 
-static inline TCGv_i32 load_reg32(int reg)
+static TCGv_i64 load_reg(int reg)
 {
-    TCGv_i32 r = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(r, regs[reg]);
+    TCGv_i64 r = tcg_temp_new_i64();
+    tcg_gen_mov_i64(r, regs[reg]);
     return r;
 }
 
-static inline TCGv_i64 load_reg32_i64(int reg)
+static TCGv_i64 load_freg32_i64(int reg)
 {
     TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_ext32s_i64(r, regs[reg]);
+    tcg_gen_shri_i64(r, fregs[reg], 32);
     return r;
 }
 
-static inline void store_reg(int reg, TCGv_i64 v)
+static void store_reg(int reg, TCGv_i64 v)
 {
     tcg_gen_mov_i64(regs[reg], v);
 }
 
-static inline void store_freg(int reg, TCGv_i64 v)
+static void store_freg(int reg, TCGv_i64 v)
 {
-    tcg_gen_st_i64(v, cpu_env, offsetof(CPUS390XState, fregs[reg].d));
-}
-
-static inline void store_reg32(int reg, TCGv_i32 v)
-{
-#if HOST_LONG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(regs[reg]), v);
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_extu_i32_i64(tmp, v);
-    /* 32 bit register writes keep the upper half */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], tmp, 0, 32);
-    tcg_temp_free_i64(tmp);
-#endif
+    tcg_gen_mov_i64(fregs[reg], v);
 }
 
-static inline void store_reg32_i64(int reg, TCGv_i64 v)
+static void store_reg32_i64(int reg, TCGv_i64 v)
 {
     /* 32 bit register writes keep the upper half */
-#if HOST_LONG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(regs[reg]), TCGV_LOW(v));
-#else
     tcg_gen_deposit_i64(regs[reg], regs[reg], v, 0, 32);
-#endif
 }
 
-static inline void store_reg16(int reg, TCGv_i32 v)
+static void store_reg32h_i64(int reg, TCGv_i64 v)
 {
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_extu_i32_i64(tmp, v);
-    /* 16 bit register writes keep the upper bytes */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], tmp, 0, 16);
-    tcg_temp_free_i64(tmp);
+    tcg_gen_deposit_i64(regs[reg], regs[reg], v, 32, 32);
 }
 
-static inline void store_reg8(int reg, TCGv_i64 v)
+static void store_freg32_i64(int reg, TCGv_i64 v)
 {
-    /* 8 bit register writes keep the upper bytes */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], v, 0, 8);
+    tcg_gen_deposit_i64(fregs[reg], fregs[reg], v, 32, 32);
 }
 
-static inline void store_freg32(int reg, TCGv_i32 v)
+static void return_low128(TCGv_i64 dest)
 {
-    tcg_gen_st_i32(v, cpu_env, offsetof(CPUS390XState, fregs[reg].l.upper));
+    tcg_gen_ld_i64(dest, cpu_env, offsetof(CPUS390XState, retxl));
 }
 
-static inline void update_psw_addr(DisasContext *s)
+static void update_psw_addr(DisasContext *s)
 {
     /* psw.addr */
     tcg_gen_movi_i64(psw_addr, s->pc);
 }
 
-static inline void potential_page_fault(DisasContext *s)
+static void update_cc_op(DisasContext *s)
+{
+    if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
+        tcg_gen_movi_i32(cc_op, s->cc_op);
+    }
+}
+
+static void potential_page_fault(DisasContext *s)
 {
-#ifndef CONFIG_USER_ONLY
     update_psw_addr(s);
-    gen_op_calc_cc(s);
-#endif
+    update_cc_op(s);
 }
 
-static inline uint64_t ld_code2(uint64_t pc)
+static inline uint64_t ld_code2(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)lduw_code(pc);
+    return (uint64_t)cpu_lduw_code(env, pc);
 }
 
-static inline uint64_t ld_code4(uint64_t pc)
+static inline uint64_t ld_code4(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)ldl_code(pc);
+    return (uint64_t)(uint32_t)cpu_ldl_code(env, pc);
 }
 
-static inline uint64_t ld_code6(uint64_t pc)
+static inline uint64_t ld_code6(CPUS390XState *env, uint64_t pc)
 {
-    uint64_t opc;
-    opc = (uint64_t)lduw_code(pc) << 32;
-    opc |= (uint64_t)(uint32_t)ldl_code(pc+2);
-    return opc;
+    return (ld_code2(env, pc) << 32) | ld_code4(env, pc + 2);
 }
 
-static inline int get_mem_index(DisasContext *s)
+static int get_mem_index(DisasContext *s)
 {
     switch (s->tb->flags & FLAG_MASK_ASC) {
     case PSW_ASC_PRIMARY >> 32:
@@ -307,177 +286,114 @@ static inline int get_mem_index(DisasContext *s)
     }
 }
 
-static inline void gen_debug(DisasContext *s)
-{
-    TCGv_i32 tmp = tcg_const_i32(EXCP_DEBUG);
-    update_psw_addr(s);
-    gen_op_calc_cc(s);
-    gen_helper_exception(tmp);
-    tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_EXCP;
-}
-
-#ifdef CONFIG_USER_ONLY
-
-static void gen_illegal_opcode(DisasContext *s, int ilc)
+static void gen_exception(int excp)
 {
-    TCGv_i32 tmp = tcg_const_i32(EXCP_SPEC);
-    update_psw_addr(s);
-    gen_op_calc_cc(s);
-    gen_helper_exception(tmp);
+    TCGv_i32 tmp = tcg_const_i32(excp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_EXCP;
-}
-
-#else /* CONFIG_USER_ONLY */
-
-static void debug_print_inst(DisasContext *s, int ilc)
-{
-#ifdef DEBUG_ILLEGAL_INSTRUCTIONS
-    uint64_t inst = 0;
-
-    switch (ilc & 3) {
-    case 1:
-        inst = ld_code2(s->pc);
-        break;
-    case 2:
-        inst = ld_code4(s->pc);
-        break;
-    case 3:
-        inst = ld_code6(s->pc);
-        break;
-    }
-
-    fprintf(stderr, "Illegal instruction [%d at %016" PRIx64 "]: 0x%016"
-            PRIx64 "\n", ilc, s->pc, inst);
-#endif
 }
 
-static void gen_program_exception(DisasContext *s, int ilc, int code)
+static void gen_program_exception(DisasContext *s, int code)
 {
     TCGv_i32 tmp;
 
-    debug_print_inst(s, ilc);
-
-    /* remember what pgm exeption this was */
+    /* Remember what pgm exeption this was.  */
     tmp = tcg_const_i32(code);
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_code));
     tcg_temp_free_i32(tmp);
 
-    tmp = tcg_const_i32(ilc);
-    tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_ilc));
+    tmp = tcg_const_i32(s->next_pc - s->pc);
+    tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_ilen));
     tcg_temp_free_i32(tmp);
 
-    /* advance past instruction */
-    s->pc += (ilc * 2);
+    /* Advance past instruction.  */
+    s->pc = s->next_pc;
     update_psw_addr(s);
 
-    /* save off cc */
-    gen_op_calc_cc(s);
-
-    /* trigger exception */
-    tmp = tcg_const_i32(EXCP_PGM);
-    gen_helper_exception(tmp);
-    tcg_temp_free_i32(tmp);
-
-    /* end TB here */
-    s->is_jmp = DISAS_EXCP;
-}
-
+    /* Save off cc.  */
+    update_cc_op(s);
 
-static void gen_illegal_opcode(DisasContext *s, int ilc)
-{
-    gen_program_exception(s, ilc, PGM_SPECIFICATION);
+    /* Trigger exception.  */
+    gen_exception(EXCP_PGM);
 }
 
-static void gen_privileged_exception(DisasContext *s, int ilc)
+static inline void gen_illegal_opcode(DisasContext *s)
 {
-    gen_program_exception(s, ilc, PGM_PRIVILEGED);
+    gen_program_exception(s, PGM_SPECIFICATION);
 }
 
-static void check_privileged(DisasContext *s, int ilc)
+static inline void check_privileged(DisasContext *s)
 {
     if (s->tb->flags & (PSW_MASK_PSTATE >> 32)) {
-        gen_privileged_exception(s, ilc);
+        gen_program_exception(s, PGM_PRIVILEGED);
     }
 }
 
-#endif /* CONFIG_USER_ONLY */
-
 static TCGv_i64 get_address(DisasContext *s, int x2, int b2, int d2)
 {
-    TCGv_i64 tmp;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    bool need_31 = !(s->tb->flags & FLAG_MASK_64);
 
-    /* 31-bitify the immediate part; register contents are dealt with below */
-    if (!(s->tb->flags & FLAG_MASK_64)) {
-        d2 &= 0x7fffffffUL;
-    }
+    /* Note that d2 is limited to 20 bits, signed.  If we crop negative
+       displacements early we create larger immedate addends.  */
 
-    if (x2) {
-        if (d2) {
-            tmp = tcg_const_i64(d2);
-            tcg_gen_add_i64(tmp, tmp, regs[x2]);
-        } else {
-            tmp = load_reg(x2);
-        }
-        if (b2) {
-            tcg_gen_add_i64(tmp, tmp, regs[b2]);
-        }
+    /* Note that addi optimizes the imm==0 case.  */
+    if (b2 && x2) {
+        tcg_gen_add_i64(tmp, regs[b2], regs[x2]);
+        tcg_gen_addi_i64(tmp, tmp, d2);
     } else if (b2) {
-        if (d2) {
-            tmp = tcg_const_i64(d2);
-            tcg_gen_add_i64(tmp, tmp, regs[b2]);
-        } else {
-            tmp = load_reg(b2);
-        }
+        tcg_gen_addi_i64(tmp, regs[b2], d2);
+    } else if (x2) {
+        tcg_gen_addi_i64(tmp, regs[x2], d2);
     } else {
-        tmp = tcg_const_i64(d2);
+        if (need_31) {
+            d2 &= 0x7fffffff;
+            need_31 = false;
+        }
+        tcg_gen_movi_i64(tmp, d2);
     }
-
-    /* 31-bit mode mask if there are values loaded from registers */
-    if (!(s->tb->flags & FLAG_MASK_64) && (x2 || b2)) {
-        tcg_gen_andi_i64(tmp, tmp, 0x7fffffffUL);
+    if (need_31) {
+        tcg_gen_andi_i64(tmp, tmp, 0x7fffffff);
     }
 
     return tmp;
 }
 
-static void gen_op_movi_cc(DisasContext *s, uint32_t val)
+static inline bool live_cc_data(DisasContext *s)
 {
-    s->cc_op = CC_OP_CONST0 + val;
+    return (s->cc_op != CC_OP_DYNAMIC
+            && s->cc_op != CC_OP_STATIC
+            && s->cc_op > 3);
 }
 
-static void gen_op_update1_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 dst)
+static inline void gen_op_movi_cc(DisasContext *s, uint32_t val)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_mov_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = op;
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_dst);
+        tcg_gen_discard_i64(cc_vr);
+    }
+    s->cc_op = CC_OP_CONST0 + val;
 }
 
-static void gen_op_update1_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 dst)
+static void gen_op_update1_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 dst)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_vr);
+    }
+    tcg_gen_mov_i64(cc_dst, dst);
     s->cc_op = op;
 }
 
 static void gen_op_update2_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
                                   TCGv_i64 dst)
 {
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_vr);
+    }
     tcg_gen_mov_i64(cc_src, src);
     tcg_gen_mov_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = op;
-}
-
-static void gen_op_update2_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 src,
-                                  TCGv_i32 dst)
-{
-    tcg_gen_extu_i32_i64(cc_src, src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
     s->cc_op = op;
 }
 
@@ -490,219 +406,71 @@ static void gen_op_update3_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
     s->cc_op = op;
 }
 
-static void gen_op_update3_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 src,
-                                  TCGv_i32 dst, TCGv_i32 vr)
-{
-    tcg_gen_extu_i32_i64(cc_src, src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_extu_i32_i64(cc_vr, vr);
-    s->cc_op = op;
-}
-
-static inline void set_cc_nz_u32(DisasContext *s, TCGv_i32 val)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NZ, val);
-}
-
-static inline void set_cc_nz_u64(DisasContext *s, TCGv_i64 val)
+static void set_cc_nz_u64(DisasContext *s, TCGv_i64 val)
 {
     gen_op_update1_cc_i64(s, CC_OP_NZ, val);
 }
 
-static inline void cmp_32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          enum cc_op cond)
-{
-    gen_op_update2_cc_i32(s, cond, v1, v2);
-}
-
-static inline void cmp_64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          enum cc_op cond)
+static void gen_set_cc_nz_f32(DisasContext *s, TCGv_i64 val)
 {
-    gen_op_update2_cc_i64(s, cond, v1, v2);
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F32, val);
 }
 
-static inline void cmp_s32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
+static void gen_set_cc_nz_f64(DisasContext *s, TCGv_i64 val)
 {
-    cmp_32(s, v1, v2, CC_OP_LTGT_32);
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, val);
 }
 
-static inline void cmp_u32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
+static void gen_set_cc_nz_f128(DisasContext *s, TCGv_i64 vh, TCGv_i64 vl)
 {
-    cmp_32(s, v1, v2, CC_OP_LTUGTU_32);
-}
-
-static inline void cmp_s32c(DisasContext *s, TCGv_i32 v1, int32_t v2)
-{
-    /* XXX optimize for the constant? put it in s? */
-    TCGv_i32 tmp = tcg_const_i32(v2);
-    cmp_32(s, v1, tmp, CC_OP_LTGT_32);
-    tcg_temp_free_i32(tmp);
-}
-
-static inline void cmp_u32c(DisasContext *s, TCGv_i32 v1, uint32_t v2)
-{
-    TCGv_i32 tmp = tcg_const_i32(v2);
-    cmp_32(s, v1, tmp, CC_OP_LTUGTU_32);
-    tcg_temp_free_i32(tmp);
-}
-
-static inline void cmp_s64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2)
-{
-    cmp_64(s, v1, v2, CC_OP_LTGT_64);
-}
-
-static inline void cmp_u64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2)
-{
-    cmp_64(s, v1, v2, CC_OP_LTUGTU_64);
-}
-
-static inline void cmp_s64c(DisasContext *s, TCGv_i64 v1, int64_t v2)
-{
-    TCGv_i64 tmp = tcg_const_i64(v2);
-    cmp_s64(s, v1, tmp);
-    tcg_temp_free_i64(tmp);
-}
-
-static inline void cmp_u64c(DisasContext *s, TCGv_i64 v1, uint64_t v2)
-{
-    TCGv_i64 tmp = tcg_const_i64(v2);
-    cmp_u64(s, v1, tmp);
-    tcg_temp_free_i64(tmp);
-}
-
-static inline void set_cc_s32(DisasContext *s, TCGv_i32 val)
-{
-    gen_op_update1_cc_i32(s, CC_OP_LTGT0_32, val);
-}
-
-static inline void set_cc_s64(DisasContext *s, TCGv_i64 val)
-{
-    gen_op_update1_cc_i64(s, CC_OP_LTGT0_64, val);
-}
-
-static void set_cc_add64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2, TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_ADD_64, v1, v2, vr);
-}
-
-static void set_cc_addu64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_ADDU_64, v1, v2, vr);
-}
-
-static void set_cc_sub64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2, TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_SUB_64, v1, v2, vr);
-}
-
-static void set_cc_subu64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_SUBU_64, v1, v2, vr);
-}
-
-static void set_cc_abs64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_ABS_64, v1);
-}
-
-static void set_cc_nabs64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_NABS_64, v1);
-}
-
-static void set_cc_add32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2, TCGv_i32 vr)
-{
-    gen_op_update3_cc_i32(s, CC_OP_ADD_32, v1, v2, vr);
-}
-
-static void set_cc_addu32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          TCGv_i32 vr)
-{
-    gen_op_update3_cc_i32(s, CC_OP_ADDU_32, v1, v2, vr);
-}
-
-static void set_cc_sub32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2, TCGv_i32 vr)
-{
-    gen_op_update3_cc_i32(s, CC_OP_SUB_32, v1, v2, vr);
-}
-
-static void set_cc_subu32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          TCGv_i32 vr)
-{
-    gen_op_update3_cc_i32(s, CC_OP_SUBU_32, v1, v2, vr);
-}
-
-static void set_cc_abs32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_ABS_32, v1);
-}
-
-static void set_cc_nabs32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NABS_32, v1);
-}
-
-static void set_cc_comp32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_COMP_32, v1);
-}
-
-static void set_cc_comp64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_COMP_64, v1);
-}
-
-static void set_cc_icm(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
-{
-    gen_op_update2_cc_i32(s, CC_OP_ICM, v1, v2);
-}
-
-static void set_cc_cmp_f32_i64(DisasContext *s, TCGv_i32 v1, TCGv_i64 v2)
-{
-    tcg_gen_extu_i32_i64(cc_src, v1);
-    tcg_gen_mov_i64(cc_dst, v2);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = CC_OP_LTGT_F32;
-}
-
-static void set_cc_nz_f32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NZ_F32, v1);
-}
-
-static inline void set_cc_nz_f64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, v1);
+    gen_op_update2_cc_i64(s, CC_OP_NZ_F128, vh, vl);
 }
 
 /* CC value is in env->cc_op */
-static inline void set_cc_static(DisasContext *s)
+static void set_cc_static(DisasContext *s)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_discard_i64(cc_dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = CC_OP_STATIC;
-}
-
-static inline void gen_op_set_cc_op(DisasContext *s)
-{
-    if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
-        tcg_gen_movi_i32(cc_op, s->cc_op);
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_dst);
+        tcg_gen_discard_i64(cc_vr);
     }
-}
-
-static inline void gen_update_cc_op(DisasContext *s)
-{
-    gen_op_set_cc_op(s);
+    s->cc_op = CC_OP_STATIC;
 }
 
 /* calculates cc into cc_op */
 static void gen_op_calc_cc(DisasContext *s)
 {
-    TCGv_i32 local_cc_op = tcg_const_i32(s->cc_op);
-    TCGv_i64 dummy = tcg_const_i64(0);
+    TCGv_i32 local_cc_op;
+    TCGv_i64 dummy;
+
+    TCGV_UNUSED_I32(local_cc_op);
+    TCGV_UNUSED_I64(dummy);
+    switch (s->cc_op) {
+    default:
+        dummy = tcg_const_i64(0);
+        /* FALLTHRU */
+    case CC_OP_ADD_64:
+    case CC_OP_ADDU_64:
+    case CC_OP_ADDC_64:
+    case CC_OP_SUB_64:
+    case CC_OP_SUBU_64:
+    case CC_OP_SUBB_64:
+    case CC_OP_ADD_32:
+    case CC_OP_ADDU_32:
+    case CC_OP_ADDC_32:
+    case CC_OP_SUB_32:
+    case CC_OP_SUBU_32:
+    case CC_OP_SUBB_32:
+        local_cc_op = tcg_const_i32(s->cc_op);
+        break;
+    case CC_OP_CONST0:
+    case CC_OP_CONST1:
+    case CC_OP_CONST2:
+    case CC_OP_CONST3:
+    case CC_OP_STATIC:
+    case CC_OP_DYNAMIC:
+        break;
+    }
 
     switch (s->cc_op) {
     case CC_OP_CONST0:
@@ -726,8 +494,9 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_COMP_64:
     case CC_OP_NZ_F32:
     case CC_OP_NZ_F64:
+    case CC_OP_FLOGR:
         /* 1 argument */
-        gen_helper_calc_cc(cc_op, local_cc_op, dummy, cc_dst, dummy);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, dummy, cc_dst, dummy);
         break;
     case CC_OP_ICM:
     case CC_OP_LTGT_32:
@@ -736,4375 +505,4233 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_LTUGTU_64:
     case CC_OP_TM_32:
     case CC_OP_TM_64:
-    case CC_OP_LTGT_F32:
-    case CC_OP_LTGT_F64:
-    case CC_OP_SLAG:
+    case CC_OP_SLA_32:
+    case CC_OP_SLA_64:
+    case CC_OP_NZ_F128:
         /* 2 arguments */
-        gen_helper_calc_cc(cc_op, local_cc_op, cc_src, cc_dst, dummy);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, dummy);
         break;
     case CC_OP_ADD_64:
     case CC_OP_ADDU_64:
+    case CC_OP_ADDC_64:
     case CC_OP_SUB_64:
     case CC_OP_SUBU_64:
+    case CC_OP_SUBB_64:
     case CC_OP_ADD_32:
     case CC_OP_ADDU_32:
+    case CC_OP_ADDC_32:
     case CC_OP_SUB_32:
     case CC_OP_SUBU_32:
+    case CC_OP_SUBB_32:
         /* 3 arguments */
-        gen_helper_calc_cc(cc_op, local_cc_op, cc_src, cc_dst, cc_vr);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, cc_vr);
         break;
     case CC_OP_DYNAMIC:
         /* unknown operation - assume 3 arguments and cc_op in env */
-        gen_helper_calc_cc(cc_op, cc_op, cc_src, cc_dst, cc_vr);
+        gen_helper_calc_cc(cc_op, cpu_env, cc_op, cc_src, cc_dst, cc_vr);
         break;
     default:
         tcg_abort();
     }
 
-    tcg_temp_free_i32(local_cc_op);
+    if (!TCGV_IS_UNUSED_I32(local_cc_op)) {
+        tcg_temp_free_i32(local_cc_op);
+    }
+    if (!TCGV_IS_UNUSED_I64(dummy)) {
+        tcg_temp_free_i64(dummy);
+    }
 
     /* We now have cc in cc_op as constant */
     set_cc_static(s);
 }
 
-static inline void decode_rr(DisasContext *s, uint64_t insn, int *r1, int *r2)
+static int use_goto_tb(DisasContext *s, uint64_t dest)
 {
-    debug_insn(insn);
-
-    *r1 = (insn >> 4) & 0xf;
-    *r2 = insn & 0xf;
-}
-
-static inline TCGv_i64 decode_rx(DisasContext *s, uint64_t insn, int *r1,
-                                 int *x2, int *b2, int *d2)
-{
-    debug_insn(insn);
-
-    *r1 = (insn >> 20) & 0xf;
-    *x2 = (insn >> 16) & 0xf;
-    *b2 = (insn >> 12) & 0xf;
-    *d2 = insn & 0xfff;
-
-    return get_address(s, *x2, *b2, *d2);
-}
-
-static inline void decode_rs(DisasContext *s, uint64_t insn, int *r1, int *r3,
-                             int *b2, int *d2)
-{
-    debug_insn(insn);
-
-    *r1 = (insn >> 20) & 0xf;
-    /* aka m3 */
-    *r3 = (insn >> 16) & 0xf;
-    *b2 = (insn >> 12) & 0xf;
-    *d2 = insn & 0xfff;
-}
-
-static inline TCGv_i64 decode_si(DisasContext *s, uint64_t insn, int *i2,
-                                 int *b1, int *d1)
-{
-    debug_insn(insn);
-
-    *i2 = (insn >> 16) & 0xff;
-    *b1 = (insn >> 12) & 0xf;
-    *d1 = insn & 0xfff;
-
-    return get_address(s, 0, *b1, *d1);
-}
-
-static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong pc)
-{
-    TranslationBlock *tb;
-
-    gen_update_cc_op(s);
-
-    tb = s->tb;
     /* NOTE: we handle the case where the TB spans two pages here */
-    if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) ||
-        (pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))  {
-        /* jump to same page: we can use a direct jump */
-        tcg_gen_goto_tb(tb_num);
-        tcg_gen_movi_i64(psw_addr, pc);
-        tcg_gen_exit_tb((tcg_target_long)tb + tb_num);
-    } else {
-        /* jump to another page: currently not optimized */
-        tcg_gen_movi_i64(psw_addr, pc);
-        tcg_gen_exit_tb(0);
-    }
+    return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK)
+             || (dest & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))
+            && !s->singlestep_enabled
+            && !(s->tb->cflags & CF_LAST_IO));
 }
 
-static inline void account_noninline_branch(DisasContext *s, int cc_op)
+static void account_noninline_branch(DisasContext *s, int cc_op)
 {
 #ifdef DEBUG_INLINE_BRANCHES
     inline_branch_miss[cc_op]++;
 #endif
 }
 
-static inline void account_inline_branch(DisasContext *s)
+static void account_inline_branch(DisasContext *s, int cc_op)
 {
 #ifdef DEBUG_INLINE_BRANCHES
-    inline_branch_hit[s->cc_op]++;
+    inline_branch_hit[cc_op]++;
 #endif
 }
 
-static void gen_jcc(DisasContext *s, uint32_t mask, int skip)
+/* Table of mask values to comparison codes, given a comparison as input.
+   For such, CC=3 should not be possible.  */
+static const TCGCond ltgt_cond[16] = {
+    TCG_COND_NEVER,  TCG_COND_NEVER,     /*    |    |    | x */
+    TCG_COND_GT,     TCG_COND_GT,        /*    |    | GT | x */
+    TCG_COND_LT,     TCG_COND_LT,        /*    | LT |    | x */
+    TCG_COND_NE,     TCG_COND_NE,        /*    | LT | GT | x */
+    TCG_COND_EQ,     TCG_COND_EQ,        /* EQ |    |    | x */
+    TCG_COND_GE,     TCG_COND_GE,        /* EQ |    | GT | x */
+    TCG_COND_LE,     TCG_COND_LE,        /* EQ | LT |    | x */
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | LT | GT | x */
+};
+
+/* Table of mask values to comparison codes, given a logic op as input.
+   For such, only CC=0 and CC=1 should be possible.  */
+static const TCGCond nz_cond[16] = {
+    TCG_COND_NEVER, TCG_COND_NEVER,      /*    |    | x | x */
+    TCG_COND_NEVER, TCG_COND_NEVER,
+    TCG_COND_NE, TCG_COND_NE,            /*    | NE | x | x */
+    TCG_COND_NE, TCG_COND_NE,
+    TCG_COND_EQ, TCG_COND_EQ,            /* EQ |    | x | x */
+    TCG_COND_EQ, TCG_COND_EQ,
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | NE | x | x */
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,
+};
+
+/* Interpret MASK in terms of S->CC_OP, and fill in C with all the
+   details required to generate a TCG comparison.  */
+static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
 {
-    TCGv_i32 tmp, tmp2, r;
-    TCGv_i64 tmp64;
-    int old_cc_op;
+    TCGCond cond;
+    enum cc_op old_cc_op = s->cc_op;
+
+    if (mask == 15 || mask == 0) {
+        c->cond = (mask ? TCG_COND_ALWAYS : TCG_COND_NEVER);
+        c->u.s32.a = cc_op;
+        c->u.s32.b = cc_op;
+        c->g1 = c->g2 = true;
+        c->is_64 = false;
+        return;
+    }
 
-    switch (s->cc_op) {
+    /* Find the TCG condition for the mask + cc op.  */
+    switch (old_cc_op) {
     case CC_OP_LTGT0_32:
-        tmp = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_dst);
-        switch (mask) {
-        case 0x8 | 0x4: /* dst <= 0 */
-            tcg_gen_brcondi_i32(TCG_COND_GT, tmp, 0, skip);
-            break;
-        case 0x8 | 0x2: /* dst >= 0 */
-            tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, skip);
-            break;
-        case 0x8: /* dst == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
-            break;
-        case 0x7: /* dst != 0 */
-        case 0x6: /* dst != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
-            break;
-        case 0x4: /* dst < 0 */
-            tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, skip);
-            break;
-        case 0x2: /* dst > 0 */
-            tcg_gen_brcondi_i32(TCG_COND_LE, tmp, 0, skip);
-            break;
-        default:
-            tcg_temp_free_i32(tmp);
-            goto do_dynamic;
-        }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        break;
     case CC_OP_LTGT0_64:
-        switch (mask) {
-        case 0x8 | 0x4: /* dst <= 0 */
-            tcg_gen_brcondi_i64(TCG_COND_GT, cc_dst, 0, skip);
-            break;
-        case 0x8 | 0x2: /* dst >= 0 */
-            tcg_gen_brcondi_i64(TCG_COND_LT, cc_dst, 0, skip);
-            break;
-        case 0x8: /* dst == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
-            break;
-        case 0x7: /* dst != 0 */
-        case 0x6: /* dst != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
-            break;
-        case 0x4: /* dst < 0 */
-            tcg_gen_brcondi_i64(TCG_COND_GE, cc_dst, 0, skip);
-            break;
-        case 0x2: /* dst > 0 */
-            tcg_gen_brcondi_i64(TCG_COND_LE, cc_dst, 0, skip);
-            break;
-        default:
+    case CC_OP_LTGT_32:
+    case CC_OP_LTGT_64:
+        cond = ltgt_cond[mask];
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTGT_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
-        switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i32(TCG_COND_GT, tmp, tmp2, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i32(TCG_COND_LT, tmp, tmp2, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i32(TCG_COND_NE, tmp, tmp2, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i32(TCG_COND_EQ, tmp, tmp2, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i32(TCG_COND_GE, tmp, tmp2, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i32(TCG_COND_LE, tmp, tmp2, skip);
-            break;
-        default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
+
+    case CC_OP_LTUGTU_32:
+    case CC_OP_LTUGTU_64:
+        cond = tcg_unsigned_cond(ltgt_cond[mask]);
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTGT_64:
-        switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i64(TCG_COND_GT, cc_src, cc_dst, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i64(TCG_COND_LT, cc_src, cc_dst, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i64(TCG_COND_NE, cc_src, cc_dst, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i64(TCG_COND_EQ, cc_src, cc_dst, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i64(TCG_COND_GE, cc_src, cc_dst, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i64(TCG_COND_LE, cc_src, cc_dst, skip);
-            break;
-        default:
+
+    case CC_OP_NZ:
+        cond = nz_cond[mask];
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTUGTU_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
+
+    case CC_OP_TM_32:
+    case CC_OP_TM_64:
         switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i32(TCG_COND_GTU, tmp, tmp2, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i32(TCG_COND_LTU, tmp, tmp2, skip);
+        case 8:
+            cond = TCG_COND_EQ;
             break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i32(TCG_COND_NE, tmp, tmp2, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i32(TCG_COND_EQ, tmp, tmp2, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i32(TCG_COND_GEU, tmp, tmp2, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i32(TCG_COND_LEU, tmp, tmp2, skip);
+        case 4 | 2 | 1:
+            cond = TCG_COND_NE;
             break;
         default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
             goto do_dynamic;
         }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTUGTU_64:
+
+    case CC_OP_ICM:
         switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i64(TCG_COND_GTU, cc_src, cc_dst, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i64(TCG_COND_LTU, cc_src, cc_dst, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i64(TCG_COND_NE, cc_src, cc_dst, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i64(TCG_COND_EQ, cc_src, cc_dst, skip);
+        case 8:
+            cond = TCG_COND_EQ;
             break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i64(TCG_COND_GEU, cc_src, cc_dst, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i64(TCG_COND_LEU, cc_src, cc_dst, skip);
+        case 4 | 2 | 1:
+        case 4 | 2:
+            cond = TCG_COND_NE;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_NZ:
-        switch (mask) {
-        /* dst == 0 || dst != 0 */
-        case 0x8 | 0x4:
-        case 0x8 | 0x4 | 0x2:
-        case 0x8 | 0x4 | 0x2 | 0x1:
-        case 0x8 | 0x4 | 0x1:
-            break;
-        /* dst == 0 */
-        case 0x8:
-        case 0x8 | 0x2:
-        case 0x8 | 0x2 | 0x1:
-        case 0x8 | 0x1:
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
+
+    case CC_OP_FLOGR:
+        switch (mask & 0xa) {
+        case 8: /* src == 0 -> no one bit found */
+            cond = TCG_COND_EQ;
             break;
-        /* dst != 0 */
-        case 0x4:
-        case 0x4 | 0x2:
-        case 0x4 | 0x2 | 0x1:
-        case 0x4 | 0x1:
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
+        case 2: /* src != 0 -> one bit found */
+            cond = TCG_COND_NE;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_TM_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
 
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
-        tcg_gen_and_i32(tmp, tmp, tmp2);
+    case CC_OP_ADDU_32:
+    case CC_OP_ADDU_64:
         switch (mask) {
-        case 0x8: /* val & mask == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
+        case 8 | 2: /* vr == 0 */
+            cond = TCG_COND_EQ;
+            break;
+        case 4 | 1: /* vr != 0 */
+            cond = TCG_COND_NE;
             break;
-        case 0x4 | 0x2 | 0x1: /* val & mask != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
+        case 8 | 4: /* no carry -> vr >= src */
+            cond = TCG_COND_GEU;
+            break;
+        case 2 | 1: /* carry -> vr < src */
+            cond = TCG_COND_LTU;
             break;
         default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
             goto do_dynamic;
         }
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_TM_64:
-        tmp64 = tcg_temp_new_i64();
 
-        tcg_gen_and_i64(tmp64, cc_src, cc_dst);
-        switch (mask) {
-        case 0x8: /* val & mask == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, tmp64, 0, skip);
+    case CC_OP_SUBU_32:
+    case CC_OP_SUBU_64:
+        /* Note that CC=0 is impossible; treat it as dont-care.  */
+        switch (mask & 7) {
+        case 2: /* zero -> op1 == op2 */
+            cond = TCG_COND_EQ;
             break;
-        case 0x4 | 0x2 | 0x1: /* val & mask != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, tmp64, 0, skip);
+        case 4 | 1: /* !zero -> op1 != op2 */
+            cond = TCG_COND_NE;
             break;
-        default:
-            tcg_temp_free_i64(tmp64);
-            goto do_dynamic;
-        }
-        tcg_temp_free_i64(tmp64);
-        account_inline_branch(s);
-        break;
-    case CC_OP_ICM:
-        switch (mask) {
-        case 0x8: /* val == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
+        case 4: /* borrow (!carry) -> op1 < op2 */
+            cond = TCG_COND_LTU;
             break;
-        case 0x4 | 0x2 | 0x1: /* val != 0 */
-        case 0x4 | 0x2: /* val != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
+        case 2 | 1: /* !borrow (carry) -> op1 >= op2 */
+            cond = TCG_COND_GEU;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_STATIC:
-        old_cc_op = s->cc_op;
-        goto do_dynamic_nocccalc;
-    case CC_OP_DYNAMIC:
+
     default:
-do_dynamic:
-        old_cc_op = s->cc_op;
-        /* calculate cc value */
+    do_dynamic:
+        /* Calculate cc value.  */
         gen_op_calc_cc(s);
+        /* FALLTHRU */
 
-do_dynamic_nocccalc:
-        /* jump based on cc */
+    case CC_OP_STATIC:
+        /* Jump based on CC.  We'll load up the real cond below;
+           the assignment here merely avoids a compiler warning.  */
         account_noninline_branch(s, old_cc_op);
+        old_cc_op = CC_OP_STATIC;
+        cond = TCG_COND_NEVER;
+        break;
+    }
+
+    /* Load up the arguments of the comparison.  */
+    c->is_64 = true;
+    c->g1 = c->g2 = false;
+    switch (old_cc_op) {
+    case CC_OP_LTGT0_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_dst);
+        c->u.s32.b = tcg_const_i32(0);
+        break;
+    case CC_OP_LTGT_32:
+    case CC_OP_LTUGTU_32:
+    case CC_OP_SUBU_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_src);
+        c->u.s32.b = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.b, cc_dst);
+        break;
+
+    case CC_OP_LTGT0_64:
+    case CC_OP_NZ:
+    case CC_OP_FLOGR:
+        c->u.s64.a = cc_dst;
+        c->u.s64.b = tcg_const_i64(0);
+        c->g1 = true;
+        break;
+    case CC_OP_LTGT_64:
+    case CC_OP_LTUGTU_64:
+    case CC_OP_SUBU_64:
+        c->u.s64.a = cc_src;
+        c->u.s64.b = cc_dst;
+        c->g1 = c->g2 = true;
+        break;
+
+    case CC_OP_TM_32:
+    case CC_OP_TM_64:
+    case CC_OP_ICM:
+        c->u.s64.a = tcg_temp_new_i64();
+        c->u.s64.b = tcg_const_i64(0);
+        tcg_gen_and_i64(c->u.s64.a, cc_src, cc_dst);
+        break;
+
+    case CC_OP_ADDU_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        c->u.s32.b = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_vr);
+        if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
+            tcg_gen_movi_i32(c->u.s32.b, 0);
+        } else {
+            tcg_gen_trunc_i64_i32(c->u.s32.b, cc_src);
+        }
+        break;
 
+    case CC_OP_ADDU_64:
+        c->u.s64.a = cc_vr;
+        c->g1 = true;
+        if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
+            c->u.s64.b = tcg_const_i64(0);
+        } else {
+            c->u.s64.b = cc_src;
+            c->g2 = true;
+        }
+        break;
+
+    case CC_OP_STATIC:
+        c->is_64 = false;
+        c->u.s32.a = cc_op;
+        c->g1 = true;
         switch (mask) {
-        case 0x8 | 0x4 | 0x2 | 0x1:
-            /* always true */
-            break;
         case 0x8 | 0x4 | 0x2: /* cc != 3 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 3, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(3);
             break;
         case 0x8 | 0x4 | 0x1: /* cc != 2 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 2, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x8 | 0x2 | 0x1: /* cc != 1 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 1, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(1);
             break;
-        case 0x8 | 0x2: /* cc == 0 || cc == 2 */
-            tmp = tcg_temp_new_i32();
-            tcg_gen_andi_i32(tmp, cc_op, 1);
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
-            tcg_temp_free_i32(tmp);
+        case 0x8 | 0x2: /* cc == 0 || cc == 2 => (cc & 1) == 0 */
+            cond = TCG_COND_EQ;
+            c->g1 = false;
+            c->u.s32.a = tcg_temp_new_i32();
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_andi_i32(c->u.s32.a, cc_op, 1);
             break;
         case 0x8 | 0x4: /* cc < 2 */
-            tcg_gen_brcondi_i32(TCG_COND_GEU, cc_op, 2, skip);
+            cond = TCG_COND_LTU;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x8: /* cc == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 0, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(0);
             break;
         case 0x4 | 0x2 | 0x1: /* cc != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 0, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(0);
             break;
-        case 0x4 | 0x1: /* cc == 1 || cc == 3 */
-            tmp = tcg_temp_new_i32();
-            tcg_gen_andi_i32(tmp, cc_op, 1);
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
-            tcg_temp_free_i32(tmp);
+        case 0x4 | 0x1: /* cc == 1 || cc == 3 => (cc & 1) != 0 */
+            cond = TCG_COND_NE;
+            c->g1 = false;
+            c->u.s32.a = tcg_temp_new_i32();
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_andi_i32(c->u.s32.a, cc_op, 1);
             break;
         case 0x4: /* cc == 1 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 1, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(1);
             break;
         case 0x2 | 0x1: /* cc > 1 */
-            tcg_gen_brcondi_i32(TCG_COND_LEU, cc_op, 1, skip);
+            cond = TCG_COND_GTU;
+            c->u.s32.b = tcg_const_i32(1);
             break;
         case 0x2: /* cc == 2 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 2, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x1: /* cc == 3 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 3, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(3);
             break;
-        default: /* cc is masked by something else */
-            tmp = tcg_const_i32(3);
-            /* 3 - cc */
-            tcg_gen_sub_i32(tmp, tmp, cc_op);
-            tmp2 = tcg_const_i32(1);
-            /* 1 << (3 - cc) */
-            tcg_gen_shl_i32(tmp2, tmp2, tmp);
-            r = tcg_const_i32(mask);
-            /* mask & (1 << (3 - cc)) */
-            tcg_gen_and_i32(r, r, tmp2);
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
-
-            tcg_gen_brcondi_i32(TCG_COND_EQ, r, 0, skip);
-            tcg_temp_free_i32(r);
+        default:
+            /* CC is masked by something else: (8 >> cc) & mask.  */
+            cond = TCG_COND_NE;
+            c->g1 = false;
+            c->u.s32.a = tcg_const_i32(8);
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_shr_i32(c->u.s32.a, c->u.s32.a, cc_op);
+            tcg_gen_andi_i32(c->u.s32.a, c->u.s32.a, mask);
             break;
         }
         break;
+
+    default:
+        abort();
     }
+    c->cond = cond;
 }
 
-static void gen_bcr(DisasContext *s, uint32_t mask, TCGv_i64 target,
-                    uint64_t offset)
+static void free_compare(DisasCompare *c)
 {
-    int skip;
-
-    if (mask == 0xf) {
-        /* unconditional */
-        tcg_gen_mov_i64(psw_addr, target);
-        tcg_gen_exit_tb(0);
-    } else if (mask == 0) {
-        /* ignore cc and never match */
-        gen_goto_tb(s, 0, offset + 2);
-    } else {
-        TCGv_i64 new_addr = tcg_temp_local_new_i64();
-
-        tcg_gen_mov_i64(new_addr, target);
-        skip = gen_new_label();
-        gen_jcc(s, mask, skip);
-        tcg_gen_mov_i64(psw_addr, new_addr);
-        tcg_temp_free_i64(new_addr);
-        tcg_gen_exit_tb(0);
-        gen_set_label(skip);
-        tcg_temp_free_i64(new_addr);
-        gen_goto_tb(s, 1, offset + 2);
+    if (!c->g1) {
+        if (c->is_64) {
+            tcg_temp_free_i64(c->u.s64.a);
+        } else {
+            tcg_temp_free_i32(c->u.s32.a);
+        }
+    }
+    if (!c->g2) {
+        if (c->is_64) {
+            tcg_temp_free_i64(c->u.s64.b);
+        } else {
+            tcg_temp_free_i32(c->u.s32.b);
+        }
     }
 }
 
-static void gen_brc(uint32_t mask, DisasContext *s, int32_t offset)
+/* ====================================================================== */
+/* Define the insn format enumeration.  */
+#define F0(N)                         FMT_##N,
+#define F1(N, X1)                     F0(N)
+#define F2(N, X1, X2)                 F0(N)
+#define F3(N, X1, X2, X3)             F0(N)
+#define F4(N, X1, X2, X3, X4)         F0(N)
+#define F5(N, X1, X2, X3, X4, X5)     F0(N)
+
+typedef enum {
+#include "insn-format.def"
+} DisasFormat;
+
+#undef F0
+#undef F1
+#undef F2
+#undef F3
+#undef F4
+#undef F5
+
+/* Define a structure to hold the decoded fields.  We'll store each inside
+   an array indexed by an enum.  In order to conserve memory, we'll arrange
+   for fields that do not exist at the same time to overlap, thus the "C"
+   for compact.  For checking purposes there is an "O" for original index
+   as well that will be applied to availability bitmaps.  */
+
+enum DisasFieldIndexO {
+    FLD_O_r1,
+    FLD_O_r2,
+    FLD_O_r3,
+    FLD_O_m1,
+    FLD_O_m3,
+    FLD_O_m4,
+    FLD_O_b1,
+    FLD_O_b2,
+    FLD_O_b4,
+    FLD_O_d1,
+    FLD_O_d2,
+    FLD_O_d4,
+    FLD_O_x2,
+    FLD_O_l1,
+    FLD_O_l2,
+    FLD_O_i1,
+    FLD_O_i2,
+    FLD_O_i3,
+    FLD_O_i4,
+    FLD_O_i5
+};
+
+enum DisasFieldIndexC {
+    FLD_C_r1 = 0,
+    FLD_C_m1 = 0,
+    FLD_C_b1 = 0,
+    FLD_C_i1 = 0,
+
+    FLD_C_r2 = 1,
+    FLD_C_b2 = 1,
+    FLD_C_i2 = 1,
+
+    FLD_C_r3 = 2,
+    FLD_C_m3 = 2,
+    FLD_C_i3 = 2,
+
+    FLD_C_m4 = 3,
+    FLD_C_b4 = 3,
+    FLD_C_i4 = 3,
+    FLD_C_l1 = 3,
+
+    FLD_C_i5 = 4,
+    FLD_C_d1 = 4,
+
+    FLD_C_d2 = 5,
+
+    FLD_C_d4 = 6,
+    FLD_C_x2 = 6,
+    FLD_C_l2 = 6,
+
+    NUM_C_FIELD = 7
+};
+
+struct DisasFields {
+    unsigned op:8;
+    unsigned op2:8;
+    unsigned presentC:16;
+    unsigned int presentO;
+    int c[NUM_C_FIELD];
+};
+
+/* This is the way fields are to be accessed out of DisasFields.  */
+#define have_field(S, F)  have_field1((S), FLD_O_##F)
+#define get_field(S, F)   get_field1((S), FLD_O_##F, FLD_C_##F)
+
+static bool have_field1(const DisasFields *f, enum DisasFieldIndexO c)
+{
+    return (f->presentO >> c) & 1;
+}
+
+static int get_field1(const DisasFields *f, enum DisasFieldIndexO o,
+                      enum DisasFieldIndexC c)
+{
+    assert(have_field1(f, o));
+    return f->c[c];
+}
+
+/* Describe the layout of each field in each format.  */
+typedef struct DisasField {
+    unsigned int beg:8;
+    unsigned int size:8;
+    unsigned int type:2;
+    unsigned int indexC:6;
+    enum DisasFieldIndexO indexO:8;
+} DisasField;
+
+typedef struct DisasFormatInfo {
+    DisasField op[NUM_C_FIELD];
+} DisasFormatInfo;
+
+#define R(N, B)       {  B,  4, 0, FLD_C_r##N, FLD_O_r##N }
+#define M(N, B)       {  B,  4, 0, FLD_C_m##N, FLD_O_m##N }
+#define BD(N, BB, BD) { BB,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { BD, 12, 0, FLD_C_d##N, FLD_O_d##N }
+#define BXD(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
+                      { 20, 12, 0, FLD_C_d##N, FLD_O_d##N }
+#define BDL(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
+#define BXDL(N)       { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
+                      { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
+#define I(N, B, S)    {  B,  S, 1, FLD_C_i##N, FLD_O_i##N }
+#define L(N, B, S)    {  B,  S, 0, FLD_C_l##N, FLD_O_l##N }
+
+#define F0(N)                     { { } },
+#define F1(N, X1)                 { { X1 } },
+#define F2(N, X1, X2)             { { X1, X2 } },
+#define F3(N, X1, X2, X3)         { { X1, X2, X3 } },
+#define F4(N, X1, X2, X3, X4)     { { X1, X2, X3, X4 } },
+#define F5(N, X1, X2, X3, X4, X5) { { X1, X2, X3, X4, X5 } },
+
+static const DisasFormatInfo format_info[] = {
+#include "insn-format.def"
+};
+
+#undef F0
+#undef F1
+#undef F2
+#undef F3
+#undef F4
+#undef F5
+#undef R
+#undef M
+#undef BD
+#undef BXD
+#undef BDL
+#undef BXDL
+#undef I
+#undef L
+
+/* Generally, we'll extract operands into this structures, operate upon
+   them, and store them back.  See the "in1", "in2", "prep", "wout" sets
+   of routines below for more details.  */
+typedef struct {
+    bool g_out, g_out2, g_in1, g_in2;
+    TCGv_i64 out, out2, in1, in2;
+    TCGv_i64 addr1;
+} DisasOps;
+
+/* Instructions can place constraints on their operands, raising specification
+   exceptions if they are violated.  To make this easy to automate, each "in1",
+   "in2", "prep", "wout" helper will have a SPEC_<name> define that equals one
+   of the following, or 0.  To make this easy to document, we'll put the
+   SPEC_<name> defines next to <name>.  */
+
+#define SPEC_r1_even    1
+#define SPEC_r2_even    2
+#define SPEC_r3_even    4
+#define SPEC_r1_f128    8
+#define SPEC_r2_f128    16
+
+/* Return values from translate_one, indicating the state of the TB.  */
+typedef enum {
+    /* Continue the TB.  */
+    NO_EXIT,
+    /* We have emitted one or more goto_tb.  No fixup required.  */
+    EXIT_GOTO_TB,
+    /* We are not using a goto_tb (for whatever reason), but have updated
+       the PC (for whatever reason), so there's no need to do it again on
+       exiting the TB.  */
+    EXIT_PC_UPDATED,
+    /* We are exiting the TB, but have neither emitted a goto_tb, nor
+       updated the PC for the next instruction to be executed.  */
+    EXIT_PC_STALE,
+    /* We are ending the TB with a noreturn function call, e.g. longjmp.
+       No following code will be executed.  */
+    EXIT_NORETURN,
+} ExitStatus;
+
+typedef enum DisasFacility {
+    FAC_Z,                  /* zarch (default) */
+    FAC_CASS,               /* compare and swap and store */
+    FAC_CASS2,              /* compare and swap and store 2*/
+    FAC_DFP,                /* decimal floating point */
+    FAC_DFPR,               /* decimal floating point rounding */
+    FAC_DO,                 /* distinct operands */
+    FAC_EE,                 /* execute extensions */
+    FAC_EI,                 /* extended immediate */
+    FAC_FPE,                /* floating point extension */
+    FAC_FPSSH,              /* floating point support sign handling */
+    FAC_FPRGR,              /* FPR-GR transfer */
+    FAC_GIE,                /* general instructions extension */
+    FAC_HFP_MA,             /* HFP multiply-and-add/subtract */
+    FAC_HW,                 /* high-word */
+    FAC_IEEEE_SIM,          /* IEEE exception sumilation */
+    FAC_LOC,                /* load/store on condition */
+    FAC_LD,                 /* long displacement */
+    FAC_PC,                 /* population count */
+    FAC_SCF,                /* store clock fast */
+    FAC_SFLE,               /* store facility list extended */
+} DisasFacility;
+
+struct DisasInsn {
+    unsigned opc:16;
+    DisasFormat fmt:8;
+    DisasFacility fac:8;
+    unsigned spec:8;
+
+    const char *name;
+
+    void (*help_in1)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_in2)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_prep)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_wout)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_cout)(DisasContext *, DisasOps *);
+    ExitStatus (*help_op)(DisasContext *, DisasOps *);
+
+    uint64_t data;
+};
+
+/* ====================================================================== */
+/* Miscelaneous helpers, used by several operations.  */
+
+static void help_l2_shift(DisasContext *s, DisasFields *f,
+                          DisasOps *o, int mask)
 {
-    int skip;
+    int b2 = get_field(f, b2);
+    int d2 = get_field(f, d2);
 
-    if (mask == 0xf) {
-        /* unconditional */
-        gen_goto_tb(s, 0, s->pc + offset);
-    } else if (mask == 0) {
-        /* ignore cc and never match */
-        gen_goto_tb(s, 0, s->pc + 4);
+    if (b2 == 0) {
+        o->in2 = tcg_const_i64(d2 & mask);
     } else {
-        skip = gen_new_label();
-        gen_jcc(s, mask, skip);
-        gen_goto_tb(s, 0, s->pc + offset);
-        gen_set_label(skip);
-        gen_goto_tb(s, 1, s->pc + 4);
+        o->in2 = get_address(s, 0, b2, d2);
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
     }
-    s->is_jmp = DISAS_TB_JUMP;
 }
 
-static void gen_op_mvc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
+static ExitStatus help_goto_direct(DisasContext *s, uint64_t dest)
 {
-    TCGv_i64 tmp, tmp2;
-    int i;
-    int l_memset = gen_new_label();
-    int l_out = gen_new_label();
-    TCGv_i64 dest = tcg_temp_local_new_i64();
-    TCGv_i64 src = tcg_temp_local_new_i64();
-    TCGv_i32 vl;
+    if (dest == s->next_pc) {
+        return NO_EXIT;
+    }
+    if (use_goto_tb(s, dest)) {
+        update_cc_op(s);
+        tcg_gen_goto_tb(0);
+        tcg_gen_movi_i64(psw_addr, dest);
+        tcg_gen_exit_tb((tcg_target_long)s->tb);
+        return EXIT_GOTO_TB;
+    } else {
+        tcg_gen_movi_i64(psw_addr, dest);
+        return EXIT_PC_UPDATED;
+    }
+}
 
-    /* Find out if we should use the inline version of mvc */
-    switch (l) {
-    case 0:
-    case 1:
-    case 2:
-    case 3:
-    case 4:
-    case 5:
-    case 6:
-    case 7:
-    case 11:
-    case 15:
-        /* use inline */
-        break;
-    default:
-        /* Fall back to helper */
-        vl = tcg_const_i32(l);
-        potential_page_fault(s);
-        gen_helper_mvc(vl, s1, s2);
-        tcg_temp_free_i32(vl);
-        return;
+static ExitStatus help_branch(DisasContext *s, DisasCompare *c,
+                              bool is_imm, int imm, TCGv_i64 cdest)
+{
+    ExitStatus ret;
+    uint64_t dest = s->pc + 2 * imm;
+    int lab;
+
+    /* Take care of the special cases first.  */
+    if (c->cond == TCG_COND_NEVER) {
+        ret = NO_EXIT;
+        goto egress;
+    }
+    if (is_imm) {
+        if (dest == s->next_pc) {
+            /* Branch to next.  */
+            ret = NO_EXIT;
+            goto egress;
+        }
+        if (c->cond == TCG_COND_ALWAYS) {
+            ret = help_goto_direct(s, dest);
+            goto egress;
+        }
+    } else {
+        if (TCGV_IS_UNUSED_I64(cdest)) {
+            /* E.g. bcr %r0 -> no branch.  */
+            ret = NO_EXIT;
+            goto egress;
+        }
+        if (c->cond == TCG_COND_ALWAYS) {
+            tcg_gen_mov_i64(psw_addr, cdest);
+            ret = EXIT_PC_UPDATED;
+            goto egress;
+        }
     }
 
-    tcg_gen_mov_i64(dest, s1);
-    tcg_gen_mov_i64(src, s2);
+    if (use_goto_tb(s, s->next_pc)) {
+        if (is_imm && use_goto_tb(s, dest)) {
+            /* Both exits can use goto_tb.  */
+            update_cc_op(s);
 
-    if (!(s->tb->flags & FLAG_MASK_64)) {
-        /* XXX what if we overflow while moving? */
-        tcg_gen_andi_i64(dest, dest, 0x7fffffffUL);
-        tcg_gen_andi_i64(src, src, 0x7fffffffUL);
+            lab = gen_new_label();
+            if (c->is_64) {
+                tcg_gen_brcond_i64(c->cond, c->u.s64.a, c->u.s64.b, lab);
+            } else {
+                tcg_gen_brcond_i32(c->cond, c->u.s32.a, c->u.s32.b, lab);
+            }
+
+            /* Branch not taken.  */
+            tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(psw_addr, s->next_pc);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 0);
+
+            /* Branch taken.  */
+            gen_set_label(lab);
+            tcg_gen_goto_tb(1);
+            tcg_gen_movi_i64(psw_addr, dest);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 1);
+
+            ret = EXIT_GOTO_TB;
+        } else {
+            /* Fallthru can use goto_tb, but taken branch cannot.  */
+            /* Store taken branch destination before the brcond.  This
+               avoids having to allocate a new local temp to hold it.
+               We'll overwrite this in the not taken case anyway.  */
+            if (!is_imm) {
+                tcg_gen_mov_i64(psw_addr, cdest);
+            }
+
+            lab = gen_new_label();
+            if (c->is_64) {
+                tcg_gen_brcond_i64(c->cond, c->u.s64.a, c->u.s64.b, lab);
+            } else {
+                tcg_gen_brcond_i32(c->cond, c->u.s32.a, c->u.s32.b, lab);
+            }
+
+            /* Branch not taken.  */
+            update_cc_op(s);
+            tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(psw_addr, s->next_pc);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 0);
+
+            gen_set_label(lab);
+            if (is_imm) {
+                tcg_gen_movi_i64(psw_addr, dest);
+            }
+            ret = EXIT_PC_UPDATED;
+        }
+    } else {
+        /* Fallthru cannot use goto_tb.  This by itself is vanishingly rare.
+           Most commonly we're single-stepping or some other condition that
+           disables all use of goto_tb.  Just update the PC and exit.  */
+
+        TCGv_i64 next = tcg_const_i64(s->next_pc);
+        if (is_imm) {
+            cdest = tcg_const_i64(dest);
+        }
+
+        if (c->is_64) {
+            tcg_gen_movcond_i64(c->cond, psw_addr, c->u.s64.a, c->u.s64.b,
+                                cdest, next);
+        } else {
+            TCGv_i32 t0 = tcg_temp_new_i32();
+            TCGv_i64 t1 = tcg_temp_new_i64();
+            TCGv_i64 z = tcg_const_i64(0);
+            tcg_gen_setcond_i32(c->cond, t0, c->u.s32.a, c->u.s32.b);
+            tcg_gen_extu_i32_i64(t1, t0);
+            tcg_temp_free_i32(t0);
+            tcg_gen_movcond_i64(TCG_COND_NE, psw_addr, t1, z, cdest, next);
+            tcg_temp_free_i64(t1);
+            tcg_temp_free_i64(z);
+        }
+
+        if (is_imm) {
+            tcg_temp_free_i64(cdest);
+        }
+        tcg_temp_free_i64(next);
+
+        ret = EXIT_PC_UPDATED;
     }
 
-    tmp = tcg_temp_new_i64();
-    tcg_gen_addi_i64(tmp, src, 1);
-    tcg_gen_brcond_i64(TCG_COND_EQ, dest, tmp, l_memset);
-    tcg_temp_free_i64(tmp);
+ egress:
+    free_compare(c);
+    return ret;
+}
 
-    switch (l) {
-    case 0:
-        tmp = tcg_temp_new_i64();
+/* ====================================================================== */
+/* The operations.  These perform the bulk of the work for any insn,
+   usually after the operands have been loaded and output initialized.  */
 
-        tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
+static ExitStatus op_abs(DisasContext *s, DisasOps *o)
+{
+    gen_helper_abs_i64(o->out, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 1:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_absf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffull);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld16u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st16(tmp, dest, get_mem_index(s));
+static ExitStatus op_absf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 3:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_absf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in1, 0x7fffffffffffffffull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld32u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st32(tmp, dest, get_mem_index(s));
+static ExitStatus op_add(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_add_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 4:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
-
-        tcg_gen_qemu_ld32u(tmp, src, get_mem_index(s));
-        tcg_gen_addi_i64(src, src, 4);
-        tcg_gen_qemu_ld8u(tmp2, src, get_mem_index(s));
-        tcg_gen_qemu_st32(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 4);
-        tcg_gen_qemu_st8(tmp2, dest, get_mem_index(s));
-
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 7:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_addc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare cmp;
+    TCGv_i64 carry;
+
+    tcg_gen_add_i64(o->out, o->in1, o->in2);
+
+    /* The carry flag is the msb of CC, therefore the branch mask that would
+       create that comparison is 3.  Feeding the generated comparison to
+       setcond produces the carry flag that we desire.  */
+    disas_jcc(s, &cmp, 3);
+    carry = tcg_temp_new_i64();
+    if (cmp.is_64) {
+        tcg_gen_setcond_i64(cmp.cond, carry, cmp.u.s64.a, cmp.u.s64.b);
+    } else {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cmp.cond, t, cmp.u.s32.a, cmp.u.s32.b);
+        tcg_gen_extu_i32_i64(carry, t);
+        tcg_temp_free_i32(t);
+    }
+    free_compare(&cmp);
 
-        tcg_gen_qemu_ld64(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
+    tcg_gen_add_i64(o->out, o->out, carry);
+    tcg_temp_free_i64(carry);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        /* The inline version can become too big for too uneven numbers, only
-           use it on known good lengths */
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_const_i64(8);
-        for (i = 0; (i + 7) <= l; i += 8) {
-            tcg_gen_qemu_ld64(tmp, src, get_mem_index(s));
-            tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
-
-            tcg_gen_add_i64(src, src, tmp2);
-            tcg_gen_add_i64(dest, dest, tmp2);
-        }
+static ExitStatus op_aeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_aeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp2);
-        tmp2 = tcg_const_i64(1);
+static ExitStatus op_adb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_adb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        for (; i <= l; i++) {
-            tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-            tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
+static ExitStatus op_axb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_axb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
 
-            tcg_gen_add_i64(src, src, tmp2);
-            tcg_gen_add_i64(dest, dest, tmp2);
-        }
+static ExitStatus op_and(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_and_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
+static ExitStatus op_andi(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_ori_i64(o->in2, o->in2, ~mask);
+    tcg_gen_and_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_bas(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_movi_i64(o->out, pc_to_link_info(s, s->next_pc));
+    if (!TCGV_IS_UNUSED_I64(o->in2)) {
+        tcg_gen_mov_i64(psw_addr, o->in2);
+        return EXIT_PC_UPDATED;
+    } else {
+        return NO_EXIT;
     }
+}
+
+static ExitStatus op_basi(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_movi_i64(o->out, pc_to_link_info(s, s->next_pc));
+    return help_goto_direct(s, s->pc + 2 * get_field(s->fields, i2));
+}
+
+static ExitStatus op_bc(DisasContext *s, DisasOps *o)
+{
+    int m1 = get_field(s->fields, m1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+
+    disas_jcc(s, &c, m1);
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
+
+static ExitStatus op_bct32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+    TCGv_i64 t;
+
+    c.cond = TCG_COND_NE;
+    c.is_64 = false;
+    c.g1 = false;
+    c.g2 = false;
+
+    t = tcg_temp_new_i64();
+    tcg_gen_subi_i64(t, regs[r1], 1);
+    store_reg32_i64(r1, t);
+    c.u.s32.a = tcg_temp_new_i32();
+    c.u.s32.b = tcg_const_i32(0);
+    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_temp_free_i64(t);
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
+
+static ExitStatus op_bct64(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
 
-    tcg_gen_br(l_out);
+    c.cond = TCG_COND_NE;
+    c.is_64 = true;
+    c.g1 = true;
+    c.g2 = false;
 
-    gen_set_label(l_memset);
-    /* memset case (dest == (src + 1)) */
+    tcg_gen_subi_i64(regs[r1], regs[r1], 1);
+    c.u.s64.a = regs[r1];
+    c.u.s64.b = tcg_const_i64(0);
 
-    tmp = tcg_temp_new_i64();
-    tmp2 = tcg_temp_new_i64();
-    /* fill tmp with the byte */
-    tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-    tcg_gen_shli_i64(tmp2, tmp, 8);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_gen_shli_i64(tmp2, tmp, 16);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_gen_shli_i64(tmp2, tmp, 32);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_temp_free_i64(tmp2);
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
 
-    tmp2 = tcg_const_i64(8);
+static ExitStatus op_bx32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+    TCGv_i64 t;
+
+    c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
+    c.is_64 = false;
+    c.g1 = false;
+    c.g2 = false;
+
+    t = tcg_temp_new_i64();
+    tcg_gen_add_i64(t, regs[r1], regs[r3]);
+    c.u.s32.a = tcg_temp_new_i32();
+    c.u.s32.b = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_trunc_i64_i32(c.u.s32.b, regs[r3 | 1]);
+    store_reg32_i64(r1, t);
+    tcg_temp_free_i64(t);
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
 
-    for (i = 0; (i + 7) <= l; i += 8) {
-        tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 8);
+static ExitStatus op_bx64(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+
+    c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
+    c.is_64 = true;
+
+    if (r1 == (r3 | 1)) {
+        c.u.s64.b = load_reg(r3 | 1);
+        c.g2 = false;
+    } else {
+        c.u.s64.b = regs[r3 | 1];
+        c.g2 = true;
     }
 
-    tcg_temp_free_i64(tmp2);
-    tmp2 = tcg_const_i64(1);
+    tcg_gen_add_i64(regs[r1], regs[r1], regs[r3]);
+    c.u.s64.a = regs[r1];
+    c.g1 = true;
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
 
-    for (; i <= l; i++) {
-        tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 1);
+static ExitStatus op_cj(DisasContext *s, DisasOps *o)
+{
+    int imm, m3 = get_field(s->fields, m3);
+    bool is_imm;
+    DisasCompare c;
+
+    c.cond = ltgt_cond[m3];
+    if (s->insn->data) {
+        c.cond = tcg_unsigned_cond(c.cond);
     }
+    c.is_64 = c.g1 = c.g2 = true;
+    c.u.s64.a = o->in1;
+    c.u.s64.b = o->in2;
 
-    tcg_temp_free_i64(tmp2);
-    tcg_temp_free_i64(tmp);
+    is_imm = have_field(s->fields, i4);
+    if (is_imm) {
+        imm = get_field(s->fields, i4);
+    } else {
+        imm = 0;
+        o->out = get_address(s, 0, get_field(s->fields, b4),
+                             get_field(s->fields, d4));
+    }
 
-    gen_set_label(l_out);
+    return help_branch(s, &c, is_imm, imm, o->out);
+}
 
-    tcg_temp_free(dest);
-    tcg_temp_free(src);
+static ExitStatus op_ceb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ceb(cc_op, cpu_env, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
 }
 
-static void gen_op_clc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
+static ExitStatus op_cdb(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i64 tmp2;
-    TCGv_i32 vl;
+    gen_helper_cdb(cc_op, cpu_env, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-    /* check for simple 32bit or 64bit match */
-    switch (l) {
-    case 0:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_cxb(cc_op, cpu_env, o->out, o->out2, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld8u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld8u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cfeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 1:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cfdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld16u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld16u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cfxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 3:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cgeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld32u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld32u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cgdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 7:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cgxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld64(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld64(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_clfeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    }
+static ExitStatus op_clfdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clfxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cegb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cegb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cdgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cxgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cxgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_celgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_celgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdlgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cdlgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cxlgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cxlgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cksm(DisasContext *s, DisasOps *o)
+{
+    int r2 = get_field(s->fields, r2);
+    TCGv_i64 len = tcg_temp_new_i64();
 
     potential_page_fault(s);
-    vl = tcg_const_i32(l);
-    gen_helper_clc(cc_op, vl, s1, s2);
-    tcg_temp_free_i32(vl);
+    gen_helper_cksm(len, cpu_env, o->in1, o->in2, regs[r2 + 1]);
     set_cc_static(s);
+    return_low128(o->out);
+
+    tcg_gen_add_i64(regs[r2], regs[r2], len);
+    tcg_gen_sub_i64(regs[r2 + 1], regs[r2 + 1], len);
+    tcg_temp_free_i64(len);
+
+    return NO_EXIT;
 }
 
-static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
+static ExitStatus op_clc(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 addr, tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    int l = get_field(s->fields, l1);
+    TCGv_i32 vl;
 
-    LOG_DISAS("disas_e3: op 0x%x r1 %d x2 %d b2 %d d2 %d\n",
-              op, r1, x2, b2, d2);
-    addr = get_address(s, x2, b2, d2);
-    switch (op) {
-    case 0x2: /* LTG R1,D2(X2,B2) [RXY] */
-    case 0x4: /* lg r1,d2(x2,b2) */
-        tcg_gen_qemu_ld64(regs[r1], addr, get_mem_index(s));
-        if (op == 0x2) {
-            set_cc_s64(s, regs[r1]);
-        }
-        break;
-    case 0x12: /* LT R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_reg32(r1, tmp32_1);
-        set_cc_s32(s, tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xc: /* MSG      R1,D2(X2,B2)     [RXY] */
-    case 0x1c: /* MSGF     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0xc) {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        }
-        tcg_gen_mul_i64(regs[r1], regs[r1], tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xd: /* DSG      R1,D2(X2,B2)     [RXY] */
-    case 0x1d: /* DSGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x1d) {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1 + 1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_div_i64(tmp3, tmp4, tmp2);
-        store_reg(r1 + 1, tmp3);
-        tcg_gen_rem_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x8: /* AG      R1,D2(X2,B2)     [RXY] */
-    case 0xa: /* ALG      R1,D2(X2,B2)     [RXY] */
-    case 0x18: /* AGF       R1,D2(X2,B2)     [RXY] */
-    case 0x1a: /* ALGF      R1,D2(X2,B2)     [RXY] */
-        if (op == 0x1a) {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        } else if (op == 0x18) {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x8:
-        case 0x18:
-            set_cc_add64(s, tmp4, tmp2, tmp3);
-            break;
-        case 0xa:
-        case 0x1a:
-            set_cc_addu64(s, tmp4, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x9: /* SG      R1,D2(X2,B2)     [RXY] */
-    case 0xb: /* SLG      R1,D2(X2,B2)     [RXY] */
-    case 0x19: /* SGF      R1,D2(X2,B2)     [RXY] */
-    case 0x1b: /* SLGF     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x19) {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else if (op == 0x1b) {
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_sub_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x9:
-        case 0x19:
-            set_cc_sub64(s, tmp4, tmp2, tmp3);
-            break;
-        case 0xb:
-        case 0x1b:
-            set_cc_subu64(s, tmp4, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0xf: /* LRVG     R1,D2(X2,B2)     [RXE] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        tcg_gen_bswap64_i64(tmp2, tmp2);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x14: /* LGF      R1,D2(X2,B2)     [RXY] */
-    case 0x16: /* LLGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        if (op == 0x14) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
-        }
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x15: /* LGH     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, addr, get_mem_index(s));
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x17: /* LLGT      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_andi_i64(tmp2, tmp2, 0x7fffffffULL);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x1e: /* LRV R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x1f: /* LRVH R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld16u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_bswap16_i32(tmp32_1, tmp32_1);
-        store_reg16(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x20: /* CG      R1,D2(X2,B2)     [RXY] */
-    case 0x21: /* CLG      R1,D2(X2,B2) */
-    case 0x30: /* CGF       R1,D2(X2,B2)     [RXY] */
-    case 0x31: /* CLGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        switch (op) {
-        case 0x20:
-        case 0x21:
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-            break;
-        case 0x30:
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-            break;
-        case 0x31:
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-            break;
-        default:
-            tcg_abort();
-        }
-        switch (op) {
-        case 0x20:
-        case 0x30:
-            cmp_s64(s, regs[r1], tmp2);
-            break;
-        case 0x21:
-        case 0x31:
-            cmp_u64(s, regs[r1], tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x24: /* stg r1, d2(x2,b2) */
-        tcg_gen_qemu_st64(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x3e: /* STRV R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_gen_qemu_st32(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x50: /* STY  R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_gen_qemu_st32(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x57: /* XY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_xor_i32(tmp32_2, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_2);
-        set_cc_nz_u32(s, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x58: /* LY R1,D2(X2,B2) [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp3, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5a: /* AY R1,D2(X2,B2) [RXY] */
-    case 0x5b: /* SY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_temp_free_i64(tmp2);
-        switch (op) {
-        case 0x5a:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        case 0x5b:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        switch (op) {
-        case 0x5a:
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5b:
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x71: /* LAY R1,D2(X2,B2) [RXY] */
-        store_reg(r1, addr);
-        break;
-    case 0x72: /* STCY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st8(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x73: /* ICY R1,D2(X2,B2) [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp3, addr, get_mem_index(s));
-        store_reg8(r1, tmp3);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x76: /* LB R1,D2(X2,B2) [RXY] */
-    case 0x77: /* LGB R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8s(tmp2, addr, get_mem_index(s));
-        switch (op) {
-        case 0x76:
-            tcg_gen_ext8s_i64(tmp2, tmp2);
-            store_reg32_i64(r1, tmp2);
-            break;
-        case 0x77:
-            tcg_gen_ext8s_i64(tmp2, tmp2);
-            store_reg(r1, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x78: /* LHY R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x80: /* NG      R1,D2(X2,B2)     [RXY] */
-    case 0x81: /* OG      R1,D2(X2,B2)     [RXY] */
-    case 0x82: /* XG      R1,D2(X2,B2)     [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp3, addr, get_mem_index(s));
-        switch (op) {
-        case 0x80:
-            tcg_gen_and_i64(regs[r1], regs[r1], tmp3);
-            break;
-        case 0x81:
-            tcg_gen_or_i64(regs[r1], regs[r1], tmp3);
-            break;
-        case 0x82:
-            tcg_gen_xor_i64(regs[r1], regs[r1], tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        set_cc_nz_u64(s, regs[r1]);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x86: /* MLG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_mlg(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x87: /* DLG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_dlg(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x88: /* ALCG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        tcg_gen_extu_i32_i64(tmp3, cc_op);
-        tcg_gen_shri_i64(tmp3, tmp3, 1);
-        tcg_gen_andi_i64(tmp3, tmp3, 1);
-        tcg_gen_add_i64(tmp3, tmp2, tmp3);
-        tcg_gen_add_i64(tmp3, regs[r1], tmp3);
-        store_reg(r1, tmp3);
-        set_cc_addu64(s, regs[r1], tmp2, tmp3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x89: /* SLBG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cc_op, tmp32_1, regs[r1], tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x90: /* LLGC      R1,D2(X2,B2)     [RXY] */
-        tcg_gen_qemu_ld8u(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x91: /* LLGH      R1,D2(X2,B2)     [RXY] */
-        tcg_gen_qemu_ld16u(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x94: /* LLC     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x95: /* LLH     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16u(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x96: /* ML      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    switch (l + 1) {
+    case 1:
+        tcg_gen_qemu_ld8u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld8u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x97: /* DL     R1,D2(X2,B2)     [RXY] */
-        /* reg(r1) = reg(r1, r1+1) % ld32(addr) */
-        /* reg(r1+1) = reg(r1, r1+1) / ld32(addr) */
-        tmp = load_reg(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp3);
-
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        tcg_gen_div_i64(tmp, tmp, tmp2);
-        store_reg32_i64((r1 + 1) & 15, tmp);
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    case 2:
+        tcg_gen_qemu_ld16u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld16u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x98: /* ALC     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_addc_u32(tmp32_3, cc_op, tmp32_1, tmp32_2);
-        set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+    case 4:
+        tcg_gen_qemu_ld32u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld32u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x99: /* SLB     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cc_op, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    case 8:
+        tcg_gen_qemu_ld64(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld64(cc_dst, o->in2, get_mem_index(s));
         break;
     default:
-        LOG_DISAS("illegal e3 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
-        break;
+        potential_page_fault(s);
+        vl = tcg_const_i32(l);
+        gen_helper_clc(cc_op, cpu_env, vl, o->addr1, o->in2);
+        tcg_temp_free_i32(vl);
+        set_cc_static(s);
+        return NO_EXIT;
+    }
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, cc_src, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clcle(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_clcle(cc_op, cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clm(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t1, o->in1);
+    potential_page_fault(s);
+    gen_helper_clm(cc_op, cpu_env, t1, m3, o->in2);
+    set_cc_static(s);
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clst(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_clst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cps(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t = tcg_temp_new_i64();
+    tcg_gen_andi_i64(t, o->in1, 0x8000000000000000ull);
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
+    tcg_gen_or_i64(o->out, o->out, t);
+    tcg_temp_free_i64(t);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cs(DisasContext *s, DisasOps *o)
+{
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    int is_64 = s->insn->data;
+    TCGv_i64 addr, mem, cc, z;
+
+    /* Note that in1 = R3 (new value) and
+       in2 = (zero-extended) R1 (expected value).  */
+
+    /* Load the memory into the (temporary) output.  While the PoO only talks
+       about moving the memory to R1 on inequality, if we include equality it
+       means that R1 is equal to the memory in all conditions.  */
+    addr = get_address(s, 0, b2, d2);
+    if (is_64) {
+        tcg_gen_qemu_ld64(o->out, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_ld32u(o->out, addr, get_mem_index(s));
+    }
+
+    /* Are the memory and expected values (un)equal?  Note that this setcond
+       produces the output CC value, thus the NE sense of the test.  */
+    cc = tcg_temp_new_i64();
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in2, o->out);
+
+    /* If the memory and expected values are equal (CC==0), copy R3 to MEM.
+       Recall that we are allowed to unconditionally issue the store (and
+       thus any possible write trap), so (re-)store the original contents
+       of MEM in case of inequality.  */
+    z = tcg_const_i64(0);
+    mem = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, mem, cc, z, o->in1, o->out);
+    if (is_64) {
+        tcg_gen_qemu_st64(mem, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(mem, addr, get_mem_index(s));
     }
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(mem);
     tcg_temp_free_i64(addr);
+
+    /* Store CC back to cc_op.  Wait until after the store so that any
+       exception gets the old cc_op value.  */
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(cc);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
+{
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    TCGv_i64 addrh, addrl, memh, meml, outh, outl, cc, z;
+
+    /* Note that R1:R1+1 = expected value and R3:R3+1 = new value.  */
+
+    addrh = get_address(s, 0, b2, d2);
+    addrl = get_address(s, 0, b2, d2 + 8);
+    outh = tcg_temp_new_i64();
+    outl = tcg_temp_new_i64();
+
+    tcg_gen_qemu_ld64(outh, addrh, get_mem_index(s));
+    tcg_gen_qemu_ld64(outl, addrl, get_mem_index(s));
+
+    /* Fold the double-word compare with arithmetic.  */
+    cc = tcg_temp_new_i64();
+    z = tcg_temp_new_i64();
+    tcg_gen_xor_i64(cc, outh, regs[r1]);
+    tcg_gen_xor_i64(z, outl, regs[r1 + 1]);
+    tcg_gen_or_i64(cc, cc, z);
+    tcg_gen_movi_i64(z, 0);
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, cc, z);
+
+    memh = tcg_temp_new_i64();
+    meml = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, memh, cc, z, regs[r3], outh);
+    tcg_gen_movcond_i64(TCG_COND_EQ, meml, cc, z, regs[r3 + 1], outl);
+    tcg_temp_free_i64(z);
+
+    tcg_gen_qemu_st64(memh, addrh, get_mem_index(s));
+    tcg_gen_qemu_st64(meml, addrl, get_mem_index(s));
+    tcg_temp_free_i64(memh);
+    tcg_temp_free_i64(meml);
+    tcg_temp_free_i64(addrh);
+    tcg_temp_free_i64(addrl);
+
+    /* Save back state now that we've passed all exceptions.  */
+    tcg_gen_mov_i64(regs[r1], outh);
+    tcg_gen_mov_i64(regs[r1 + 1], outl);
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(outh);
+    tcg_temp_free_i64(outl);
+    tcg_temp_free_i64(cc);
+    set_cc_static(s);
+    return NO_EXIT;
 }
 
 #ifndef CONFIG_USER_ONLY
-static void disas_e5(DisasContext* s, uint64_t insn)
+static ExitStatus op_csp(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    int op = (insn >> 32) & 0xff;
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    check_privileged(s);
+    gen_helper_csp(cc_op, cpu_env, r1, o->in2);
+    tcg_temp_free_i32(r1);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+#endif
 
-    tmp = get_address(s, 0, (insn >> 28) & 0xf, (insn >> 16) & 0xfff);
-    tmp2 = get_address(s, 0, (insn >> 12) & 0xf, insn & 0xfff);
+static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i32 t2 = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t2, o->in1);
+    gen_helper_cvd(t1, t2);
+    tcg_temp_free_i32(t2);
+    tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s));
+    tcg_temp_free_i64(t1);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_e5: insn %" PRIx64 "\n", insn);
-    switch (op) {
-    case 0x01: /* TPROT    D1(B1),D2(B2)  [SSE] */
-        /* Test Protection */
-        potential_page_fault(s);
-        gen_helper_tprot(cc_op, tmp, tmp2);
-        set_cc_static(s);
-        break;
-    default:
-        LOG_DISAS("illegal e5 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
-        break;
+static ExitStatus op_ct(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int lab = gen_new_label();
+    TCGv_i32 t;
+    TCGCond c;
+
+    c = tcg_invert_cond(ltgt_cond[m3]);
+    if (s->insn->data) {
+        c = tcg_unsigned_cond(c);
     }
+    tcg_gen_brcond_i64(c, o->in1, o->in2, lab);
 
-    tcg_temp_free_i64(tmp);
-    tcg_temp_free_i64(tmp2);
+    /* Set DXC to 0xff.  */
+    t = tcg_temp_new_i32();
+    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_gen_ori_i32(t, t, 0xff00);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_temp_free_i32(t);
+
+    /* Trap.  */
+    gen_program_exception(s, PGM_DATA);
+
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_diag(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 tmp;
+
+    check_privileged(s);
+    potential_page_fault(s);
+
+    /* We pretend the format is RX_a so that D2 is the field we want.  */
+    tmp = tcg_const_i32(get_field(s->fields, d2) & 0xfff);
+    gen_helper_diag(regs[2], cpu_env, tmp, regs[2], regs[1]);
+    tcg_temp_free_i32(tmp);
+    return NO_EXIT;
 }
 #endif
 
-static void disas_eb(DisasContext *s, int op, int r1, int r3, int b2, int d2)
+static ExitStatus op_divs32(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2;
-    int i, stm_len;
-    int ilc = 3;
+    gen_helper_divs32(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_eb: op 0x%x r1 %d r3 %d b2 %d d2 0x%x\n",
-              op, r1, r3, b2, d2);
-    switch (op) {
-    case 0xc: /* SRLG     R1,R3,D2(B2)     [RSY] */
-    case 0xd: /* SLLG     R1,R3,D2(B2)     [RSY] */
-    case 0xa: /* SRAG     R1,R3,D2(B2)     [RSY] */
-    case 0xb: /* SLAG     R1,R3,D2(B2)     [RSY] */
-    case 0x1c: /* RLLG     R1,R3,D2(B2)     [RSY] */
-        if (b2) {
-            tmp = get_address(s, 0, b2, d2);
-            tcg_gen_andi_i64(tmp, tmp, 0x3f);
-        } else {
-            tmp = tcg_const_i64(d2 & 0x3f);
-        }
-        switch (op) {
-        case 0xc:
-            tcg_gen_shr_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xd:
-            tcg_gen_shl_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xa:
-            tcg_gen_sar_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xb:
-            tmp2 = tcg_temp_new_i64();
-            tmp3 = tcg_temp_new_i64();
-            gen_op_update2_cc_i64(s, CC_OP_SLAG, regs[r3], tmp);
-            tcg_gen_shl_i64(tmp2, regs[r3], tmp);
-            /* override sign bit with source sign */
-            tcg_gen_andi_i64(tmp2, tmp2, ~0x8000000000000000ULL);
-            tcg_gen_andi_i64(tmp3, regs[r3], 0x8000000000000000ULL);
-            tcg_gen_or_i64(regs[r1], tmp2, tmp3);
-            tcg_temp_free_i64(tmp2);
-            tcg_temp_free_i64(tmp3);
-            break;
-        case 0x1c:
-            tcg_gen_rotl_i64(regs[r1], regs[r3], tmp);
-            break;
-        default:
-            tcg_abort();
-            break;
-        }
-        if (op == 0xa) {
-            set_cc_s64(s, regs[r1]);
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1d: /* RLL    R1,R3,D2(B2)        [RSY] */
-        if (b2) {
-            tmp = get_address(s, 0, b2, d2);
-            tcg_gen_andi_i64(tmp, tmp, 0x3f);
-        } else {
-            tmp = tcg_const_i64(d2 & 0x3f);
-        }
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r3);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        switch (op) {
-        case 0x1d:
-            tcg_gen_rotl_i32(tmp32_1, tmp32_2, tmp32_1);
-            break;
-        default:
-            tcg_abort();
-            break;
-        }
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+static ExitStatus op_divu32(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divu32(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_divs64(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divs64(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_divu64(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divu64(o->out2, cpu_env, o->out, o->out2, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_deb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_deb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ddb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ddb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_dxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_dxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ear(DisasContext *s, DisasOps *o)
+{
+    int r2 = get_field(s->fields, r2);
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, aregs[r2]));
+    return NO_EXIT;
+}
+
+static ExitStatus op_efpc(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, fpc));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ex(DisasContext *s, DisasOps *o)
+{
+    /* ??? Perhaps a better way to implement EXECUTE is to set a bit in
+       tb->flags, (ab)use the tb->cs_base field as the address of
+       the template in memory, and grab 8 bits of tb->flags/cflags for
+       the contents of the register.  We would then recognize all this
+       in gen_intermediate_code_internal, generating code for exactly
+       one instruction.  This new TB then gets executed normally.
+
+       On the other hand, this seems to be mostly used for modifying
+       MVC inside of memcpy, which needs a helper call anyway.  So
+       perhaps this doesn't bear thinking about any further.  */
+
+    TCGv_i64 tmp;
+
+    update_psw_addr(s);
+    update_cc_op(s);
+
+    tmp = tcg_const_i64(s->next_pc);
+    gen_helper_ex(cc_op, cpu_env, cc_op, o->in1, o->in2, tmp);
+    tcg_temp_free_i64(tmp);
+
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_flogr(DisasContext *s, DisasOps *o)
+{
+    /* We'll use the original input for cc computation, since we get to
+       compare that against 0, which ought to be better than comparing
+       the real output against 64.  It also lets cc_dst be a convenient
+       temporary during our computation.  */
+    gen_op_update1_cc_i64(s, CC_OP_FLOGR, o->in2);
+
+    /* R1 = IN ? CLZ(IN) : 64.  */
+    gen_helper_clz(o->out, o->in2);
+
+    /* R1+1 = IN & ~(found bit).  Note that we may attempt to shift this
+       value by 64, which is undefined.  But since the shift is 64 iff the
+       input is zero, we still get the correct result after and'ing.  */
+    tcg_gen_movi_i64(o->out2, 0x8000000000000000ull);
+    tcg_gen_shr_i64(o->out2, o->out2, o->out);
+    tcg_gen_andc_i64(o->out2, cc_dst, o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_icm(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int pos, len, base = s->insn->data;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    uint64_t ccm;
+
+    switch (m3) {
+    case 0xf:
+        /* Effectively a 32-bit load.  */
+        tcg_gen_qemu_ld32u(tmp, o->in2, get_mem_index(s));
+        len = 32;
+        goto one_insert;
+
+    case 0xc:
+    case 0x6:
+    case 0x3:
+        /* Effectively a 16-bit load.  */
+        tcg_gen_qemu_ld16u(tmp, o->in2, get_mem_index(s));
+        len = 16;
+        goto one_insert;
+
+    case 0x8:
+    case 0x4:
+    case 0x2:
+    case 0x1:
+        /* Effectively an 8-bit load.  */
+        tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s));
+        len = 8;
+        goto one_insert;
+
+    one_insert:
+        pos = base + ctz32(m3) * 8;
+        tcg_gen_deposit_i64(o->out, o->out, tmp, pos, len);
+        ccm = ((1ull << len) - 1) << pos;
         break;
-    case 0x4:  /* LMG      R1,R3,D2(B2)     [RSE] */
-    case 0x24: /* STMG     R1,R3,D2(B2)     [RSE] */
-        stm_len = 8;
-        goto do_mh;
-    case 0x26: /* STMH     R1,R3,D2(B2)     [RSE] */
-    case 0x96: /* LMH      R1,R3,D2(B2)     [RSE] */
-        stm_len = 4;
-do_mh:
-        /* Apparently, unrolling lmg/stmg of any size gains performance -
-           even for very long ones... */
-        tmp = get_address(s, 0, b2, d2);
-        tmp3 = tcg_const_i64(stm_len);
-        tmp4 = tcg_const_i64(op == 0x26 ? 32 : 4);
-        for (i = r1;; i = (i + 1) % 16) {
-            switch (op) {
-            case 0x4:
-                tcg_gen_qemu_ld64(regs[i], tmp, get_mem_index(s));
-                break;
-            case 0x96:
-                tmp2 = tcg_temp_new_i64();
-#if HOST_LONG_BITS == 32
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_trunc_i64_i32(TCGV_HIGH(regs[i]), tmp2);
-#else
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_shl_i64(tmp2, tmp2, tmp4);
-                tcg_gen_ext32u_i64(regs[i], regs[i]);
-                tcg_gen_or_i64(regs[i], regs[i], tmp2);
-#endif
-                tcg_temp_free_i64(tmp2);
-                break;
-            case 0x24:
-                tcg_gen_qemu_st64(regs[i], tmp, get_mem_index(s));
-                break;
-            case 0x26:
-                tmp2 = tcg_temp_new_i64();
-                tcg_gen_shr_i64(tmp2, regs[i], tmp4);
-                tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-                tcg_temp_free_i64(tmp2);
-                break;
-            default:
-                tcg_abort();
-            }
-            if (i == r3) {
-                break;
+
+    default:
+        /* This is going to be a sequence of loads and inserts.  */
+        pos = base + 32 - 8;
+        ccm = 0;
+        while (m3) {
+            if (m3 & 0x8) {
+                tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s));
+                tcg_gen_addi_i64(o->in2, o->in2, 1);
+                tcg_gen_deposit_i64(o->out, o->out, tmp, pos, 8);
+                ccm |= 0xff << pos;
             }
-            tcg_gen_add_i64(tmp, tmp, tmp3);
+            m3 = (m3 << 1) & 0xf;
+            pos -= 8;
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x2c: /* STCMH R1,M3,D2(B2) [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stcmh(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
         break;
+    }
+
+    tcg_gen_movi_i64(tmp, ccm);
+    gen_op_update2_cc_i64(s, CC_OP_ICM, tmp, o->out);
+    tcg_temp_free_i64(tmp);
+    return NO_EXIT;
+}
+
+static ExitStatus op_insi(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    tcg_gen_deposit_i64(o->out, o->in1, o->in2, shift, size);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ipm(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1;
+
+    gen_op_calc_cc(s);
+    tcg_gen_andi_i64(o->out, o->out, ~0xff000000ull);
+
+    t1 = tcg_temp_new_i64();
+    tcg_gen_shli_i64(t1, psw_mask, 20);
+    tcg_gen_shri_i64(t1, t1, 36);
+    tcg_gen_or_i64(o->out, o->out, t1);
+
+    tcg_gen_extu_i32_i64(t1, cc_op);
+    tcg_gen_shli_i64(t1, t1, 28);
+    tcg_gen_or_i64(o->out, o->out, t1);
+    tcg_temp_free_i64(t1);
+    return NO_EXIT;
+}
+
 #ifndef CONFIG_USER_ONLY
-    case 0x2f: /* LCTLG     R1,R3,D2(B2)     [RSE] */
-        /* Load Control */
-        check_privileged(s, ilc);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lctlg(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x25: /* STCTG     R1,R3,D2(B2)     [RSE] */
-        /* Store Control */
-        check_privileged(s, ilc);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stctg(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
+static ExitStatus op_ipte(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_ipte(cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_iske(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_iske(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
 #endif
-    case 0x30: /* CSG     R1,R3,D2(B2)     [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX rewrite in tcg */
-        gen_helper_csg(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x3e: /* CDSG R1,R3,D2(B2) [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX rewrite in tcg */
-        gen_helper_cdsg(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x51: /* TMY D1(B1),I2 [SIY] */
-        tmp = get_address(s, 0, b2, d2); /* SIY -> this is the destination */
-        tmp2 = tcg_const_i64((r1 << 4) | r3);
-        tcg_gen_qemu_ld8u(tmp, tmp, get_mem_index(s));
-        /* yes, this is a 32 bit operation with 64 bit tcg registers, because
-           that incurs less conversions */
-        cmp_64(s, tmp, tmp2, CC_OP_TM_32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x52: /* MVIY D1(B1),I2 [SIY] */
-        tmp = get_address(s, 0, b2, d2); /* SIY -> this is the destination */
-        tmp2 = tcg_const_i64((r1 << 4) | r3);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x55: /* CLIY D1(B1),I2 [SIY] */
-        tmp3 = get_address(s, 0, b2, d2); /* SIY -> this is the 1st operand */
-        tmp = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld8u(tmp, tmp3, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        cmp_u32c(s, tmp32_1, (r1 << 4) | r3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x80: /* ICMH      R1,M3,D2(B2)     [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX split CC calculation out */
-        gen_helper_icmh(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    default:
-        LOG_DISAS("illegal eb operation 0x%x\n", op);
-        gen_illegal_opcode(s, ilc);
-        break;
-    }
+
+static ExitStatus op_ldeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ldeb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
 }
 
-static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
-                     int r1b)
+static ExitStatus op_ledb(DisasContext *s, DisasOps *o)
 {
-    TCGv_i32 tmp_r1, tmp32;
-    TCGv_i64 addr, tmp;
-    addr = get_address(s, x2, b2, d2);
-    tmp_r1 = tcg_const_i32(r1);
-    switch (op) {
-    case 0x4: /* LDEB R1,D2(X2,B2) [RXE] */
-        potential_page_fault(s);
-        gen_helper_ldeb(tmp_r1, addr);
-        break;
-    case 0x5: /* LXDB R1,D2(X2,B2) [RXE] */
-        potential_page_fault(s);
-        gen_helper_lxdb(tmp_r1, addr);
-        break;
-    case 0x9: /* CEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = load_freg32(r1);
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        set_cc_cmp_f32_i64(s, tmp32, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xa: /* AEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_aeb(tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-
-        tmp32 = load_freg32(r1);
-        set_cc_nz_f32(s, tmp32);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xb: /* SEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_seb(tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-
-        tmp32 = load_freg32(r1);
-        set_cc_nz_f32(s, tmp32);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xd: /* DEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_deb(tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0x10: /* TCEB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tceb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x11: /* TCDB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tcdb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x12: /* TCXB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tcxb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x17: /* MEEB   R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_meeb(tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0x19: /* CDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_cdb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1a: /* ADB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_adb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1b: /* SDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_sdb(cc_op, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1c: /* MDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_mdb(tmp_r1, addr);
-        break;
-    case 0x1d: /* DDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_ddb(tmp_r1, addr);
-        break;
-    case 0x1e: /* MADB  R1,R3,D2(X2,B2) [RXF] */
-        /* for RXF insns, r1 is R3 and r1b is R1 */
-        tmp32 = tcg_const_i32(r1b);
-        potential_page_fault(s);
-        gen_helper_madb(tmp32, addr, tmp_r1);
-        tcg_temp_free_i32(tmp32);
-        break;
-    default:
-        LOG_DISAS("illegal ed operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
-        return;
+    gen_helper_ledb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ldxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ldxb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lexb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lexb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lxdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lxdb(o->out, cpu_env, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lxeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lxeb(o->out, cpu_env, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_llgt(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffff);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld8s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld8s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld8u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld8u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld16s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld16s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld16u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld16u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld32s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld32s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld32u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld32u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld64(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_loc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare c;
+
+    disas_jcc(s, &c, get_field(s->fields, m3));
+
+    if (c.is_64) {
+        tcg_gen_movcond_i64(c.cond, o->out, c.u.s64.a, c.u.s64.b,
+                            o->in2, o->in1);
+        free_compare(&c);
+    } else {
+        TCGv_i32 t32 = tcg_temp_new_i32();
+        TCGv_i64 t, z;
+
+        tcg_gen_setcond_i32(c.cond, t32, c.u.s32.a, c.u.s32.b);
+        free_compare(&c);
+
+        t = tcg_temp_new_i64();
+        tcg_gen_extu_i32_i64(t, t32);
+        tcg_temp_free_i32(t32);
+
+        z = tcg_const_i64(0);
+        tcg_gen_movcond_i64(TCG_COND_NE, o->out, t, z, o->in2, o->in1);
+        tcg_temp_free_i64(t);
+        tcg_temp_free_i64(z);
     }
-    tcg_temp_free_i32(tmp_r1);
-    tcg_temp_free_i64(addr);
+
+    return NO_EXIT;
 }
 
-static void disas_a5(DisasContext *s, int op, int r1, int i2)
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_lctl(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    TCGv_i32 tmp32;
-    LOG_DISAS("disas_a5: op 0x%x r1 %d i2 0x%x\n", op, r1, i2);
-    switch (op) {
-    case 0x0: /* IIHH     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 48, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1: /* IIHL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 32, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x2: /* IILH     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 16, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x3: /* IILL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 0, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* NIHH     R1,I2     [RI] */
-    case 0x8: /* OIHH     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x4:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 48)
-                               | 0x0000ffffffffffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0x8:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 48);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp2, tmp, 48);
-        tcg_gen_trunc_i64_i32(tmp32, tmp2);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x5: /* NIHL     R1,I2     [RI] */
-    case 0x9: /* OIHL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x5:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 32)
-                               | 0xffff0000ffffffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0x9:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 32);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp2, tmp, 32);
-        tcg_gen_trunc_i64_i32(tmp32, tmp2);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x6: /* NILH     R1,I2     [RI] */
-    case 0xa: /* OILH     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x6:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 16)
-                               | 0xffffffff0000ffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0xa:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 16);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lctl(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lctlg(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lctlg(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+static ExitStatus op_lra(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lra(o->out, cpu_env, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lpsw(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1, t2;
+
+    check_privileged(s);
+
+    t1 = tcg_temp_new_i64();
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 4);
+    tcg_gen_qemu_ld32u(t2, o->in2, get_mem_index(s));
+    /* Convert the 32-bit PSW_MASK into the 64-bit PSW_MASK.  */
+    tcg_gen_shli_i64(t1, t1, 32);
+    gen_helper_load_psw(cpu_env, t1, t2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    return EXIT_NORETURN;
+}
+
+static ExitStatus op_lpswe(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1, t2;
+
+    check_privileged(s);
+
+    t1 = tcg_temp_new_i64();
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 8);
+    tcg_gen_qemu_ld64(t2, o->in2, get_mem_index(s));
+    gen_helper_load_psw(cpu_env, t1, t2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    return EXIT_NORETURN;
+}
+#endif
+
+static ExitStatus op_lam(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_lam(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lm32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+
+    while (1) {
+        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
+        store_reg32_i64(r1, t);
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp, tmp, 16);
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x7: /* NILL     R1,I2     [RI] */
-    case 0xb: /* OILL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x7:
-            tmp2 = tcg_const_i64(i2 | 0xffffffffffff0000ULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0xb:
-            tmp2 = tcg_const_i64(i2);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
+    }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lmh(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+
+    while (1) {
+        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
+        store_reg32h_i64(r1, t);
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);        /* signedness should not matter here */
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xc: /* LLIHH     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 48 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xd: /* LLIHL     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 32 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* LLILH     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 16 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xf: /* LLILL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        LOG_DISAS("illegal a5 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
-        return;
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    return NO_EXIT;
 }
 
-static void disas_a7(DisasContext *s, int op, int r1, int i2)
+static ExitStatus op_lm64(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    int l1;
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t8 = tcg_const_i64(8);
 
-    LOG_DISAS("disas_a7: op 0x%x r1 %d i2 0x%x\n", op, r1, i2);
-    switch (op) {
-    case 0x0: /* TMLH or TMH     R1,I2     [RI] */
-    case 0x1: /* TMLL or TML     R1,I2     [RI] */
-    case 0x2: /* TMHH     R1,I2     [RI] */
-    case 0x3: /* TMHL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64((uint16_t)i2);
-        switch (op) {
-        case 0x0:
-            tcg_gen_shri_i64(tmp, tmp, 16);
-            break;
-        case 0x1:
-            break;
-        case 0x2:
-            tcg_gen_shri_i64(tmp, tmp, 48);
-            break;
-        case 0x3:
-            tcg_gen_shri_i64(tmp, tmp, 32);
+    while (1) {
+        tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s));
+        if (r1 == r3) {
             break;
         }
-        tcg_gen_andi_i64(tmp, tmp, 0xffff);
-        cmp_64(s, tmp, tmp2, CC_OP_TM_64);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4: /* brc m1, i2 */
-        gen_brc(r1, s, i2 * 2LL);
-        return;
-    case 0x5: /* BRAS     R1,I2     [RI] */
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 4));
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        gen_goto_tb(s, 0, s->pc + i2 * 2LL);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x6: /* BRCT     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp32_1, 0, l1);
-        gen_goto_tb(s, 0, s->pc + (i2 * 2LL));
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 4);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x7: /* BRCTG     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tcg_gen_subi_i64(tmp, tmp, 1);
-        store_reg(r1, tmp);
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i64(TCG_COND_EQ, tmp, 0, l1);
-        gen_goto_tb(s, 0, s->pc + (i2 * 2LL));
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 4);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x8: /* lhi r1, i2 */
-        tmp32_1 = tcg_const_i32(i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x9: /* lghi r1, i2 */
-        tmp = tcg_const_i64(i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa: /* AHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_const_i32(i2);
+        tcg_gen_add_i64(o->in2, o->in2, t8);
+        r1 = (r1 + 1) & 15;
+    }
 
-        if (i2 < 0) {
-            tcg_gen_subi_i32(tmp32_2, tmp32_1, -i2);
-        } else {
-            tcg_gen_add_i32(tmp32_2, tmp32_1, tmp32_3);
-        }
+    tcg_temp_free_i64(t8);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mov2(DisasContext *s, DisasOps *o)
+{
+    o->out = o->in2;
+    o->g_out = o->g_in2;
+    TCGV_UNUSED_I64(o->in2);
+    o->g_in2 = false;
+    return NO_EXIT;
+}
+
+static ExitStatus op_movx(DisasContext *s, DisasOps *o)
+{
+    o->out = o->in1;
+    o->out2 = o->in2;
+    o->g_out = o->g_in1;
+    o->g_out2 = o->g_in2;
+    TCGV_UNUSED_I64(o->in1);
+    TCGV_UNUSED_I64(o->in2);
+    o->g_in1 = o->g_in2 = false;
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_mvc(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+    potential_page_fault(s);
+    gen_helper_mvcl(cc_op, cpu_env, r1, r2);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcle(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_mvcle(cc_op, cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_mvcp(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, l1);
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_mvcp(cc_op, cpu_env, regs[r1], o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcs(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, l1);
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_mvcs(cc_op, cpu_env, regs[r1], o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_mvpg(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_mvpg(cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvst(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_mvst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mul(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_mul_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mul128(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mul128(o->out, cpu_env, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_meeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_meeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mdeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mdeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mdb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mxdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mxdb(o->out, cpu_env, o->out, o->out2, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_maeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 r3 = load_freg32_i64(get_field(s->fields, r3));
+    gen_helper_maeb(o->out, cpu_env, o->in1, o->in2, r3);
+    tcg_temp_free_i64(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_madb(DisasContext *s, DisasOps *o)
+{
+    int r3 = get_field(s->fields, r3);
+    gen_helper_madb(o->out, cpu_env, o->in1, o->in2, fregs[r3]);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mseb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 r3 = load_freg32_i64(get_field(s->fields, r3));
+    gen_helper_mseb(o->out, cpu_env, o->in1, o->in2, r3);
+    tcg_temp_free_i64(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_msdb(DisasContext *s, DisasOps *o)
+{
+    int r3 = get_field(s->fields, r3);
+    gen_helper_msdb(o->out, cpu_env, o->in1, o->in2, fregs[r3]);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabs(DisasContext *s, DisasOps *o)
+{
+    gen_helper_nabs_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in2, 0x80000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in2, 0x8000000000000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in1, 0x8000000000000000ull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_nc(cc_op, cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_neg(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_neg_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in2, 0x80000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in2, 0x8000000000000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in1, 0x8000000000000000ull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_oc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_oc(cc_op, cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_or(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_or_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ori(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_or_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_popcnt(DisasContext *s, DisasOps *o)
+{
+    gen_helper_popcnt(o->out, o->in2);
+    return NO_EXIT;
+}
 
-        store_reg32(r1, tmp32_2);
-        set_cc_add32(s, tmp32_1, tmp32_3, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_ptlb(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_ptlb(cpu_env);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_risbg(DisasContext *s, DisasOps *o)
+{
+    int i3 = get_field(s->fields, i3);
+    int i4 = get_field(s->fields, i4);
+    int i5 = get_field(s->fields, i5);
+    int do_zero = i4 & 0x80;
+    uint64_t mask, imask, pmask;
+    int pos, len, rot;
+
+    /* Adjust the arguments for the specific insn.  */
+    switch (s->fields->op2) {
+    case 0x55: /* risbg */
+        i3 &= 63;
+        i4 &= 63;
+        pmask = ~0;
+        break;
+    case 0x5d: /* risbhg */
+        i3 &= 31;
+        i4 &= 31;
+        pmask = 0xffffffff00000000ull;
+        break;
+    case 0x51: /* risblg */
+        i3 &= 31;
+        i4 &= 31;
+        pmask = 0x00000000ffffffffull;
         break;
-    case 0xb: /* aghi r1, i2 */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64(i2);
+    default:
+        abort();
+    }
 
-        if (i2 < 0) {
-            tcg_gen_subi_i64(regs[r1], tmp, -i2);
+    /* MASK is the set of bits to be inserted from R2.
+       Take care for I3/I4 wraparound.  */
+    mask = pmask >> i3;
+    if (i3 <= i4) {
+        mask ^= pmask >> i4 >> 1;
+    } else {
+        mask |= ~(pmask >> i4 >> 1);
+    }
+    mask &= pmask;
+
+    /* IMASK is the set of bits to be kept from R1.  In the case of the high/low
+       insns, we need to keep the other half of the register.  */
+    imask = ~mask | ~pmask;
+    if (do_zero) {
+        if (s->fields->op2 == 0x55) {
+            imask = 0;
         } else {
-            tcg_gen_add_i64(regs[r1], tmp, tmp2);
+            imask = ~pmask;
         }
-        set_cc_add64(s, tmp, tmp2, regs[r1]);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xc: /* MHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_muli_i32(tmp32_1, tmp32_1, i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xd: /* MGHI     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tcg_gen_muli_i64(tmp, tmp, i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
+    }
+
+    /* In some cases we can implement this with deposit, which can be more
+       efficient on some hosts.  */
+    if (~mask == imask && i3 <= i4) {
+        if (s->fields->op2 == 0x5d) {
+            i3 += 32, i4 += 32;
+        }
+        /* Note that we rotate the bits to be inserted to the lsb, not to
+           the position as described in the PoO.  */
+        len = i4 - i3 + 1;
+        pos = 63 - i4;
+        rot = (i5 - pos) & 63;
+    } else {
+        pos = len = -1;
+        rot = i5 & 63;
+    }
+
+    /* Rotate the input as necessary.  */
+    tcg_gen_rotli_i64(o->in2, o->in2, rot);
+
+    /* Insert the selected bits into the output.  */
+    if (pos >= 0) {
+        tcg_gen_deposit_i64(o->out, o->out, o->in2, pos, len);
+    } else if (imask == 0) {
+        tcg_gen_andi_i64(o->out, o->in2, mask);
+    } else {
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_andi_i64(o->out, o->out, imask);
+        tcg_gen_or_i64(o->out, o->out, o->in2);
+    }
+    return NO_EXIT;
+}
+
+static ExitStatus op_rosbg(DisasContext *s, DisasOps *o)
+{
+    int i3 = get_field(s->fields, i3);
+    int i4 = get_field(s->fields, i4);
+    int i5 = get_field(s->fields, i5);
+    uint64_t mask;
+
+    /* If this is a test-only form, arrange to discard the result.  */
+    if (i3 & 0x80) {
+        o->out = tcg_temp_new_i64();
+        o->g_out = false;
+    }
+
+    i3 &= 63;
+    i4 &= 63;
+    i5 &= 63;
+
+    /* MASK is the set of bits to be operated on from R2.
+       Take care for I3/I4 wraparound.  */
+    mask = ~0ull >> i3;
+    if (i3 <= i4) {
+        mask ^= ~0ull >> i4 >> 1;
+    } else {
+        mask |= ~(~0ull >> i4 >> 1);
+    }
+
+    /* Rotate the input as necessary.  */
+    tcg_gen_rotli_i64(o->in2, o->in2, i5);
+
+    /* Operate.  */
+    switch (s->fields->op2) {
+    case 0x55: /* AND */
+        tcg_gen_ori_i64(o->in2, o->in2, ~mask);
+        tcg_gen_and_i64(o->out, o->out, o->in2);
         break;
-    case 0xe: /* CHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        cmp_s32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
+    case 0x56: /* OR */
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_or_i64(o->out, o->out, o->in2);
         break;
-    case 0xf: /* CGHI     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        cmp_s64c(s, tmp, i2);
-        tcg_temp_free_i64(tmp);
+    case 0x57: /* XOR */
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_xor_i64(o->out, o->out, o->in2);
         break;
     default:
-        LOG_DISAS("illegal a7 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
-        return;
+        abort();
     }
+
+    /* Set the CC.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rev16(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_bswap16_i64(o->out, o->in2);
+    return NO_EXIT;
 }
 
-static void disas_b2(DisasContext *s, int op, uint32_t insn)
+static ExitStatus op_rev32(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    int r1, r2;
-    int ilc = 2;
+    tcg_gen_bswap32_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rev64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_bswap64_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rll32(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    TCGv_i32 t2 = tcg_temp_new_i32();
+    TCGv_i32 to = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t1, o->in1);
+    tcg_gen_trunc_i64_i32(t2, o->in2);
+    tcg_gen_rotl_i32(to, t1, t2);
+    tcg_gen_extu_i32_i64(o->out, to);
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(t2);
+    tcg_temp_free_i32(to);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rll64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_rotl_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
 #ifndef CONFIG_USER_ONLY
-    int r3, d2, b2;
+static ExitStatus op_rrbe(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_rrbe(cc_op, cpu_env, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sacf(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sacf(cpu_env, o->in2);
+    /* Addressing mode has changed, so end the block.  */
+    return EXIT_PC_STALE;
+}
 #endif
 
-    r1 = (insn >> 4) & 0xf;
-    r2 = insn & 0xf;
+static ExitStatus op_sar(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    tcg_gen_st32_i64(o->in2, cpu_env, offsetof(CPUS390XState, aregs[r1]));
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_b2: op 0x%x r1 %d r2 %d\n", op, r1, r2);
+static ExitStatus op_seb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_seb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x22: /* IPM    R1               [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_ipm(cc_op, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x41: /* CKSM    R1,R2     [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_cksm(tmp32_1, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        gen_op_movi_cc(s, 0);
-        break;
-    case 0x4e: /* SAR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, aregs[r1]));
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x4f: /* EAR     R1,R2     [RRE] */
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, aregs[r2]));
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x52: /* MSR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tcg_gen_mul_i32(tmp32_1, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x54: /* MVPG     R1,R2     [RRE] */
-        tmp = load_reg(0);
-        tmp2 = load_reg(r1);
-        tmp3 = load_reg(r2);
-        potential_page_fault(s);
-        gen_helper_mvpg(tmp, tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        /* XXX check CCO bit and set CC accordingly */
-        gen_op_movi_cc(s, 0);
-        break;
-    case 0x55: /* MVST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_mvst(tmp32_1, tmp32_2, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        gen_op_movi_cc(s, 1);
-        break;
-    case 0x5d: /* CLST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_clst(cc_op, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x5e: /* SRST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_srst(cc_op, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
+static ExitStatus op_sdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sdb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqeb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqdb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqxb(o->out, cpu_env, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
 
 #ifndef CONFIG_USER_ONLY
-    case 0x02: /* STIDP     D2(B2)     [S] */
-        /* Store CPU ID */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stidp(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x04: /* SCK       D2(B2)     [S] */
-        /* Set Clock */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sck(cc_op, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x05: /* STCK     D2(B2)     [S] */
-        /* Store Clock */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stck(cc_op, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x06: /* SCKC     D2(B2)     [S] */
-        /* Set Clock Comparator */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sckc(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x07: /* STCKC    D2(B2)     [S] */
-        /* Store Clock Comparator */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stckc(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x08: /* SPT      D2(B2)     [S] */
-        /* Set CPU Timer */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_spt(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x09: /* STPT     D2(B2)     [S] */
-        /* Store CPU Timer */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stpt(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x0a: /* SPKA     D2(B2)     [S] */
-        /* Set PSW Key from Address */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_andi_i64(tmp2, psw_mask, ~PSW_MASK_KEY);
-        tcg_gen_shli_i64(tmp, tmp, PSW_SHIFT_KEY - 4);
-        tcg_gen_or_i64(psw_mask, tmp2, tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x0d: /* PTLB                [S] */
-        /* Purge TLB */
-        check_privileged(s, ilc);
-        gen_helper_ptlb();
-        break;
-    case 0x10: /* SPX      D2(B2)     [S] */
-        /* Set Prefix Register */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_spx(tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x11: /* STPX     D2(B2)     [S] */
-        /* Store Prefix */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_ld_i64(tmp2, cpu_env, offsetof(CPUS390XState, psa));
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x12: /* STAP     D2(B2)     [S] */
-        /* Store CPU Address */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, cpu_num));
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x21: /* IPTE     R1,R2      [RRE] */
-        /* Invalidate PTE */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        gen_helper_ipte(tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x29: /* ISKE     R1,R2      [RRE] */
-        /* Insert Storage Key Extended */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp = load_reg(r2);
-        tmp2 = tcg_temp_new_i64();
-        gen_helper_iske(tmp2, tmp);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x2a: /* RRBE     R1,R2      [RRE] */
-        /* Set Storage Key Extended */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        gen_helper_rrbe(cc_op, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x2b: /* SSKE     R1,R2      [RRE] */
-        /* Set Storage Key Extended */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        gen_helper_sske(tmp32_1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x34: /* STCH ? */
-        /* Store Subchannel */
-        check_privileged(s, ilc);
-        gen_op_movi_cc(s, 3);
-        break;
-    case 0x46: /* STURA    R1,R2      [RRE] */
-        /* Store Using Real Address */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        potential_page_fault(s);
-        gen_helper_stura(tmp, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x50: /* CSP      R1,R2      [RRE] */
-        /* Compare And Swap And Purge */
-        check_privileged(s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        gen_helper_csp(cc_op, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5f: /* CHSC ? */
-        /* Channel Subsystem Call */
-        check_privileged(s, ilc);
-        gen_op_movi_cc(s, 3);
-        break;
-    case 0x78: /* STCKE    D2(B2)     [S] */
-        /* Store Clock Extended */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stcke(cc_op, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x79: /* SACF    D2(B2)     [S] */
-        /* Store Clock Extended */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sacf(tmp);
-        tcg_temp_free_i64(tmp);
-        /* addressing mode has changed, so end the block */
-        s->pc += ilc * 2;
-        update_psw_addr(s);
-        s->is_jmp = DISAS_EXCP;
-        break;
-    case 0x7d: /* STSI     D2,(B2)     [S] */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = load_reg32(1);
-        potential_page_fault(s);
-        gen_helper_stsi(cc_op, tmp, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x9d: /* LFPC      D2(B2)   [S] */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xb1: /* STFL     D2(B2)     [S] */
-        /* Store Facility List (CPU features) at 200 */
-        check_privileged(s, ilc);
-        tmp2 = tcg_const_i64(0xc0000000);
-        tmp = tcg_const_i64(200);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xb2: /* LPSWE    D2(B2)     [S] */
-        /* Load PSW Extended */
-        check_privileged(s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
-        tcg_gen_addi_i64(tmp, tmp, 8);
-        tcg_gen_qemu_ld64(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(tmp2, tmp3);
-        /* we need to keep cc_op intact */
-        s->is_jmp = DISAS_JUMP;
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x20: /* SERVC     R1,R2     [RRE] */
-        /* SCLP Service call (PV hypercall) */
-        check_privileged(s, ilc);
-        potential_page_fault(s);
-        tmp32_1 = load_reg32(r2);
-        tmp = load_reg(r1);
-        gen_helper_servc(cc_op, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
+static ExitStatus op_servc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_servc(cc_op, cpu_env, o->in2, o->in1);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sigp(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_sigp(cc_op, cpu_env, o->in2, r1, o->in1);
+    tcg_temp_free_i32(r1);
+    return NO_EXIT;
+}
 #endif
-    default:
-        LOG_DISAS("illegal b2 operation 0x%x\n", op);
-        gen_illegal_opcode(s, ilc);
-        break;
+
+static ExitStatus op_soc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare c;
+    TCGv_i64 a;
+    int lab, r1;
+
+    disas_jcc(s, &c, get_field(s->fields, m3));
+
+    lab = gen_new_label();
+    if (c.is_64) {
+        tcg_gen_brcond_i64(c.cond, c.u.s64.a, c.u.s64.b, lab);
+    } else {
+        tcg_gen_brcond_i32(c.cond, c.u.s32.a, c.u.s32.b, lab);
     }
+    free_compare(&c);
+
+    r1 = get_field(s->fields, r1);
+    a = get_address(s, 0, get_field(s->fields, b2), get_field(s->fields, d2));
+    if (s->insn->data) {
+        tcg_gen_qemu_st64(regs[r1], a, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(regs[r1], a, get_mem_index(s));
+    }
+    tcg_temp_free_i64(a);
+
+    gen_set_label(lab);
+    return NO_EXIT;
 }
 
-static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
+static ExitStatus op_sla(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    LOG_DISAS("disas_b3: op 0x%x m3 0x%x r1 %d r2 %d\n", op, m3, r1, r2);
-#define FP_HELPER(i) \
-    tmp32_1 = tcg_const_i32(r1); \
-    tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i (tmp32_1, tmp32_2); \
-    tcg_temp_free_i32(tmp32_1); \
-    tcg_temp_free_i32(tmp32_2);
-
-#define FP_HELPER_CC(i) \
-    tmp32_1 = tcg_const_i32(r1); \
-    tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i (cc_op, tmp32_1, tmp32_2); \
-    set_cc_static(s); \
-    tcg_temp_free_i32(tmp32_1); \
-    tcg_temp_free_i32(tmp32_2);
+    uint64_t sign = 1ull << s->insn->data;
+    enum cc_op cco = s->insn->data == 31 ? CC_OP_SLA_32 : CC_OP_SLA_64;
+    gen_op_update2_cc_i64(s, cco, o->in1, o->in2);
+    tcg_gen_shl_i64(o->out, o->in1, o->in2);
+    /* The arithmetic left shift is curious in that it does not affect
+       the sign bit.  Copy that over from the source unchanged.  */
+    tcg_gen_andi_i64(o->out, o->out, ~sign);
+    tcg_gen_andi_i64(o->in1, o->in1, sign);
+    tcg_gen_or_i64(o->out, o->out, o->in1);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x0: /* LPEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpebr);
-        break;
-    case 0x2: /* LTEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltebr);
-        break;
-    case 0x3: /* LCEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcebr);
-        break;
-    case 0x4: /* LDEBR       R1,R2             [RRE] */
-        FP_HELPER(ldebr);
-        break;
-    case 0x5: /* LXDBR       R1,R2             [RRE] */
-        FP_HELPER(lxdbr);
-        break;
-    case 0x9: /* CEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cebr);
-        break;
-    case 0xa: /* AEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(aebr);
-        break;
-    case 0xb: /* SEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sebr);
-        break;
-    case 0xd: /* DEBR        R1,R2             [RRE] */
-        FP_HELPER(debr);
-        break;
-    case 0x10: /* LPDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpdbr);
-        break;
-    case 0x12: /* LTDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltdbr);
-        break;
-    case 0x13: /* LCDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcdbr);
-        break;
-    case 0x15: /* SQBDR       R1,R2             [RRE] */
-        FP_HELPER(sqdbr);
-        break;
-    case 0x17: /* MEEBR       R1,R2             [RRE] */
-        FP_HELPER(meebr);
-        break;
-    case 0x19: /* CDBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cdbr);
-        break;
-    case 0x1a: /* ADBR        R1,R2             [RRE] */
-        FP_HELPER_CC(adbr);
-        break;
-    case 0x1b: /* SDBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sdbr);
-        break;
-    case 0x1c: /* MDBR        R1,R2             [RRE] */
-        FP_HELPER(mdbr);
-        break;
-    case 0x1d: /* DDBR        R1,R2             [RRE] */
-        FP_HELPER(ddbr);
-        break;
-    case 0xe: /* MAEBR  R1,R3,R2 [RRF] */
-    case 0x1e: /* MADBR R1,R3,R2 [RRF] */
-    case 0x1f: /* MSDBR R1,R3,R2 [RRF] */
-        /* for RRF insns, m3 is R1, r1 is R3, and r2 is R2 */
-        tmp32_1 = tcg_const_i32(m3);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(r1);
-        switch (op) {
-        case 0xe:
-            gen_helper_maebr(tmp32_1, tmp32_3, tmp32_2);
-            break;
-        case 0x1e:
-            gen_helper_madbr(tmp32_1, tmp32_3, tmp32_2);
-            break;
-        case 0x1f:
-            gen_helper_msdbr(tmp32_1, tmp32_3, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x40: /* LPXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpxbr);
-        break;
-    case 0x42: /* LTXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltxbr);
-        break;
-    case 0x43: /* LCXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcxbr);
-        break;
-    case 0x44: /* LEDBR       R1,R2             [RRE] */
-        FP_HELPER(ledbr);
-        break;
-    case 0x45: /* LDXBR       R1,R2             [RRE] */
-        FP_HELPER(ldxbr);
-        break;
-    case 0x46: /* LEXBR       R1,R2             [RRE] */
-        FP_HELPER(lexbr);
-        break;
-    case 0x49: /* CXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cxbr);
-        break;
-    case 0x4a: /* AXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(axbr);
-        break;
-    case 0x4b: /* SXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sxbr);
-        break;
-    case 0x4c: /* MXBR        R1,R2             [RRE] */
-        FP_HELPER(mxbr);
-        break;
-    case 0x4d: /* DXBR        R1,R2             [RRE] */
-        FP_HELPER(dxbr);
-        break;
-    case 0x65: /* LXR         R1,R2             [RRE] */
-        tmp = load_freg(r2);
-        store_freg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        tmp = load_freg(r2 + 2);
-        store_freg(r1 + 2, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x74: /* LZER        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzer(tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x75: /* LZDR        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzdr(tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x76: /* LZXR        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzxr(tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x84: /* SFPC        R1                [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x8c: /* EFPC        R1                [RRE] */
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x94: /* CEFBR       R1,R2             [RRE] */
-    case 0x95: /* CDFBR       R1,R2             [RRE] */
-    case 0x96: /* CXFBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = load_reg32(r2);
-        switch (op) {
-        case 0x94:
-            gen_helper_cefbr(tmp32_1, tmp32_2);
-            break;
-        case 0x95:
-            gen_helper_cdfbr(tmp32_1, tmp32_2);
-            break;
-        case 0x96:
-            gen_helper_cxfbr(tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x98: /* CFEBR       R1,R2             [RRE] */
-    case 0x99: /* CFDBR              R1,R2             [RRE] */
-    case 0x9a: /* CFXBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        switch (op) {
-        case 0x98:
-            gen_helper_cfebr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x99:
-            gen_helper_cfdbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x9a:
-            gen_helper_cfxbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xa4: /* CEGBR       R1,R2             [RRE] */
-    case 0xa5: /* CDGBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        switch (op) {
-        case 0xa4:
-            gen_helper_cegbr(tmp32_1, tmp);
-            break;
-        case 0xa5:
-            gen_helper_cdgbr(tmp32_1, tmp);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa6: /* CXGBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        gen_helper_cxgbr(tmp32_1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa8: /* CGEBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgebr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xa9: /* CGDBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgdbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+static ExitStatus op_sll(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_shl_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sra(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_sar_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_srl(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_shr_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sfpc(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sfpc(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sfas(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sfas(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_srnm(DisasContext *s, DisasOps *o)
+{
+    int b2 = get_field(s->fields, b2);
+    int d2 = get_field(s->fields, d2);
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    int mask, pos, len;
+
+    switch (s->fields->op2) {
+    case 0x99: /* SRNM */
+        pos = 0, len = 2;
         break;
-    case 0xaa: /* CGXBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgxbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+    case 0xb8: /* SRNMB */
+        pos = 0, len = 3;
         break;
+    case 0xb9: /* SRNMT */
+        pos = 4, len = 3;
     default:
-        LOG_DISAS("illegal b3 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
-        break;
+        tcg_abort();
     }
+    mask = (1 << len) - 1;
 
-#undef FP_HELPER_CC
-#undef FP_HELPER
+    /* Insert the value into the appropriate field of the FPC.  */
+    if (b2 == 0) {
+        tcg_gen_movi_i64(t1, d2 & mask);
+    } else {
+        tcg_gen_addi_i64(t1, regs[b2], d2);
+        tcg_gen_andi_i64(t1, t1, mask);
+    }
+    tcg_gen_ld32u_i64(t2, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_gen_deposit_i64(t2, t2, t1, pos, len);
+    tcg_temp_free_i64(t1);
+
+    /* Then install the new FPC to set the rounding mode in fpu_status.  */
+    gen_helper_sfpc(cpu_env, t2);
+    tcg_temp_free_i64(t2);
+    return NO_EXIT;
 }
 
-static void disas_b9(DisasContext *s, int op, int r1, int r2)
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_spka(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    check_privileged(s);
+    tcg_gen_shri_i64(o->in2, o->in2, 4);
+    tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, PSW_SHIFT_KEY - 4, 4);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_b9: op 0x%x r1 %d r2 %d\n", op, r1, r2);
-    switch (op) {
-    case 0x0: /* LPGR     R1,R2     [RRE] */
-    case 0x1: /* LNGR     R1,R2     [RRE] */
-    case 0x2: /* LTGR R1,R2 [RRE] */
-    case 0x3: /* LCGR     R1,R2     [RRE] */
-    case 0x10: /* LPGFR R1,R2 [RRE] */
-    case 0x11: /* LNFGR     R1,R2     [RRE] */
-    case 0x12: /* LTGFR R1,R2 [RRE] */
-    case 0x13: /* LCGFR    R1,R2     [RRE] */
-        if (op & 0x10) {
-            tmp = load_reg32_i64(r2);
-        } else {
-            tmp = load_reg(r2);
-        }
-        switch (op & 0xf) {
-        case 0x0: /* LP?GR */
-            set_cc_abs64(s, tmp);
-            gen_helper_abs_i64(tmp, tmp);
-            store_reg(r1, tmp);
-            break;
-        case 0x1: /* LN?GR */
-            set_cc_nabs64(s, tmp);
-            gen_helper_nabs_i64(tmp, tmp);
-            store_reg(r1, tmp);
-            break;
-        case 0x2: /* LT?GR */
-            if (r1 != r2) {
-                store_reg(r1, tmp);
-            }
-            set_cc_s64(s, tmp);
-            break;
-        case 0x3: /* LC?GR */
-            tcg_gen_neg_i64(regs[r1], tmp);
-            set_cc_comp64(s, regs[r1]);
-            break;
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* LGR R1,R2 [RRE] */
-        store_reg(r1, regs[r2]);
-        break;
-    case 0x6: /* LGBR R1,R2 [RRE] */
-        tmp2 = load_reg(r2);
-        tcg_gen_ext8s_i64(tmp2, tmp2);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x8: /* AGR     R1,R2     [RRE] */
-    case 0xa: /* ALGR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x8:
-            set_cc_add64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xa:
-            set_cc_addu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+static ExitStatus op_sske(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sske(cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ssm(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, 56, 8);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stap(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    /* ??? Surely cpu address != cpu number.  In any case the previous
+       version of this stored more than the required half-word, so it
+       is unlikely this has ever been tested.  */
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, cpu_num));
+    return NO_EXIT;
+}
+
+static ExitStatus op_stck(DisasContext *s, DisasOps *o)
+{
+    gen_helper_stck(o->out, cpu_env);
+    /* ??? We don't implement clock states.  */
+    gen_op_movi_cc(s, 0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stcke(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 c1 = tcg_temp_new_i64();
+    TCGv_i64 c2 = tcg_temp_new_i64();
+    gen_helper_stck(c1, cpu_env);
+    /* Shift the 64-bit value into its place as a zero-extended
+       104-bit value.  Note that "bit positions 64-103 are always
+       non-zero so that they compare differently to STCK"; we set
+       the least significant bit to 1.  */
+    tcg_gen_shli_i64(c2, c1, 56);
+    tcg_gen_shri_i64(c1, c1, 8);
+    tcg_gen_ori_i64(c2, c2, 0x10000);
+    tcg_gen_qemu_st64(c1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 8);
+    tcg_gen_qemu_st64(c2, o->in2, get_mem_index(s));
+    tcg_temp_free_i64(c1);
+    tcg_temp_free_i64(c2);
+    /* ??? We don't implement clock states.  */
+    gen_op_movi_cc(s, 0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sckc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sckc(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stckc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_stckc(o->out, cpu_env);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stctg(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stctg(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stctl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stctl(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stidp(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, cpu_num));
+    return NO_EXIT;
+}
+
+static ExitStatus op_spt(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_spt(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stfl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 f, a;
+    /* We really ought to have more complete indication of facilities
+       that we implement.  Address this when STFLE is implemented.  */
+    check_privileged(s);
+    f = tcg_const_i64(0xc0000000);
+    a = tcg_const_i64(200);
+    tcg_gen_qemu_st32(f, a, get_mem_index(s));
+    tcg_temp_free_i64(f);
+    tcg_temp_free_i64(a);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stpt(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_stpt(o->out, cpu_env);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stsi(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stsi(cc_op, cpu_env, o->in2, regs[0], regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_spx(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_spx(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_subchannel(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    /* Not operational.  */
+    gen_op_movi_cc(s, 3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stpx(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_ld_i64(o->out, cpu_env, offsetof(CPUS390XState, psa));
+    tcg_gen_andi_i64(o->out, o->out, 0x7fffe000);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stnosm(DisasContext *s, DisasOps *o)
+{
+    uint64_t i2 = get_field(s->fields, i2);
+    TCGv_i64 t;
+
+    check_privileged(s);
+
+    /* It is important to do what the instruction name says: STORE THEN.
+       If we let the output hook perform the store then if we fault and
+       restart, we'll have the wrong SYSTEM MASK in place.  */
+    t = tcg_temp_new_i64();
+    tcg_gen_shri_i64(t, psw_mask, 56);
+    tcg_gen_qemu_st8(t, o->addr1, get_mem_index(s));
+    tcg_temp_free_i64(t);
+
+    if (s->fields->op == 0xac) {
+        tcg_gen_andi_i64(psw_mask, psw_mask,
+                         (i2 << 56) | 0x00ffffffffffffffull);
+    } else {
+        tcg_gen_ori_i64(psw_mask, psw_mask, i2 << 56);
+    }
+    return NO_EXIT;
+}
+
+static ExitStatus op_stura(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stura(cpu_env, o->in2, o->in1);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_st8(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st8(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st16(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st16(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st64(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_stam(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_stam(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stcm(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int pos, base = s->insn->data;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+
+    pos = base + ctz32(m3) * 8;
+    switch (m3) {
+    case 0xf:
+        /* Effectively a 32-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st32(tmp, o->in2, get_mem_index(s));
         break;
-    case 0x9: /* SGR     R1,R2     [RRE] */
-    case 0xb: /* SLGR     R1,R2     [RRE] */
-    case 0x1b: /* SLGFR     R1,R2     [RRE] */
-    case 0x19: /* SGFR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        switch (op) {
-        case 0x1b:
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
-            break;
-        case 0x19:
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
-            break;
-        default:
-            tmp2 = load_reg(r2);
-            break;
-        }
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_sub_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x9:
-        case 0x19:
-            set_cc_sub64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xb:
-        case 0x1b:
-            set_cc_subu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+
+    case 0xc:
+    case 0x6:
+    case 0x3:
+        /* Effectively a 16-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st16(tmp, o->in2, get_mem_index(s));
         break;
-    case 0xc: /* MSGR      R1,R2     [RRE] */
-    case 0x1c: /* MSGFR      R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        if (op == 0x1c) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
-        }
-        tcg_gen_mul_i64(tmp, tmp, tmp2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
+
+    case 0x8:
+    case 0x4:
+    case 0x2:
+    case 0x1:
+        /* Effectively an 8-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s));
         break;
-    case 0xd: /* DSGR      R1,R2     [RRE] */
-    case 0x1d: /* DSGFR      R1,R2     [RRE] */
-        tmp = load_reg(r1 + 1);
-        if (op == 0xd) {
-            tmp2 = load_reg(r2);
-        } else {
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
+
+    default:
+        /* This is going to be a sequence of shifts and stores.  */
+        pos = base + 32 - 8;
+        while (m3) {
+            if (m3 & 0x8) {
+                tcg_gen_shri_i64(tmp, o->in1, pos);
+                tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s));
+                tcg_gen_addi_i64(o->in2, o->in2, 1);
+            }
+            m3 = (m3 << 1) & 0xf;
+            pos -= 8;
         }
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        store_reg(r1 + 1, tmp3);
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x14: /* LGFR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_ext_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
         break;
-    case 0x16: /* LLGFR      R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x17: /* LLGTR      R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0x7fffffffUL);
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x18: /* AGFR     R1,R2     [RRE] */
-    case 0x1a: /* ALGFR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x18) {
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-        } else {
-            tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tmp = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        if (op == 0x18) {
-            set_cc_add64(s, tmp, tmp2, tmp3);
+    }
+    tcg_temp_free_i64(tmp);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stm(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    int size = s->insn->data;
+    TCGv_i64 tsize = tcg_const_i64(size);
+
+    while (1) {
+        if (size == 8) {
+            tcg_gen_qemu_st64(regs[r1], o->in2, get_mem_index(s));
         } else {
-            set_cc_addu64(s, tmp, tmp2, tmp3);
+            tcg_gen_qemu_st32(regs[r1], o->in2, get_mem_index(s));
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x0f: /* LRVGR    R1,R2     [RRE] */
-        tcg_gen_bswap64_i64(regs[r1], regs[r2]);
-        break;
-    case 0x1f: /* LRVR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x20: /* CGR      R1,R2     [RRE] */
-    case 0x30: /* CGFR     R1,R2     [RRE] */
-        tmp2 = load_reg(r2);
-        if (op == 0x30) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
-        }
-        tmp = load_reg(r1);
-        cmp_s64(s, tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x21: /* CLGR     R1,R2     [RRE] */
-    case 0x31: /* CLGFR    R1,R2     [RRE] */
-        tmp2 = load_reg(r2);
-        if (op == 0x31) {
-            tcg_gen_ext32u_i64(tmp2, tmp2);
-        }
-        tmp = load_reg(r1);
-        cmp_u64(s, tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x26: /* LBR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_ext8s_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x27: /* LHR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_ext16s_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x80: /* NGR R1,R2 [RRE] */
-    case 0x81: /* OGR R1,R2 [RRE] */
-    case 0x82: /* XGR R1,R2 [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        switch (op) {
-        case 0x80:
-            tcg_gen_and_i64(tmp, tmp, tmp2);
+        if (r1 == r3) {
             break;
-        case 0x81:
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        case 0x82:
-            tcg_gen_xor_i64(tmp, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        set_cc_nz_u64(s, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x83: /* FLOGR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_flogr(cc_op, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x84: /* LLGCR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tcg_gen_andi_i64(tmp, tmp, 0xff);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x85: /* LLGHR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tcg_gen_andi_i64(tmp, tmp, 0xffff);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x87: /* DLGR      R1,R2     [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        gen_helper_dlg(tmp32_1, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x88: /* ALCGR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = tcg_temp_new_i64();
-        gen_op_calc_cc(s);
-        tcg_gen_extu_i32_i64(tmp3, cc_op);
-        tcg_gen_shri_i64(tmp3, tmp3, 1);
-        tcg_gen_andi_i64(tmp3, tmp3, 1);
-        tcg_gen_add_i64(tmp3, tmp2, tmp3);
-        tcg_gen_add_i64(tmp3, tmp, tmp3);
-        store_reg(r1, tmp3);
-        set_cc_addu64(s, tmp, tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x89: /* SLBGR   R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp32_1 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cc_op, tmp32_1, tmp, tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x94: /* LLCR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0xff);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x95: /* LLHR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0xffff);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x96: /* MLR     R1,R2     [RRE] */
-        /* reg(r1, r1+1) = reg(r1+1) * reg(r2) */
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x97: /* DLR     R1,R2     [RRE] */
-        /* reg(r1) = reg(r1, r1+1) % reg(r2) */
-        /* reg(r1+1) = reg(r1, r1+1) / reg(r2) */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp3);
-
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        tcg_gen_div_i64(tmp, tmp, tmp2);
-        store_reg32_i64((r1 + 1) & 15, tmp);
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x98: /* ALCR    R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_addc_u32(tmp32_3, cc_op, tmp32_1, tmp32_2);
-        set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x99: /* SLBR    R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp32_2 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cc_op, tmp32_2, tmp32_1);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    default:
-        LOG_DISAS("illegal b9 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
-        break;
+        tcg_gen_add_i64(o->in2, o->in2, tsize);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(tsize);
+    return NO_EXIT;
 }
 
-static void disas_c0(DisasContext *s, int op, int r1, int i2)
+static ExitStatus op_stmh(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i32 tmp32_1, tmp32_2;
-    uint64_t target = s->pc + i2 * 2LL;
-    int l1;
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+    TCGv_i64 t32 = tcg_const_i64(32);
 
-    LOG_DISAS("disas_c0: op 0x%x r1 %d i2 %d\n", op, r1, i2);
-
-    switch (op) {
-    case 0: /* larl r1, i2 */
-        tmp = tcg_const_i64(target);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1: /* LGFI R1,I2 [RIL] */
-        tmp = tcg_const_i64((int64_t)i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* BRCL     M1,I2     [RIL] */
-        /* m1 & (1 << (3 - cc)) */
-        tmp32_1 = tcg_const_i32(3);
-        tmp32_2 = tcg_const_i32(1);
-        gen_op_calc_cc(s);
-        tcg_gen_sub_i32(tmp32_1, tmp32_1, cc_op);
-        tcg_gen_shl_i32(tmp32_2, tmp32_2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tmp32_1 = tcg_const_i32(r1); /* m1 == r1 */
-        tcg_gen_and_i32(tmp32_1, tmp32_1, tmp32_2);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp32_1, 0, l1);
-        gen_goto_tb(s, 0, target);
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 6);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5: /* brasl r1, i2 */
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 6));
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        gen_goto_tb(s, 0, target);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x7: /* XILF R1,I2 [RIL] */
-    case 0xb: /* NILF R1,I2 [RIL] */
-    case 0xd: /* OILF R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        switch (op) {
-        case 0x7:
-            tcg_gen_xori_i32(tmp32_1, tmp32_1, (uint32_t)i2);
-            break;
-        case 0xb:
-            tcg_gen_andi_i32(tmp32_1, tmp32_1, (uint32_t)i2);
-            break;
-        case 0xd:
-            tcg_gen_ori_i32(tmp32_1, tmp32_1, (uint32_t)i2);
+    while (1) {
+        tcg_gen_shl_i64(t, regs[r1], t32);
+        tcg_gen_qemu_st32(t, o->in2, get_mem_index(s));
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x9: /* IILF R1,I2 [RIL] */
-        tmp32_1 = tcg_const_i32((uint32_t)i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xa: /* NIHF R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_andi_i64(tmp, tmp, (((uint64_t)((uint32_t)i2)) << 32)
-                                   | 0xffffffffULL);
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp, tmp, 32);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xe: /* LLIHF R1,I2 [RIL] */
-        tmp = tcg_const_i64(((uint64_t)(uint32_t)i2) << 32);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xf: /* LLILF R1,I2 [RIL] */
-        tmp = tcg_const_i64((uint32_t)i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        LOG_DISAS("illegal c0 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
-        break;
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    tcg_temp_free_i64(t32);
+    return NO_EXIT;
 }
 
-static void disas_c2(DisasContext *s, int op, int r1, int i2)
+static ExitStatus op_srst(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    potential_page_fault(s);
+    gen_helper_srst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x4: /* SLGFI R1,I2 [RIL] */
-    case 0xa: /* ALGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64((uint64_t)(uint32_t)i2);
-        tmp3 = tcg_temp_new_i64();
-        switch (op) {
-        case 0x4:
-            tcg_gen_sub_i64(tmp3, tmp, tmp2);
-            set_cc_subu64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xa:
-            tcg_gen_add_i64(tmp3, tmp, tmp2);
-            set_cc_addu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5: /* SLFI R1,I2 [RIL] */
-    case 0xb: /* ALFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(i2);
-        tmp32_3 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x5:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0xb:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xc: /* CGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        cmp_s64c(s, tmp, (int64_t)i2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* CLGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        cmp_u64c(s, tmp, (uint64_t)(uint32_t)i2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xd: /* CFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        cmp_s32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xf: /* CLFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        cmp_u32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    default:
-        LOG_DISAS("illegal c2 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
-        break;
-    }
+static ExitStatus op_sub(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_sub_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
 }
 
-static void gen_and_or_xor_i32(int opc, TCGv_i32 tmp, TCGv_i32 tmp2)
+static ExitStatus op_subb(DisasContext *s, DisasOps *o)
 {
-    switch (opc & 0xf) {
-    case 0x4:
-        tcg_gen_and_i32(tmp, tmp, tmp2);
-        break;
-    case 0x6:
-        tcg_gen_or_i32(tmp, tmp, tmp2);
-        break;
-    case 0x7:
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        break;
-    default:
-        tcg_abort();
+    DisasCompare cmp;
+    TCGv_i64 borrow;
+
+    tcg_gen_sub_i64(o->out, o->in1, o->in2);
+
+    /* The !borrow flag is the msb of CC.  Since we want the inverse of
+       that, we ask for a comparison of CC=0 | CC=1 -> mask of 8 | 4.  */
+    disas_jcc(s, &cmp, 8 | 4);
+    borrow = tcg_temp_new_i64();
+    if (cmp.is_64) {
+        tcg_gen_setcond_i64(cmp.cond, borrow, cmp.u.s64.a, cmp.u.s64.b);
+    } else {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cmp.cond, t, cmp.u.s32.a, cmp.u.s32.b);
+        tcg_gen_extu_i32_i64(borrow, t);
+        tcg_temp_free_i32(t);
     }
+    free_compare(&cmp);
+
+    tcg_gen_sub_i64(o->out, o->out, borrow);
+    tcg_temp_free_i64(borrow);
+    return NO_EXIT;
 }
 
-static void disas_s390_insn(DisasContext *s)
+static ExitStatus op_svc(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3, tmp32_4;
-    unsigned char opc;
-    uint64_t insn;
-    int op, r1, r2, r3, d1, d2, x2, b1, b2, i, i2, r1b;
-    TCGv_i32 vl;
-    int ilc;
-    int l1;
+    TCGv_i32 t;
+
+    update_psw_addr(s);
+    update_cc_op(s);
 
-    opc = ldub_code(s->pc);
-    LOG_DISAS("opc 0x%x\n", opc);
+    t = tcg_const_i32(get_field(s->fields, i1) & 0xff);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, int_svc_code));
+    tcg_temp_free_i32(t);
 
-    ilc = get_ilc(opc);
+    t = tcg_const_i32(s->next_pc - s->pc);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, int_svc_ilen));
+    tcg_temp_free_i32(t);
+
+    gen_exception(EXCP_SVC);
+    return EXIT_NORETURN;
+}
+
+static ExitStatus op_tceb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tceb(cc_op, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_tcdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tcdb(cc_op, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_tcxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tcxb(cc_op, o->out, o->out2, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-    switch (opc) {
 #ifndef CONFIG_USER_ONLY
-    case 0x01: /* SAM */
-        insn = ld_code2(s->pc);
-        /* set addressing mode, but we only do 64bit anyways */
-        break;
+static ExitStatus op_tprot(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_tprot(cc_op, o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 #endif
-    case 0x6: /* BCTR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-
-        if (r2) {
-            gen_update_cc_op(s);
-            l1 = gen_new_label();
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp32_1, 0, l1);
-
-            /* not taking the branch, jump to after the instruction */
-            gen_goto_tb(s, 0, s->pc + 2);
-            gen_set_label(l1);
-
-            /* take the branch, move R2 into psw.addr */
-            tmp32_1 = load_reg32(r2);
-            tmp = tcg_temp_new_i64();
-            tcg_gen_extu_i32_i64(tmp, tmp32_1);
-            tcg_gen_mov_i64(psw_addr, tmp);
-            s->is_jmp = DISAS_JUMP;
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i64(tmp);
-        }
-        break;
-    case 0x7: /* BCR    M1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        if (r2) {
-            tmp = load_reg(r2);
-            gen_bcr(s, r1, tmp, s->pc);
-            tcg_temp_free_i64(tmp);
-            s->is_jmp = DISAS_TB_JUMP;
-        } else {
-            /* XXX: "serialization and checkpoint-synchronization function"? */
-        }
-        break;
-    case 0xa: /* SVC    I         [RR] */
-        insn = ld_code2(s->pc);
-        debug_insn(insn);
-        i = insn & 0xff;
-        update_psw_addr(s);
-        gen_op_calc_cc(s);
-        tmp32_1 = tcg_const_i32(i);
-        tmp32_2 = tcg_const_i32(ilc * 2);
-        tmp32_3 = tcg_const_i32(EXCP_SVC);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, int_svc_code));
-        tcg_gen_st_i32(tmp32_2, cpu_env, offsetof(CPUS390XState, int_svc_ilc));
-        gen_helper_exception(tmp32_3);
-        s->is_jmp = DISAS_EXCP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xd: /* BASR   R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 2));
-        store_reg(r1, tmp);
-        if (r2) {
-            tmp2 = load_reg(r2);
-            tcg_gen_mov_i64(psw_addr, tmp2);
-            tcg_temp_free_i64(tmp2);
-            s->is_jmp = DISAS_JUMP;
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* MVCL   R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_mvcl(cc_op, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x10: /* LPR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        set_cc_abs32(s, tmp32_1);
-        gen_helper_abs_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x11: /* LNR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        set_cc_nabs32(s, tmp32_1);
-        gen_helper_nabs_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x12: /* LTR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        if (r1 != r2) {
-            store_reg32(r1, tmp32_1);
-        }
-        set_cc_s32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x13: /* LCR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_neg_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        set_cc_comp32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x14: /* NR     R1,R2     [RR] */
-    case 0x16: /* OR     R1,R2     [RR] */
-    case 0x17: /* XR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_2 = load_reg32(r2);
-        tmp32_1 = load_reg32(r1);
-        gen_and_or_xor_i32(opc, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x18: /* LR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x15: /* CLR    R1,R2     [RR] */
-    case 0x19: /* CR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        if (opc == 0x15) {
-            cmp_u32(s, tmp32_1, tmp32_2);
-        } else {
-            cmp_s32(s, tmp32_1, tmp32_2);
+
+static ExitStatus op_tr(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_tr(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_unpk(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_unpk(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xc(DisasContext *s, DisasOps *o)
+{
+    int d1 = get_field(s->fields, d1);
+    int d2 = get_field(s->fields, d2);
+    int b1 = get_field(s->fields, b1);
+    int b2 = get_field(s->fields, b2);
+    int l = get_field(s->fields, l1);
+    TCGv_i32 t32;
+
+    o->addr1 = get_address(s, 0, b1, d1);
+
+    /* If the addresses are identical, this is a store/memset of zero.  */
+    if (b1 == b2 && d1 == d2 && (l + 1) <= 32) {
+        o->in2 = tcg_const_i64(0);
+
+        l++;
+        while (l >= 8) {
+            tcg_gen_qemu_st64(o->in2, o->addr1, get_mem_index(s));
+            l -= 8;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 8);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x1a: /* AR     R1,R2     [RR] */
-    case 0x1e: /* ALR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_3);
-        if (opc == 0x1a) {
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-        } else {
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
+        if (l >= 4) {
+            tcg_gen_qemu_st32(o->in2, o->addr1, get_mem_index(s));
+            l -= 4;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 4);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x1b: /* SR     R1,R2     [RR] */
-    case 0x1f: /* SLR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_3);
-        if (opc == 0x1b) {
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-        } else {
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
+        if (l >= 2) {
+            tcg_gen_qemu_st16(o->in2, o->addr1, get_mem_index(s));
+            l -= 2;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 2);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x1c: /* MR     R1,R2     [RR] */
-        /* reg(r1, r1+1) = reg(r1+1) * reg(r2) */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32s_i64(tmp2, tmp2);
-        tcg_gen_ext32s_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x1d: /* DR     R1,R2               [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-        tmp32_3 = load_reg32(r2);
-
-        tmp = tcg_temp_new_i64(); /* dividend */
-        tmp2 = tcg_temp_new_i64(); /* divisor */
-        tmp3 = tcg_temp_new_i64();
-
-        /* dividend is r(r1 << 32) | r(r1 + 1) */
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_2);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp2);
-
-        /* divisor is r(r2) */
-        tcg_gen_ext_i32_i64(tmp2, tmp32_3);
-
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        tcg_gen_rem_i64(tmp, tmp, tmp2);
-
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp3);
-
-        store_reg32(r1, tmp32_1); /* remainder */
-        store_reg32(r1 + 1, tmp32_2); /* quotient */
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x28: /* LDR    R1,R2               [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp = load_freg(r2);
-        store_freg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x38: /* LER    R1,R2               [RR] */
-        insn = ld_code2(s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_freg32(r2);
-        store_freg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x40: /* STH    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st16(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x41:        /* la */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        store_reg(r1, tmp); /* FIXME: 31/24-bit addressing */
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x42: /* STC    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x43: /* IC     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        store_reg8(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x44: /* EX     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tmp3 = tcg_const_i64(s->pc + 4);
-        update_psw_addr(s);
-        gen_op_calc_cc(s);
-        gen_helper_ex(cc_op, cc_op, tmp2, tmp, tmp3);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x46: /* BCT    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tcg_temp_free_i64(tmp);
-
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp32_1, 0, l1);
-
-        /* not taking the branch, jump to after the instruction */
-        gen_goto_tb(s, 0, s->pc + 4);
-        gen_set_label(l1);
-
-        /* take the branch, move R2 into psw.addr */
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tcg_gen_mov_i64(psw_addr, tmp);
-        s->is_jmp = DISAS_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x47: /* BC     M1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        gen_bcr(s, r1, tmp, s->pc + 4);
-        tcg_temp_free_i64(tmp);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x48: /* LH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x49: /* CH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        cmp_s32(s, tmp32_1, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4a: /* AH     R1,D2(X2,B2)     [RX] */
-    case 0x4b: /* SH     R1,D2(X2,B2)     [RX] */
-    case 0x4c: /* MH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        switch (opc) {
-        case 0x4a:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x4b:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x4c:
-            tcg_gen_mul_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
+        if (l) {
+            tcg_gen_qemu_st8(o->in2, o->addr1, get_mem_index(s));
         }
-        store_reg32(r1, tmp32_3);
+        gen_op_movi_cc(s, 0);
+        return NO_EXIT;
+    }
 
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4d: /* BAS    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_const_i64(pc_to_link_info(s, s->pc + 4));
-        store_reg(r1, tmp2);
-        tcg_gen_mov_i64(psw_addr, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        s->is_jmp = DISAS_JUMP;
-        break;
-    case 0x4e: /* CVD    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp32_1, regs[r1]);
-        gen_helper_cvd(tmp2, tmp32_1);
-        tcg_gen_qemu_st64(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x50: /* st r1, d2(x2, b2) */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x55: /* CL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r1);
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        cmp_u32(s, tmp32_2, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x54: /* N      R1,D2(X2,B2)     [RX] */
-    case 0x56: /* O      R1,D2(X2,B2)     [RX] */
-    case 0x57: /* X      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        gen_and_or_xor_i32(opc, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x58: /* l r1, d2(x2, b2) */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x59: /* C      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r1);
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        cmp_s32(s, tmp32_2, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5a: /* A      R1,D2(X2,B2)     [RX] */
-    case 0x5b: /* S      R1,D2(X2,B2)     [RX] */
-    case 0x5e: /* AL     R1,D2(X2,B2)     [RX] */
-    case 0x5f: /* SL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp);
-        switch (opc) {
-        case 0x5a:
-        case 0x5e:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        case 0x5b:
-        case 0x5f:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        switch (opc) {
-        case 0x5a:
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5e:
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5b:
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5f:
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x5c: /* M      R1,D2(X2,B2)        [RX] */
-        /* reg(r1, r1+1) = reg(r1+1) * *(s32*)addr */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32s_i64(tmp2, tmp2);
-        tcg_gen_ext32s_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5d: /* D      R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
-        tmp3 = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
-
-        /* dividend is r(r1 << 32) | r(r1 + 1) */
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_2);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp2);
-
-        /* divisor is in memory */
-        tcg_gen_qemu_ld32s(tmp2, tmp3, get_mem_index(s));
-
-        /* XXX divisor == 0 -> FixP divide exception */
-
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        tcg_gen_rem_i64(tmp, tmp, tmp2);
-
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp3);
-
-        store_reg32(r1, tmp32_1); /* remainder */
-        store_reg32(r1 + 1, tmp32_2); /* quotient */
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x60: /* STD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_freg(r1);
-        tcg_gen_qemu_st64(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x68: /* LD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
-        store_freg(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
+    /* But in general we'll defer to a helper.  */
+    o->in2 = get_address(s, 0, b2, d2);
+    t32 = tcg_const_i32(l);
+    potential_page_fault(s);
+    gen_helper_xc(cc_op, cpu_env, t32, o->addr1, o->in2);
+    tcg_temp_free_i32(t32);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xor(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xor_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xori(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_xor_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_zero(DisasContext *s, DisasOps *o)
+{
+    o->out = tcg_const_i64(0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_zero2(DisasContext *s, DisasOps *o)
+{
+    o->out = tcg_const_i64(0);
+    o->out2 = o->out;
+    o->g_out2 = true;
+    return NO_EXIT;
+}
+
+/* ====================================================================== */
+/* The "Cc OUTput" generators.  Given the generated output (and in some cases
+   the original inputs), update the various cc data structures in order to
+   be able to compute the new condition code.  */
+
+static void cout_abs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_ABS_32, o->out);
+}
+
+static void cout_abs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_ABS_64, o->out);
+}
+
+static void cout_adds32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADD_32, o->in1, o->in2, o->out);
+}
+
+static void cout_adds64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADD_64, o->in1, o->in2, o->out);
+}
+
+static void cout_addu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDU_32, o->in1, o->in2, o->out);
+}
+
+static void cout_addu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDU_64, o->in1, o->in2, o->out);
+}
+
+static void cout_addc32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDC_32, o->in1, o->in2, o->out);
+}
+
+static void cout_addc64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDC_64, o->in1, o->in2, o->out);
+}
+
+static void cout_cmps32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTGT_32, o->in1, o->in2);
+}
+
+static void cout_cmps64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTGT_64, o->in1, o->in2);
+}
+
+static void cout_cmpu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_32, o->in1, o->in2);
+}
+
+static void cout_cmpu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, o->in1, o->in2);
+}
+
+static void cout_f32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F32, o->out);
+}
+
+static void cout_f64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, o->out);
+}
+
+static void cout_f128(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_NZ_F128, o->out, o->out2);
+}
+
+static void cout_nabs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NABS_32, o->out);
+}
+
+static void cout_nabs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NABS_64, o->out);
+}
+
+static void cout_neg32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_COMP_32, o->out);
+}
+
+static void cout_neg64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_COMP_64, o->out);
+}
+
+static void cout_nz32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ext32u_i64(cc_dst, o->out);
+    gen_op_update1_cc_i64(s, CC_OP_NZ, cc_dst);
+}
+
+static void cout_nz64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ, o->out);
+}
+
+static void cout_s32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_LTGT0_32, o->out);
+}
+
+static void cout_s64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_LTGT0_64, o->out);
+}
+
+static void cout_subs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUB_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUB_64, o->in1, o->in2, o->out);
+}
+
+static void cout_subu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBU_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBU_64, o->in1, o->in2, o->out);
+}
+
+static void cout_subb32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBB_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subb64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBB_64, o->in1, o->in2, o->out);
+}
+
+static void cout_tm32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_TM_32, o->in1, o->in2);
+}
+
+static void cout_tm64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_TM_64, o->in1, o->in2);
+}
+
+/* ====================================================================== */
+/* The "PREPeration" generators.  These initialize the DisasOps.OUT fields
+   with the TCG register to which we will write.  Used in combination with
+   the "wout" generators, in some cases we need a new temporary, and in
+   some cases we can write to a TCG global.  */
+
+static void prep_new(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = tcg_temp_new_i64();
+}
+#define SPEC_prep_new 0
+
+static void prep_new_P(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = tcg_temp_new_i64();
+    o->out2 = tcg_temp_new_i64();
+}
+#define SPEC_prep_new_P 0
+
+static void prep_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = regs[get_field(f, r1)];
+    o->g_out = true;
+}
+#define SPEC_prep_r1 0
+
+static void prep_r1_P(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = regs[r1];
+    o->out2 = regs[r1 + 1];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_prep_r1_P SPEC_r1_even
+
+static void prep_f1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = fregs[get_field(f, r1)];
+    o->g_out = true;
+}
+#define SPEC_prep_f1 0
+
+static void prep_x1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = fregs[r1];
+    o->out2 = fregs[r1 + 2];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_prep_x1 SPEC_r1_f128
+
+/* ====================================================================== */
+/* The "Write OUTput" generators.  These generally perform some non-trivial
+   copy of data to TCG globals, or to main memory.  The trivial cases are
+   generally handled by having a "prep" generator install the TCG global
+   as the destination of the operation.  */
+
+static void wout_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_reg(get_field(f, r1), o->out);
+}
+#define SPEC_wout_r1 0
+
+static void wout_r1_8(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 8);
+}
+#define SPEC_wout_r1_8 0
+
+static void wout_r1_16(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 16);
+}
+#define SPEC_wout_r1_16 0
+
+static void wout_r1_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_reg32_i64(get_field(f, r1), o->out);
+}
+#define SPEC_wout_r1_32 0
+
+static void wout_r1_P32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    store_reg32_i64(r1, o->out);
+    store_reg32_i64(r1 + 1, o->out2);
+}
+#define SPEC_wout_r1_P32 SPEC_r1_even
+
+static void wout_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    store_reg32_i64(r1 + 1, o->out);
+    tcg_gen_shri_i64(o->out, o->out, 32);
+    store_reg32_i64(r1, o->out);
+}
+#define SPEC_wout_r1_D32 SPEC_r1_even
+
+static void wout_e1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_freg32_i64(get_field(f, r1), o->out);
+}
+#define SPEC_wout_e1 0
+
+static void wout_f1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_freg(get_field(f, r1), o->out);
+}
+#define SPEC_wout_f1 0
+
+static void wout_x1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int f1 = get_field(s->fields, r1);
+    store_freg(f1, o->out);
+    store_freg(f1 + 2, o->out2);
+}
+#define SPEC_wout_x1 SPEC_r1_f128
+
+static void wout_cond_r1r2_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    if (get_field(f, r1) != get_field(f, r2)) {
+        store_reg32_i64(get_field(f, r1), o->out);
+    }
+}
+#define SPEC_wout_cond_r1r2_32 0
+
+static void wout_cond_e1e2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    if (get_field(f, r1) != get_field(f, r2)) {
+        store_freg32_i64(get_field(f, r1), o->out);
+    }
+}
+#define SPEC_wout_cond_e1e2 0
+
+static void wout_m1_8(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st8(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_8 0
+
+static void wout_m1_16(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st16(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_16 0
+
+static void wout_m1_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_32 0
+
+static void wout_m1_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st64(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_64 0
+
+static void wout_m2_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->out, o->in2, get_mem_index(s));
+}
+#define SPEC_wout_m2_32 0
+
+/* ====================================================================== */
+/* The "INput 1" generators.  These load the first operand to an insn.  */
+
+static void in1_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r1));
+}
+#define SPEC_in1_r1 0
+
+static void in1_r1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = regs[get_field(f, r1)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_r1_o 0
+
+static void in1_r1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r1)]);
+}
+#define SPEC_in1_r1_32s 0
+
+static void in1_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r1)]);
+}
+#define SPEC_in1_r1_32u 0
+
+static void in1_r1_sr32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(o->in1, regs[get_field(f, r1)], 32);
+}
+#define SPEC_in1_r1_sr32 0
+
+static void in1_r1p1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r1) + 1);
+}
+#define SPEC_in1_r1p1 SPEC_r1_even
+
+static void in1_r1p1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r1) + 1]);
+}
+#define SPEC_in1_r1p1_32s SPEC_r1_even
+
+static void in1_r1p1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r1) + 1]);
+}
+#define SPEC_in1_r1p1_32u SPEC_r1_even
+
+static void in1_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in1_r1_D32 SPEC_r1_even
+
+static void in1_r2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r2));
+}
+#define SPEC_in1_r2 0
+
+static void in1_r3(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r3));
+}
+#define SPEC_in1_r3 0
+
+static void in1_r3_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = regs[get_field(f, r3)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_r3_o 0
+
+static void in1_r3_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r3)]);
+}
+#define SPEC_in1_r3_32s 0
+
+static void in1_r3_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r3)]);
+}
+#define SPEC_in1_r3_32u 0
+
+static void in1_r3_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r3 = get_field(f, r3);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r3 + 1], regs[r3]);
+}
+#define SPEC_in1_r3_D32 SPEC_r3_even
+
+static void in1_e1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_freg32_i64(get_field(f, r1));
+}
+#define SPEC_in1_e1 0
+
+static void in1_f1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = fregs[get_field(f, r1)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_f1_o 0
+
+static void in1_x1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = fregs[r1];
+    o->out2 = fregs[r1 + 2];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_in1_x1_o SPEC_r1_f128
+
+static void in1_f3_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = fregs[get_field(f, r3)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_f3_o 0
+
+static void in1_la1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->addr1 = get_address(s, 0, get_field(f, b1), get_field(f, d1));
+}
+#define SPEC_in1_la1 0
+
+static void in1_la2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int x2 = have_field(f, x2) ? get_field(f, x2) : 0;
+    o->addr1 = get_address(s, x2, get_field(f, b2), get_field(f, d2));
+}
+#define SPEC_in1_la2 0
+
+static void in1_m1_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld8u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_8u 0
+
+static void in1_m1_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld16s(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_16s 0
+
+static void in1_m1_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld16u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_16u 0
+
+static void in1_m1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32s(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_32s 0
+
+static void in1_m1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_32u 0
+
+static void in1_m1_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_64 0
+
+/* ====================================================================== */
+/* The "INput 2" generators.  These load the second operand to an insn.  */
+
+static void in2_r1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = regs[get_field(f, r1)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_r1_o 0
+
+static void in2_r1_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16u_i64(o->in2, regs[get_field(f, r1)]);
+}
+#define SPEC_in2_r1_16u 0
+
+static void in2_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in2, regs[get_field(f, r1)]);
+}
+#define SPEC_in2_r1_32u 0
+
+static void in2_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in2, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in2_r1_D32 SPEC_r1_even
+
+static void in2_r2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_reg(get_field(f, r2));
+}
+#define SPEC_in2_r2 0
+
+static void in2_r2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = regs[get_field(f, r2)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_r2_o 0
+
+static void in2_r2_nz(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r2 = get_field(f, r2);
+    if (r2 != 0) {
+        o->in2 = load_reg(r2);
+    }
+}
+#define SPEC_in2_r2_nz 0
+
+static void in2_r2_8s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext8s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_8s 0
+
+static void in2_r2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext8u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_8u 0
+
+static void in2_r2_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_16s 0
+
+static void in2_r2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_16u 0
+
+static void in2_r3(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_reg(get_field(f, r3));
+}
+#define SPEC_in2_r3 0
+
+static void in2_r2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_32s 0
+
+static void in2_r2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_32u 0
+
+static void in2_e2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_freg32_i64(get_field(f, r2));
+}
+#define SPEC_in2_e2 0
+
+static void in2_f2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = fregs[get_field(f, r2)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_f2_o 0
+
+static void in2_x2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r2 = get_field(f, r2);
+    o->in1 = fregs[r2];
+    o->in2 = fregs[r2 + 2];
+    o->g_in1 = o->g_in2 = true;
+}
+#define SPEC_in2_x2_o SPEC_r2_f128
+
+static void in2_ra2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = get_address(s, 0, get_field(f, r2), 0);
+}
+#define SPEC_in2_ra2 0
+
+static void in2_a2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int x2 = have_field(f, x2) ? get_field(f, x2) : 0;
+    o->in2 = get_address(s, x2, get_field(f, b2), get_field(f, d2));
+}
+#define SPEC_in2_a2 0
+
+static void in2_ri2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64(s->pc + (int64_t)get_field(f, i2) * 2);
+}
+#define SPEC_in2_ri2 0
+
+static void in2_sh32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    help_l2_shift(s, f, o, 31);
+}
+#define SPEC_in2_sh32 0
+
+static void in2_sh64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    help_l2_shift(s, f, o, 63);
+}
+#define SPEC_in2_sh64 0
+
+static void in2_m2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld8u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_8u 0
+
+static void in2_m2_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld16s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_16s 0
+
+static void in2_m2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld16u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_16u 0
+
+static void in2_m2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld32s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_32s 0
+
+static void in2_m2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld32u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_32u 0
+
+static void in2_m2_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_64 0
+
+static void in2_mri2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld16u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_16u 0
+
+static void in2_mri2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld32s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_32s 0
+
+static void in2_mri2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld32u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_32u 0
+
+static void in2_mri2_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_64 0
+
+static void in2_i2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64(get_field(f, i2));
+}
+#define SPEC_in2_i2 0
+
+static void in2_i2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint8_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_8u 0
+
+static void in2_i2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint16_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_16u 0
+
+static void in2_i2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint32_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_32u 0
+
+static void in2_i2_16u_shl(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    uint64_t i2 = (uint16_t)get_field(f, i2);
+    o->in2 = tcg_const_i64(i2 << s->insn->data);
+}
+#define SPEC_in2_i2_16u_shl 0
+
+static void in2_i2_32u_shl(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    uint64_t i2 = (uint32_t)get_field(f, i2);
+    o->in2 = tcg_const_i64(i2 << s->insn->data);
+}
+#define SPEC_in2_i2_32u_shl 0
+
+/* ====================================================================== */
+
+/* Find opc within the table of insns.  This is formulated as a switch
+   statement so that (1) we get compile-time notice of cut-paste errors
+   for duplicated opcodes, and (2) the compiler generates the binary
+   search tree, rather than us having to post-process the table.  */
+
+#define C(OPC, NM, FT, FC, I1, I2, P, W, OP, CC) \
+    D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, 0)
+
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) insn_ ## NM,
+
+enum DisasInsnEnum {
+#include "insn-data.def"
+};
+
+#undef D
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) {                       \
+    .opc = OPC,                                                             \
+    .fmt = FMT_##FT,                                                        \
+    .fac = FAC_##FC,                                                        \
+    .spec = SPEC_in1_##I1 | SPEC_in2_##I2 | SPEC_prep_##P | SPEC_wout_##W,  \
+    .name = #NM,                                                            \
+    .help_in1 = in1_##I1,                                                   \
+    .help_in2 = in2_##I2,                                                   \
+    .help_prep = prep_##P,                                                  \
+    .help_wout = wout_##W,                                                  \
+    .help_cout = cout_##CC,                                                 \
+    .help_op = op_##OP,                                                     \
+    .data = D                                                               \
+ },
+
+/* Allow 0 to be used for NULL in the table below.  */
+#define in1_0  NULL
+#define in2_0  NULL
+#define prep_0  NULL
+#define wout_0  NULL
+#define cout_0  NULL
+#define op_0  NULL
+
+#define SPEC_in1_0 0
+#define SPEC_in2_0 0
+#define SPEC_prep_0 0
+#define SPEC_wout_0 0
+
+static const DisasInsn insn_info[] = {
+#include "insn-data.def"
+};
+
+#undef D
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) \
+    case OPC: return &insn_info[insn_ ## NM];
+
+static const DisasInsn *lookup_opc(uint16_t opc)
+{
+    switch (opc) {
+#include "insn-data.def"
+    default:
+        return NULL;
+    }
+}
+
+#undef D
+#undef C
+
+/* Extract a field from the insn.  The INSN should be left-aligned in
+   the uint64_t so that we can more easily utilize the big-bit-endian
+   definitions we extract from the Principals of Operation.  */
+
+static void extract_field(DisasFields *o, const DisasField *f, uint64_t insn)
+{
+    uint32_t r, m;
+
+    if (f->size == 0) {
+        return;
+    }
+
+    /* Zero extract the field from the insn.  */
+    r = (insn << f->beg) >> (64 - f->size);
+
+    /* Sign-extend, or un-swap the field as necessary.  */
+    switch (f->type) {
+    case 0: /* unsigned */
         break;
-    case 0x70: /* STE R1,D2(X2,B2) [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_freg32(r1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
+    case 1: /* signed */
+        assert(f->size <= 32);
+        m = 1u << (f->size - 1);
+        r = (r ^ m) - m;
         break;
-    case 0x71: /* MS      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_gen_mul_i32(tmp32_1, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    case 2: /* dl+dh split, signed 20 bit. */
+        r = ((int8_t)r << 12) | (r >> 8);
         break;
-    case 0x78: /* LE     R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_freg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
+    default:
+        abort();
+    }
+
+    /* Validate that the "compressed" encoding we selected above is valid.
+       I.e. we havn't make two different original fields overlap.  */
+    assert(((o->presentC >> f->indexC) & 1) == 0);
+    o->presentC |= 1 << f->indexC;
+    o->presentO |= 1 << f->indexO;
+
+    o->c[f->indexC] = r;
+}
+
+/* Lookup the insn at the current PC, extracting the operands into O and
+   returning the info struct for the insn.  Returns NULL for invalid insn.  */
+
+static const DisasInsn *extract_insn(CPUS390XState *env, DisasContext *s,
+                                     DisasFields *f)
+{
+    uint64_t insn, pc = s->pc;
+    int op, op2, ilen;
+    const DisasInsn *info;
+
+    insn = ld_code2(env, pc);
+    op = (insn >> 8) & 0xff;
+    ilen = get_ilen(op);
+    s->next_pc = s->pc + ilen;
+
+    switch (ilen) {
+    case 2:
+        insn = insn << 48;
         break;
-#ifndef CONFIG_USER_ONLY
-    case 0x80: /* SSM      D2(B2)       [S] */
-        /* Set System Mask */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_andi_i64(tmp3, psw_mask, ~0xff00000000000000ULL);
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_shli_i64(tmp2, tmp2, 56);
-        tcg_gen_or_i64(psw_mask, tmp3, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    case 4:
+        insn = ld_code4(env, pc) << 32;
         break;
-    case 0x82: /* LPSW     D2(B2)       [S] */
-        /* Load PSW */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_addi_i64(tmp, tmp, 4);
-        tcg_gen_qemu_ld32u(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        /* we need to keep cc_op intact */
-        s->is_jmp = DISAS_JUMP;
+    case 6:
+        insn = (insn << 48) | (ld_code4(env, pc + 2) << 16);
         break;
-    case 0x83: /* DIAG     R1,R3,D2     [RS] */
-        /* Diagnose call (KVM hypercall) */
-        check_privileged(s, ilc);
-        potential_page_fault(s);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp32_1 = tcg_const_i32(insn & 0xfff);
-        tmp2 = load_reg(2);
-        tmp3 = load_reg(1);
-        gen_helper_diag(tmp2, tmp32_1, tmp2, tmp3);
-        store_reg(2, tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    default:
+        abort();
+    }
+
+    /* We can't actually determine the insn format until we've looked up
+       the full insn opcode.  Which we can't do without locating the
+       secondary opcode.  Assume by default that OP2 is at bit 40; for
+       those smaller insns that don't actually have a secondary opcode
+       this will correctly result in OP2 = 0. */
+    switch (op) {
+    case 0x01: /* E */
+    case 0x80: /* S */
+    case 0x82: /* S */
+    case 0x93: /* S */
+    case 0xb2: /* S, RRF, RRE */
+    case 0xb3: /* RRE, RRD, RRF */
+    case 0xb9: /* RRE, RRF */
+    case 0xe5: /* SSE, SIL */
+        op2 = (insn << 8) >> 56;
+        break;
+    case 0xa5: /* RI */
+    case 0xa7: /* RI */
+    case 0xc0: /* RIL */
+    case 0xc2: /* RIL */
+    case 0xc4: /* RIL */
+    case 0xc6: /* RIL */
+    case 0xc8: /* SSF */
+    case 0xcc: /* RIL */
+        op2 = (insn << 12) >> 60;
+        break;
+    case 0xd0 ... 0xdf: /* SS */
+    case 0xe1: /* SS */
+    case 0xe2: /* SS */
+    case 0xe8: /* SS */
+    case 0xe9: /* SS */
+    case 0xea: /* SS */
+    case 0xee ... 0xf3: /* SS */
+    case 0xf8 ... 0xfd: /* SS */
+        op2 = 0;
         break;
-#endif
-    case 0x88: /* SRL    R1,D2(B2)        [RS] */
-    case 0x89: /* SLL    R1,D2(B2)        [RS] */
-    case 0x8a: /* SRA    R1,D2(B2)        [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp);
-        tcg_gen_andi_i32(tmp32_2, tmp32_2, 0x3f);
-        switch (opc) {
-        case 0x88:
-            tcg_gen_shr_i32(tmp32_1, tmp32_1, tmp32_2);
-            break;
-        case 0x89:
-            tcg_gen_shl_i32(tmp32_1, tmp32_1, tmp32_2);
-            break;
-        case 0x8a:
-            tcg_gen_sar_i32(tmp32_1, tmp32_1, tmp32_2);
-            set_cc_s32(s, tmp32_1);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    default:
+        op2 = (insn << 40) >> 56;
         break;
-    case 0x8c: /* SRDL   R1,D2(B2)        [RS] */
-    case 0x8d: /* SLDL   R1,D2(B2)        [RS] */
-    case 0x8e: /* SRDA   R1,D2(B2)        [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2); /* shift */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-        tcg_gen_concat_i32_i64(tmp2, tmp32_2, tmp32_1); /* operand */
-        switch (opc) {
-        case 0x8c:
-            tcg_gen_shr_i64(tmp2, tmp2, tmp);
-            break;
-        case 0x8d:
-            tcg_gen_shl_i64(tmp2, tmp2, tmp);
-            break;
-        case 0x8e:
-            tcg_gen_sar_i64(tmp2, tmp2, tmp);
-            set_cc_s64(s, tmp2);
-            break;
+    }
+
+    memset(f, 0, sizeof(*f));
+    f->op = op;
+    f->op2 = op2;
+
+    /* Lookup the instruction.  */
+    info = lookup_opc(op << 8 | op2);
+
+    /* If we found it, extract the operands.  */
+    if (info != NULL) {
+        DisasFormat fmt = info->fmt;
+        int i;
+
+        for (i = 0; i < NUM_C_FIELD; ++i) {
+            extract_field(f, &format_info[fmt].op[i], insn);
         }
-        tcg_gen_shri_i64(tmp, tmp2, 32);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        store_reg32(r1, tmp32_1);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        store_reg32(r1 + 1, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x98: /* LM     R1,R3,D2(B2)     [RS] */
-    case 0x90: /* STM    R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_const_i64(4);
-        tmp4 = tcg_const_i64(0xffffffff00000000ULL);
-        for (i = r1;; i = (i + 1) % 16) {
-            if (opc == 0x98) {
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_and_i64(regs[i], regs[i], tmp4);
-                tcg_gen_or_i64(regs[i], regs[i], tmp2);
-            } else {
-                tcg_gen_qemu_st32(regs[i], tmp, get_mem_index(s));
-            }
-            if (i == r3) {
-                break;
+    }
+    return info;
+}
+
+static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
+{
+    const DisasInsn *insn;
+    ExitStatus ret = NO_EXIT;
+    DisasFields f;
+    DisasOps o;
+
+    /* Search for the insn in the table.  */
+    insn = extract_insn(env, s, &f);
+
+    /* Not found means unimplemented/illegal opcode.  */
+    if (insn == NULL) {
+        qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%02x%02x\n",
+                      f.op, f.op2);
+        gen_illegal_opcode(s);
+        return EXIT_NORETURN;
+    }
+
+    /* Check for insn specification exceptions.  */
+    if (insn->spec) {
+        int spec = insn->spec, excp = 0, r;
+
+        if (spec & SPEC_r1_even) {
+            r = get_field(&f, r1);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
             }
-            tcg_gen_add_i64(tmp, tmp, tmp3);
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x91: /* TM     D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_const_i64(i2);
-        tcg_gen_qemu_ld8u(tmp, tmp, get_mem_index(s));
-        cmp_64(s, tmp, tmp2, CC_OP_TM_32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x92: /* MVI    D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_const_i64(i2);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x94: /* NI     D1(B1),I2        [SI] */
-    case 0x96: /* OI     D1(B1),I2        [SI] */
-    case 0x97: /* XI     D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        switch (opc) {
-        case 0x94:
-            tcg_gen_andi_i64(tmp2, tmp2, i2);
-            break;
-        case 0x96:
-            tcg_gen_ori_i64(tmp2, tmp2, i2);
-            break;
-        case 0x97:
-            tcg_gen_xori_i64(tmp2, tmp2, i2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        set_cc_nz_u64(s, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x95: /* CLI    D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        cmp_u64c(s, tmp2, i2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x9a: /* LAM      R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lam(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x9b: /* STAM     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stam(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xa5:
-        insn = ld_code4(s->pc);
-        r1 = (insn >> 20) & 0xf;
-        op = (insn >> 16) & 0xf;
-        i2 = insn & 0xffff;
-        disas_a5(s, op, r1, i2);
-        break;
-    case 0xa7:
-        insn = ld_code4(s->pc);
-        r1 = (insn >> 20) & 0xf;
-        op = (insn >> 16) & 0xf;
-        i2 = (short)insn;
-        disas_a7(s, op, r1, i2);
-        break;
-    case 0xa8: /* MVCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_mvcle(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xa9: /* CLCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_clcle(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xac: /* STNSM   D1(B1),I2     [SI] */
-    case 0xad: /* STOSM   D1(B1),I2     [SI] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_shri_i64(tmp2, psw_mask, 56);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        if (opc == 0xac) {
-            tcg_gen_andi_i64(psw_mask, psw_mask,
-                    ((uint64_t)i2 << 56) | 0x00ffffffffffffffULL);
-        } else {
-            tcg_gen_ori_i64(psw_mask, psw_mask, (uint64_t)i2 << 56);
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xae: /* SIGP   R1,R3,D2(B2)     [RS] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = load_reg(r3);
-        tmp32_1 = tcg_const_i32(r1);
-        potential_page_fault(s);
-        gen_helper_sigp(cc_op, tmp, tmp32_1, tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xb1: /* LRA    R1,D2(X2, B2)     [RX] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = tcg_const_i32(r1);
-        potential_page_fault(s);
-        gen_helper_lra(cc_op, tmp, tmp32_1);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-#endif
-    case 0xb2:
-        insn = ld_code4(s->pc);
-        op = (insn >> 16) & 0xff;
-        switch (op) {
-        case 0x9c: /* STFPC    D2(B2) [S] */
-            d2 = insn & 0xfff;
-            b2 = (insn >> 12) & 0xf;
-            tmp32_1 = tcg_temp_new_i32();
-            tmp = tcg_temp_new_i64();
-            tmp2 = get_address(s, 0, b2, d2);
-            tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-            tcg_gen_extu_i32_i64(tmp, tmp32_1);
-            tcg_gen_qemu_st32(tmp, tmp2, get_mem_index(s));
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i64(tmp2);
-            break;
-        default:
-            disas_b2(s, op, insn);
-            break;
+        if (spec & SPEC_r2_even) {
+            r = get_field(&f, r2);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        break;
-    case 0xb3:
-        insn = ld_code4(s->pc);
-        op = (insn >> 16) & 0xff;
-        r3 = (insn >> 12) & 0xf; /* aka m3 */
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        disas_b3(s, op, r3, r1, r2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xb6: /* STCTL     R1,R3,D2(B2)     [RS] */
-        /* Store Control */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stctl(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xb7: /* LCTL      R1,R3,D2(B2)     [RS] */
-        /* Load Control */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lctl(tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-#endif
-    case 0xb9:
-        insn = ld_code4(s->pc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        op = (insn >> 16) & 0xff;
-        disas_b9(s, op, r1, r2);
-        break;
-    case 0xba: /* CS     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_cs(cc_op, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbd: /* CLM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_clm(cc_op, tmp32_1, tmp32_2, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbe: /* STCM R1,M3,D2(B2) [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stcm(tmp32_1, tmp32_2, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbf: /* ICM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        if (r3 == 15) {
-            /* effectively a 32-bit load */
-            tmp = get_address(s, 0, b2, d2);
-            tmp32_1 = tcg_temp_new_i32();
-            tmp32_2 = tcg_const_i32(r3);
-            tcg_gen_qemu_ld32u(tmp, tmp, get_mem_index(s));
-            store_reg32_i64(r1, tmp);
-            tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-            set_cc_icm(s, tmp32_2, tmp32_1);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i32(tmp32_2);
-        } else if (r3) {
-            uint32_t mask = 0x00ffffffUL;
-            uint32_t shift = 24;
-            int m3 = r3;
-            tmp = get_address(s, 0, b2, d2);
-            tmp2 = tcg_temp_new_i64();
-            tmp32_1 = load_reg32(r1);
-            tmp32_2 = tcg_temp_new_i32();
-            tmp32_3 = tcg_const_i32(r3);
-            tmp32_4 = tcg_const_i32(0);
-            while (m3) {
-                if (m3 & 8) {
-                    tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-                    tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-                    if (shift) {
-                        tcg_gen_shli_i32(tmp32_2, tmp32_2, shift);
-                    }
-                    tcg_gen_andi_i32(tmp32_1, tmp32_1, mask);
-                    tcg_gen_or_i32(tmp32_1, tmp32_1, tmp32_2);
-                    tcg_gen_or_i32(tmp32_4, tmp32_4, tmp32_2);
-                    tcg_gen_addi_i64(tmp, tmp, 1);
-                }
-                m3 = (m3 << 1) & 0xf;
-                mask = (mask >> 8) | 0xff000000UL;
-                shift -= 8;
+        if (spec & SPEC_r3_even) {
+            r = get_field(&f, r3);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
             }
-            store_reg32(r1, tmp32_1);
-            set_cc_icm(s, tmp32_3, tmp32_4);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i64(tmp2);
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i32(tmp32_2);
-            tcg_temp_free_i32(tmp32_3);
-            tcg_temp_free_i32(tmp32_4);
-        } else {
-            /* i.e. env->cc = 0 */
-            gen_op_movi_cc(s, 0);
         }
-        break;
-    case 0xc0:
-    case 0xc2:
-        insn = ld_code6(s->pc);
-        r1 = (insn >> 36) & 0xf;
-        op = (insn >> 32) & 0xf;
-        i2 = (int)insn;
-        switch (opc) {
-        case 0xc0:
-            disas_c0(s, op, r1, i2);
-            break;
-        case 0xc2:
-            disas_c2(s, op, r1, i2);
-            break;
-        default:
-            tcg_abort();
+        if (spec & SPEC_r1_f128) {
+            r = get_field(&f, r1);
+            if (r > 13) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        break;
-    case 0xd2: /* MVC    D1(L,B1),D2(B2)         [SS] */
-    case 0xd4: /* NC     D1(L,B1),D2(B2)         [SS] */
-    case 0xd5: /* CLC    D1(L,B1),D2(B2)         [SS] */
-    case 0xd6: /* OC     D1(L,B1),D2(B2)         [SS] */
-    case 0xd7: /* XC     D1(L,B1),D2(B2)         [SS] */
-    case 0xdc: /* TR     D1(L,B1),D2(B2)         [SS] */
-    case 0xf3: /* UNPK   D1(L1,B1),D2(L2,B2)     [SS] */
-        insn = ld_code6(s->pc);
-        vl = tcg_const_i32((insn >> 32) & 0xff);
-        b1 = (insn >> 28) & 0xf;
-        b2 = (insn >> 12) & 0xf;
-        d1 = (insn >> 16) & 0xfff;
-        d2 = insn & 0xfff;
-        tmp = get_address(s, 0, b1, d1);
-        tmp2 = get_address(s, 0, b2, d2);
-        switch (opc) {
-        case 0xd2:
-            gen_op_mvc(s, (insn >> 32) & 0xff, tmp, tmp2);
-            break;
-        case 0xd4:
-            potential_page_fault(s);
-            gen_helper_nc(cc_op, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xd5:
-            gen_op_clc(s, (insn >> 32) & 0xff, tmp, tmp2);
-            break;
-        case 0xd6:
-            potential_page_fault(s);
-            gen_helper_oc(cc_op, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xd7:
-            potential_page_fault(s);
-            gen_helper_xc(cc_op, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xdc:
-            potential_page_fault(s);
-            gen_helper_tr(vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xf3:
-            potential_page_fault(s);
-            gen_helper_unpk(vl, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
+        if (spec & SPEC_r2_f128) {
+            r = get_field(&f, r2);
+            if (r > 13) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xda: /* MVCP     D1(R1,B1),D2(B2),R3   [SS] */
-    case 0xdb: /* MVCS     D1(R1,B1),D2(B2),R3   [SS] */
-        check_privileged(s, ilc);
-        potential_page_fault(s);
-        insn = ld_code6(s->pc);
-        r1 = (insn >> 36) & 0xf;
-        r3 = (insn >> 32) & 0xf;
-        b1 = (insn >> 28) & 0xf;
-        d1 = (insn >> 16) & 0xfff;
-        b2 = (insn >> 12) & 0xf;
-        d2 = insn & 0xfff;
-        tmp = load_reg(r1);
-        /* XXX key in r3 */
-        tmp2 = get_address(s, 0, b1, d1);
-        tmp3 = get_address(s, 0, b2, d2);
-        if (opc == 0xda) {
-            gen_helper_mvcp(cc_op, tmp, tmp2, tmp3);
-        } else {
-            gen_helper_mvcs(cc_op, tmp, tmp2, tmp3);
+        if (excp) {
+            gen_program_exception(s, excp);
+            return EXIT_NORETURN;
         }
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-#endif
-    case 0xe3:
-        insn = ld_code6(s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        x2 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = ((int)((((insn >> 16) & 0xfff)
-           | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_e3(s, op,  r1, x2, b2, d2 );
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xe5:
-        /* Test Protection */
-        check_privileged(s, ilc);
-        insn = ld_code6(s->pc);
-        debug_insn(insn);
-        disas_e5(s, insn);
-        break;
-#endif
-    case 0xeb:
-        insn = ld_code6(s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        r3 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = ((int)((((insn >> 16) & 0xfff)
-           | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_eb(s, op, r1, r3, b2, d2);
-        break;
-    case 0xed:
-        insn = ld_code6(s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        x2 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = (short)((insn >> 16) & 0xfff);
-        r1b = (insn >> 12) & 0xf;
-        disas_ed(s, op, r1, x2, b2, d2, r1b);
-        break;
-    default:
-        qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%x\n", opc);
-        gen_illegal_opcode(s, ilc);
-        break;
     }
 
-    /* Instruction length is encoded in the opcode */
-    s->pc += (ilc * 2);
+    /* Set up the strutures we use to communicate with the helpers. */
+    s->insn = insn;
+    s->fields = &f;
+    o.g_out = o.g_out2 = o.g_in1 = o.g_in2 = false;
+    TCGV_UNUSED_I64(o.out);
+    TCGV_UNUSED_I64(o.out2);
+    TCGV_UNUSED_I64(o.in1);
+    TCGV_UNUSED_I64(o.in2);
+    TCGV_UNUSED_I64(o.addr1);
+
+    /* Implement the instruction.  */
+    if (insn->help_in1) {
+        insn->help_in1(s, &f, &o);
+    }
+    if (insn->help_in2) {
+        insn->help_in2(s, &f, &o);
+    }
+    if (insn->help_prep) {
+        insn->help_prep(s, &f, &o);
+    }
+    if (insn->help_op) {
+        ret = insn->help_op(s, &o);
+    }
+    if (insn->help_wout) {
+        insn->help_wout(s, &f, &o);
+    }
+    if (insn->help_cout) {
+        insn->help_cout(s, &o);
+    }
+
+    /* Free any temporaries created by the helpers.  */
+    if (!TCGV_IS_UNUSED_I64(o.out) && !o.g_out) {
+        tcg_temp_free_i64(o.out);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.out2) && !o.g_out2) {
+        tcg_temp_free_i64(o.out2);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.in1) && !o.g_in1) {
+        tcg_temp_free_i64(o.in1);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.in2) && !o.g_in2) {
+        tcg_temp_free_i64(o.in2);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.addr1)) {
+        tcg_temp_free_i64(o.addr1);
+    }
+
+    /* Advance to the next instruction.  */
+    s->pc = s->next_pc;
+    return ret;
 }
 
 static inline void gen_intermediate_code_internal(CPUS390XState *env,
@@ -5118,6 +4745,8 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
     int j, lj = -1;
     int num_insns, max_insns;
     CPUBreakpoint *bp;
+    ExitStatus status;
+    bool do_debug;
 
     pc_start = tb->pc;
 
@@ -5126,12 +4755,12 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
         pc_start &= 0x7fffffff;
     }
 
-    dc.pc = pc_start;
-    dc.is_jmp = DISAS_NEXT;
     dc.tb = tb;
+    dc.pc = pc_start;
     dc.cc_op = CC_OP_DYNAMIC;
+    do_debug = dc.singlestep_enabled = env->singlestep_enabled;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
 
@@ -5144,80 +4773,96 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
     gen_icount_start();
 
     do {
-        if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
-            QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
-                if (bp->pc == dc.pc) {
-                    gen_debug(&dc);
-                    break;
-                }
-            }
-        }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc.pc;
+            tcg_ctx.gen_opc_pc[lj] = dc.pc;
             gen_opc_cc_op[lj] = dc.cc_op;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (++num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
-#if defined(S390X_DEBUG_DISAS_VERBOSE)
-        LOG_DISAS("pc " TARGET_FMT_lx "\n",
-                  dc.pc);
-#endif
-        disas_s390_insn(&dc);
 
-        num_insns++;
-        if (env->singlestep_enabled) {
-            gen_debug(&dc);
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+            tcg_gen_debug_insn_start(dc.pc);
         }
-    } while (!dc.is_jmp && gen_opc_ptr < gen_opc_end && dc.pc < next_page_start
-             && num_insns < max_insns && !env->singlestep_enabled
-             && !singlestep);
 
-    if (!dc.is_jmp) {
-        update_psw_addr(&dc);
-    }
+        status = NO_EXIT;
+        if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
+            QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
+                if (bp->pc == dc.pc) {
+                    status = EXIT_PC_STALE;
+                    do_debug = true;
+                    break;
+                }
+            }
+        }
+        if (status == NO_EXIT) {
+            status = translate_one(env, &dc);
+        }
 
-    if (singlestep && dc.cc_op != CC_OP_DYNAMIC) {
-        gen_op_calc_cc(&dc);
-    } else {
-        /* next TB starts off with CC_OP_DYNAMIC, so make sure the cc op type
-           is in env */
-        gen_op_set_cc_op(&dc);
-    }
+        /* If we reach a page boundary, are single stepping,
+           or exhaust instruction count, stop generation.  */
+        if (status == NO_EXIT
+            && (dc.pc >= next_page_start
+                || tcg_ctx.gen_opc_ptr >= gen_opc_end
+                || num_insns >= max_insns
+                || singlestep
+                || env->singlestep_enabled)) {
+            status = EXIT_PC_STALE;
+        }
+    } while (status == NO_EXIT);
 
     if (tb->cflags & CF_LAST_IO) {
         gen_io_end();
     }
-    /* Generate the return instruction */
-    if (dc.is_jmp != DISAS_TB_JUMP) {
-        tcg_gen_exit_tb(0);
+
+    switch (status) {
+    case EXIT_GOTO_TB:
+    case EXIT_NORETURN:
+        break;
+    case EXIT_PC_STALE:
+        update_psw_addr(&dc);
+        /* FALLTHRU */
+    case EXIT_PC_UPDATED:
+        /* Next TB starts off with CC_OP_DYNAMIC, so make sure the
+           cc op type is in env */
+        update_cc_op(&dc);
+        /* Exit the TB, either by raising a debug exception or by return.  */
+        if (do_debug) {
+            gen_exception(EXCP_DEBUG);
+        } else {
+            tcg_gen_exit_tb(0);
+        }
+        break;
+    default:
+        abort();
     }
+
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc.pc - pc_start;
         tb->icount = num_insns;
     }
+
 #if defined(S390X_DEBUG_DISAS)
-    log_cpu_state_mask(CPU_LOG_TB_CPU, env, 0);
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc.pc - pc_start, 1);
+        log_target_disas(env, pc_start, dc.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
@@ -5236,7 +4881,7 @@ void gen_intermediate_code_pc (CPUS390XState *env, struct TranslationBlock *tb)
 void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, int pc_pos)
 {
     int cc_op;
-    env->psw.addr = gen_opc_pc[pc_pos];
+    env->psw.addr = tcg_ctx.gen_opc_pc[pc_pos];
     cc_op = gen_opc_cc_op[pc_pos];
     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
         env->cc_op = cc_op;
diff --git a/target-sh4/Makefile.objs b/target-sh4/Makefile.objs
index 2e0e093e1f9..ca20f21443b 100644
--- a/target-sh4/Makefile.objs
+++ b/target-sh4/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-sh4/cpu-qom.h b/target-sh4/cpu-qom.h
index c41164aa221..09573c9c34f 100644
--- a/target-sh4/cpu-qom.h
+++ b/target-sh4/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_SUPERH_CPU_QOM_H
 #define QEMU_SUPERH_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_SUPERH_CPU "superh-cpu"
 
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index a1a177fa88a..e4858a03ed2 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -31,7 +31,7 @@ static void superh_cpu_reset(CPUState *s)
     CPUSH4State *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index bf592227ee1..34e9b0acf7b 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -39,9 +39,9 @@
 
 #define CPUArchState struct CPUSH4State
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_PAGE_BITS 12	/* 4k XXXXX */
 
@@ -204,20 +204,20 @@ void sh4_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #if !defined(CONFIG_USER_ONLY)
 void cpu_sh4_invalidate_tlb(CPUSH4State *s);
 uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 #endif
 
@@ -230,8 +230,6 @@ static inline void cpu_set_tls(CPUSH4State *env, target_ulong newtls)
 
 void cpu_load_tlb(CPUSH4State * env);
 
-#include "softfloat.h"
-
 static inline CPUSH4State *cpu_init(const char *cpu_model)
 {
     SuperHCPU *cpu = cpu_sh4_init(cpu_model);
@@ -264,7 +262,7 @@ static inline void cpu_clone_regs(CPUSH4State *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Memory access type */
 enum {
@@ -371,12 +369,14 @@ static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc,
             | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 4 */
 }
 
-static inline bool cpu_has_work(CPUSH4State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUSH4State *env = &SUPERH_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUSH4State *env, TranslationBlock *tb)
 {
diff --git a/target-sh4/helper.c b/target-sh4/helper.c
index 5c57380f128..ddebc789646 100644
--- a/target-sh4/helper.c
+++ b/target-sh4/helper.c
@@ -503,7 +503,7 @@ int cpu_sh4_handle_mmu_fault(CPUSH4State * env, target_ulong address, int rw,
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUSH4State * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUSH4State * env, target_ulong addr)
 {
     target_ulong physical;
     int prot;
@@ -574,7 +574,7 @@ void cpu_load_tlb(CPUSH4State * env)
 }
 
 uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int index = (addr & 0x00000300) >> 8;
     tlb_t * entry = &s->itlb[index];
@@ -584,7 +584,7 @@ uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
            (entry->asid);
 }
 
-void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, hwaddr addr,
 				    uint32_t mem_value)
 {
     uint32_t vpn = (mem_value & 0xfffffc00) >> 10;
@@ -604,7 +604,7 @@ void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int array = (addr & 0x00800000) >> 23;
     int index = (addr & 0x00000300) >> 8;
@@ -626,7 +626,7 @@ uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
     }
 }
 
-void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value)
 {
     int array = (addr & 0x00800000) >> 23;
@@ -655,7 +655,7 @@ void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int index = (addr & 0x00003f00) >> 8;
     tlb_t * entry = &s->utlb[index];
@@ -667,7 +667,7 @@ uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
            (entry->asid);
 }
 
-void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, hwaddr addr,
 				    uint32_t mem_value)
 {
     int associate = addr & 0x0000080;
@@ -740,7 +740,7 @@ void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int array = (addr & 0x00800000) >> 23;
     int index = (addr & 0x00003f00) >> 8;
@@ -766,7 +766,7 @@ uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
     }
 }
 
-void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value)
 {
     int array = (addr & 0x00800000) >> 23;
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index 95e3c7c8f77..7162448497f 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -1,54 +1,50 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_0(ldtlb, void)
-DEF_HELPER_0(raise_illegal_instruction, void)
-DEF_HELPER_0(raise_slot_illegal_instruction, void)
-DEF_HELPER_0(raise_fpu_disable, void)
-DEF_HELPER_0(raise_slot_fpu_disable, void)
-DEF_HELPER_0(debug, void)
-DEF_HELPER_1(sleep, void, i32)
-DEF_HELPER_1(trapa, void, i32)
+DEF_HELPER_1(ldtlb, void, env)
+DEF_HELPER_1(raise_illegal_instruction, noreturn, env)
+DEF_HELPER_1(raise_slot_illegal_instruction, noreturn, env)
+DEF_HELPER_1(raise_fpu_disable, noreturn, env)
+DEF_HELPER_1(raise_slot_fpu_disable, noreturn, env)
+DEF_HELPER_1(debug, noreturn, env)
+DEF_HELPER_1(sleep, noreturn, env)
+DEF_HELPER_2(trapa, noreturn, env, i32)
 
-DEF_HELPER_2(movcal, void, i32, i32)
-DEF_HELPER_0(discard_movcal_backup, void)
-DEF_HELPER_1(ocbi, void, i32)
+DEF_HELPER_3(movcal, void, env, i32, i32)
+DEF_HELPER_1(discard_movcal_backup, void, env)
+DEF_HELPER_2(ocbi, void, env, i32)
 
-DEF_HELPER_2(addv, i32, i32, i32)
-DEF_HELPER_2(addc, i32, i32, i32)
-DEF_HELPER_2(subv, i32, i32, i32)
-DEF_HELPER_2(subc, i32, i32, i32)
-DEF_HELPER_2(div1, i32, i32, i32)
-DEF_HELPER_2(macl, void, i32, i32)
-DEF_HELPER_2(macw, void, i32, i32)
+DEF_HELPER_3(div1, i32, env, i32, i32)
+DEF_HELPER_3(macl, void, env, i32, i32)
+DEF_HELPER_3(macw, void, env, i32, i32)
 
-DEF_HELPER_1(ld_fpscr, void, i32)
+DEF_HELPER_2(ld_fpscr, void, env, i32)
 
-DEF_HELPER_1(fabs_FT, f32, f32)
-DEF_HELPER_1(fabs_DT, f64, f64)
-DEF_HELPER_2(fadd_FT, f32, f32, f32)
-DEF_HELPER_2(fadd_DT, f64, f64, f64)
-DEF_HELPER_1(fcnvsd_FT_DT, f64, f32)
-DEF_HELPER_1(fcnvds_DT_FT, f32, f64)
+DEF_HELPER_FLAGS_1(fabs_FT, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_FLAGS_1(fabs_DT, TCG_CALL_NO_RWG_SE, f64, f64)
+DEF_HELPER_3(fadd_FT, f32, env, f32, f32)
+DEF_HELPER_3(fadd_DT, f64, env, f64, f64)
+DEF_HELPER_2(fcnvsd_FT_DT, f64, env, f32)
+DEF_HELPER_2(fcnvds_DT_FT, f32, env, f64)
 
-DEF_HELPER_2(fcmp_eq_FT, void, f32, f32)
-DEF_HELPER_2(fcmp_eq_DT, void, f64, f64)
-DEF_HELPER_2(fcmp_gt_FT, void, f32, f32)
-DEF_HELPER_2(fcmp_gt_DT, void, f64, f64)
-DEF_HELPER_2(fdiv_FT, f32, f32, f32)
-DEF_HELPER_2(fdiv_DT, f64, f64, f64)
-DEF_HELPER_1(float_FT, f32, i32)
-DEF_HELPER_1(float_DT, f64, i32)
-DEF_HELPER_3(fmac_FT, f32, f32, f32, f32)
-DEF_HELPER_2(fmul_FT, f32, f32, f32)
-DEF_HELPER_2(fmul_DT, f64, f64, f64)
-DEF_HELPER_1(fneg_T, f32, f32)
-DEF_HELPER_2(fsub_FT, f32, f32, f32)
-DEF_HELPER_2(fsub_DT, f64, f64, f64)
-DEF_HELPER_1(fsqrt_FT, f32, f32)
-DEF_HELPER_1(fsqrt_DT, f64, f64)
-DEF_HELPER_1(ftrc_FT, i32, f32)
-DEF_HELPER_1(ftrc_DT, i32, f64)
-DEF_HELPER_2(fipr, void, i32, i32)
-DEF_HELPER_1(ftrv, void, i32)
+DEF_HELPER_3(fcmp_eq_FT, void, env, f32, f32)
+DEF_HELPER_3(fcmp_eq_DT, void, env, f64, f64)
+DEF_HELPER_3(fcmp_gt_FT, void, env, f32, f32)
+DEF_HELPER_3(fcmp_gt_DT, void, env, f64, f64)
+DEF_HELPER_3(fdiv_FT, f32, env, f32, f32)
+DEF_HELPER_3(fdiv_DT, f64, env, f64, f64)
+DEF_HELPER_2(float_FT, f32, env, i32)
+DEF_HELPER_2(float_DT, f64, env, i32)
+DEF_HELPER_4(fmac_FT, f32, env, f32, f32, f32)
+DEF_HELPER_3(fmul_FT, f32, env, f32, f32)
+DEF_HELPER_3(fmul_DT, f64, env, f64, f64)
+DEF_HELPER_FLAGS_1(fneg_T, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_3(fsub_FT, f32, env, f32, f32)
+DEF_HELPER_3(fsub_DT, f64, env, f64, f64)
+DEF_HELPER_2(fsqrt_FT, f32, env, f32)
+DEF_HELPER_2(fsqrt_DT, f64, env, f64)
+DEF_HELPER_2(ftrc_FT, i32, env, f32)
+DEF_HELPER_2(ftrc_DT, i32, env, f64)
+DEF_HELPER_3(fipr, void, env, i32, i32)
+DEF_HELPER_2(ftrv, void, env, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index 40547911cd7..09e3d23aff1 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -19,60 +19,43 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
-static void cpu_restore_state_from_retaddr(uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-
-    if (retaddr) {
-        tb = tb_find_pc(retaddr);
-        if (tb) {
-            /* the PC is inside the translated code. It means that we have
-               a virtual CPU fault */
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
 #ifndef CONFIG_USER_ONLY
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
-void tlb_fill(CPUSH4State *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUSH4State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    CPUSH4State *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_sh4_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
         /* now we have a real cpu fault */
-        cpu_restore_state_from_retaddr(retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
 #endif
 
-void helper_ldtlb(void)
+void helper_ldtlb(CPUSH4State *env)
 {
 #ifdef CONFIG_USER_ONLY
     /* XXXXX */
@@ -82,55 +65,55 @@ void helper_ldtlb(void)
 #endif
 }
 
-static inline void raise_exception(int index, uintptr_t retaddr)
+static inline void QEMU_NORETURN raise_exception(CPUSH4State *env, int index,
+                                                 uintptr_t retaddr)
 {
     env->exception_index = index;
-    cpu_restore_state_from_retaddr(retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     cpu_loop_exit(env);
 }
 
-void helper_raise_illegal_instruction(void)
+void helper_raise_illegal_instruction(CPUSH4State *env)
 {
-    raise_exception(0x180, GETPC());
+    raise_exception(env, 0x180, 0);
 }
 
-void helper_raise_slot_illegal_instruction(void)
+void helper_raise_slot_illegal_instruction(CPUSH4State *env)
 {
-    raise_exception(0x1a0, GETPC());
+    raise_exception(env, 0x1a0, 0);
 }
 
-void helper_raise_fpu_disable(void)
+void helper_raise_fpu_disable(CPUSH4State *env)
 {
-    raise_exception(0x800, GETPC());
+    raise_exception(env, 0x800, 0);
 }
 
-void helper_raise_slot_fpu_disable(void)
+void helper_raise_slot_fpu_disable(CPUSH4State *env)
 {
-    raise_exception(0x820, GETPC());
+    raise_exception(env, 0x820, 0);
 }
 
-void helper_debug(void)
+void helper_debug(CPUSH4State *env)
 {
-    env->exception_index = EXCP_DEBUG;
-    cpu_loop_exit(env);
+    raise_exception(env, EXCP_DEBUG, 0);
 }
 
-void helper_sleep(uint32_t next_pc)
+void helper_sleep(CPUSH4State *env)
 {
     env->halted = 1;
     env->in_sleep = 1;
-    env->exception_index = EXCP_HLT;
-    env->pc = next_pc;
-    cpu_loop_exit(env);
+    raise_exception(env, EXCP_HLT, 0);
 }
 
-void helper_trapa(uint32_t tra)
+void helper_trapa(CPUSH4State *env, uint32_t tra)
 {
     env->tra = tra << 2;
-    raise_exception(0x160, GETPC());
+    raise_exception(env, 0x160, 0);
 }
 
-void helper_movcal(uint32_t address, uint32_t value)
+void helper_movcal(CPUSH4State *env, uint32_t address, uint32_t value)
 {
     if (cpu_sh4_is_cached (env, address))
     {
@@ -144,7 +127,7 @@ void helper_movcal(uint32_t address, uint32_t value)
     }
 }
 
-void helper_discard_movcal_backup(void)
+void helper_discard_movcal_backup(CPUSH4State *env)
 {
     memory_content *current = env->movcal_backup;
 
@@ -158,7 +141,7 @@ void helper_discard_movcal_backup(void)
     } 
 }
 
-void helper_ocbi(uint32_t address)
+void helper_ocbi(CPUSH4State *env, uint32_t address)
 {
     memory_content **current = &(env->movcal_backup);
     while (*current)
@@ -167,7 +150,7 @@ void helper_ocbi(uint32_t address)
 	if ((a & ~0x1F) == (address & ~0x1F))
 	{
 	    memory_content *next = (*current)->next;
-	    stl(a, (*current)->value);
+            cpu_stl_data(env, a, (*current)->value);
 	    
 	    if (next == NULL)
 	    {
@@ -181,51 +164,6 @@ void helper_ocbi(uint32_t address)
     }
 }
 
-uint32_t helper_addc(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t tmp0, tmp1;
-
-    tmp1 = arg0 + arg1;
-    tmp0 = arg1;
-    arg1 = tmp1 + (env->sr & 1);
-    if (tmp0 > tmp1)
-	env->sr |= SR_T;
-    else
-	env->sr &= ~SR_T;
-    if (tmp1 > arg1)
-	env->sr |= SR_T;
-    return arg1;
-}
-
-uint32_t helper_addv(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t dest, src, ans;
-
-    if ((int32_t) arg1 >= 0)
-	dest = 0;
-    else
-	dest = 1;
-    if ((int32_t) arg0 >= 0)
-	src = 0;
-    else
-	src = 1;
-    src += dest;
-    arg1 += arg0;
-    if ((int32_t) arg1 >= 0)
-	ans = 0;
-    else
-	ans = 1;
-    ans += dest;
-    if (src == 0 || src == 2) {
-	if (ans == 1)
-	    env->sr |= SR_T;
-	else
-	    env->sr &= ~SR_T;
-    } else
-	env->sr &= ~SR_T;
-    return arg1;
-}
-
 #define T (env->sr & SR_T)
 #define Q (env->sr & SR_Q ? 1 : 0)
 #define M (env->sr & SR_M ? 1 : 0)
@@ -236,7 +174,7 @@ uint32_t helper_addv(uint32_t arg0, uint32_t arg1)
 #define SETM env->sr |= SR_M
 #define CLRM env->sr &= ~SR_M
 
-uint32_t helper_div1(uint32_t arg0, uint32_t arg1)
+uint32_t helper_div1(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     uint32_t tmp0, tmp2;
     uint8_t old_q, tmp1 = 0xff;
@@ -344,7 +282,7 @@ uint32_t helper_div1(uint32_t arg0, uint32_t arg1)
     return arg1;
 }
 
-void helper_macl(uint32_t arg0, uint32_t arg1)
+void helper_macl(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     int64_t res;
 
@@ -360,7 +298,7 @@ void helper_macl(uint32_t arg0, uint32_t arg1)
     }
 }
 
-void helper_macw(uint32_t arg0, uint32_t arg1)
+void helper_macw(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     int64_t res;
 
@@ -379,62 +317,17 @@ void helper_macw(uint32_t arg0, uint32_t arg1)
     }
 }
 
-uint32_t helper_subc(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t tmp0, tmp1;
-
-    tmp1 = arg1 - arg0;
-    tmp0 = arg1;
-    arg1 = tmp1 - (env->sr & SR_T);
-    if (tmp0 < tmp1)
-	env->sr |= SR_T;
-    else
-	env->sr &= ~SR_T;
-    if (tmp1 < arg1)
-	env->sr |= SR_T;
-    return arg1;
-}
-
-uint32_t helper_subv(uint32_t arg0, uint32_t arg1)
-{
-    int32_t dest, src, ans;
-
-    if ((int32_t) arg1 >= 0)
-	dest = 0;
-    else
-	dest = 1;
-    if ((int32_t) arg0 >= 0)
-	src = 0;
-    else
-	src = 1;
-    src += dest;
-    arg1 -= arg0;
-    if ((int32_t) arg1 >= 0)
-	ans = 0;
-    else
-	ans = 1;
-    ans += dest;
-    if (src == 1) {
-	if (ans == 1)
-	    env->sr |= SR_T;
-	else
-	    env->sr &= ~SR_T;
-    } else
-	env->sr &= ~SR_T;
-    return arg1;
-}
-
-static inline void set_t(void)
+static inline void set_t(CPUSH4State *env)
 {
     env->sr |= SR_T;
 }
 
-static inline void clr_t(void)
+static inline void clr_t(CPUSH4State *env)
 {
     env->sr &= ~SR_T;
 }
 
-void helper_ld_fpscr(uint32_t val)
+void helper_ld_fpscr(CPUSH4State *env, uint32_t val)
 {
     env->fpscr = val & FPSCR_MASK;
     if ((val & FPSCR_RM_MASK) == FPSCR_RM_ZERO) {
@@ -445,7 +338,7 @@ void helper_ld_fpscr(uint32_t val)
     set_flush_to_zero((val & FPSCR_DN) != 0, &env->fp_status);
 }
 
-static void update_fpscr(uintptr_t retaddr)
+static void update_fpscr(CPUSH4State *env, uintptr_t retaddr)
 {
     int xcpt, cause, enable;
 
@@ -479,9 +372,7 @@ static void update_fpscr(uintptr_t retaddr)
         cause = (env->fpscr & FPSCR_CAUSE_MASK) >> FPSCR_CAUSE_SHIFT;
         enable = (env->fpscr & FPSCR_ENABLE_MASK) >> FPSCR_ENABLE_SHIFT;
         if (cause & enable) {
-            cpu_restore_state_from_retaddr(retaddr);
-            env->exception_index = 0x120;
-            cpu_loop_exit(env);
+            raise_exception(env, 0x120, retaddr);
         }
     }
 }
@@ -496,156 +387,155 @@ float64 helper_fabs_DT(float64 t0)
     return float64_abs(t0);
 }
 
-float32 helper_fadd_FT(float32 t0, float32 t1)
+float32 helper_fadd_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_add(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fadd_DT(float64 t0, float64 t1)
+float64 helper_fadd_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_add(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-void helper_fcmp_eq_FT(float32 t0, float32 t1)
+void helper_fcmp_eq_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_equal) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_eq_DT(float64 t0, float64 t1)
+void helper_fcmp_eq_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_equal) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_gt_FT(float32 t0, float32 t1)
+void helper_fcmp_gt_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_greater) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_gt_DT(float64 t0, float64 t1)
+void helper_fcmp_gt_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_greater) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-float64 helper_fcnvsd_FT_DT(float32 t0)
+float64 helper_fcnvsd_FT_DT(CPUSH4State *env, float32 t0)
 {
     float64 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float32_to_float64(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fcnvds_DT_FT(float64 t0)
+float32 helper_fcnvds_DT_FT(CPUSH4State *env, float64 t0)
 {
     float32 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float64_to_float32(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fdiv_FT(float32 t0, float32 t1)
+float32 helper_fdiv_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_div(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fdiv_DT(float64 t0, float64 t1)
+float64 helper_fdiv_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_div(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_float_FT(uint32_t t0)
+float32 helper_float_FT(CPUSH4State *env, uint32_t t0)
 {
     float32 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = int32_to_float32(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float64 helper_float_DT(uint32_t t0)
+float64 helper_float_DT(CPUSH4State *env, uint32_t t0)
 {
     float64 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = int32_to_float64(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fmac_FT(float32 t0, float32 t1, float32 t2)
+float32 helper_fmac_FT(CPUSH4State *env, float32 t0, float32 t1, float32 t2)
 {
     set_float_exception_flags(0, &env->fp_status);
-    t0 = float32_mul(t0, t1, &env->fp_status);
-    t0 = float32_add(t0, t2, &env->fp_status);
-    update_fpscr(GETPC());
+    t0 = float32_muladd(t0, t1, t2, 0, &env->fp_status);
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_fmul_FT(float32 t0, float32 t1)
+float32 helper_fmul_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_mul(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fmul_DT(float64 t0, float64 t1)
+float64 helper_fmul_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_mul(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
@@ -654,57 +544,57 @@ float32 helper_fneg_T(float32 t0)
     return float32_chs(t0);
 }
 
-float32 helper_fsqrt_FT(float32 t0)
+float32 helper_fsqrt_FT(CPUSH4State *env, float32 t0)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_sqrt(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fsqrt_DT(float64 t0)
+float64 helper_fsqrt_DT(CPUSH4State *env, float64 t0)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_sqrt(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_fsub_FT(float32 t0, float32 t1)
+float32 helper_fsub_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_sub(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fsub_DT(float64 t0, float64 t1)
+float64 helper_fsub_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_sub(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-uint32_t helper_ftrc_FT(float32 t0)
+uint32_t helper_ftrc_FT(CPUSH4State *env, float32 t0)
 {
     uint32_t ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float32_to_int32_round_to_zero(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-uint32_t helper_ftrc_DT(float64 t0)
+uint32_t helper_ftrc_DT(CPUSH4State *env, float64 t0)
 {
     uint32_t ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float64_to_int32_round_to_zero(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-void helper_fipr(uint32_t m, uint32_t n)
+void helper_fipr(CPUSH4State *env, uint32_t m, uint32_t n)
 {
     int bank, i;
     float32 r, p;
@@ -719,12 +609,12 @@ void helper_fipr(uint32_t m, uint32_t n)
                         &env->fp_status);
         r = float32_add(r, p, &env->fp_status);
     }
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
 
     env->fregs[bank + n + 3] = r;
 }
 
-void helper_ftrv(uint32_t n)
+void helper_ftrv(CPUSH4State *env, uint32_t n)
 {
     int bank_matrix, bank_vector;
     int i, j;
@@ -743,7 +633,7 @@ void helper_ftrv(uint32_t n)
             r[i] = float32_add(r[i], p, &env->fp_status);
         }
     }
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
 
     for (i = 0 ; i < 4 ; i++) {
         env->fregs[bank_vector + i] = r[i];
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 6532ad2ade7..260aaab5596 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -18,11 +18,10 @@
  */
 
 #define DEBUG_DISAS
-#define SH4_DEBUG_DISAS
 //#define SH4_SINGLE_STEP
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -32,8 +31,6 @@
 typedef struct DisasContext {
     struct TranslationBlock *tb;
     target_ulong pc;
-    uint32_t sr;
-    uint32_t fpscr;
     uint16_t opcode;
     uint32_t flags;
     int bstate;
@@ -47,7 +44,7 @@ typedef struct DisasContext {
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(ctx) 1
 #else
-#define IS_USER(ctx) (!(ctx->sr & SR_MD))
+#define IS_USER(ctx) (!(ctx->flags & SR_MD))
 #endif
 
 enum {
@@ -72,7 +69,7 @@ static TCGv cpu_flags, cpu_delayed_pc;
 
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static void sh4_translate_init(void)
 {
@@ -276,7 +273,7 @@ static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest)
     } else {
         tcg_gen_movi_i32(cpu_pc, dest);
         if (ctx->singlestep_enabled)
-            gen_helper_debug();
+            gen_helper_debug(cpu_env);
         tcg_gen_exit_tb(0);
     }
 }
@@ -288,7 +285,7 @@ static void gen_jump(DisasContext * ctx)
 	   delayed jump as immediate jump are conditinal jumps */
 	tcg_gen_mov_i32(cpu_pc, cpu_delayed_pc);
 	if (ctx->singlestep_enabled)
-	    gen_helper_debug();
+            gen_helper_debug(cpu_env);
 	tcg_gen_exit_tb(0);
     } else {
 	gen_goto_tb(ctx, 0, ctx->delayed_pc);
@@ -339,16 +336,6 @@ static void gen_delayed_conditional_jump(DisasContext * ctx)
     gen_jump(ctx);
 }
 
-static inline void gen_set_t(void)
-{
-    tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_T);
-}
-
-static inline void gen_clr_t(void)
-{
-    tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-}
-
 static inline void gen_cmp(int cond, TCGv t0, TCGv t1)
 {
     TCGv t;
@@ -423,44 +410,47 @@ static inline void gen_store_fpr64 (TCGv_i64 t, int reg)
 #define B11_8 ((ctx->opcode >> 8) & 0xf)
 #define B15_12 ((ctx->opcode >> 12) & 0xf)
 
-#define REG(x) ((x) < 8 && (ctx->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB) ? \
-		(cpu_gregs[x + 16]) : (cpu_gregs[x]))
+#define REG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) == (SR_MD | SR_RB) \
+                ? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
-#define ALTREG(x) ((x) < 8 && (ctx->sr & (SR_MD | SR_RB)) != (SR_MD | SR_RB) \
+#define ALTREG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) != (SR_MD | SR_RB)\
 		? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
-#define FREG(x) (ctx->fpscr & FPSCR_FR ? (x) ^ 0x10 : (x))
+#define FREG(x) (ctx->flags & FPSCR_FR ? (x) ^ 0x10 : (x))
 #define XHACK(x) ((((x) & 1 ) << 4) | ((x) & 0xe))
-#define XREG(x) (ctx->fpscr & FPSCR_FR ? XHACK(x) ^ 0x10 : XHACK(x))
+#define XREG(x) (ctx->flags & FPSCR_FR ? XHACK(x) ^ 0x10 : XHACK(x))
 #define DREG(x) FREG(x) /* Assumes lsb of (x) is always 0 */
 
 #define CHECK_NOT_DELAY_SLOT \
   if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL))     \
   {                                                           \
-      gen_helper_raise_slot_illegal_instruction();            \
-      ctx->bstate = BS_EXCP;                                  \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                      \
+      gen_helper_raise_slot_illegal_instruction(cpu_env);     \
+      ctx->bstate = BS_BRANCH;                                \
       return;                                                 \
   }
 
 #define CHECK_PRIVILEGED                                        \
   if (IS_USER(ctx)) {                                           \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                        \
       if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) { \
-         gen_helper_raise_slot_illegal_instruction();           \
+          gen_helper_raise_slot_illegal_instruction(cpu_env);   \
       } else {                                                  \
-         gen_helper_raise_illegal_instruction();                \
+          gen_helper_raise_illegal_instruction(cpu_env);        \
       }                                                         \
-      ctx->bstate = BS_EXCP;                                    \
+      ctx->bstate = BS_BRANCH;                                  \
       return;                                                   \
   }
 
 #define CHECK_FPU_ENABLED                                       \
   if (ctx->flags & SR_FD) {                                     \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                        \
       if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) { \
-          gen_helper_raise_slot_fpu_disable();                  \
+          gen_helper_raise_slot_fpu_disable(cpu_env);           \
       } else {                                                  \
-          gen_helper_raise_fpu_disable();                       \
+          gen_helper_raise_fpu_disable(cpu_env);                \
       }                                                         \
-      ctx->bstate = BS_EXCP;                                    \
+      ctx->bstate = BS_BRANCH;                                  \
       return;                                                   \
   }
 
@@ -492,7 +482,7 @@ static void _decode_opc(DisasContext * ctx)
 	  if (opcode != 0x0093 /* ocbi */
 	      && opcode != 0x00c3 /* movca.l */)
 	      {
-		  gen_helper_discard_movcal_backup ();
+                  gen_helper_discard_movcal_backup(cpu_env);
 		  ctx->has_movcal = 0;
 	      }
 	}
@@ -519,11 +509,11 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_S);
 	return;
     case 0x0008:		/* clrt */
-	gen_clr_t();
+        tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
 	return;
     case 0x0038:		/* ldtlb */
 	CHECK_PRIVILEGED
-	gen_helper_ldtlb();
+        gen_helper_ldtlb(cpu_env);
 	return;
     case 0x002b:		/* rte */
 	CHECK_PRIVILEGED
@@ -537,21 +527,22 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_S);
 	return;
     case 0x0018:		/* sett */
-	gen_set_t();
+        tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_T);
 	return;
     case 0xfbfd:		/* frchg */
 	tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_FR);
 	ctx->bstate = BS_STOP;
 	return;
     case 0xf3fd:		/* fschg */
-	tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_SZ);
+        tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_SZ);
 	ctx->bstate = BS_STOP;
 	return;
     case 0x0009:		/* nop */
 	return;
     case 0x001b:		/* sleep */
 	CHECK_PRIVILEGED
-	gen_helper_sleep(tcg_const_i32(ctx->pc + 2));
+        tcg_gen_movi_i32(cpu_pc, ctx->pc + 2);
+        gen_helper_sleep(cpu_env);
 	return;
     }
 
@@ -732,17 +723,7 @@ static void _decode_opc(DisasContext * ctx)
 	}
 	return;
     case 0x6009:		/* swap.w Rm,Rn */
-	{
-	    TCGv high, low;
-	    high = tcg_temp_new();
-	    tcg_gen_shli_i32(high, REG(B7_4), 16);
-	    low = tcg_temp_new();
-	    tcg_gen_shri_i32(low, REG(B7_4), 16);
-	    tcg_gen_ext16u_i32(low, low);
-	    tcg_gen_or_i32(REG(B11_8), high, low);
-	    tcg_temp_free(low);
-	    tcg_temp_free(high);
-	}
+        tcg_gen_rotli_i32(REG(B11_8), REG(B7_4), 16);
 	return;
     case 0x200d:		/* xtrct Rm,Rn */
 	{
@@ -751,7 +732,6 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_shli_i32(high, REG(B7_4), 16);
 	    low = tcg_temp_new();
 	    tcg_gen_shri_i32(low, REG(B11_8), 16);
-	    tcg_gen_ext16u_i32(low, low);
 	    tcg_gen_or_i32(REG(B11_8), high, low);
 	    tcg_temp_free(low);
 	    tcg_temp_free(high);
@@ -761,10 +741,43 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_add_i32(REG(B11_8), REG(B11_8), REG(B7_4));
 	return;
     case 0x300e:		/* addc Rm,Rn */
-	gen_helper_addc(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            t1 = tcg_temp_new();
+            tcg_gen_add_i32(t1, REG(B7_4), REG(B11_8));
+            tcg_gen_add_i32(t0, t0, t1);
+            t2 = tcg_temp_new();
+            tcg_gen_setcond_i32(TCG_COND_GTU, t2, REG(B11_8), t1);
+            tcg_gen_setcond_i32(TCG_COND_GTU, t1, t1, t0);
+            tcg_gen_or_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x300f:		/* addv Rm,Rn */
-	gen_helper_addv(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_add_i32(t0, REG(B7_4), REG(B11_8));
+            t1 = tcg_temp_new();
+            tcg_gen_xor_i32(t1, t0, REG(B11_8));
+            t2 = tcg_temp_new();
+            tcg_gen_xor_i32(t2, REG(B7_4), REG(B11_8));
+            tcg_gen_andc_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_shri_i32(t1, t1, 31);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B7_4), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x2009:		/* and Rm,Rn */
 	tcg_gen_and_i32(REG(B11_8), REG(B11_8), REG(B7_4));
@@ -817,7 +830,7 @@ static void _decode_opc(DisasContext * ctx)
 	}
 	return;
     case 0x3004:		/* div1 Rm,Rn */
-	gen_helper_div1(REG(B11_8), REG(B7_4), REG(B11_8));
+        gen_helper_div1(REG(B11_8), cpu_env, REG(B7_4), REG(B11_8));
 	return;
     case 0x300d:		/* dmuls.l Rm,Rn */
 	{
@@ -870,7 +883,7 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_qemu_ld32s(arg0, REG(B7_4), ctx->memidx);
 	    arg1 = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(arg1, REG(B11_8), ctx->memidx);
-	    gen_helper_macl(arg0, arg1);
+            gen_helper_macl(cpu_env, arg0, arg1);
 	    tcg_temp_free(arg1);
 	    tcg_temp_free(arg0);
 	    tcg_gen_addi_i32(REG(B7_4), REG(B7_4), 4);
@@ -884,7 +897,7 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_qemu_ld32s(arg0, REG(B7_4), ctx->memidx);
 	    arg1 = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(arg1, REG(B11_8), ctx->memidx);
-	    gen_helper_macw(arg0, arg1);
+            gen_helper_macw(cpu_env, arg0, arg1);
 	    tcg_temp_free(arg1);
 	    tcg_temp_free(arg0);
 	    tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 2);
@@ -1013,10 +1026,43 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_sub_i32(REG(B11_8), REG(B11_8), REG(B7_4));
 	return;
     case 0x300a:		/* subc Rm,Rn */
-	gen_helper_subc(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            t1 = tcg_temp_new();
+            tcg_gen_sub_i32(t1, REG(B11_8), REG(B7_4));
+            tcg_gen_sub_i32(t0, t1, t0);
+            t2 = tcg_temp_new();
+            tcg_gen_setcond_i32(TCG_COND_LTU, t2, REG(B11_8), t1);
+            tcg_gen_setcond_i32(TCG_COND_LTU, t1, t1, t0);
+            tcg_gen_or_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x300b:		/* subv Rm,Rn */
-	gen_helper_subv(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_sub_i32(t0, REG(B11_8), REG(B7_4));
+            t1 = tcg_temp_new();
+            tcg_gen_xor_i32(t1, t0, REG(B7_4));
+            t2 = tcg_temp_new();
+            tcg_gen_xor_i32(t2, REG(B11_8), REG(B7_4));
+            tcg_gen_and_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_shri_i32(t1, t1, 31);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x2008:		/* tst Rm,Rn */
 	{
@@ -1031,7 +1077,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00c: /* fmov {F,D,X}Rm,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, XREG(B7_4));
 	    gen_store_fpr64(fp, XREG(B11_8));
@@ -1042,7 +1088,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00a: /* fmov {F,D,X}Rm,@Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B7_4);
 	    tcg_gen_addi_i32(addr_hi, REG(B11_8), 4);
@@ -1055,7 +1101,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf008: /* fmov @Rm,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B11_8);
 	    tcg_gen_addi_i32(addr_hi, REG(B7_4), 4);
@@ -1068,7 +1114,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf009: /* fmov @Rm+,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B11_8);
 	    tcg_gen_addi_i32(addr_hi, REG(B7_4), 4);
@@ -1083,7 +1129,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00b: /* fmov {F,D,X}Rm,@-Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr = tcg_temp_new_i32();
 	    int fr = XREG(B7_4);
 	    tcg_gen_subi_i32(addr, REG(B11_8), 4);
@@ -1106,7 +1152,7 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv addr = tcg_temp_new_i32();
 	    tcg_gen_add_i32(addr, REG(B7_4), REG(0));
-	    if (ctx->fpscr & FPSCR_SZ) {
+            if (ctx->flags & FPSCR_SZ) {
 		int fr = XREG(B11_8);
 		tcg_gen_qemu_ld32u(cpu_fregs[fr	 ], addr, ctx->memidx);
 		tcg_gen_addi_i32(addr, addr, 4);
@@ -1122,7 +1168,7 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv addr = tcg_temp_new();
 	    tcg_gen_add_i32(addr, REG(B11_8), REG(0));
-	    if (ctx->fpscr & FPSCR_SZ) {
+            if (ctx->flags & FPSCR_SZ) {
 		int fr = XREG(B7_4);
 		tcg_gen_qemu_ld32u(cpu_fregs[fr	 ], addr, ctx->memidx);
 		tcg_gen_addi_i32(addr, addr, 4);
@@ -1141,7 +1187,7 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf005: /* fcmp/gt Rm,Rn - FPSCR: R[PR,Enable.V]/W[Cause,Flag] */
 	{
 	    CHECK_FPU_ENABLED
-	    if (ctx->fpscr & FPSCR_PR) {
+            if (ctx->flags & FPSCR_PR) {
                 TCGv_i64 fp0, fp1;
 
 		if (ctx->opcode & 0x0110)
@@ -1152,22 +1198,22 @@ static void _decode_opc(DisasContext * ctx)
 		gen_load_fpr64(fp1, DREG(B7_4));
                 switch (ctx->opcode & 0xf00f) {
                 case 0xf000:		/* fadd Rm,Rn */
-                    gen_helper_fadd_DT(fp0, fp0, fp1);
+                    gen_helper_fadd_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf001:		/* fsub Rm,Rn */
-                    gen_helper_fsub_DT(fp0, fp0, fp1);
+                    gen_helper_fsub_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf002:		/* fmul Rm,Rn */
-                    gen_helper_fmul_DT(fp0, fp0, fp1);
+                    gen_helper_fmul_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf003:		/* fdiv Rm,Rn */
-                    gen_helper_fdiv_DT(fp0, fp0, fp1);
+                    gen_helper_fdiv_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf004:		/* fcmp/eq Rm,Rn */
-                    gen_helper_fcmp_eq_DT(fp0, fp1);
+                    gen_helper_fcmp_eq_DT(cpu_env, fp0, fp1);
                     return;
                 case 0xf005:		/* fcmp/gt Rm,Rn */
-                    gen_helper_fcmp_gt_DT(fp0, fp1);
+                    gen_helper_fcmp_gt_DT(cpu_env, fp0, fp1);
                     return;
                 }
 		gen_store_fpr64(fp0, DREG(B11_8));
@@ -1176,22 +1222,32 @@ static void _decode_opc(DisasContext * ctx)
 	    } else {
                 switch (ctx->opcode & 0xf00f) {
                 case 0xf000:		/* fadd Rm,Rn */
-                    gen_helper_fadd_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fadd_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf001:		/* fsub Rm,Rn */
-                    gen_helper_fsub_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fsub_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf002:		/* fmul Rm,Rn */
-                    gen_helper_fmul_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fmul_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf003:		/* fdiv Rm,Rn */
-                    gen_helper_fdiv_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fdiv_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf004:		/* fcmp/eq Rm,Rn */
-                    gen_helper_fcmp_eq_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fcmp_eq_FT(cpu_env, cpu_fregs[FREG(B11_8)],
+                                          cpu_fregs[FREG(B7_4)]);
                     return;
                 case 0xf005:		/* fcmp/gt Rm,Rn */
-                    gen_helper_fcmp_gt_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fcmp_gt_FT(cpu_env, cpu_fregs[FREG(B11_8)],
+                                          cpu_fregs[FREG(B7_4)]);
                     return;
                 }
 	    }
@@ -1200,11 +1256,12 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf00e: /* fmac FR0,RM,Rn */
         {
             CHECK_FPU_ENABLED
-            if (ctx->fpscr & FPSCR_PR) {
+            if (ctx->flags & FPSCR_PR) {
                 break; /* illegal instruction */
             } else {
-                gen_helper_fmac_FT(cpu_fregs[FREG(B11_8)],
-                                   cpu_fregs[FREG(0)], cpu_fregs[FREG(B7_4)], cpu_fregs[FREG(B11_8)]);
+                gen_helper_fmac_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                   cpu_fregs[FREG(0)], cpu_fregs[FREG(B7_4)],
+                                   cpu_fregs[FREG(B11_8)]);
                 return;
             }
         }
@@ -1355,8 +1412,9 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv imm;
 	    CHECK_NOT_DELAY_SLOT
+            tcg_gen_movi_i32(cpu_pc, ctx->pc);
 	    imm = tcg_const_i32(B7_0);
-	    gen_helper_trapa(imm);
+            gen_helper_trapa(cpu_env, imm);
 	    tcg_temp_free(imm);
 	    ctx->bstate = BS_BRANCH;
 	}
@@ -1531,7 +1589,7 @@ static void _decode_opc(DisasContext * ctx)
 	LDST(fpul, 0x405a, 0x4056, 0x005a, 0x4052, {CHECK_FPU_ENABLED})
     case 0x406a:		/* lds Rm,FPSCR */
 	CHECK_FPU_ENABLED
-	gen_helper_ld_fpscr(REG(B11_8));
+        gen_helper_ld_fpscr(cpu_env, REG(B11_8));
 	ctx->bstate = BS_STOP;
 	return;
     case 0x4066:		/* lds.l @Rm+,FPSCR */
@@ -1540,7 +1598,7 @@ static void _decode_opc(DisasContext * ctx)
 	    TCGv addr = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(addr, REG(B11_8), ctx->memidx);
 	    tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 4);
-	    gen_helper_ld_fpscr(addr);
+            gen_helper_ld_fpscr(cpu_env, addr);
 	    tcg_temp_free(addr);
 	    ctx->bstate = BS_STOP;
 	}
@@ -1567,7 +1625,7 @@ static void _decode_opc(DisasContext * ctx)
         {
             TCGv val = tcg_temp_new();
             tcg_gen_qemu_ld32u(val, REG(B11_8), ctx->memidx);
-            gen_helper_movcal (REG(B11_8), val);            
+            gen_helper_movcal(cpu_env, REG(B11_8), val);
             tcg_gen_qemu_st32(REG(0), REG(B11_8), ctx->memidx);
         }
         ctx->has_movcal = 1;
@@ -1594,7 +1652,7 @@ static void _decode_opc(DisasContext * ctx)
         */
         if (ctx->features & SH_FEATURE_SH4A) {
 	    int label = gen_new_label();
-	    gen_clr_t();
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
 	    tcg_gen_or_i32(cpu_sr, cpu_sr, cpu_ldst);
 	    tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ldst, 0, label);
 	    tcg_gen_qemu_st32(REG(0), REG(B11_8), ctx->memidx);
@@ -1619,7 +1677,7 @@ static void _decode_opc(DisasContext * ctx)
 	    break;
     case 0x0093:		/* ocbi @Rn */
 	{
-	    gen_helper_ocbi (REG(B11_8));
+            gen_helper_ocbi(cpu_env, REG(B11_8));
 	}
 	return;
     case 0x00a3:		/* ocbp @Rn */
@@ -1728,32 +1786,32 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf02d: /* float FPUL,FRn/DRn - FPSCR: R[PR,Enable.I]/W[Cause,Flag] */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    TCGv_i64 fp;
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    fp = tcg_temp_new_i64();
-	    gen_helper_float_DT(fp, cpu_fpul);
+            gen_helper_float_DT(fp, cpu_env, cpu_fpul);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	}
 	else {
-	    gen_helper_float_FT(cpu_fregs[FREG(B11_8)], cpu_fpul);
+            gen_helper_float_FT(cpu_fregs[FREG(B11_8)], cpu_env, cpu_fpul);
 	}
 	return;
     case 0xf03d: /* ftrc FRm/DRm,FPUL - FPSCR: R[PR,Enable.V]/W[Cause,Flag] */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    TCGv_i64 fp;
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_ftrc_DT(cpu_fpul, fp);
+            gen_helper_ftrc_DT(cpu_fpul, cpu_env, fp);
 	    tcg_temp_free_i64(fp);
 	}
 	else {
-	    gen_helper_ftrc_FT(cpu_fpul, cpu_fregs[FREG(B11_8)]);
+            gen_helper_ftrc_FT(cpu_fpul, cpu_env, cpu_fregs[FREG(B11_8)]);
 	}
 	return;
     case 0xf04d: /* fneg FRn/DRn - FPSCR: Nothing */
@@ -1764,7 +1822,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf05d: /* fabs FRn/DRn */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    TCGv_i64 fp = tcg_temp_new_i64();
@@ -1778,16 +1836,17 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf06d: /* fsqrt FRn */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_fsqrt_DT(fp, fp);
+            gen_helper_fsqrt_DT(fp, cpu_env, fp);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	} else {
-	    gen_helper_fsqrt_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)]);
+            gen_helper_fsqrt_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                cpu_fregs[FREG(B11_8)]);
 	}
 	return;
     case 0xf07d: /* fsrra FRn */
@@ -1795,13 +1854,13 @@ static void _decode_opc(DisasContext * ctx)
 	break;
     case 0xf08d: /* fldi0 FRn - FPSCR: R[PR] */
 	CHECK_FPU_ENABLED
-	if (!(ctx->fpscr & FPSCR_PR)) {
+        if (!(ctx->flags & FPSCR_PR)) {
 	    tcg_gen_movi_i32(cpu_fregs[FREG(B11_8)], 0);
 	}
 	return;
     case 0xf09d: /* fldi1 FRn - FPSCR: R[PR] */
 	CHECK_FPU_ENABLED
-	if (!(ctx->fpscr & FPSCR_PR)) {
+        if (!(ctx->flags & FPSCR_PR)) {
 	    tcg_gen_movi_i32(cpu_fregs[FREG(B11_8)], 0x3f800000);
 	}
 	return;
@@ -1809,7 +1868,7 @@ static void _decode_opc(DisasContext * ctx)
 	CHECK_FPU_ENABLED
 	{
 	    TCGv_i64 fp = tcg_temp_new_i64();
-	    gen_helper_fcnvsd_FT_DT(fp, cpu_fpul);
+            gen_helper_fcnvsd_FT_DT(fp, cpu_env, cpu_fpul);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	}
@@ -1819,17 +1878,17 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_fcnvds_DT_FT(cpu_fpul, fp);
+            gen_helper_fcnvds_DT_FT(cpu_fpul, cpu_env, fp);
 	    tcg_temp_free_i64(fp);
 	}
 	return;
     case 0xf0ed: /* fipr FVm,FVn */
         CHECK_FPU_ENABLED
-        if ((ctx->fpscr & FPSCR_PR) == 0) {
+        if ((ctx->flags & FPSCR_PR) == 0) {
             TCGv m, n;
             m = tcg_const_i32((ctx->opcode >> 8) & 3);
             n = tcg_const_i32((ctx->opcode >> 10) & 3);
-            gen_helper_fipr(m, n);
+            gen_helper_fipr(cpu_env, m, n);
             tcg_temp_free(m);
             tcg_temp_free(n);
             return;
@@ -1838,10 +1897,10 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf0fd: /* ftrv XMTRX,FVn */
         CHECK_FPU_ENABLED
         if ((ctx->opcode & 0x0300) == 0x0100 &&
-            (ctx->fpscr & FPSCR_PR) == 0) {
+            (ctx->flags & FPSCR_PR) == 0) {
             TCGv n;
             n = tcg_const_i32((ctx->opcode >> 10) & 3);
-            gen_helper_ftrv(n);
+            gen_helper_ftrv(cpu_env, n);
             tcg_temp_free(n);
             return;
         }
@@ -1852,19 +1911,20 @@ static void _decode_opc(DisasContext * ctx)
 	    ctx->opcode, ctx->pc);
     fflush(stderr);
 #endif
+    tcg_gen_movi_i32(cpu_pc, ctx->pc);
     if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
-       gen_helper_raise_slot_illegal_instruction();
+        gen_helper_raise_slot_illegal_instruction(cpu_env);
     } else {
-       gen_helper_raise_illegal_instruction();
+        gen_helper_raise_illegal_instruction(cpu_env);
     }
-    ctx->bstate = BS_EXCP;
+    ctx->bstate = BS_BRANCH;
 }
 
 static void decode_opc(DisasContext * ctx)
 {
     uint32_t old_flags = ctx->flags;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(ctx->pc);
     }
 
@@ -1907,20 +1967,18 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.pc = pc_start;
     ctx.flags = (uint32_t)tb->flags;
     ctx.bstate = BS_NONE;
-    ctx.sr = env->sr;
-    ctx.fpscr = env->fpscr;
-    ctx.memidx = (env->sr & SR_MD) == 0 ? 1 : 0;
+    ctx.memidx = (ctx.flags & SR_MD) == 0 ? 1 : 0;
     /* We don't know if the delayed pc came from a dynamic or static branch,
        so assume it is a dynamic branch.  */
     ctx.delayed_pc = -1; /* use delayed pc from env pointer */
     ctx.tb = tb;
     ctx.singlestep_enabled = env->singlestep_enabled;
     ctx.features = env->features;
-    ctx.has_movcal = (tb->flags & TB_FLAG_PENDING_MOVCA);
+    ctx.has_movcal = (ctx.flags & TB_FLAG_PENDING_MOVCA);
 
     ii = -1;
     num_insns = 0;
@@ -1928,29 +1986,29 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     if (max_insns == 0)
         max_insns = CF_COUNT_MASK;
     gen_icount_start();
-    while (ctx.bstate == BS_NONE && gen_opc_ptr < gen_opc_end) {
+    while (ctx.bstate == BS_NONE && tcg_ctx.gen_opc_ptr < gen_opc_end) {
         if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (ctx.pc == bp->pc) {
 		    /* We have hit a breakpoint - make sure PC is up-to-date */
 		    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-		    gen_helper_debug();
-		    ctx.bstate = BS_EXCP;
+                    gen_helper_debug(cpu_env);
+                    ctx.bstate = BS_BRANCH;
 		    break;
 		}
 	    }
 	}
         if (search_pc) {
-            i = gen_opc_ptr - gen_opc_buf;
+            i = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (ii < i) {
                 ii++;
                 while (ii < i)
-                    gen_opc_instr_start[ii++] = 0;
+                    tcg_ctx.gen_opc_instr_start[ii++] = 0;
             }
-            gen_opc_pc[ii] = ctx.pc;
+            tcg_ctx.gen_opc_pc[ii] = ctx.pc;
             gen_opc_hflags[ii] = ctx.flags;
-            gen_opc_instr_start[ii] = 1;
-            gen_opc_icount[ii] = num_insns;
+            tcg_ctx.gen_opc_instr_start[ii] = 1;
+            tcg_ctx.gen_opc_icount[ii] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -1958,7 +2016,7 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
 	fprintf(stderr, "Loading opcode at address 0x%08x\n", ctx.pc);
 	fflush(stderr);
 #endif
-	ctx.opcode = lduw_code(ctx.pc);
+        ctx.opcode = cpu_lduw_code(env, ctx.pc);
 	decode_opc(&ctx);
         num_insns++;
 	ctx.pc += 2;
@@ -1975,7 +2033,7 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
         gen_io_end();
     if (env->singlestep_enabled) {
         tcg_gen_movi_i32(cpu_pc, ctx.pc);
-        gen_helper_debug();
+        gen_helper_debug(cpu_env);
     } else {
 	switch (ctx.bstate) {
         case BS_STOP:
@@ -1998,24 +2056,21 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        i = gen_opc_ptr - gen_opc_buf;
+        i = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         ii++;
         while (ii <= i)
-            gen_opc_instr_start[ii++] = 0;
+            tcg_ctx.gen_opc_instr_start[ii++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
     }
 
 #ifdef DEBUG_DISAS
-#ifdef SH4_DEBUG_DISAS
-    qemu_log_mask(CPU_LOG_TB_IN_ASM, "\n");
-#endif
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
 	qemu_log("IN:\n");	/* , lookup_symbol(pc_start)); */
-	log_target_disas(pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
 	qemu_log("\n");
     }
 #endif
@@ -2033,6 +2088,6 @@ void gen_intermediate_code_pc(CPUSH4State * env, struct TranslationBlock *tb)
 
 void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
     env->flags = gen_opc_hflags[pc_pos];
 }
diff --git a/target-sparc/Makefile.objs b/target-sparc/Makefile.objs
index a93e07deb11..9fc42ea9b02 100644
--- a/target-sparc/Makefile.objs
+++ b/target-sparc/Makefile.objs
@@ -4,5 +4,3 @@ obj-y += fop_helper.o cc_helper.o win_helper.o mmu_helper.o ldst_helper.o
 obj-$(TARGET_SPARC) += int32_helper.o
 obj-$(TARGET_SPARC64) += int64_helper.o
 obj-$(TARGET_SPARC64) += vis_helper.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-sparc/cpu-qom.h b/target-sparc/cpu-qom.h
index 3d3ac0fcefb..2a738ae360e 100644
--- a/target-sparc/cpu-qom.h
+++ b/target-sparc/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_SPARC_CPU_QOM_H
 #define QEMU_SPARC_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #ifdef TARGET_SPARC64
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index f7c004c7d86..f404aa8b5fb 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -31,7 +31,7 @@ static void sparc_cpu_reset(CPUState *s)
     CPUSPARCState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -643,7 +643,7 @@ static int cpu_sparc_find_by_name(sparc_def_t *cpu_def, const char *cpu_model)
 {
     unsigned int i;
     const sparc_def_t *def = NULL;
-    char *s = strdup(cpu_model);
+    char *s = g_strdup(cpu_model);
     char *featurestr, *name = strtok(s, ",");
     uint32_t plus_features = 0;
     uint32_t minus_features = 0;
@@ -735,7 +735,7 @@ static int cpu_sparc_find_by_name(sparc_def_t *cpu_def, const char *cpu_model)
 #ifdef DEBUG_FEATURES
     print_features(stderr, fprintf, cpu_def->features, NULL);
 #endif
-    free(s);
+    g_free(s);
     return 0;
 
  error:
@@ -792,7 +792,6 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
 
     cpu_fprintf(f, "pc: " TARGET_FMT_lx "  npc: " TARGET_FMT_lx "\n", env->pc,
                 env->npc);
-    cpu_fprintf(f, "General Registers:\n");
 
     for (i = 0; i < 8; i++) {
         if (i % REGS_PER_LINE == 0) {
@@ -803,7 +802,6 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
             cpu_fprintf(f, "\n");
         }
     }
-    cpu_fprintf(f, "\nCurrent Register Window:\n");
     for (x = 0; x < 3; x++) {
         for (i = 0; i < 8; i++) {
             if (i % REGS_PER_LINE == 0) {
@@ -817,10 +815,10 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
             }
         }
     }
-    cpu_fprintf(f, "\nFloating Point Registers:\n");
+
     for (i = 0; i < TARGET_DPREGS; i++) {
         if ((i & 3) == 0) {
-            cpu_fprintf(f, "%%f%02d:", i * 2);
+            cpu_fprintf(f, "%%f%02d: ", i * 2);
         }
         cpu_fprintf(f, " %016" PRIx64, env->fpr[i].ll);
         if ((i & 3) == 3) {
@@ -850,6 +848,7 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "fsr: " TARGET_FMT_lx " y: " TARGET_FMT_lx "\n",
                 env->fsr, env->y);
 #endif
+    cpu_fprintf(f, "\n");
 }
 
 static void sparc_cpu_initfn(Object *obj)
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index e16b7b35152..7389b03514e 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -3,7 +3,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 #if !defined(TARGET_SPARC64)
 #define TARGET_LONG_BITS 32
@@ -25,9 +25,9 @@
 
 #define CPUArchState struct CPUSPARCState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -392,7 +392,6 @@ struct CPUSPARCState {
     target_ulong cc_dst;
     uint32_t cc_op;
 
-    target_ulong t0, t1; /* temporaries live across basic blocks */
     target_ulong cond; /* conditional branch result (XXX: save it in a
                           temporary register when possible) */
 
@@ -583,10 +582,10 @@ static inline int tlb_compare_context(const SparcTLBEntry *tlb,
 
 /* cpu-exec.c */
 #if !defined(CONFIG_USER_ONLY)
-void cpu_unassigned_access(CPUSPARCState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env1, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size);
 #if defined(TARGET_SPARC64)
-target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
+hwaddr cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
                                            int mmu_idx);
 #endif
 #endif
@@ -702,7 +701,7 @@ static inline void cpu_clone_regs(CPUSPARCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 #ifdef TARGET_SPARC64
 /* sun4u.c */
@@ -711,9 +710,6 @@ uint64_t cpu_tick_get_count(CPUTimer *timer);
 void cpu_tick_set_limit(CPUTimer *timer, uint64_t limit);
 trap_state* cpu_tsptr(CPUSPARCState* env);
 #endif
-void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env, target_ulong addr,
-                                       int is_write, int is_user,
-                                       uintptr_t retaddr);
 
 #define TB_FLAG_FPU_ENABLED (1 << 4)
 #define TB_FLAG_AM_ENABLED (1 << 5)
@@ -763,13 +759,15 @@ static inline bool tb_am_enabled(int tb_flags)
 #endif
 }
 
-static inline bool cpu_has_work(CPUSPARCState *env1)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUSPARCState *env1 = &SPARC_CPU(cpu)->env;
+
     return (env1->interrupt_request & CPU_INTERRUPT_HARD) &&
            cpu_interrupts_enabled(env1);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUSPARCState *env, TranslationBlock *tb)
 {
diff --git a/target-sparc/fop_helper.c b/target-sparc/fop_helper.c
index 9c64ef89fe2..f4b62a5ba28 100644
--- a/target-sparc/fop_helper.c
+++ b/target-sparc/fop_helper.c
@@ -334,34 +334,28 @@ void helper_fsqrtq(CPUSPARCState *env)
 }
 
 #define GEN_FCMP(name, size, reg1, reg2, FS, E)                         \
-    void glue(helper_, name) (CPUSPARCState *env)                            \
+    void glue(helper_, name) (CPUSPARCState *env)                       \
     {                                                                   \
-        env->fsr &= FSR_FTT_NMASK;                                      \
-        if (E && (glue(size, _is_any_nan)(reg1) ||                      \
-                  glue(size, _is_any_nan)(reg2)) &&                     \
-            (env->fsr & FSR_NVM)) {                                     \
-            env->fsr |= FSR_NVC;                                        \
-            env->fsr |= FSR_FTT_IEEE_EXCP;                              \
-            helper_raise_exception(env, TT_FP_EXCP);                    \
+        int ret;                                                        \
+        clear_float_exceptions(env);                                    \
+        if (E) {                                                        \
+            ret = glue(size, _compare)(reg1, reg2, &env->fp_status);    \
+        } else {                                                        \
+            ret = glue(size, _compare_quiet)(reg1, reg2,                \
+                                             &env->fp_status);          \
         }                                                               \
-        switch (glue(size, _compare) (reg1, reg2, &env->fp_status)) {   \
+        check_ieee_exceptions(env);                                     \
+        switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            if ((env->fsr & FSR_NVM)) {                                 \
-                env->fsr |= FSR_NVC;                                    \
-                env->fsr |= FSR_FTT_IEEE_EXCP;                          \
-                helper_raise_exception(env, TT_FP_EXCP);                \
-            } else {                                                    \
-                env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);             \
-                env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                \
-                env->fsr |= FSR_NVA;                                    \
-            }                                                           \
+            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
+            env->fsr |= FSR_NVA;                                        \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC1) << FS;                              \
             env->fsr |= FSR_FCC0 << FS;                                 \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC0) << FS;                              \
             env->fsr |= FSR_FCC1 << FS;                                 \
             break;                                                      \
         default:                                                        \
@@ -370,34 +364,27 @@ void helper_fsqrtq(CPUSPARCState *env)
         }                                                               \
     }
 #define GEN_FCMP_T(name, size, FS, E)                                   \
-    void glue(helper_, name)(CPUSPARCState *env, size src1, size src2)       \
+    void glue(helper_, name)(CPUSPARCState *env, size src1, size src2)  \
     {                                                                   \
-        env->fsr &= FSR_FTT_NMASK;                                      \
-        if (E && (glue(size, _is_any_nan)(src1) ||                      \
-                  glue(size, _is_any_nan)(src2)) &&                     \
-            (env->fsr & FSR_NVM)) {                                     \
-            env->fsr |= FSR_NVC;                                        \
-            env->fsr |= FSR_FTT_IEEE_EXCP;                              \
-            helper_raise_exception(env, TT_FP_EXCP);                    \
+        int ret;                                                        \
+        clear_float_exceptions(env);                                    \
+        if (E) {                                                        \
+            ret = glue(size, _compare)(src1, src2, &env->fp_status);    \
+        } else {                                                        \
+            ret = glue(size, _compare_quiet)(src1, src2,                \
+                                             &env->fp_status);          \
         }                                                               \
-        switch (glue(size, _compare) (src1, src2, &env->fp_status)) {   \
+        check_ieee_exceptions(env);                                     \
+        switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            if ((env->fsr & FSR_NVM)) {                                 \
-                env->fsr |= FSR_NVC;                                    \
-                env->fsr |= FSR_FTT_IEEE_EXCP;                          \
-                helper_raise_exception(env, TT_FP_EXCP);                \
-            } else {                                                    \
-                env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);             \
-                env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                \
-                env->fsr |= FSR_NVA;                                    \
-            }                                                           \
+            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC1 << FS);                              \
             env->fsr |= FSR_FCC0 << FS;                                 \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC0 << FS);                              \
             env->fsr |= FSR_FCC1 << FS;                                 \
             break;                                                      \
         default:                                                        \
diff --git a/target-sparc/helper.c b/target-sparc/helper.c
index 65e1740e197..91ecfc7aa85 100644
--- a/target-sparc/helper.c
+++ b/target-sparc/helper.c
@@ -18,9 +18,9 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 void helper_raise_exception(CPUSPARCState *env, int tt)
 {
@@ -75,6 +75,7 @@ static target_ulong helper_udiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -113,6 +114,7 @@ static target_ulong helper_sdiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -139,3 +141,87 @@ target_ulong helper_sdiv_cc(CPUSPARCState *env, target_ulong a, target_ulong b)
 {
     return helper_sdiv_common(env, a, b, 1);
 }
+
+#ifdef TARGET_SPARC64
+int64_t helper_sdivx(CPUSPARCState *env, int64_t a, int64_t b)
+{
+    if (b == 0) {
+        /* Raise divide by zero trap.  */
+        cpu_restore_state(env, GETPC());
+        helper_raise_exception(env, TT_DIV_ZERO);
+    } else if (b == -1) {
+        /* Avoid overflow trap with i386 divide insn.  */
+        return -a;
+    } else {
+        return a / b;
+    }
+}
+
+uint64_t helper_udivx(CPUSPARCState *env, uint64_t a, uint64_t b)
+{
+    if (b == 0) {
+        /* Raise divide by zero trap.  */
+        cpu_restore_state(env, GETPC());
+        helper_raise_exception(env, TT_DIV_ZERO);
+    }
+    return a / b;
+}
+#endif
+
+target_ulong helper_taddcctv(CPUSPARCState *env, target_ulong src1,
+                             target_ulong src2)
+{
+    target_ulong dst;
+
+    /* Tag overflow occurs if either input has bits 0 or 1 set.  */
+    if ((src1 | src2) & 3) {
+        goto tag_overflow;
+    }
+
+    dst = src1 + src2;
+
+    /* Tag overflow occurs if the addition overflows.  */
+    if (~(src1 ^ src2) & (src1 ^ dst) & (1u << 31)) {
+        goto tag_overflow;
+    }
+
+    /* Only modify the CC after any exceptions have been generated.  */
+    env->cc_op = CC_OP_TADDTV;
+    env->cc_src = src1;
+    env->cc_src2 = src2;
+    env->cc_dst = dst;
+    return dst;
+
+ tag_overflow:
+    cpu_restore_state(env, GETPC());
+    helper_raise_exception(env, TT_TOVF);
+}
+
+target_ulong helper_tsubcctv(CPUSPARCState *env, target_ulong src1,
+                             target_ulong src2)
+{
+    target_ulong dst;
+
+    /* Tag overflow occurs if either input has bits 0 or 1 set.  */
+    if ((src1 | src2) & 3) {
+        goto tag_overflow;
+    }
+
+    dst = src1 - src2;
+
+    /* Tag overflow occurs if the subtraction overflows.  */
+    if ((src1 ^ src2) & (src1 ^ dst) & (1u << 31)) {
+        goto tag_overflow;
+    }
+
+    /* Only modify the CC after any exceptions have been generated.  */
+    env->cc_op = CC_OP_TSUBTV;
+    env->cc_src = src1;
+    env->cc_src2 = src2;
+    env->cc_dst = dst;
+    return dst;
+
+ tag_overflow:
+    cpu_restore_state(env, GETPC());
+    helper_raise_exception(env, TT_TOVF);
+}
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index e3c7fddbe8e..cfcdab1ea4f 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 #ifndef TARGET_SPARC64
 DEF_HELPER_1(rett, void, env)
@@ -16,7 +16,7 @@ DEF_HELPER_1(rdccr, tl, env)
 DEF_HELPER_2(wrccr, void, env, tl)
 DEF_HELPER_1(rdcwp, tl, env)
 DEF_HELPER_2(wrcwp, void, env, tl)
-DEF_HELPER_FLAGS_2(array8, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(array8, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 DEF_HELPER_1(popc, tl, tl)
 DEF_HELPER_4(ldda_asi, void, env, tl, int, int)
 DEF_HELPER_5(ldf_asi, void, env, tl, int, int, int)
@@ -38,6 +38,12 @@ DEF_HELPER_3(udiv, tl, env, tl, tl)
 DEF_HELPER_3(udiv_cc, tl, env, tl, tl)
 DEF_HELPER_3(sdiv, tl, env, tl, tl)
 DEF_HELPER_3(sdiv_cc, tl, env, tl, tl)
+DEF_HELPER_3(taddcctv, tl, env, tl, tl)
+DEF_HELPER_3(tsubcctv, tl, env, tl, tl)
+#ifdef TARGET_SPARC64
+DEF_HELPER_3(sdivx, s64, env, s64, s64)
+DEF_HELPER_3(udivx, i64, env, i64, i64)
+#endif
 DEF_HELPER_3(ldqf, void, env, tl, int)
 DEF_HELPER_3(stqf, void, env, tl, int)
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
@@ -45,7 +51,7 @@ DEF_HELPER_5(ld_asi, i64, env, tl, int, int, int)
 DEF_HELPER_5(st_asi, void, env, tl, i64, int, int)
 #endif
 DEF_HELPER_2(ldfsr, void, env, i32)
-DEF_HELPER_FLAGS_1(fabss, TCG_CALL_CONST | TCG_CALL_PURE, f32, f32)
+DEF_HELPER_FLAGS_1(fabss, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_2(fsqrts, f32, env, f32)
 DEF_HELPER_2(fsqrtd, f64, env, f64)
 DEF_HELPER_3(fcmps, void, env, f32, f32)
@@ -57,7 +63,7 @@ DEF_HELPER_1(fcmpq, void, env)
 DEF_HELPER_1(fcmpeq, void, env)
 #ifdef TARGET_SPARC64
 DEF_HELPER_2(ldxfsr, void, env, i64)
-DEF_HELPER_FLAGS_1(fabsd, TCG_CALL_CONST | TCG_CALL_PURE, f64, f64)
+DEF_HELPER_FLAGS_1(fabsd, TCG_CALL_NO_RWG_SE, f64, f64)
 DEF_HELPER_3(fcmps_fcc1, void, env, f32, f32)
 DEF_HELPER_3(fcmps_fcc2, void, env, f32, f32)
 DEF_HELPER_3(fcmps_fcc3, void, env, f32, f32)
@@ -98,14 +104,14 @@ DEF_HELPER_3(fdivs, f32, env, f32, f32)
 DEF_HELPER_3(fsmuld, f64, env, f32, f32)
 DEF_HELPER_3(fdmulq, void, env, f64, f64);
 
-DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_CONST | TCG_CALL_PURE, f32, f32)
+DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_2(fitod, f64, env, s32)
 DEF_HELPER_2(fitoq, void, env, s32)
 
 DEF_HELPER_2(fitos, f32, env, s32)
 
 #ifdef TARGET_SPARC64
-DEF_HELPER_FLAGS_1(fnegd, TCG_CALL_CONST | TCG_CALL_PURE, f64, f64)
+DEF_HELPER_FLAGS_1(fnegd, TCG_CALL_NO_RWG_SE, f64, f64)
 DEF_HELPER_1(fnegq, void, env)
 DEF_HELPER_2(fxtos, f32, env, s64)
 DEF_HELPER_2(fxtod, f64, env, s64)
@@ -125,36 +131,36 @@ DEF_HELPER_2(fstox, s64, env, f32)
 DEF_HELPER_2(fdtox, s64, env, f64)
 DEF_HELPER_1(fqtox, s64, env)
 
-DEF_HELPER_FLAGS_2(fpmerge, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16al, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16au, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8sux16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8ulx16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmuld8sux16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmuld8ulx16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fexpand, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_3(pdist, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fpack16, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64, i64)
-DEF_HELPER_FLAGS_3(fpack32, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fpackfix, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64, i64)
-DEF_HELPER_FLAGS_3(bshuffle, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpmerge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16al, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16au, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8sux16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8ulx16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmuld8sux16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmuld8ulx16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fexpand, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_3(pdist, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpack16, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(fpack32, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpackfix, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(bshuffle, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
 #define VIS_HELPER(name)                                                 \
-    DEF_HELPER_FLAGS_2(f ## name ## 16, TCG_CALL_CONST | TCG_CALL_PURE,  \
+    DEF_HELPER_FLAGS_2(f ## name ## 16, TCG_CALL_NO_RWG_SE,  \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 16s, TCG_CALL_CONST | TCG_CALL_PURE, \
+    DEF_HELPER_FLAGS_2(f ## name ## 16s, TCG_CALL_NO_RWG_SE, \
                        i32, i32, i32)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 32, TCG_CALL_CONST | TCG_CALL_PURE,  \
+    DEF_HELPER_FLAGS_2(f ## name ## 32, TCG_CALL_NO_RWG_SE,  \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 32s, TCG_CALL_CONST | TCG_CALL_PURE, \
+    DEF_HELPER_FLAGS_2(f ## name ## 32s, TCG_CALL_NO_RWG_SE, \
                        i32, i32, i32)
 
 VIS_HELPER(padd);
 VIS_HELPER(psub);
 #define VIS_CMPHELPER(name)                                              \
-    DEF_HELPER_FLAGS_2(f##name##16, TCG_CALL_CONST | TCG_CALL_PURE,      \
+    DEF_HELPER_FLAGS_2(f##name##16, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f##name##32, TCG_CALL_CONST | TCG_CALL_PURE,      \
+    DEF_HELPER_FLAGS_2(f##name##32, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)
 VIS_CMPHELPER(cmpgt);
 VIS_CMPHELPER(cmpeq);
@@ -167,4 +173,4 @@ VIS_CMPHELPER(cmpne);
 DEF_HELPER_1(compute_psr, void, env);
 DEF_HELPER_1(compute_C_icc, i32, env);
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-sparc/int32_helper.c b/target-sparc/int32_helper.c
index 5e33d50e23d..c35f522e0f4 100644
--- a/target-sparc/int32_helper.c
+++ b/target-sparc/int32_helper.c
@@ -19,9 +19,9 @@
 
 #include "cpu.h"
 #include "trace.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
-//#define DEBUG_PCALL
+#define DEBUG_PCALL
 
 #ifdef DEBUG_PCALL
 static const char * const excp_names[0x80] = {
@@ -62,6 +62,11 @@ void do_interrupt(CPUSPARCState *env)
 {
     int cwp, intno = env->exception_index;
 
+    /* Compute PSR before exposing state.  */
+    if (env->cc_op != CC_OP_FLAGS) {
+        cpu_get_psr(env);
+    }
+
 #ifdef DEBUG_PCALL
     if (qemu_loglevel_mask(CPU_LOG_INT)) {
         static int count;
@@ -78,10 +83,7 @@ void do_interrupt(CPUSPARCState *env)
             }
         }
 
-        qemu_log("%6d: %s (v=%02x) pc=%08x npc=%08x SP=%08x\n",
-                count, name, intno,
-                env->pc,
-                env->npc, env->regwptr[6]);
+        qemu_log("%6d: %s (v=%02x)\n", count, name, intno);
         log_cpu_state(env, 0);
 #if 0
         {
diff --git a/target-sparc/int64_helper.c b/target-sparc/int64_helper.c
index 5e3eff76dd4..df37aa1d14a 100644
--- a/target-sparc/int64_helper.c
+++ b/target-sparc/int64_helper.c
@@ -21,7 +21,7 @@
 #include "helper.h"
 #include "trace.h"
 
-//#define DEBUG_PCALL
+#define DEBUG_PCALL
 
 #ifdef DEBUG_PCALL
 static const char * const excp_names[0x80] = {
@@ -64,6 +64,11 @@ void do_interrupt(CPUSPARCState *env)
     int intno = env->exception_index;
     trap_state *tsptr;
 
+    /* Compute PSR before exposing state.  */
+    if (env->cc_op != CC_OP_FLAGS) {
+        cpu_get_psr(env);
+    }
+
 #ifdef DEBUG_PCALL
     if (qemu_loglevel_mask(CPU_LOG_INT)) {
         static int count;
@@ -84,11 +89,7 @@ void do_interrupt(CPUSPARCState *env)
             }
         }
 
-        qemu_log("%6d: %s (v=%04x) pc=%016" PRIx64 " npc=%016" PRIx64
-                " SP=%016" PRIx64 "\n",
-                count, name, intno,
-                env->pc,
-                env->npc, env->regwptr[6]);
+        qemu_log("%6d: %s (v=%04x)\n", count, name, intno);
         log_cpu_state(env, 0);
 #if 0
         {
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index 9bec7a92f7a..cf1bddf2db4 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -65,21 +65,24 @@
 #define QT1 (env->qt1)
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
+                                              target_ulong addr, int is_write,
+                                              int is_user, uintptr_t retaddr);
+#include "exec/softmmu_exec.h"
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #endif
 
 #if defined(TARGET_SPARC64) && !defined(CONFIG_USER_ONLY)
@@ -619,21 +622,21 @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
     case 0x21 ... 0x2f: /* MMU passthrough, 0x100000000 to 0xfffffffff */
         switch (size) {
         case 1:
-            ret = ldub_phys((target_phys_addr_t)addr
-                            | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldub_phys((hwaddr)addr
+                            | ((hwaddr)(asi & 0xf) << 32));
             break;
         case 2:
-            ret = lduw_phys((target_phys_addr_t)addr
-                            | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = lduw_phys((hwaddr)addr
+                            | ((hwaddr)(asi & 0xf) << 32));
             break;
         default:
         case 4:
-            ret = ldl_phys((target_phys_addr_t)addr
-                           | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldl_phys((hwaddr)addr
+                           | ((hwaddr)(asi & 0xf) << 32));
             break;
         case 8:
-            ret = ldq_phys((target_phys_addr_t)addr
-                           | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldq_phys((hwaddr)addr
+                           | ((hwaddr)(asi & 0xf) << 32));
             break;
         }
         break;
@@ -1015,21 +1018,21 @@ void helper_st_asi(CPUSPARCState *env, target_ulong addr, uint64_t val, int asi,
         {
             switch (size) {
             case 1:
-                stb_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stb_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 2:
-                stw_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stw_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 4:
             default:
-                stl_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stl_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 8:
-                stq_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stq_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             }
         }
@@ -2313,7 +2316,7 @@ void helper_stqf(CPUSPARCState *env, target_ulong addr, int mem_idx)
 
 #if !defined(CONFIG_USER_ONLY)
 #ifndef TARGET_SPARC64
-void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
     int fault_type;
@@ -2373,7 +2376,7 @@ void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
     }
 }
 #else
-void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2391,30 +2394,17 @@ void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-/* XXX: make it generic ? */
-static void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-
-    if (retaddr) {
-        /* now we have a real cpu fault */
-        tb = tb_find_pc(retaddr);
-        if (tb) {
-            /* the PC is inside the translated code. It means that we have
-               a virtual CPU fault */
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
-void do_unaligned_access(CPUSPARCState *env, target_ulong addr, int is_write,
-                         int is_user, uintptr_t retaddr)
+static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
+                                              target_ulong addr, int is_write,
+                                              int is_user, uintptr_t retaddr)
 {
 #ifdef DEBUG_UNALIGNED
     printf("Unaligned access to 0x" TARGET_FMT_lx " from 0x" TARGET_FMT_lx
            "\n", addr, env->pc);
 #endif
-    cpu_restore_state2(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     helper_raise_exception(env, TT_UNALIGNED);
 }
 
@@ -2429,7 +2419,9 @@ void tlb_fill(CPUSPARCState *env, target_ulong addr, int is_write, int mmu_idx,
 
     ret = cpu_sparc_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
-        cpu_restore_state2(env, retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         cpu_loop_exit(env);
     }
 }
diff --git a/target-sparc/machine.c b/target-sparc/machine.c
index eb4d87f1589..a353dabdd96 100644
--- a/target-sparc/machine.c
+++ b/target-sparc/machine.c
@@ -1,6 +1,6 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #include "cpu.h"
 
diff --git a/target-sparc/mmu_helper.c b/target-sparc/mmu_helper.c
index cb73c449f0e..a9649ae064e 100644
--- a/target-sparc/mmu_helper.c
+++ b/target-sparc/mmu_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 /* Sparc MMU emulation */
 
@@ -76,13 +76,13 @@ static const int perm_table[2][8] = {
     }
 };
 
-static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical,
+static int get_physical_address(CPUSPARCState *env, hwaddr *physical,
                                 int *prot, int *access_index,
                                 target_ulong address, int rw, int mmu_idx,
                                 target_ulong *page_size)
 {
     int access_perms = 0;
-    target_phys_addr_t pde_ptr;
+    hwaddr pde_ptr;
     uint32_t pde;
     int error_code = 0, is_dirty, is_user;
     unsigned long page_offset;
@@ -192,7 +192,7 @@ static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical
 
     /* Even if large ptes, we map only one 4KB page in the cache to
        avoid filling it too fast */
-    *physical = ((target_phys_addr_t)(pde & PTE_ADDR_MASK) << 4) + page_offset;
+    *physical = ((hwaddr)(pde & PTE_ADDR_MASK) << 4) + page_offset;
     return error_code;
 }
 
@@ -200,7 +200,7 @@ static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical
 int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong vaddr;
     target_ulong page_size;
     int error_code = 0, prot, access_index;
@@ -244,11 +244,11 @@ int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
 
 target_ulong mmu_probe(CPUSPARCState *env, target_ulong address, int mmulev)
 {
-    target_phys_addr_t pde_ptr;
+    hwaddr pde_ptr;
     uint32_t pde;
 
     /* Context base + context number */
-    pde_ptr = (target_phys_addr_t)(env->mmuregs[1] << 4) +
+    pde_ptr = (hwaddr)(env->mmuregs[1] << 4) +
         (env->mmuregs[2] << 2);
     pde = ldl_phys(pde_ptr);
 
@@ -312,13 +312,13 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUSPARCState *env)
 {
     target_ulong va, va1, va2;
     unsigned int n, m, o;
-    target_phys_addr_t pde_ptr, pa;
+    hwaddr pde_ptr, pa;
     uint32_t pde;
 
     pde_ptr = (env->mmuregs[1] << 4) + (env->mmuregs[2] << 2);
     pde = ldl_phys(pde_ptr);
     (*cpu_fprintf)(f, "Root ptr: " TARGET_FMT_plx ", ctx: %d\n",
-                   (target_phys_addr_t)env->mmuregs[1] << 4, env->mmuregs[2]);
+                   (hwaddr)env->mmuregs[1] << 4, env->mmuregs[2]);
     for (n = 0, va = 0; n < 256; n++, va += 16 * 1024 * 1024) {
         pde = mmu_probe(env, va, 2);
         if (pde) {
@@ -431,7 +431,7 @@ int target_memory_rw_debug(CPUSPARCState *env, target_ulong addr,
 #else /* !TARGET_SPARC64 */
 
 /* 41 bit physical address space */
-static inline target_phys_addr_t ultrasparc_truncate_physical(uint64_t x)
+static inline hwaddr ultrasparc_truncate_physical(uint64_t x)
 {
     return x & 0x1ffffffffffULL;
 }
@@ -445,7 +445,7 @@ static inline target_phys_addr_t ultrasparc_truncate_physical(uint64_t x)
    entry size */
 static inline int ultrasparc_tag_match(SparcTLBEntry *tlb,
                                        uint64_t address, uint64_t context,
-                                       target_phys_addr_t *physical)
+                                       hwaddr *physical)
 {
     uint64_t mask;
 
@@ -478,7 +478,7 @@ static inline int ultrasparc_tag_match(SparcTLBEntry *tlb,
 }
 
 static int get_physical_address_data(CPUSPARCState *env,
-                                     target_phys_addr_t *physical, int *prot,
+                                     hwaddr *physical, int *prot,
                                      target_ulong address, int rw, int mmu_idx)
 {
     unsigned int i;
@@ -597,7 +597,7 @@ static int get_physical_address_data(CPUSPARCState *env,
 }
 
 static int get_physical_address_code(CPUSPARCState *env,
-                                     target_phys_addr_t *physical, int *prot,
+                                     hwaddr *physical, int *prot,
                                      target_ulong address, int mmu_idx)
 {
     unsigned int i;
@@ -665,7 +665,7 @@ static int get_physical_address_code(CPUSPARCState *env,
     return 1;
 }
 
-static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical,
+static int get_physical_address(CPUSPARCState *env, hwaddr *physical,
                                 int *prot, int *access_index,
                                 target_ulong address, int rw, int mmu_idx,
                                 target_ulong *page_size)
@@ -703,7 +703,7 @@ int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
     target_ulong vaddr;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong page_size;
     int error_code = 0, prot, access_index;
 
@@ -810,7 +810,7 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUSPARCState *env)
 
 #endif /* TARGET_SPARC64 */
 
-static int cpu_sparc_get_phys_page(CPUSPARCState *env, target_phys_addr_t *phys,
+static int cpu_sparc_get_phys_page(CPUSPARCState *env, hwaddr *phys,
                                    target_ulong addr, int rw, int mmu_idx)
 {
     target_ulong page_size;
@@ -821,10 +821,10 @@ static int cpu_sparc_get_phys_page(CPUSPARCState *env, target_phys_addr_t *phys,
 }
 
 #if defined(TARGET_SPARC64)
-target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
+hwaddr cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
                                            int mmu_idx)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     if (cpu_sparc_get_phys_page(env, &phys_addr, addr, 4, mmu_idx) != 0) {
         return -1;
@@ -833,9 +833,9 @@ target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong ad
 }
 #endif
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUSPARCState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUSPARCState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int mmu_idx = cpu_mmu_index(env);
     MemoryRegionSection section;
 
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index b95f91cd3ba..ca75e1aa484 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -25,7 +25,7 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "helper.h"
 #include "tcg-op.h"
 
@@ -48,7 +48,7 @@ static TCGv cpu_y;
 #ifndef CONFIG_USER_ONLY
 static TCGv cpu_tbr;
 #endif
-static TCGv cpu_cond, cpu_dst, cpu_addr, cpu_val;
+static TCGv cpu_cond;
 #ifdef TARGET_SPARC64
 static TCGv_i32 cpu_xcc, cpu_asi, cpu_fprs;
 static TCGv cpu_gsr;
@@ -58,17 +58,13 @@ static TCGv_i32 cpu_softint;
 #else
 static TCGv cpu_wim;
 #endif
-/* local register indexes (only used inside old micro ops) */
-static TCGv cpu_tmp0;
-static TCGv_i32 cpu_tmp32;
-static TCGv_i64 cpu_tmp64;
 /* Floating point registers */
 static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 
 static target_ulong gen_opc_npc[OPC_BUF_SIZE];
 static target_ulong gen_opc_jump_pc[2];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 typedef struct DisasContext {
     target_ulong pc;    /* current Program Counter: integer or DYNAMIC_PC */
@@ -83,9 +79,18 @@ typedef struct DisasContext {
     struct TranslationBlock *tb;
     sparc_def_t *def;
     TCGv_i32 t32[3];
+    TCGv ttl[5];
     int n_t32;
+    int n_ttl;
 } DisasContext;
 
+typedef struct {
+    TCGCond cond;
+    bool is_bool;
+    bool g1, g2;
+    TCGv c1, c2;
+} DisasCompare;
+
 // This function uses non-native bit order
 #define GET_FIELD(X, FROM, TO)                                  \
     ((X) >> (31 - (TO)) & ((1 << ((TO) - (FROM) + 1)) - 1))
@@ -116,6 +121,22 @@ static int sign_extend(int x, int len)
 
 #define IS_IMM (insn & (1<<13))
 
+static inline TCGv_i32 get_temp_i32(DisasContext *dc)
+{
+    TCGv_i32 t;
+    assert(dc->n_t32 < ARRAY_SIZE(dc->t32));
+    dc->t32[dc->n_t32++] = t = tcg_temp_new_i32();
+    return t;
+}
+
+static inline TCGv get_temp_tl(DisasContext *dc)
+{
+    TCGv t;
+    assert(dc->n_ttl < ARRAY_SIZE(dc->ttl));
+    dc->ttl[dc->n_ttl++] = t = tcg_temp_new();
+    return t;
+}
+
 static inline void gen_update_fprs_dirty(int rd)
 {
 #if defined(TARGET_SPARC64)
@@ -136,16 +157,13 @@ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src)
     if (src & 1) {
         return MAKE_TCGV_I32(GET_TCGV_I64(cpu_fpr[src / 2]));
     } else {
-        TCGv_i32 ret = tcg_temp_local_new_i32();
+        TCGv_i32 ret = get_temp_i32(dc);
         TCGv_i64 t = tcg_temp_new_i64();
 
         tcg_gen_shri_i64(t, cpu_fpr[src / 2], 32);
         tcg_gen_trunc_i64_i32(ret, t);
         tcg_temp_free_i64(t);
 
-        dc->t32[dc->n_t32++] = ret;
-        assert(dc->n_t32 <= ARRAY_SIZE(dc->t32));
-
         return ret;
     }
 #endif
@@ -167,9 +185,9 @@ static void gen_store_fpr_F(DisasContext *dc, unsigned int dst, TCGv_i32 v)
     gen_update_fprs_dirty(dst);
 }
 
-static TCGv_i32 gen_dest_fpr_F(void)
+static TCGv_i32 gen_dest_fpr_F(DisasContext *dc)
 {
-    return cpu_tmp32;
+    return get_temp_i32(dc);
 }
 
 static TCGv_i64 gen_load_fpr_D(DisasContext *dc, unsigned int src)
@@ -185,9 +203,9 @@ static void gen_store_fpr_D(DisasContext *dc, unsigned int dst, TCGv_i64 v)
     gen_update_fprs_dirty(dst);
 }
 
-static TCGv_i64 gen_dest_fpr_D(void)
+static TCGv_i64 gen_dest_fpr_D(DisasContext *dc, unsigned int dst)
 {
-    return cpu_tmp64;
+    return cpu_fpr[DFPREG(dst) / 2];
 }
 
 static void gen_op_load_fpr_QT0(unsigned int src)
@@ -256,25 +274,38 @@ static inline void gen_address_mask(DisasContext *dc, TCGv addr)
 #endif
 }
 
-static inline void gen_movl_reg_TN(int reg, TCGv tn)
+static inline TCGv gen_load_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0)
-        tcg_gen_movi_tl(tn, 0);
-    else if (reg < 8)
-        tcg_gen_mov_tl(tn, cpu_gregs[reg]);
-    else {
-        tcg_gen_ld_tl(tn, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+    if (reg == 0 || reg >= 8) {
+        TCGv t = get_temp_tl(dc);
+        if (reg == 0) {
+            tcg_gen_movi_tl(t, 0);
+        } else {
+            tcg_gen_ld_tl(t, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+        }
+        return t;
+    } else {
+        return cpu_gregs[reg];
     }
 }
 
-static inline void gen_movl_TN_reg(int reg, TCGv tn)
+static inline void gen_store_gpr(DisasContext *dc, int reg, TCGv v)
 {
-    if (reg == 0)
-        return;
-    else if (reg < 8)
-        tcg_gen_mov_tl(cpu_gregs[reg], tn);
-    else {
-        tcg_gen_st_tl(tn, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+    if (reg > 0) {
+        if (reg < 8) {
+            tcg_gen_mov_tl(cpu_gregs[reg], v);
+        } else {
+            tcg_gen_st_tl(v, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+        }
+    }
+}
+
+static inline TCGv gen_dest_gpr(DisasContext *dc, int reg)
+{
+    if (reg == 0 || reg >= 8) {
+        return get_temp_tl(dc);
+    } else {
+        return cpu_gregs[reg];
     }
 }
 
@@ -329,43 +360,6 @@ static inline void gen_mov_reg_C(TCGv reg, TCGv_i32 src)
     tcg_gen_andi_tl(reg, reg, 0x1);
 }
 
-static inline void gen_add_tv(TCGv dst, TCGv src1, TCGv src2)
-{
-    TCGv r_temp;
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-
-    r_temp = tcg_temp_new();
-    tcg_gen_xor_tl(r_temp, src1, src2);
-    tcg_gen_not_tl(r_temp, r_temp);
-    tcg_gen_xor_tl(cpu_tmp0, src1, dst);
-    tcg_gen_and_tl(r_temp, r_temp, cpu_tmp0);
-    tcg_gen_andi_tl(r_temp, r_temp, (1ULL << 31));
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_temp, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-    tcg_temp_free(r_temp);
-}
-
-static inline void gen_tag_tv(TCGv src1, TCGv src2)
-{
-    int l1;
-    TCGv_i32 r_const;
-
-    l1 = gen_new_label();
-    tcg_gen_or_tl(cpu_tmp0, src1, src2);
-    tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0x3);
-    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_tmp0, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-}
-
 static inline void gen_op_addi_cc(TCGv dst, TCGv src1, target_long src2)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -510,45 +504,6 @@ static void gen_op_addx_int(DisasContext *dc, TCGv dst, TCGv src1,
     }
 }
 
-static inline void gen_op_tadd_cc(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_op_tadd_ccTV(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    gen_tag_tv(cpu_cc_src, cpu_cc_src2);
-    tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    gen_add_tv(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_sub_tv(TCGv dst, TCGv src1, TCGv src2)
-{
-    TCGv r_temp;
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-
-    r_temp = tcg_temp_new();
-    tcg_gen_xor_tl(r_temp, src1, src2);
-    tcg_gen_xor_tl(cpu_tmp0, src1, dst);
-    tcg_gen_and_tl(r_temp, r_temp, cpu_tmp0);
-    tcg_gen_andi_tl(r_temp, r_temp, (1ULL << 31));
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_temp, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-    tcg_temp_free(r_temp);
-}
-
 static inline void gen_op_subi_cc(TCGv dst, TCGv src1, target_long src2, DisasContext *dc)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -649,63 +604,46 @@ static void gen_op_subx_int(DisasContext *dc, TCGv dst, TCGv src1,
     }
 }
 
-static inline void gen_op_tsub_cc(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    tcg_gen_sub_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_op_tsub_ccTV(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    gen_tag_tv(cpu_cc_src, cpu_cc_src2);
-    tcg_gen_sub_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    gen_sub_tv(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
 static inline void gen_op_mulscc(TCGv dst, TCGv src1, TCGv src2)
 {
-    TCGv r_temp;
-    int l1;
+    TCGv r_temp, zero, t0;
 
-    l1 = gen_new_label();
     r_temp = tcg_temp_new();
+    t0 = tcg_temp_new();
 
     /* old op:
     if (!(env->y & 1))
         T1 = 0;
     */
+    zero = tcg_const_tl(0);
     tcg_gen_andi_tl(cpu_cc_src, src1, 0xffffffff);
     tcg_gen_andi_tl(r_temp, cpu_y, 0x1);
     tcg_gen_andi_tl(cpu_cc_src2, src2, 0xffffffff);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp, 0, l1);
-    tcg_gen_movi_tl(cpu_cc_src2, 0);
-    gen_set_label(l1);
+    tcg_gen_movcond_tl(TCG_COND_EQ, cpu_cc_src2, r_temp, zero,
+                       zero, cpu_cc_src2);
+    tcg_temp_free(zero);
 
     // b2 = T0 & 1;
     // env->y = (b2 << 31) | (env->y >> 1);
     tcg_gen_andi_tl(r_temp, cpu_cc_src, 0x1);
     tcg_gen_shli_tl(r_temp, r_temp, 31);
-    tcg_gen_shri_tl(cpu_tmp0, cpu_y, 1);
-    tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0x7fffffff);
-    tcg_gen_or_tl(cpu_tmp0, cpu_tmp0, r_temp);
-    tcg_gen_andi_tl(cpu_y, cpu_tmp0, 0xffffffff);
+    tcg_gen_shri_tl(t0, cpu_y, 1);
+    tcg_gen_andi_tl(t0, t0, 0x7fffffff);
+    tcg_gen_or_tl(t0, t0, r_temp);
+    tcg_gen_andi_tl(cpu_y, t0, 0xffffffff);
 
     // b1 = N ^ V;
-    gen_mov_reg_N(cpu_tmp0, cpu_psr);
+    gen_mov_reg_N(t0, cpu_psr);
     gen_mov_reg_V(r_temp, cpu_psr);
-    tcg_gen_xor_tl(cpu_tmp0, cpu_tmp0, r_temp);
+    tcg_gen_xor_tl(t0, t0, r_temp);
     tcg_temp_free(r_temp);
 
     // T0 = (b1 << 31) | (T0 >> 1);
     // src1 = T0;
-    tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, 31);
+    tcg_gen_shli_tl(t0, t0, 31);
     tcg_gen_shri_tl(cpu_cc_src, cpu_cc_src, 1);
-    tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_tmp0);
+    tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, t0);
+    tcg_temp_free(t0);
 
     tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
 
@@ -737,9 +675,9 @@ static inline void gen_op_multiply(TCGv dst, TCGv src1, TCGv src2, int sign_ext)
     tcg_gen_mul_i64(r_temp2, r_temp, r_temp2);
 
     tcg_gen_shri_i64(r_temp, r_temp2, 32);
-    tcg_gen_trunc_i64_tl(cpu_tmp0, r_temp);
+    tcg_gen_trunc_i64_tl(cpu_y, r_temp);
     tcg_temp_free_i64(r_temp);
-    tcg_gen_andi_tl(cpu_y, cpu_tmp0, 0xffffffff);
+    tcg_gen_andi_tl(cpu_y, cpu_y, 0xffffffff);
 
     tcg_gen_trunc_i64_tl(dst, r_temp2);
 
@@ -761,44 +699,6 @@ static inline void gen_op_smul(TCGv dst, TCGv src1, TCGv src2)
     gen_op_multiply(dst, src1, src2, 1);
 }
 
-#ifdef TARGET_SPARC64
-static inline void gen_trap_ifdivzero_tl(TCGv divisor)
-{
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-    tcg_gen_brcondi_tl(TCG_COND_NE, divisor, 0, l1);
-    r_const = tcg_const_i32(TT_DIV_ZERO);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-}
-
-static inline void gen_op_sdivx(TCGv dst, TCGv src1, TCGv src2)
-{
-    int l1, l2;
-    TCGv r_temp1, r_temp2;
-
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-    r_temp1 = tcg_temp_local_new();
-    r_temp2 = tcg_temp_local_new();
-    tcg_gen_mov_tl(r_temp1, src1);
-    tcg_gen_mov_tl(r_temp2, src2);
-    gen_trap_ifdivzero_tl(r_temp2);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp1, INT64_MIN, l1);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp2, -1, l1);
-    tcg_gen_movi_i64(dst, INT64_MIN);
-    tcg_gen_br(l2);
-    gen_set_label(l1);
-    tcg_gen_div_i64(dst, r_temp1, r_temp2);
-    gen_set_label(l2);
-    tcg_temp_free(r_temp1);
-    tcg_temp_free(r_temp2);
-}
-#endif
-
 // 1
 static inline void gen_op_eval_ba(TCGv dst)
 {
@@ -814,27 +714,33 @@ static inline void gen_op_eval_be(TCGv dst, TCGv_i32 src)
 // Z | (N ^ V)
 static inline void gen_op_eval_ble(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_N(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_N(t0, src);
     gen_mov_reg_V(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
-    gen_mov_reg_Z(cpu_tmp0, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    tcg_gen_xor_tl(dst, dst, t0);
+    gen_mov_reg_Z(t0, src);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // N ^ V
 static inline void gen_op_eval_bl(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_V(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_V(t0, src);
     gen_mov_reg_N(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    tcg_gen_xor_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // C | Z
 static inline void gen_op_eval_bleu(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_Z(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_Z(t0, src);
     gen_mov_reg_C(dst, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // C
@@ -871,29 +777,21 @@ static inline void gen_op_eval_bne(TCGv dst, TCGv_i32 src)
 // !(Z | (N ^ V))
 static inline void gen_op_eval_bg(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_N(cpu_tmp0, src);
-    gen_mov_reg_V(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
-    gen_mov_reg_Z(cpu_tmp0, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_ble(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
 // !(N ^ V)
 static inline void gen_op_eval_bge(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_V(cpu_tmp0, src);
-    gen_mov_reg_N(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_bl(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
 // !(C | Z)
 static inline void gen_op_eval_bgu(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_Z(cpu_tmp0, src);
-    gen_mov_reg_C(dst, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_bleu(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
@@ -943,18 +841,22 @@ static inline void gen_mov_reg_FCC1(TCGv reg, TCGv src,
 static inline void gen_op_eval_fbne(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 1 or 2: FCC0 ^ FCC1
 static inline void gen_op_eval_fblg(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_xor_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 1 or 3: FCC0
@@ -968,10 +870,11 @@ static inline void gen_op_eval_fbul(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbl(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xori_tl(cpu_tmp0, cpu_tmp0, 0x1);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 2 or 3: FCC1
@@ -985,39 +888,46 @@ static inline void gen_op_eval_fbug(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbg(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    tcg_gen_xori_tl(dst, dst, 0x1);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, t0, dst);
+    tcg_temp_free(t0);
 }
 
 // 3: FCC0 & FCC1
 static inline void gen_op_eval_fbu(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_and_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 0: !(FCC0 | FCC1)
 static inline void gen_op_eval_fbe(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_or_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 3: !(FCC0 ^ FCC1)
 static inline void gen_op_eval_fbue(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_xor_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 2: !FCC0
@@ -1032,11 +942,12 @@ static inline void gen_op_eval_fbge(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbuge(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xori_tl(cpu_tmp0, cpu_tmp0, 0x1);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 1: !FCC1
@@ -1051,21 +962,24 @@ static inline void gen_op_eval_fble(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbule(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, t0, dst);
     tcg_gen_xori_tl(dst, dst, 0x1);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
-    tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // !3: !(FCC0 & FCC1)
 static inline void gen_op_eval_fbo(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_and_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 static inline void gen_branch2(DisasContext *dc, target_ulong pc1,
@@ -1098,59 +1012,57 @@ static inline void gen_branch_a(DisasContext *dc, target_ulong pc1,
     gen_goto_tb(dc, 1, pc2 + 4, pc2 + 8);
 }
 
-static inline void gen_generic_branch(target_ulong npc1, target_ulong npc2,
-                                      TCGv r_cond)
+static inline void gen_generic_branch(DisasContext *dc)
 {
-    int l1, l2;
-
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+    TCGv npc0 = tcg_const_tl(dc->jump_pc[0]);
+    TCGv npc1 = tcg_const_tl(dc->jump_pc[1]);
+    TCGv zero = tcg_const_tl(0);
 
-    tcg_gen_movi_tl(cpu_npc, npc1);
-    tcg_gen_br(l2);
+    tcg_gen_movcond_tl(TCG_COND_NE, cpu_npc, cpu_cond, zero, npc0, npc1);
 
-    gen_set_label(l1);
-    tcg_gen_movi_tl(cpu_npc, npc2);
-    gen_set_label(l2);
+    tcg_temp_free(npc0);
+    tcg_temp_free(npc1);
+    tcg_temp_free(zero);
 }
 
 /* call this function before using the condition register as it may
    have been set for a jump */
-static inline void flush_cond(DisasContext *dc, TCGv cond)
+static inline void flush_cond(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         dc->npc = DYNAMIC_PC;
     }
 }
 
-static inline void save_npc(DisasContext *dc, TCGv cond)
+static inline void save_npc(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         dc->npc = DYNAMIC_PC;
     } else if (dc->npc != DYNAMIC_PC) {
         tcg_gen_movi_tl(cpu_npc, dc->npc);
     }
 }
 
-static inline void save_state(DisasContext *dc, TCGv cond)
+static inline void update_psr(DisasContext *dc)
 {
-    tcg_gen_movi_tl(cpu_pc, dc->pc);
-    /* flush pending conditional evaluations before exposing cpu state */
     if (dc->cc_op != CC_OP_FLAGS) {
         dc->cc_op = CC_OP_FLAGS;
         gen_helper_compute_psr(cpu_env);
     }
-    save_npc(dc, cond);
 }
 
-static inline void gen_mov_pc_npc(DisasContext *dc, TCGv cond)
+static inline void save_state(DisasContext *dc)
+{
+    tcg_gen_movi_tl(cpu_pc, dc->pc);
+    save_npc(dc);
+}
+
+static inline void gen_mov_pc_npc(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         tcg_gen_mov_tl(cpu_pc, cpu_npc);
         dc->pc = DYNAMIC_PC;
     } else if (dc->npc == DYNAMIC_PC) {
@@ -1167,82 +1079,201 @@ static inline void gen_op_next_insn(void)
     tcg_gen_addi_tl(cpu_npc, cpu_npc, 4);
 }
 
-static inline void gen_cond(TCGv r_dst, unsigned int cc, unsigned int cond,
-                            DisasContext *dc)
+static void free_compare(DisasCompare *cmp)
 {
+    if (!cmp->g1) {
+        tcg_temp_free(cmp->c1);
+    }
+    if (!cmp->g2) {
+        tcg_temp_free(cmp->c2);
+    }
+}
+
+static void gen_compare(DisasCompare *cmp, bool xcc, unsigned int cond,
+                        DisasContext *dc)
+{
+    static int subcc_cond[16] = {
+        TCG_COND_NEVER,
+        TCG_COND_EQ,
+        TCG_COND_LE,
+        TCG_COND_LT,
+        TCG_COND_LEU,
+        TCG_COND_LTU,
+        -1, /* neg */
+        -1, /* overflow */
+        TCG_COND_ALWAYS,
+        TCG_COND_NE,
+        TCG_COND_GT,
+        TCG_COND_GE,
+        TCG_COND_GTU,
+        TCG_COND_GEU,
+        -1, /* pos */
+        -1, /* no overflow */
+    };
+
+    static int logic_cond[16] = {
+        TCG_COND_NEVER,
+        TCG_COND_EQ,     /* eq:  Z */
+        TCG_COND_LE,     /* le:  Z | (N ^ V) -> Z | N */
+        TCG_COND_LT,     /* lt:  N ^ V -> N */
+        TCG_COND_EQ,     /* leu: C | Z -> Z */
+        TCG_COND_NEVER,  /* ltu: C -> 0 */
+        TCG_COND_LT,     /* neg: N */
+        TCG_COND_NEVER,  /* vs:  V -> 0 */
+        TCG_COND_ALWAYS,
+        TCG_COND_NE,     /* ne:  !Z */
+        TCG_COND_GT,     /* gt:  !(Z | (N ^ V)) -> !(Z | N) */
+        TCG_COND_GE,     /* ge:  !(N ^ V) -> !N */
+        TCG_COND_NE,     /* gtu: !(C | Z) -> !Z */
+        TCG_COND_ALWAYS, /* geu: !C -> 1 */
+        TCG_COND_GE,     /* pos: !N */
+        TCG_COND_ALWAYS, /* vc:  !V -> 1 */
+    };
+
     TCGv_i32 r_src;
+    TCGv r_dst;
 
 #ifdef TARGET_SPARC64
-    if (cc)
+    if (xcc) {
         r_src = cpu_xcc;
-    else
+    } else {
         r_src = cpu_psr;
+    }
 #else
     r_src = cpu_psr;
 #endif
+
     switch (dc->cc_op) {
-    case CC_OP_FLAGS:
+    case CC_OP_LOGIC:
+        cmp->cond = logic_cond[cond];
+    do_compare_dst_0:
+        cmp->is_bool = false;
+        cmp->g2 = false;
+        cmp->c2 = tcg_const_tl(0);
+#ifdef TARGET_SPARC64
+        if (!xcc) {
+            cmp->g1 = false;
+            cmp->c1 = tcg_temp_new();
+            tcg_gen_ext32s_tl(cmp->c1, cpu_cc_dst);
+            break;
+        }
+#endif
+        cmp->g1 = true;
+        cmp->c1 = cpu_cc_dst;
+        break;
+
+    case CC_OP_SUB:
+        switch (cond) {
+        case 6:  /* neg */
+        case 14: /* pos */
+            cmp->cond = (cond == 6 ? TCG_COND_LT : TCG_COND_GE);
+            goto do_compare_dst_0;
+
+        case 7: /* overflow */
+        case 15: /* !overflow */
+            goto do_dynamic;
+
+        default:
+            cmp->cond = subcc_cond[cond];
+            cmp->is_bool = false;
+#ifdef TARGET_SPARC64
+            if (!xcc) {
+                /* Note that sign-extension works for unsigned compares as
+                   long as both operands are sign-extended.  */
+                cmp->g1 = cmp->g2 = false;
+                cmp->c1 = tcg_temp_new();
+                cmp->c2 = tcg_temp_new();
+                tcg_gen_ext32s_tl(cmp->c1, cpu_cc_src);
+                tcg_gen_ext32s_tl(cmp->c2, cpu_cc_src2);
+                break;
+            }
+#endif
+            cmp->g1 = cmp->g2 = true;
+            cmp->c1 = cpu_cc_src;
+            cmp->c2 = cpu_cc_src2;
+            break;
+        }
         break;
+
     default:
+    do_dynamic:
         gen_helper_compute_psr(cpu_env);
         dc->cc_op = CC_OP_FLAGS;
-        break;
-    }
-    switch (cond) {
-    case 0x0:
-        gen_op_eval_bn(r_dst);
-        break;
-    case 0x1:
-        gen_op_eval_be(r_dst, r_src);
-        break;
-    case 0x2:
-        gen_op_eval_ble(r_dst, r_src);
-        break;
-    case 0x3:
-        gen_op_eval_bl(r_dst, r_src);
-        break;
-    case 0x4:
-        gen_op_eval_bleu(r_dst, r_src);
-        break;
-    case 0x5:
-        gen_op_eval_bcs(r_dst, r_src);
-        break;
-    case 0x6:
-        gen_op_eval_bneg(r_dst, r_src);
-        break;
-    case 0x7:
-        gen_op_eval_bvs(r_dst, r_src);
-        break;
-    case 0x8:
-        gen_op_eval_ba(r_dst);
-        break;
-    case 0x9:
-        gen_op_eval_bne(r_dst, r_src);
-        break;
-    case 0xa:
-        gen_op_eval_bg(r_dst, r_src);
-        break;
-    case 0xb:
-        gen_op_eval_bge(r_dst, r_src);
-        break;
-    case 0xc:
-        gen_op_eval_bgu(r_dst, r_src);
-        break;
-    case 0xd:
-        gen_op_eval_bcc(r_dst, r_src);
-        break;
-    case 0xe:
-        gen_op_eval_bpos(r_dst, r_src);
-        break;
-    case 0xf:
-        gen_op_eval_bvc(r_dst, r_src);
+        /* FALLTHRU */
+
+    case CC_OP_FLAGS:
+        /* We're going to generate a boolean result.  */
+        cmp->cond = TCG_COND_NE;
+        cmp->is_bool = true;
+        cmp->g1 = cmp->g2 = false;
+        cmp->c1 = r_dst = tcg_temp_new();
+        cmp->c2 = tcg_const_tl(0);
+
+        switch (cond) {
+        case 0x0:
+            gen_op_eval_bn(r_dst);
+            break;
+        case 0x1:
+            gen_op_eval_be(r_dst, r_src);
+            break;
+        case 0x2:
+            gen_op_eval_ble(r_dst, r_src);
+            break;
+        case 0x3:
+            gen_op_eval_bl(r_dst, r_src);
+            break;
+        case 0x4:
+            gen_op_eval_bleu(r_dst, r_src);
+            break;
+        case 0x5:
+            gen_op_eval_bcs(r_dst, r_src);
+            break;
+        case 0x6:
+            gen_op_eval_bneg(r_dst, r_src);
+            break;
+        case 0x7:
+            gen_op_eval_bvs(r_dst, r_src);
+            break;
+        case 0x8:
+            gen_op_eval_ba(r_dst);
+            break;
+        case 0x9:
+            gen_op_eval_bne(r_dst, r_src);
+            break;
+        case 0xa:
+            gen_op_eval_bg(r_dst, r_src);
+            break;
+        case 0xb:
+            gen_op_eval_bge(r_dst, r_src);
+            break;
+        case 0xc:
+            gen_op_eval_bgu(r_dst, r_src);
+            break;
+        case 0xd:
+            gen_op_eval_bcc(r_dst, r_src);
+            break;
+        case 0xe:
+            gen_op_eval_bpos(r_dst, r_src);
+            break;
+        case 0xf:
+            gen_op_eval_bvc(r_dst, r_src);
+            break;
+        }
         break;
     }
 }
 
-static inline void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
+static void gen_fcompare(DisasCompare *cmp, unsigned int cc, unsigned int cond)
 {
     unsigned int offset;
+    TCGv r_dst;
+
+    /* For now we still generate a straight boolean result.  */
+    cmp->cond = TCG_COND_NE;
+    cmp->is_bool = true;
+    cmp->g1 = cmp->g2 = false;
+    cmp->c1 = r_dst = tcg_temp_new();
+    cmp->c2 = tcg_const_tl(0);
 
     switch (cc) {
     default:
@@ -1312,6 +1343,37 @@ static inline void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
     }
 }
 
+static void gen_cond(TCGv r_dst, unsigned int cc, unsigned int cond,
+                     DisasContext *dc)
+{
+    DisasCompare cmp;
+    gen_compare(&cmp, cc, cond, dc);
+
+    /* The interface is to return a boolean in r_dst.  */
+    if (cmp.is_bool) {
+        tcg_gen_mov_tl(r_dst, cmp.c1);
+    } else {
+        tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+    }
+
+    free_compare(&cmp);
+}
+
+static void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
+{
+    DisasCompare cmp;
+    gen_fcompare(&cmp, cc, cond);
+
+    /* The interface is to return a boolean in r_dst.  */
+    if (cmp.is_bool) {
+        tcg_gen_mov_tl(r_dst, cmp.c1);
+    } else {
+        tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+    }
+
+    free_compare(&cmp);
+}
+
 #ifdef TARGET_SPARC64
 // Inverted logic
 static const int gen_tcg_cond_reg[8] = {
@@ -1325,20 +1387,29 @@ static const int gen_tcg_cond_reg[8] = {
     TCG_COND_LT,
 };
 
+static void gen_compare_reg(DisasCompare *cmp, int cond, TCGv r_src)
+{
+    cmp->cond = tcg_invert_cond(gen_tcg_cond_reg[cond]);
+    cmp->is_bool = false;
+    cmp->g1 = true;
+    cmp->g2 = false;
+    cmp->c1 = r_src;
+    cmp->c2 = tcg_const_tl(0);
+}
+
 static inline void gen_cond_reg(TCGv r_dst, int cond, TCGv r_src)
 {
-    int l1;
+    DisasCompare cmp;
+    gen_compare_reg(&cmp, cond, r_src);
 
-    l1 = gen_new_label();
-    tcg_gen_movi_tl(r_dst, 0);
-    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], r_src, 0, l1);
-    tcg_gen_movi_tl(r_dst, 1);
-    gen_set_label(l1);
+    /* The interface is to return a boolean in r_dst.  */
+    tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+
+    free_compare(&cmp);
 }
 #endif
 
-static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
-                      TCGv r_cond)
+static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
 {
     unsigned int cond = GET_FIELD(insn, 3, 6), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1368,10 +1439,10 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
             tcg_gen_mov_tl(cpu_pc, cpu_npc);
         }
     } else {
-        flush_cond(dc, r_cond);
-        gen_cond(r_cond, cc, cond, dc);
+        flush_cond(dc);
+        gen_cond(cpu_cond, cc, cond, dc);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, r_cond);
+            gen_branch_a(dc, target, dc->npc, cpu_cond);
             dc->is_br = 1;
         } else {
             dc->pc = dc->npc;
@@ -1387,8 +1458,7 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
     }
 }
 
-static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
-                      TCGv r_cond)
+static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
 {
     unsigned int cond = GET_FIELD(insn, 3, 6), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1418,10 +1488,10 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
             tcg_gen_mov_tl(cpu_pc, cpu_npc);
         }
     } else {
-        flush_cond(dc, r_cond);
-        gen_fcond(r_cond, cc, cond);
+        flush_cond(dc);
+        gen_fcond(cpu_cond, cc, cond);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, r_cond);
+            gen_branch_a(dc, target, dc->npc, cpu_cond);
             dc->is_br = 1;
         } else {
             dc->pc = dc->npc;
@@ -1439,7 +1509,7 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
 
 #ifdef TARGET_SPARC64
 static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
-                          TCGv r_cond, TCGv r_reg)
+                          TCGv r_reg)
 {
     unsigned int cond = GET_FIELD_SP(insn, 25, 27), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1447,10 +1517,10 @@ static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
     if (unlikely(AM_CHECK(dc))) {
         target &= 0xffffffffULL;
     }
-    flush_cond(dc, r_cond);
-    gen_cond_reg(r_cond, cond, r_reg);
+    flush_cond(dc);
+    gen_cond_reg(cpu_cond, cond, r_reg);
     if (a) {
-        gen_branch_a(dc, target, dc->npc, r_cond);
+        gen_branch_a(dc, target, dc->npc, cpu_cond);
         dc->is_br = 1;
     } else {
         dc->pc = dc->npc;
@@ -1617,13 +1687,13 @@ static inline void gen_op_fpexception_im(int fsr_flags)
     tcg_temp_free_i32(r_const);
 }
 
-static int gen_trap_ifnofpu(DisasContext *dc, TCGv r_cond)
+static int gen_trap_ifnofpu(DisasContext *dc)
 {
 #if !defined(CONFIG_USER_ONLY)
     if (!dc->fpu_enabled) {
         TCGv_i32 r_const;
 
-        save_state(dc, r_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_NFPU_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -1645,7 +1715,7 @@ static inline void gen_fop_FF(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst, src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
 
@@ -1658,7 +1728,7 @@ static inline void gen_ne_fop_FF(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst, src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, src);
 
@@ -1672,7 +1742,7 @@ static inline void gen_fop_FFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1687,7 +1757,7 @@ static inline void gen_ne_fop_FFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, src1, src2);
 
@@ -1701,7 +1771,7 @@ static inline void gen_fop_DD(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst, src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1715,7 +1785,7 @@ static inline void gen_ne_fop_DD(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst, src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src);
 
@@ -1730,7 +1800,7 @@ static inline void gen_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1745,7 +1815,7 @@ static inline void gen_ne_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src1, src2);
 
@@ -1759,7 +1829,7 @@ static inline void gen_gsr_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_gsr, src1, src2);
 
@@ -1774,7 +1844,7 @@ static inline void gen_ne_fop_DDDD(DisasContext *dc, int rd, int rs1, int rs2,
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
     src0 = gen_load_fpr_D(dc, rd);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src0, src1, src2);
 
@@ -1826,7 +1896,7 @@ static inline void gen_fop_DFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1855,7 +1925,7 @@ static inline void gen_fop_DF(DisasContext *dc, int rd, int rs,
     TCGv_i32 src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1870,7 +1940,7 @@ static inline void gen_ne_fop_DF(DisasContext *dc, int rd, int rs,
     TCGv_i32 src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1884,7 +1954,7 @@ static inline void gen_fop_FD(DisasContext *dc, int rd, int rs,
     TCGv_i64 src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
 
@@ -1897,7 +1967,7 @@ static inline void gen_fop_FQ(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst;
 
     gen_op_load_fpr_QT1(QFPREG(rs));
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env);
 
@@ -1910,7 +1980,7 @@ static inline void gen_fop_DQ(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst;
 
     gen_op_load_fpr_QT1(QFPREG(rs));
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env);
 
@@ -2011,22 +2081,25 @@ static inline void gen_stf_asi(TCGv addr, int insn, int size, int rd)
     tcg_temp_free_i32(r_asi);
 }
 
-static inline void gen_swap_asi(TCGv dst, TCGv addr, int insn)
+static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
     r_asi = gen_get_asi(insn, addr);
     r_size = tcg_const_i32(4);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
     tcg_temp_free_i32(r_sign);
-    gen_helper_st_asi(cpu_env, addr, dst, r_asi, r_size);
+    gen_helper_st_asi(cpu_env, addr, src, r_asi, r_size);
     tcg_temp_free_i32(r_size);
     tcg_temp_free_i32(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_rd;
 
@@ -2037,42 +2110,44 @@ static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
     tcg_temp_free_i32(r_asi);
 }
 
-static inline void gen_stda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv lo = gen_load_gpr(dc, rd + 1);
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    gen_movl_reg_TN(rd + 1, cpu_tmp0);
-    tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, hi);
+    tcg_gen_concat_tl_i64(t64, lo, hi);
     r_asi = gen_get_asi(insn, addr);
     r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
     tcg_temp_free_i32(r_size);
     tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_cas_asi(TCGv dst, TCGv addr, TCGv val2, int insn,
-                               int rd)
+static inline void gen_cas_asi(DisasContext *dc, TCGv addr,
+                               TCGv val2, int insn, int rd)
 {
-    TCGv r_val1;
-    TCGv_i32 r_asi;
+    TCGv val1 = gen_load_gpr(dc, rd);
+    TCGv dst = gen_dest_gpr(dc, rd);
+    TCGv_i32 r_asi = gen_get_asi(insn, addr);
 
-    r_val1 = tcg_temp_new();
-    gen_movl_reg_TN(rd, r_val1);
-    r_asi = gen_get_asi(insn, addr);
-    gen_helper_cas_asi(dst, cpu_env, addr, r_val1, val2, r_asi);
+    gen_helper_cas_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
-    tcg_temp_free(r_val1);
+    gen_store_gpr(dc, rd, dst);
 }
 
-static inline void gen_casx_asi(TCGv dst, TCGv addr, TCGv val2, int insn,
-                                int rd)
+static inline void gen_casx_asi(DisasContext *dc, TCGv addr,
+                                TCGv val2, int insn, int rd)
 {
-    TCGv_i32 r_asi;
+    TCGv val1 = gen_load_gpr(dc, rd);
+    TCGv dst = gen_dest_gpr(dc, rd);
+    TCGv_i32 r_asi = gen_get_asi(insn, addr);
 
-    gen_movl_reg_TN(rd, cpu_tmp64);
-    r_asi = gen_get_asi(insn, addr);
-    gen_helper_casx_asi(dst, cpu_env, addr, cpu_tmp64, val2, r_asi);
+    gen_helper_casx_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
+    gen_store_gpr(dc, rd, dst);
 }
 
 #elif !defined(CONFIG_USER_ONLY)
@@ -2081,77 +2156,94 @@ static inline void gen_ld_asi(TCGv dst, TCGv addr, int insn, int size,
                               int sign)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(size);
     r_sign = tcg_const_i32(sign);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free(r_sign);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
+    tcg_temp_free_i32(r_sign);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
 static inline void gen_st_asi(TCGv src, TCGv addr, int insn, int size)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    tcg_gen_extu_tl_i64(cpu_tmp64, src);
+    tcg_gen_extu_tl_i64(t64, src);
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(size);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_swap_asi(TCGv dst, TCGv addr, int insn)
+static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
 {
     TCGv_i32 r_asi, r_size, r_sign;
-    TCGv_i64 r_val;
+    TCGv_i64 r_val, t64;
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(4);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
+    t64 = tcg_temp_new_i64();
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
     tcg_temp_free(r_sign);
     r_val = tcg_temp_new_i64();
-    tcg_gen_extu_tl_i64(r_val, dst);
+    tcg_gen_extu_tl_i64(r_val, src);
     gen_helper_st_asi(cpu_env, addr, r_val, r_asi, r_size);
     tcg_temp_free_i64(r_val);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv t;
+    TCGv_i64 t64;
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(8);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free(r_sign);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(cpu_tmp0, cpu_tmp64);
-    gen_movl_TN_reg(rd + 1, cpu_tmp0);
-    tcg_gen_shri_i64(cpu_tmp64, cpu_tmp64, 32);
-    tcg_gen_trunc_i64_tl(hi, cpu_tmp64);
-    gen_movl_TN_reg(rd, hi);
+    t64 = tcg_temp_new_i64();
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
+    tcg_temp_free_i32(r_sign);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+
+    t = gen_dest_gpr(dc, rd + 1);
+    tcg_gen_trunc_i64_tl(t, t64);
+    gen_store_gpr(dc, rd + 1, t);
+
+    tcg_gen_shri_i64(t64, t64, 32);
+    tcg_gen_trunc_i64_tl(hi, t64);
+    tcg_temp_free_i64(t64);
+    gen_store_gpr(dc, rd, hi);
 }
 
-static inline void gen_stda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv lo = gen_load_gpr(dc, rd + 1);
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    gen_movl_reg_TN(rd + 1, cpu_tmp0);
-    tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, hi);
+    tcg_gen_concat_tl_i64(t64, lo, hi);
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 #endif
 
@@ -2173,43 +2265,77 @@ static inline void gen_ldstub_asi(TCGv dst, TCGv addr, int insn)
 }
 #endif
 
-static inline TCGv get_src1(unsigned int insn, TCGv def)
+static TCGv get_src1(DisasContext *dc, unsigned int insn)
 {
-    TCGv r_rs1 = def;
-    unsigned int rs1;
-
-    rs1 = GET_FIELD(insn, 13, 17);
-    if (rs1 == 0) {
-        tcg_gen_movi_tl(def, 0);
-    } else if (rs1 < 8) {
-        r_rs1 = cpu_gregs[rs1];
-    } else {
-        tcg_gen_ld_tl(def, cpu_regwptr, (rs1 - 8) * sizeof(target_ulong));
-    }
-    return r_rs1;
+    unsigned int rs1 = GET_FIELD(insn, 13, 17);
+    return gen_load_gpr(dc, rs1);
 }
 
-static inline TCGv get_src2(unsigned int insn, TCGv def)
+static TCGv get_src2(DisasContext *dc, unsigned int insn)
 {
-    TCGv r_rs2 = def;
-
     if (IS_IMM) { /* immediate */
         target_long simm = GET_FIELDs(insn, 19, 31);
-        tcg_gen_movi_tl(def, simm);
-    } else { /* register */
+        TCGv t = get_temp_tl(dc);
+        tcg_gen_movi_tl(t, simm);
+        return t;
+    } else {      /* register */
         unsigned int rs2 = GET_FIELD(insn, 27, 31);
-        if (rs2 == 0) {
-            tcg_gen_movi_tl(def, 0);
-        } else if (rs2 < 8) {
-            r_rs2 = cpu_gregs[rs2];
-        } else {
-            tcg_gen_ld_tl(def, cpu_regwptr, (rs2 - 8) * sizeof(target_ulong));
-        }
+        return gen_load_gpr(dc, rs2);
     }
-    return r_rs2;
 }
 
 #ifdef TARGET_SPARC64
+static void gen_fmovs(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    TCGv_i32 c32, zero, dst, s1, s2;
+
+    /* We have two choices here: extend the 32 bit data and use movcond_i64,
+       or fold the comparison down to 32 bits and use movcond_i32.  Choose
+       the later.  */
+    c32 = tcg_temp_new_i32();
+    if (cmp->is_bool) {
+        tcg_gen_trunc_i64_i32(c32, cmp->c1);
+    } else {
+        TCGv_i64 c64 = tcg_temp_new_i64();
+        tcg_gen_setcond_i64(cmp->cond, c64, cmp->c1, cmp->c2);
+        tcg_gen_trunc_i64_i32(c32, c64);
+        tcg_temp_free_i64(c64);
+    }
+
+    s1 = gen_load_fpr_F(dc, rs);
+    s2 = gen_load_fpr_F(dc, rd);
+    dst = gen_dest_fpr_F(dc);
+    zero = tcg_const_i32(0);
+
+    tcg_gen_movcond_i32(TCG_COND_NE, dst, c32, zero, s1, s2);
+
+    tcg_temp_free_i32(c32);
+    tcg_temp_free_i32(zero);
+    gen_store_fpr_F(dc, rd, dst);
+}
+
+static void gen_fmovd(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    TCGv_i64 dst = gen_dest_fpr_D(dc, rd);
+    tcg_gen_movcond_i64(cmp->cond, dst, cmp->c1, cmp->c2,
+                        gen_load_fpr_D(dc, rs),
+                        gen_load_fpr_D(dc, rd));
+    gen_store_fpr_D(dc, rd, dst);
+}
+
+static void gen_fmovq(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    int qd = QFPREG(rd);
+    int qs = QFPREG(rs);
+
+    tcg_gen_movcond_i64(cmp->cond, cpu_fpr[qd / 2], cmp->c1, cmp->c2,
+                        cpu_fpr[qs / 2], cpu_fpr[qd / 2]);
+    tcg_gen_movcond_i64(cmp->cond, cpu_fpr[qd / 2 + 1], cmp->c1, cmp->c2,
+                        cpu_fpr[qs / 2 + 1], cpu_fpr[qd / 2 + 1]);
+
+    gen_update_fprs_dirty(qd);
+}
+
 static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_ptr cpu_env)
 {
     TCGv_i32 r_tl = tcg_temp_new_i32();
@@ -2389,21 +2515,18 @@ static void gen_faligndata(TCGv dst, TCGv gsr, TCGv s1, TCGv s2)
 static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 {
     unsigned int opc, rs1, rs2, rd;
-    TCGv cpu_src1, cpu_src2, cpu_tmp1, cpu_tmp2;
+    TCGv cpu_src1, cpu_src2;
     TCGv_i32 cpu_src1_32, cpu_src2_32, cpu_dst_32;
     TCGv_i64 cpu_src1_64, cpu_src2_64, cpu_dst_64;
     target_long simm;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
+    }
 
     opc = GET_FIELD(insn, 0, 1);
-
     rd = GET_FIELD(insn, 2, 6);
 
-    cpu_tmp1 = cpu_src1 = tcg_temp_new();
-    cpu_tmp2 = cpu_src2 = tcg_temp_new();
-
     switch (opc) {
     case 0:                     /* branches/sethi */
         {
@@ -2420,9 +2543,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     target <<= 2;
                     cc = GET_FIELD_SP(insn, 20, 21);
                     if (cc == 0)
-                        do_branch(dc, target, insn, 0, cpu_cond);
+                        do_branch(dc, target, insn, 0);
                     else if (cc == 2)
-                        do_branch(dc, target, insn, 1, cpu_cond);
+                        do_branch(dc, target, insn, 1);
                     else
                         goto illegal_insn;
                     goto jmp_insn;
@@ -2433,19 +2556,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         (GET_FIELD_SP(insn, 20, 21) << 14);
                     target = sign_extend(target, 16);
                     target <<= 2;
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    do_branch_reg(dc, target, insn, cpu_cond, cpu_src1);
+                    cpu_src1 = get_src1(dc, insn);
+                    do_branch_reg(dc, target, insn, cpu_src1);
                     goto jmp_insn;
                 }
             case 0x5:           /* V9 FBPcc */
                 {
                     int cc = GET_FIELD_SP(insn, 20, 21);
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     target = GET_FIELD_SP(insn, 0, 18);
                     target = sign_extend(target, 19);
                     target <<= 2;
-                    do_fbranch(dc, target, insn, cc, cpu_cond);
+                    do_fbranch(dc, target, insn, cc);
                     goto jmp_insn;
                 }
 #else
@@ -2459,27 +2583,27 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     target = GET_FIELD(insn, 10, 31);
                     target = sign_extend(target, 22);
                     target <<= 2;
-                    do_branch(dc, target, insn, 0, cpu_cond);
+                    do_branch(dc, target, insn, 0);
                     goto jmp_insn;
                 }
             case 0x6:           /* FBN+x */
                 {
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     target = GET_FIELD(insn, 10, 31);
                     target = sign_extend(target, 22);
                     target <<= 2;
-                    do_fbranch(dc, target, insn, 0, cpu_cond);
+                    do_fbranch(dc, target, insn, 0);
                     goto jmp_insn;
                 }
             case 0x4:           /* SETHI */
-                if (rd) { // nop
+                /* Special-case %g0 because that's the canonical nop.  */
+                if (rd) {
                     uint32_t value = GET_FIELD(insn, 10, 31);
-                    TCGv r_const;
-
-                    r_const = tcg_const_tl(value << 10);
-                    gen_movl_TN_reg(rd, r_const);
-                    tcg_temp_free(r_const);
+                    TCGv t = gen_dest_gpr(dc, rd);
+                    tcg_gen_movi_tl(t, value << 10);
+                    gen_store_gpr(dc, rd, t);
                 }
                 break;
             case 0x0:           /* UNIMPL */
@@ -2492,13 +2616,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 1:                     /*CALL*/
         {
             target_long target = GET_FIELDs(insn, 2, 31) << 2;
-            TCGv r_const;
+            TCGv o7 = gen_dest_gpr(dc, 15);
 
-            r_const = tcg_const_tl(dc->pc);
-            gen_movl_TN_reg(15, r_const);
-            tcg_temp_free(r_const);
+            tcg_gen_movi_tl(o7, dc->pc);
+            gen_store_gpr(dc, 15, o7);
             target += dc->pc;
-            gen_mov_pc_npc(dc, cpu_cond);
+            gen_mov_pc_npc(dc);
 #ifdef TARGET_SPARC64
             if (unlikely(AM_CHECK(dc))) {
                 target &= 0xffffffffULL;
@@ -2510,71 +2633,88 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 2:                     /* FPU & Logical Operations */
         {
             unsigned int xop = GET_FIELD(insn, 7, 12);
+            TCGv cpu_dst = get_temp_tl(dc);
+            TCGv cpu_tmp0;
+
             if (xop == 0x3a) {  /* generate trap */
-                int cond;
+                int cond = GET_FIELD(insn, 3, 6);
+                TCGv_i32 trap;
+                int l1 = -1, mask;
 
-                cpu_src1 = get_src1(insn, cpu_src1);
-                if (IS_IMM) {
-                    rs2 = GET_FIELD(insn, 25, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, rs2);
-                } else {
-                    rs2 = GET_FIELD(insn, 27, 31);
-                    if (rs2 != 0) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                if (cond == 0) {
+                    /* Trap never.  */
+                    break;
                 }
 
-                cond = GET_FIELD(insn, 3, 6);
-                if (cond == 0x8) { /* Trap Always */
-                    save_state(dc, cpu_cond);
-                    if ((dc->def->features & CPU_FEATURE_HYPV) &&
-                        supervisor(dc))
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, UA2005_HTRAP_MASK);
-                    else
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, V8_TRAP_MASK);
-                    tcg_gen_addi_tl(cpu_dst, cpu_dst, TT_TRAP);
-                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_dst);
-                    gen_helper_raise_exception(cpu_env, cpu_tmp32);
-
-                } else if (cond != 0) {
-                    TCGv r_cond = tcg_temp_new();
-                    int l1;
+                save_state(dc);
+
+                if (cond != 8) {
+                    /* Conditional trap.  */
+                    DisasCompare cmp;
 #ifdef TARGET_SPARC64
                     /* V9 icc/xcc */
                     int cc = GET_FIELD_SP(insn, 11, 12);
-
-                    save_state(dc, cpu_cond);
-                    if (cc == 0)
-                        gen_cond(r_cond, 0, cond, dc);
-                    else if (cc == 2)
-                        gen_cond(r_cond, 1, cond, dc);
-                    else
+                    if (cc == 0) {
+                        gen_compare(&cmp, 0, cond, dc);
+                    } else if (cc == 2) {
+                        gen_compare(&cmp, 1, cond, dc);
+                    } else {
                         goto illegal_insn;
+                    }
 #else
-                    save_state(dc, cpu_cond);
-                    gen_cond(r_cond, 0, cond, dc);
+                    gen_compare(&cmp, 0, cond, dc);
 #endif
                     l1 = gen_new_label();
-                    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+                    tcg_gen_brcond_tl(tcg_invert_cond(cmp.cond),
+                                      cmp.c1, cmp.c2, l1);
+                    free_compare(&cmp);
+                }
 
-                    if ((dc->def->features & CPU_FEATURE_HYPV) &&
-                        supervisor(dc))
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, UA2005_HTRAP_MASK);
-                    else
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, V8_TRAP_MASK);
-                    tcg_gen_addi_tl(cpu_dst, cpu_dst, TT_TRAP);
-                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_dst);
-                    gen_helper_raise_exception(cpu_env, cpu_tmp32);
+                mask = ((dc->def->features & CPU_FEATURE_HYPV) && supervisor(dc)
+                        ? UA2005_HTRAP_MASK : V8_TRAP_MASK);
 
+                /* Don't use the normal temporaries, as they may well have
+                   gone out of scope with the branch above.  While we're
+                   doing that we might as well pre-truncate to 32-bit.  */
+                trap = tcg_temp_new_i32();
+
+                rs1 = GET_FIELD_SP(insn, 14, 18);
+                if (IS_IMM) {
+                    rs2 = GET_FIELD_SP(insn, 0, 6);
+                    if (rs1 == 0) {
+                        tcg_gen_movi_i32(trap, (rs2 & mask) + TT_TRAP);
+                        /* Signal that the trap value is fully constant.  */
+                        mask = 0;
+                    } else {
+                        TCGv t1 = gen_load_gpr(dc, rs1);
+                        tcg_gen_trunc_tl_i32(trap, t1);
+                        tcg_gen_addi_i32(trap, trap, rs2);
+                    }
+                } else {
+                    TCGv t1, t2;
+                    rs2 = GET_FIELD_SP(insn, 0, 4);
+                    t1 = gen_load_gpr(dc, rs1);
+                    t2 = gen_load_gpr(dc, rs2);
+                    tcg_gen_add_tl(t1, t1, t2);
+                    tcg_gen_trunc_tl_i32(trap, t1);
+                }
+                if (mask != 0) {
+                    tcg_gen_andi_i32(trap, trap, mask);
+                    tcg_gen_addi_i32(trap, trap, TT_TRAP);
+                }
+
+                gen_helper_raise_exception(cpu_env, trap);
+                tcg_temp_free_i32(trap);
+
+                if (cond == 8) {
+                    /* An unconditional trap ends the TB.  */
+                    dc->is_br = 1;
+                    goto jmp_insn;
+                } else {
+                    /* A conditional trap falls through to the next insn.  */
                     gen_set_label(l1);
-                    tcg_temp_free(r_cond);
+                    break;
                 }
-                gen_op_next_insn();
-                tcg_gen_exit_tb(0);
-                dc->is_br = 1;
-                goto jmp_insn;
             } else if (xop == 0x28) {
                 rs1 = GET_FIELD(insn, 13, 17);
                 switch(rs1) {
@@ -2590,27 +2730,24 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        microSPARC II */
                     /* Read Asr17 */
                     if (rs1 == 0x11 && dc->def->features & CPU_FEATURE_ASR17) {
-                        TCGv r_const;
-
+                        TCGv t = gen_dest_gpr(dc, rd);
                         /* Read Asr17 for a Leon3 monoprocessor */
-                        r_const = tcg_const_tl((1 << 8)
-                                               | (dc->def->nwindows - 1));
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        tcg_gen_movi_tl(t, (1 << 8) | (dc->def->nwindows - 1));
+                        gen_store_gpr(dc, rd, t);
                         break;
                     }
 #endif
-                    gen_movl_TN_reg(rd, cpu_y);
+                    gen_store_gpr(dc, rd, cpu_y);
                     break;
 #ifdef TARGET_SPARC64
                 case 0x2: /* V9 rdccr */
-                    gen_helper_compute_psr(cpu_env);
+                    update_psr(dc);
                     gen_helper_rdccr(cpu_dst, cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x3: /* V9 rdasi */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_asi);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x4: /* V9 rdtick */
                     {
@@ -2621,39 +2758,38 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        offsetof(CPUSPARCState, tick));
                         gen_helper_tick_get_count(cpu_dst, r_tickptr);
                         tcg_temp_free_ptr(r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
                 case 0x5: /* V9 rdpc */
                     {
-                        TCGv r_const;
-
+                        TCGv t = gen_dest_gpr(dc, rd);
                         if (unlikely(AM_CHECK(dc))) {
-                            r_const = tcg_const_tl(dc->pc & 0xffffffffULL);
+                            tcg_gen_movi_tl(t, dc->pc & 0xffffffffULL);
                         } else {
-                           r_const = tcg_const_tl(dc->pc);
+                            tcg_gen_movi_tl(t, dc->pc);
                         }
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        gen_store_gpr(dc, rd, t);
                     }
                     break;
                 case 0x6: /* V9 rdfprs */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_fprs);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0xf: /* V9 membar */
                     break; /* no effect */
                 case 0x13: /* Graphics Status */
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
-                    gen_movl_TN_reg(rd, cpu_gsr);
+                    }
+                    gen_store_gpr(dc, rd, cpu_gsr);
                     break;
                 case 0x16: /* Softint */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_softint);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x17: /* Tick compare */
-                    gen_movl_TN_reg(rd, cpu_tick_cmpr);
+                    gen_store_gpr(dc, rd, cpu_tick_cmpr);
                     break;
                 case 0x18: /* System tick */
                     {
@@ -2664,11 +2800,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        offsetof(CPUSPARCState, stick));
                         gen_helper_tick_get_count(cpu_dst, r_tickptr);
                         tcg_temp_free_ptr(r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
                 case 0x19: /* System tick compare */
-                    gen_movl_TN_reg(rd, cpu_stick_cmpr);
+                    gen_store_gpr(dc, rd, cpu_stick_cmpr);
                     break;
                 case 0x10: /* Performance Control */
                 case 0x11: /* Performance Instrumentation Counter */
@@ -2682,10 +2818,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #if !defined(CONFIG_USER_ONLY)
             } else if (xop == 0x29) { /* rdpsr / UA2005 rdhpr */
 #ifndef TARGET_SPARC64
-                if (!supervisor(dc))
+                if (!supervisor(dc)) {
                     goto priv_insn;
-                gen_helper_compute_psr(cpu_env);
-                dc->cc_op = CC_OP_FLAGS;
+                }
+                update_psr(dc);
                 gen_helper_rdpsr(cpu_dst, cpu_env);
 #else
                 CHECK_IU_FEATURE(dc, HYPV);
@@ -2715,11 +2851,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
 #endif
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
                 break;
             } else if (xop == 0x2a) { /* rdwim / V9 rdpr */
-                if (!supervisor(dc))
+                if (!supervisor(dc)) {
                     goto priv_insn;
+                }
+                cpu_tmp0 = get_temp_tl(dc);
 #ifdef TARGET_SPARC64
                 rs1 = GET_FIELD(insn, 13, 17);
                 switch (rs1) {
@@ -2758,14 +2896,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 3: // tt
                     {
-                        TCGv_ptr r_tsptr;
+                        TCGv_ptr r_tsptr = tcg_temp_new_ptr();
 
-                        r_tsptr = tcg_temp_new_ptr();
                         gen_load_trap_state_at_tl(r_tsptr, cpu_env);
-                        tcg_gen_ld_i32(cpu_tmp32, r_tsptr,
-                                       offsetof(trap_state, tt));
+                        tcg_gen_ld32s_tl(cpu_tmp0, r_tsptr,
+                                         offsetof(trap_state, tt));
                         tcg_temp_free_ptr(r_tsptr);
-                        tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
                     }
                     break;
                 case 4: // tick
@@ -2776,7 +2912,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
                         gen_helper_tick_get_count(cpu_tmp0, r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_tmp0);
                         tcg_temp_free_ptr(r_tickptr);
                     }
                     break;
@@ -2784,53 +2919,44 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_mov_tl(cpu_tmp0, cpu_tbr);
                     break;
                 case 6: // pstate
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, pstate));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, pstate));
                     break;
                 case 7: // tl
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, tl));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, tl));
                     break;
                 case 8: // pil
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, psrpil));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, psrpil));
                     break;
                 case 9: // cwp
                     gen_helper_rdcwp(cpu_tmp0, cpu_env);
                     break;
                 case 10: // cansave
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, cansave));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, cansave));
                     break;
                 case 11: // canrestore
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, canrestore));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, canrestore));
                     break;
                 case 12: // cleanwin
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, cleanwin));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, cleanwin));
                     break;
                 case 13: // otherwin
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, otherwin));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, otherwin));
                     break;
                 case 14: // wstate
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, wstate));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, wstate));
                     break;
                 case 16: // UA2005 gl
                     CHECK_IU_FEATURE(dc, GL);
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, gl));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, gl));
                     break;
                 case 26: // UA2005 strand status
                     CHECK_IU_FEATURE(dc, HYPV);
@@ -2848,27 +2974,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #else
                 tcg_gen_ext_i32_tl(cpu_tmp0, cpu_wim);
 #endif
-                gen_movl_TN_reg(rd, cpu_tmp0);
+                gen_store_gpr(dc, rd, cpu_tmp0);
                 break;
             } else if (xop == 0x2b) { /* rdtbr / V9 flushw */
 #ifdef TARGET_SPARC64
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 gen_helper_flushw(cpu_env);
 #else
                 if (!supervisor(dc))
                     goto priv_insn;
-                gen_movl_TN_reg(rd, cpu_tbr);
+                gen_store_gpr(dc, rd, cpu_tbr);
 #endif
                 break;
 #endif
             } else if (xop == 0x34) {   /* FPU Operations */
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
                 gen_op_clear_ieee_excp_and_FTT();
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 switch (xop) {
                 case 0x1: /* fmovs */
                     cpu_src1_32 = gen_load_fpr_F(dc, rs2);
@@ -3036,216 +3163,121 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #ifdef TARGET_SPARC64
                 int cond;
 #endif
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
                 gen_op_clear_ieee_excp_and_FTT();
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc, cpu_cond);
-#ifdef TARGET_SPARC64
-                if ((xop & 0x11f) == 0x005) { // V9 fmovsr
-                    int l1;
+                save_state(dc);
 
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    cpu_src1_32 = gen_load_fpr_F(dc, rs2);
-                    gen_store_fpr_F(dc, rd, cpu_src1_32);
-                    gen_set_label(l1);
+#ifdef TARGET_SPARC64
+#define FMOVR(sz)                                                  \
+                do {                                               \
+                    DisasCompare cmp;                              \
+                    cond = GET_FIELD_SP(insn, 10, 12);             \
+                    cpu_src1 = get_src1(dc, insn);                 \
+                    gen_compare_reg(&cmp, cond, cpu_src1);         \
+                    gen_fmov##sz(dc, &cmp, rd, rs2);               \
+                    free_compare(&cmp);                            \
+                } while (0)
+
+                if ((xop & 0x11f) == 0x005) { /* V9 fmovsr */
+                    FMOVR(s);
                     break;
                 } else if ((xop & 0x11f) == 0x006) { // V9 fmovdr
-                    int l1;
-
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    gen_store_fpr_D(dc, rd, cpu_src1_64);
-                    gen_set_label(l1);
+                    FMOVR(d);
                     break;
                 } else if ((xop & 0x11f) == 0x007) { // V9 fmovqr
-                    int l1;
-
                     CHECK_FPU_FEATURE(dc, FLOAT128);
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    gen_move_Q(rd, rs2);
-                    gen_set_label(l1);
+                    FMOVR(q);
                     break;
                 }
+#undef FMOVR
 #endif
                 switch (xop) {
 #ifdef TARGET_SPARC64
-#define FMOVSCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
+#define FMOVCC(fcc, sz)                                                 \
+                    do {                                                \
+                        DisasCompare cmp;                               \
                         cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_32 = gen_load_fpr_F(dc, rs2);          \
-                        gen_store_fpr_F(dc, rd, cpu_src1_32);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVDCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_64 = gen_load_fpr_D(dc, rs2);          \
-                        gen_store_fpr_D(dc, rd, cpu_src1_64);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVQCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        gen_move_Q(rd, rs2);                            \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
+                        gen_fcompare(&cmp, fcc, cond);                  \
+                        gen_fmov##sz(dc, &cmp, rd, rs2);                \
+                        free_compare(&cmp);                             \
+                    } while (0)
+
                     case 0x001: /* V9 fmovscc %fcc0 */
-                        FMOVSCC(0);
+                        FMOVCC(0, s);
                         break;
                     case 0x002: /* V9 fmovdcc %fcc0 */
-                        FMOVDCC(0);
+                        FMOVCC(0, d);
                         break;
                     case 0x003: /* V9 fmovqcc %fcc0 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(0);
+                        FMOVCC(0, q);
                         break;
                     case 0x041: /* V9 fmovscc %fcc1 */
-                        FMOVSCC(1);
+                        FMOVCC(1, s);
                         break;
                     case 0x042: /* V9 fmovdcc %fcc1 */
-                        FMOVDCC(1);
+                        FMOVCC(1, d);
                         break;
                     case 0x043: /* V9 fmovqcc %fcc1 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(1);
+                        FMOVCC(1, q);
                         break;
                     case 0x081: /* V9 fmovscc %fcc2 */
-                        FMOVSCC(2);
+                        FMOVCC(2, s);
                         break;
                     case 0x082: /* V9 fmovdcc %fcc2 */
-                        FMOVDCC(2);
+                        FMOVCC(2, d);
                         break;
                     case 0x083: /* V9 fmovqcc %fcc2 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(2);
+                        FMOVCC(2, q);
                         break;
                     case 0x0c1: /* V9 fmovscc %fcc3 */
-                        FMOVSCC(3);
+                        FMOVCC(3, s);
                         break;
                     case 0x0c2: /* V9 fmovdcc %fcc3 */
-                        FMOVDCC(3);
+                        FMOVCC(3, d);
                         break;
                     case 0x0c3: /* V9 fmovqcc %fcc3 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(3);
+                        FMOVCC(3, q);
                         break;
-#undef FMOVSCC
-#undef FMOVDCC
-#undef FMOVQCC
-#define FMOVSCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_32 = gen_load_fpr_F(dc, rs2);          \
-                        gen_store_fpr_F(dc, rd, cpu_src1_32);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVDCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_64 = gen_load_fpr_D(dc, rs2);          \
-                        gen_store_fpr_D(dc, rd, cpu_src1_64);           \
-                        gen_update_fprs_dirty(DFPREG(rd));              \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVQCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
+#undef FMOVCC
+#define FMOVCC(xcc, sz)                                                 \
+                    do {                                                \
+                        DisasCompare cmp;                               \
                         cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        gen_move_Q(rd, rs2);                            \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
+                        gen_compare(&cmp, xcc, cond, dc);               \
+                        gen_fmov##sz(dc, &cmp, rd, rs2);                \
+                        free_compare(&cmp);                             \
+                    } while (0)
 
                     case 0x101: /* V9 fmovscc %icc */
-                        FMOVSCC(0);
+                        FMOVCC(0, s);
                         break;
                     case 0x102: /* V9 fmovdcc %icc */
-                        FMOVDCC(0);
+                        FMOVCC(0, d);
                         break;
                     case 0x103: /* V9 fmovqcc %icc */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(0);
+                        FMOVCC(0, q);
                         break;
                     case 0x181: /* V9 fmovscc %xcc */
-                        FMOVSCC(1);
+                        FMOVCC(1, s);
                         break;
                     case 0x182: /* V9 fmovdcc %xcc */
-                        FMOVDCC(1);
+                        FMOVCC(1, d);
                         break;
                     case 0x183: /* V9 fmovqcc %xcc */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(1);
+                        FMOVCC(1, q);
                         break;
-#undef FMOVSCC
-#undef FMOVDCC
-#undef FMOVQCC
+#undef FMOVCC
 #endif
                     case 0x51: /* fcmps, V9 %fcc */
                         cpu_src1_32 = gen_load_fpr_F(dc, rs1);
@@ -3283,43 +3315,45 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                 }
             } else if (xop == 0x2) {
-                // clr/mov shortcut
-
+                TCGv dst = gen_dest_gpr(dc, rd);
                 rs1 = GET_FIELD(insn, 13, 17);
                 if (rs1 == 0) {
-                    // or %g0, x, y -> mov T0, x; mov y, T0
+                    /* clr/mov shortcut : or %g0, x, y -> mov x, y */
                     if (IS_IMM) {       /* immediate */
-                        TCGv r_const;
-
                         simm = GET_FIELDs(insn, 19, 31);
-                        r_const = tcg_const_tl(simm);
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        tcg_gen_movi_tl(dst, simm);
+                        gen_store_gpr(dc, rd, dst);
                     } else {            /* register */
                         rs2 = GET_FIELD(insn, 27, 31);
-                        gen_movl_reg_TN(rs2, cpu_dst);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        if (rs2 == 0) {
+                            tcg_gen_movi_tl(dst, 0);
+                            gen_store_gpr(dc, rd, dst);
+                        } else {
+                            cpu_src2 = gen_load_gpr(dc, rs2);
+                            gen_store_gpr(dc, rd, cpu_src2);
+                        }
                     }
                 } else {
-                    cpu_src1 = get_src1(insn, cpu_src1);
+                    cpu_src1 = get_src1(dc, insn);
                     if (IS_IMM) {       /* immediate */
                         simm = GET_FIELDs(insn, 19, 31);
-                        tcg_gen_ori_tl(cpu_dst, cpu_src1, simm);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        tcg_gen_ori_tl(dst, cpu_src1, simm);
+                        gen_store_gpr(dc, rd, dst);
                     } else {            /* register */
-                        // or x, %g0, y -> mov T1, x; mov y, T1
                         rs2 = GET_FIELD(insn, 27, 31);
-                        if (rs2 != 0) {
-                            gen_movl_reg_TN(rs2, cpu_src2);
-                            tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
-                            gen_movl_TN_reg(rd, cpu_dst);
-                        } else
-                            gen_movl_TN_reg(rd, cpu_src1);
+                        if (rs2 == 0) {
+                            /* mov shortcut:  or x, %g0, y -> mov x, y */
+                            gen_store_gpr(dc, rd, cpu_src1);
+                        } else {
+                            cpu_src2 = gen_load_gpr(dc, rs2);
+                            tcg_gen_or_tl(dst, cpu_src1, cpu_src2);
+                            gen_store_gpr(dc, rd, dst);
+                        }
                     }
                 }
 #ifdef TARGET_SPARC64
             } else if (xop == 0x25) { /* sll, V9 sllx */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
@@ -3329,7 +3363,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                     } else {
@@ -3337,9 +3372,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                     tcg_gen_shl_i64(cpu_dst, cpu_src1, cpu_tmp0);
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
             } else if (xop == 0x26) { /* srl, V9 srlx */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
@@ -3350,7 +3385,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                         tcg_gen_shr_i64(cpu_dst, cpu_src1, cpu_tmp0);
@@ -3360,65 +3396,48 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_gen_shr_i64(cpu_dst, cpu_dst, cpu_tmp0);
                     }
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
             } else if (xop == 0x27) { /* sra, V9 srax */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
                         tcg_gen_sari_i64(cpu_dst, cpu_src1, simm & 0x3f);
                     } else {
-                        tcg_gen_andi_i64(cpu_dst, cpu_src1, 0xffffffffULL);
-                        tcg_gen_ext32s_i64(cpu_dst, cpu_dst);
+                        tcg_gen_ext32s_i64(cpu_dst, cpu_src1);
                         tcg_gen_sari_i64(cpu_dst, cpu_dst, simm & 0x1f);
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                         tcg_gen_sar_i64(cpu_dst, cpu_src1, cpu_tmp0);
                     } else {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x1f);
-                        tcg_gen_andi_i64(cpu_dst, cpu_src1, 0xffffffffULL);
-                        tcg_gen_ext32s_i64(cpu_dst, cpu_dst);
+                        tcg_gen_ext32s_i64(cpu_dst, cpu_src1);
                         tcg_gen_sar_i64(cpu_dst, cpu_dst, cpu_tmp0);
                     }
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
 #endif
             } else if (xop < 0x36) {
                 if (xop < 0x20) {
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src2(insn, cpu_src2);
+                    cpu_src1 = get_src1(dc, insn);
+                    cpu_src2 = get_src2(dc, insn);
                     switch (xop & ~0x10) {
                     case 0x0: /* add */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            if (xop & 0x10) {
-                                gen_op_addi_cc(cpu_dst, cpu_src1, simm);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
-                                dc->cc_op = CC_OP_ADD;
-                            } else {
-                                tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
-                            }
+                        if (xop & 0x10) {
+                            gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
+                            tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
+                            dc->cc_op = CC_OP_ADD;
                         } else {
-                            if (xop & 0x10) {
-                                gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
-                                dc->cc_op = CC_OP_ADD;
-                            } else {
-                                tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                            }
+                            tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
                         }
                         break;
                     case 0x1: /* and */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_andi_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_and_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_and_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3426,12 +3445,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x2: /* or */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_ori_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3439,12 +3453,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x3: /* xor */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_xori_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3452,30 +3461,16 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x4: /* sub */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            if (xop & 0x10) {
-                                gen_op_subi_cc(cpu_dst, cpu_src1, simm, dc);
-                            } else {
-                                tcg_gen_subi_tl(cpu_dst, cpu_src1, simm);
-                            }
+                        if (xop & 0x10) {
+                            gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
+                            tcg_gen_movi_i32(cpu_cc_op, CC_OP_SUB);
+                            dc->cc_op = CC_OP_SUB;
                         } else {
-                            if (xop & 0x10) {
-                                gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_SUB);
-                                dc->cc_op = CC_OP_SUB;
-                            } else {
-                                tcg_gen_sub_tl(cpu_dst, cpu_src1, cpu_src2);
-                            }
+                            tcg_gen_sub_tl(cpu_dst, cpu_src1, cpu_src2);
                         }
                         break;
                     case 0x5: /* andn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_andi_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_andc_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_andc_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3483,12 +3478,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x6: /* orn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_ori_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_orc_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_orc_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3496,13 +3486,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x7: /* xorn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_xori_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_not_tl(cpu_tmp0, cpu_src2);
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_tmp0);
-                        }
+                        tcg_gen_eqv_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3515,12 +3499,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         break;
 #ifdef TARGET_SPARC64
                     case 0x9: /* V9 mulx */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_muli_i64(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_mul_i64(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_mul_i64(cpu_dst, cpu_src1, cpu_src2);
                         break;
 #endif
                     case 0xa: /* umul */
@@ -3547,17 +3526,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         break;
 #ifdef TARGET_SPARC64
                     case 0xd: /* V9 udivx */
-                        {
-                            TCGv r_temp1, r_temp2;
-                            r_temp1 = tcg_temp_local_new();
-                            r_temp2 = tcg_temp_local_new();
-                            tcg_gen_mov_tl(r_temp1, cpu_src1);
-                            tcg_gen_mov_tl(r_temp2, cpu_src2);
-                            gen_trap_ifdivzero_tl(r_temp2);
-                            tcg_gen_divu_i64(cpu_dst, r_temp1, r_temp2);
-                            tcg_temp_free(r_temp1);
-                            tcg_temp_free(r_temp2);
-                        }
+                        gen_helper_udivx(cpu_dst, cpu_env, cpu_src1, cpu_src2);
                         break;
 #endif
                     case 0xe: /* udiv */
@@ -3585,41 +3554,39 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     default:
                         goto illegal_insn;
                     }
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                 } else {
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src2(insn, cpu_src2);
+                    cpu_src1 = get_src1(dc, insn);
+                    cpu_src2 = get_src2(dc, insn);
                     switch (xop) {
                     case 0x20: /* taddcc */
-                        gen_op_tadd_cc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_TADD);
                         dc->cc_op = CC_OP_TADD;
                         break;
                     case 0x21: /* tsubcc */
-                        gen_op_tsub_cc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_TSUB);
                         dc->cc_op = CC_OP_TSUB;
                         break;
                     case 0x22: /* taddcctv */
-                        save_state(dc, cpu_cond);
-                        gen_op_tadd_ccTV(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
-                        tcg_gen_movi_i32(cpu_cc_op, CC_OP_TADDTV);
+                        gen_helper_taddcctv(cpu_dst, cpu_env,
+                                            cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         dc->cc_op = CC_OP_TADDTV;
                         break;
                     case 0x23: /* tsubcctv */
-                        save_state(dc, cpu_cond);
-                        gen_op_tsub_ccTV(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
-                        tcg_gen_movi_i32(cpu_cc_op, CC_OP_TSUBTV);
+                        gen_helper_tsubcctv(cpu_dst, cpu_env,
+                                            cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         dc->cc_op = CC_OP_TSUBTV;
                         break;
                     case 0x24: /* mulscc */
-                        gen_helper_compute_psr(cpu_env);
+                        update_psr(dc);
                         gen_op_mulscc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
                         dc->cc_op = CC_OP_ADD;
                         break;
@@ -3629,34 +3596,38 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_shli_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_shl_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x26:  /* srl */
                         if (IS_IMM) { /* immediate */
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_shri_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_shr_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x27:  /* sra */
                         if (IS_IMM) { /* immediate */
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_sari_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_sar_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
 #endif
                     case 0x30:
                         {
+                            cpu_tmp0 = get_temp_tl(dc);
                             switch(rd) {
                             case 0: /* wry */
                                 tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
@@ -3674,20 +3645,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 break;
 #else
                             case 0x2: /* V9 wrccr */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                gen_helper_wrccr(cpu_env, cpu_dst);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_wrccr(cpu_env, cpu_tmp0);
                                 tcg_gen_movi_i32(cpu_cc_op, CC_OP_FLAGS);
                                 dc->cc_op = CC_OP_FLAGS;
                                 break;
                             case 0x3: /* V9 wrasi */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_andi_tl(cpu_dst, cpu_dst, 0xff);
-                                tcg_gen_trunc_tl_i32(cpu_asi, cpu_dst);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0xff);
+                                tcg_gen_trunc_tl_i32(cpu_asi, cpu_tmp0);
                                 break;
                             case 0x6: /* V9 wrfprs */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_trunc_tl_i32(cpu_fprs, cpu_dst);
-                                save_state(dc, cpu_cond);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                tcg_gen_trunc_tl_i32(cpu_fprs, cpu_tmp0);
+                                save_state(dc);
                                 gen_op_next_insn();
                                 tcg_gen_exit_tb(0);
                                 dc->is_br = 1;
@@ -3700,27 +3671,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #endif
                                 break;
                             case 0x13: /* Graphics Status */
-                                if (gen_trap_ifnofpu(dc, cpu_cond))
+                                if (gen_trap_ifnofpu(dc)) {
                                     goto jmp_insn;
+                                }
                                 tcg_gen_xor_tl(cpu_gsr, cpu_src1, cpu_src2);
                                 break;
                             case 0x14: /* Softint set */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_set_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_set_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x15: /* Softint clear */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_clear_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_clear_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x16: /* Softint write */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_write_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_write_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x17: /* Tick compare */
 #if !defined(CONFIG_USER_ONLY)
@@ -3748,13 +3720,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 {
                                     TCGv_ptr r_tickptr;
 
-                                    tcg_gen_xor_tl(cpu_dst, cpu_src1,
+                                    tcg_gen_xor_tl(cpu_tmp0, cpu_src1,
                                                    cpu_src2);
                                     r_tickptr = tcg_temp_new_ptr();
                                     tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                                    offsetof(CPUSPARCState, stick));
                                     gen_helper_tick_set_count(r_tickptr,
-                                                              cpu_dst);
+                                                              cpu_tmp0);
                                     tcg_temp_free_ptr(r_tickptr);
                                 }
                                 break;
@@ -3809,11 +3781,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 goto illegal_insn;
                             }
 #else
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                            gen_helper_wrpsr(cpu_env, cpu_dst);
+                            cpu_tmp0 = get_temp_tl(dc);
+                            tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                            gen_helper_wrpsr(cpu_env, cpu_tmp0);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_FLAGS);
                             dc->cc_op = CC_OP_FLAGS;
-                            save_state(dc, cpu_cond);
+                            save_state(dc);
                             gen_op_next_insn();
                             tcg_gen_exit_tb(0);
                             dc->is_br = 1;
@@ -3824,6 +3797,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         {
                             if (!supervisor(dc))
                                 goto priv_insn;
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
 #ifdef TARGET_SPARC64
                             switch (rd) {
@@ -3867,9 +3841,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 
                                     r_tsptr = tcg_temp_new_ptr();
                                     gen_load_trap_state_at_tl(r_tsptr, cpu_env);
-                                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                    tcg_gen_st_i32(cpu_tmp32, r_tsptr,
-                                                   offsetof(trap_state, tt));
+                                    tcg_gen_st32_tl(cpu_tmp0, r_tsptr,
+                                                    offsetof(trap_state, tt));
                                     tcg_temp_free_ptr(r_tsptr);
                                 }
                                 break;
@@ -3889,28 +3862,15 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 tcg_gen_mov_tl(cpu_tbr, cpu_tmp0);
                                 break;
                             case 6: // pstate
-                                {
-                                    TCGv r_tmp = tcg_temp_local_new();
-
-                                    tcg_gen_mov_tl(r_tmp, cpu_tmp0);
-                                    save_state(dc, cpu_cond);
-                                    gen_helper_wrpstate(cpu_env, r_tmp);
-                                    tcg_temp_free(r_tmp);
-                                    dc->npc = DYNAMIC_PC;
-                                }
+                                save_state(dc);
+                                gen_helper_wrpstate(cpu_env, cpu_tmp0);
+                                dc->npc = DYNAMIC_PC;
                                 break;
                             case 7: // tl
-                                {
-                                    TCGv r_tmp = tcg_temp_local_new();
-
-                                    tcg_gen_mov_tl(r_tmp, cpu_tmp0);
-                                    save_state(dc, cpu_cond);
-                                    tcg_gen_trunc_tl_i32(cpu_tmp32, r_tmp);
-                                    tcg_temp_free(r_tmp);
-                                    tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                                   offsetof(CPUSPARCState, tl));
-                                    dc->npc = DYNAMIC_PC;
-                                }
+                                save_state(dc);
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                               offsetof(CPUSPARCState, tl));
+                                dc->npc = DYNAMIC_PC;
                                 break;
                             case 8: // pil
                                 gen_helper_wrpil(cpu_env, cpu_tmp0);
@@ -3919,40 +3879,34 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 gen_helper_wrcwp(cpu_env, cpu_tmp0);
                                 break;
                             case 10: // cansave
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        cansave));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         cansave));
                                 break;
                             case 11: // canrestore
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        canrestore));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         canrestore));
                                 break;
                             case 12: // cleanwin
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        cleanwin));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         cleanwin));
                                 break;
                             case 13: // otherwin
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        otherwin));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         otherwin));
                                 break;
                             case 14: // wstate
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        wstate));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         wstate));
                                 break;
                             case 16: // UA2005 gl
                                 CHECK_IU_FEATURE(dc, GL);
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState, gl));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState, gl));
                                 break;
                             case 26: // UA2005 strand status
                                 CHECK_IU_FEATURE(dc, HYPV);
@@ -3964,11 +3918,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 goto illegal_insn;
                             }
 #else
-                            tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                            if (dc->def->nwindows != 32)
-                                tcg_gen_andi_tl(cpu_tmp32, cpu_tmp32,
+                            tcg_gen_trunc_tl_i32(cpu_wim, cpu_tmp0);
+                            if (dc->def->nwindows != 32) {
+                                tcg_gen_andi_tl(cpu_wim, cpu_wim,
                                                 (1 << dc->def->nwindows) - 1);
-                            tcg_gen_mov_i32(cpu_wim, cpu_tmp32);
+                            }
 #endif
                         }
                         break;
@@ -3982,11 +3936,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                             CHECK_IU_FEATURE(dc, HYPV);
                             if (!hypervisor(dc))
                                 goto priv_insn;
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
                             switch (rd) {
                             case 0: // hpstate
                                 // XXX gen_op_wrhpstate();
-                                save_state(dc, cpu_cond);
+                                save_state(dc);
                                 gen_op_next_insn();
                                 tcg_gen_exit_tb(0);
                                 dc->is_br = 1;
@@ -4026,74 +3981,67 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         {
                             int cc = GET_FIELD_SP(insn, 11, 12);
                             int cond = GET_FIELD_SP(insn, 14, 17);
-                            TCGv r_cond;
-                            int l1;
+                            DisasCompare cmp;
+                            TCGv dst;
 
-                            r_cond = tcg_temp_new();
                             if (insn & (1 << 18)) {
-                                if (cc == 0)
-                                    gen_cond(r_cond, 0, cond, dc);
-                                else if (cc == 2)
-                                    gen_cond(r_cond, 1, cond, dc);
-                                else
+                                if (cc == 0) {
+                                    gen_compare(&cmp, 0, cond, dc);
+                                } else if (cc == 2) {
+                                    gen_compare(&cmp, 1, cond, dc);
+                                } else {
                                     goto illegal_insn;
+                                }
                             } else {
-                                gen_fcond(r_cond, cc, cond);
+                                gen_fcompare(&cmp, cc, cond);
                             }
 
-                            l1 = gen_new_label();
-
-                            tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
-                            if (IS_IMM) {       /* immediate */
-                                TCGv r_const;
-
+                            /* The get_src2 above loaded the normal 13-bit
+                               immediate field, not the 11-bit field we have
+                               in movcc.  But it did handle the reg case.  */
+                            if (IS_IMM) {
                                 simm = GET_FIELD_SPs(insn, 0, 10);
-                                r_const = tcg_const_tl(simm);
-                                gen_movl_TN_reg(rd, r_const);
-                                tcg_temp_free(r_const);
-                            } else {
-                                rs2 = GET_FIELD_SP(insn, 0, 4);
-                                gen_movl_reg_TN(rs2, cpu_tmp0);
-                                gen_movl_TN_reg(rd, cpu_tmp0);
+                                tcg_gen_movi_tl(cpu_src2, simm);
                             }
-                            gen_set_label(l1);
-                            tcg_temp_free(r_cond);
+
+                            dst = gen_load_gpr(dc, rd);
+                            tcg_gen_movcond_tl(cmp.cond, dst,
+                                               cmp.c1, cmp.c2,
+                                               cpu_src2, dst);
+                            free_compare(&cmp);
+                            gen_store_gpr(dc, rd, dst);
                             break;
                         }
                     case 0x2d: /* V9 sdivx */
-                        gen_op_sdivx(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_helper_sdivx(cpu_dst, cpu_env, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x2e: /* V9 popc */
-                        {
-                            cpu_src2 = get_src2(insn, cpu_src2);
-                            gen_helper_popc(cpu_dst, cpu_src2);
-                            gen_movl_TN_reg(rd, cpu_dst);
-                        }
+                        gen_helper_popc(cpu_dst, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
+                        break;
                     case 0x2f: /* V9 movr */
                         {
                             int cond = GET_FIELD_SP(insn, 10, 12);
-                            int l1;
-
-                            cpu_src1 = get_src1(insn, cpu_src1);
-
-                            l1 = gen_new_label();
+                            DisasCompare cmp;
+                            TCGv dst;
 
-                            tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond],
-                                              cpu_src1, 0, l1);
-                            if (IS_IMM) {       /* immediate */
-                                TCGv r_const;
+                            gen_compare_reg(&cmp, cond, cpu_src1);
 
+                            /* The get_src2 above loaded the normal 13-bit
+                               immediate field, not the 10-bit field we have
+                               in movr.  But it did handle the reg case.  */
+                            if (IS_IMM) {
                                 simm = GET_FIELD_SPs(insn, 0, 9);
-                                r_const = tcg_const_tl(simm);
-                                gen_movl_TN_reg(rd, r_const);
-                                tcg_temp_free(r_const);
-                            } else {
-                                rs2 = GET_FIELD_SP(insn, 0, 4);
-                                gen_movl_reg_TN(rs2, cpu_tmp0);
-                                gen_movl_TN_reg(rd, cpu_tmp0);
+                                tcg_gen_movi_tl(cpu_src2, simm);
                             }
-                            gen_set_label(l1);
+
+                            dst = gen_load_gpr(dc, rd);
+                            tcg_gen_movcond_tl(cmp.cond, dst,
+                                               cmp.c1, cmp.c2,
+                                               cpu_src2, dst);
+                            free_compare(&cmp);
+                            gen_store_gpr(dc, rd, dst);
                             break;
                         }
 #endif
@@ -4106,194 +4054,195 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 int opf = GET_FIELD_SP(insn, 5, 13);
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
 
                 switch (opf) {
                 case 0x000: /* VIS I edge8cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x001: /* VIS II edge8n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x002: /* VIS I edge8lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x003: /* VIS II edge8ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x004: /* VIS I edge16cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x005: /* VIS II edge16n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x006: /* VIS I edge16lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x007: /* VIS II edge16ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x008: /* VIS I edge32cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x009: /* VIS II edge32n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x00a: /* VIS I edge32lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x00b: /* VIS II edge32ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x010: /* VIS I array8 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x012: /* VIS I array16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_shli_i64(cpu_dst, cpu_dst, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x014: /* VIS I array32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_shli_i64(cpu_dst, cpu_dst, 2);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x018: /* VIS I alignaddr */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_alignaddr(cpu_dst, cpu_src1, cpu_src2, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x01a: /* VIS I alignaddrl */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_alignaddr(cpu_dst, cpu_src1, cpu_src2, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x019: /* VIS II bmask */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src1(insn, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_deposit_tl(cpu_gsr, cpu_gsr, cpu_dst, 32, 32);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x020: /* VIS I fcmple16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmple16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x022: /* VIS I fcmpne16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpne16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x024: /* VIS I fcmple32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmple32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x026: /* VIS I fcmpne32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpne32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x028: /* VIS I fcmpgt16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpgt16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02a: /* VIS I fcmpeq16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpeq16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02c: /* VIS I fcmpgt32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpgt32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02e: /* VIS I fcmpeq32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpeq32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x031: /* VIS I fmul8x16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
@@ -4330,14 +4279,14 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 case 0x03b: /* VIS I fpack16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     gen_helper_fpack16(cpu_dst_32, cpu_gsr, cpu_src1_64);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
                 case 0x03d: /* VIS I fpackfix */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     gen_helper_fpackfix(cpu_dst_32, cpu_gsr, cpu_src1_64);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4395,13 +4344,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x060: /* VIS I fzero */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_movi_i64(cpu_dst_64, 0);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
                 case 0x061: /* VIS I fzeros */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     tcg_gen_movi_i32(cpu_dst_32, 0);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4523,13 +4472,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x07e: /* VIS I fone */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_movi_i64(cpu_dst_64, -1);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
                 case 0x07f: /* VIS I fones */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     tcg_gen_movi_i32(cpu_dst_32, -1);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4553,56 +4502,60 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
             } else if (xop == 0x39) { /* V9 return */
                 TCGv_i32 r_const;
 
-                save_state(dc, cpu_cond);
-                cpu_src1 = get_src1(insn, cpu_src1);
+                save_state(dc);
+                cpu_src1 = get_src1(dc, insn);
+                cpu_tmp0 = get_temp_tl(dc);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 19, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
+                    tcg_gen_addi_tl(cpu_tmp0, cpu_src1, simm);
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
                     if (rs2) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                        cpu_src2 = gen_load_gpr(dc, rs2);
+                        tcg_gen_add_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                    } else {
+                        tcg_gen_mov_tl(cpu_tmp0, cpu_src1);
+                    }
                 }
                 gen_helper_restore(cpu_env);
-                gen_mov_pc_npc(dc, cpu_cond);
+                gen_mov_pc_npc(dc);
                 r_const = tcg_const_i32(3);
-                gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                 tcg_temp_free_i32(r_const);
-                tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                 dc->npc = DYNAMIC_PC;
                 goto jmp_insn;
 #endif
             } else {
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
+                cpu_tmp0 = get_temp_tl(dc);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 19, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
+                    tcg_gen_addi_tl(cpu_tmp0, cpu_src1, simm);
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
                     if (rs2) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                        cpu_src2 = gen_load_gpr(dc, rs2);
+                        tcg_gen_add_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                    } else {
+                        tcg_gen_mov_tl(cpu_tmp0, cpu_src1);
+                    }
                 }
                 switch (xop) {
                 case 0x38:      /* jmpl */
                     {
-                        TCGv r_pc;
+                        TCGv t;
                         TCGv_i32 r_const;
 
-                        r_pc = tcg_const_tl(dc->pc);
-                        gen_movl_TN_reg(rd, r_pc);
-                        tcg_temp_free(r_pc);
-                        gen_mov_pc_npc(dc, cpu_cond);
+                        t = gen_dest_gpr(dc, rd);
+                        tcg_gen_movi_tl(t, dc->pc);
+                        gen_store_gpr(dc, rd, t);
+                        gen_mov_pc_npc(dc);
                         r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                         tcg_temp_free_i32(r_const);
-                        gen_address_mask(dc, cpu_dst);
-                        tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                        gen_address_mask(dc, cpu_tmp0);
+                        tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
                     }
                     goto jmp_insn;
@@ -4613,11 +4566,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 
                         if (!supervisor(dc))
                             goto priv_insn;
-                        gen_mov_pc_npc(dc, cpu_cond);
+                        gen_mov_pc_npc(dc);
                         r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                         tcg_temp_free_i32(r_const);
-                        tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                        tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
                         gen_helper_rett(cpu_env);
                     }
@@ -4629,14 +4582,14 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     /* nop */
                     break;
                 case 0x3c:      /* save */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_helper_save(cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_tmp0);
                     break;
                 case 0x3d:      /* restore */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_helper_restore(cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_tmp0);
                     break;
 #if !defined(CONFIG_USER_ONLY) && defined(TARGET_SPARC64)
                 case 0x3e:      /* V9 done/retry */
@@ -4672,32 +4625,29 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 3:                     /* load/store instructions */
         {
             unsigned int xop = GET_FIELD(insn, 7, 12);
+            /* ??? gen_address_mask prevents us from using a source
+               register directly.  Always generate a temporary.  */
+            TCGv cpu_addr = get_temp_tl(dc);
 
-            /* flush pending conditional evaluations before exposing
-               cpu state */
-            if (dc->cc_op != CC_OP_FLAGS) {
-                dc->cc_op = CC_OP_FLAGS;
-                gen_helper_compute_psr(cpu_env);
-            }
-            cpu_src1 = get_src1(insn, cpu_src1);
-            if (xop == 0x3c || xop == 0x3e) { // V9 casa/casxa
-                rs2 = GET_FIELD(insn, 27, 31);
-                gen_movl_reg_TN(rs2, cpu_src2);
-                tcg_gen_mov_tl(cpu_addr, cpu_src1);
+            tcg_gen_mov_tl(cpu_addr, get_src1(dc, insn));
+            if (xop == 0x3c || xop == 0x3e) {
+                /* V9 casa/casxa : no offset */
             } else if (IS_IMM) {     /* immediate */
                 simm = GET_FIELDs(insn, 19, 31);
-                tcg_gen_addi_tl(cpu_addr, cpu_src1, simm);
+                if (simm != 0) {
+                    tcg_gen_addi_tl(cpu_addr, cpu_addr, simm);
+                }
             } else {            /* register */
                 rs2 = GET_FIELD(insn, 27, 31);
                 if (rs2 != 0) {
-                    gen_movl_reg_TN(rs2, cpu_src2);
-                    tcg_gen_add_tl(cpu_addr, cpu_src1, cpu_src2);
-                } else
-                    tcg_gen_mov_tl(cpu_addr, cpu_src1);
+                    tcg_gen_add_tl(cpu_addr, cpu_addr, gen_load_gpr(dc, rs2));
+                }
             }
             if (xop < 4 || (xop > 7 && xop < 0x14 && xop != 0x0e) ||
                 (xop > 0x17 && xop <= 0x1d ) ||
                 (xop > 0x2c && xop <= 0x33) || xop == 0x1f || xop == 0x3d) {
+                TCGv cpu_val = gen_dest_gpr(dc, rd);
+
                 switch (xop) {
                 case 0x0:       /* ld, V9 lduw, load unsigned word */
                     gen_address_mask(dc, cpu_addr);
@@ -4716,20 +4666,23 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                     else {
                         TCGv_i32 r_const;
+                        TCGv_i64 t64;
 
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                         r_const = tcg_const_i32(7);
                         /* XXX remove alignment check */
                         gen_helper_check_align(cpu_env, cpu_addr, r_const);
                         tcg_temp_free_i32(r_const);
                         gen_address_mask(dc, cpu_addr);
-                        tcg_gen_qemu_ld64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                        tcg_gen_trunc_i64_tl(cpu_tmp0, cpu_tmp64);
-                        tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0xffffffffULL);
-                        gen_movl_TN_reg(rd + 1, cpu_tmp0);
-                        tcg_gen_shri_i64(cpu_tmp64, cpu_tmp64, 32);
-                        tcg_gen_trunc_i64_tl(cpu_val, cpu_tmp64);
-                        tcg_gen_andi_tl(cpu_val, cpu_val, 0xffffffffULL);
+                        t64 = tcg_temp_new_i64();
+                        tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
+                        tcg_gen_trunc_i64_tl(cpu_val, t64);
+                        tcg_gen_ext32u_tl(cpu_val, cpu_val);
+                        gen_store_gpr(dc, rd + 1, cpu_val);
+                        tcg_gen_shri_i64(t64, t64, 32);
+                        tcg_gen_trunc_i64_tl(cpu_val, t64);
+                        tcg_temp_free_i64(t64);
+                        tcg_gen_ext32u_tl(cpu_val, cpu_val);
                     }
                     break;
                 case 0x9:       /* ldsb, load signed byte */
@@ -4751,14 +4704,17 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_temp_free(r_const);
                     }
                     break;
-                case 0x0f:      /* swap, swap register with memory. Also
-                                   atomically */
-                    CHECK_IU_FEATURE(dc, SWAP);
-                    gen_movl_reg_TN(rd, cpu_val);
-                    gen_address_mask(dc, cpu_addr);
-                    tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                    tcg_gen_qemu_st32(cpu_val, cpu_addr, dc->mem_idx);
-                    tcg_gen_mov_tl(cpu_val, cpu_tmp0);
+                case 0x0f:
+                    /* swap, swap register with memory. Also atomically */
+                    {
+                        TCGv t0 = get_temp_tl(dc);
+                        CHECK_IU_FEATURE(dc, SWAP);
+                        cpu_src1 = gen_load_gpr(dc, rd);
+                        gen_address_mask(dc, cpu_addr);
+                        tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                        tcg_gen_qemu_st32(cpu_src1, cpu_addr, dc->mem_idx);
+                        tcg_gen_mov_tl(cpu_val, t0);
+                    }
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
                 case 0x10:      /* lda, V9 lduwa, load word alternate */
@@ -4768,7 +4724,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 4, 0);
                     break;
                 case 0x11:      /* lduba, load unsigned byte alternate */
@@ -4778,7 +4734,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 1, 0);
                     break;
                 case 0x12:      /* lduha, load unsigned halfword alternate */
@@ -4788,7 +4744,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 2, 0);
                     break;
                 case 0x13:      /* ldda, load double word alternate */
@@ -4800,8 +4756,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #endif
                     if (rd & 1)
                         goto illegal_insn;
-                    save_state(dc, cpu_cond);
-                    gen_ldda_asi(cpu_val, cpu_addr, insn, rd);
+                    save_state(dc);
+                    gen_ldda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     goto skip_move;
                 case 0x19:      /* ldsba, load signed byte alternate */
 #ifndef TARGET_SPARC64
@@ -4810,7 +4766,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 1, 1);
                     break;
                 case 0x1a:      /* ldsha, load signed halfword alternate */
@@ -4820,7 +4776,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 2, 1);
                     break;
                 case 0x1d:      /* ldstuba -- XXX: should be atomically */
@@ -4830,7 +4786,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldstub_asi(cpu_val, cpu_addr, insn);
                     break;
                 case 0x1f:      /* swapa, swap reg with alt. memory. Also
@@ -4842,9 +4798,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
-                    gen_movl_reg_TN(rd, cpu_val);
-                    gen_swap_asi(cpu_val, cpu_addr, insn);
+                    save_state(dc);
+                    cpu_src1 = gen_load_gpr(dc, rd);
+                    gen_swap_asi(cpu_val, cpu_src1, cpu_addr, insn);
                     break;
 
 #ifndef TARGET_SPARC64
@@ -4864,28 +4820,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_qemu_ld64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x18: /* V9 ldswa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 4, 1);
                     break;
                 case 0x1b: /* V9 ldxa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 8, 0);
                     break;
                 case 0x2d: /* V9 prefetch, no effect */
                     goto skip_move;
                 case 0x30: /* V9 ldfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 4, rd);
                     gen_update_fprs_dirty(rd);
                     goto skip_move;
                 case 0x33: /* V9 lddfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 8, DFPREG(rd));
                     gen_update_fprs_dirty(DFPREG(rd));
                     goto skip_move;
@@ -4893,10 +4849,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto skip_move;
                 case 0x32: /* V9 ldqfa */
                     CHECK_FPU_FEATURE(dc, FLOAT128);
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 16, QFPREG(rd));
                     gen_update_fprs_dirty(QFPREG(rd));
                     goto skip_move;
@@ -4904,39 +4860,42 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 default:
                     goto illegal_insn;
                 }
-                gen_movl_TN_reg(rd, cpu_val);
+                gen_store_gpr(dc, rd, cpu_val);
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
             skip_move: ;
 #endif
             } else if (xop >= 0x20 && xop < 0x24) {
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                TCGv t0;
+
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
-                save_state(dc, cpu_cond);
+                }
+                save_state(dc);
                 switch (xop) {
                 case 0x20:      /* ldf, load fpreg */
                     gen_address_mask(dc, cpu_addr);
-                    tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                    cpu_dst_32 = gen_dest_fpr_F();
-                    tcg_gen_trunc_tl_i32(cpu_dst_32, cpu_tmp0);
+                    t0 = get_temp_tl(dc);
+                    tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
+                    tcg_gen_trunc_tl_i32(cpu_dst_32, t0);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
                 case 0x21:      /* ldfsr, V9 ldxfsr */
 #ifdef TARGET_SPARC64
                     gen_address_mask(dc, cpu_addr);
                     if (rd == 1) {
-                        tcg_gen_qemu_ld64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                        gen_helper_ldxfsr(cpu_env, cpu_tmp64);
-                    } else {
-                        tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                        tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                        gen_helper_ldfsr(cpu_env, cpu_tmp32);
-                    }
-#else
-                    {
-                        tcg_gen_qemu_ld32u(cpu_tmp32, cpu_addr, dc->mem_idx);
-                        gen_helper_ldfsr(cpu_env, cpu_tmp32);
+                        TCGv_i64 t64 = tcg_temp_new_i64();
+                        tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
+                        gen_helper_ldxfsr(cpu_env, t64);
+                        tcg_temp_free_i64(t64);
+                        break;
                     }
 #endif
+                    cpu_dst_32 = get_temp_i32(dc);
+                    t0 = get_temp_tl(dc);
+                    tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                    tcg_gen_trunc_tl_i32(cpu_dst_32, t0);
+                    gen_helper_ldfsr(cpu_env, cpu_dst_32);
                     break;
                 case 0x22:      /* ldqf, load quad fpreg */
                     {
@@ -4953,7 +4912,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x23:      /* lddf, load double fpreg */
                     gen_address_mask(dc, cpu_addr);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_qemu_ld64(cpu_dst_64, cpu_addr, dc->mem_idx);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
@@ -4962,7 +4921,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 }
             } else if (xop < 8 || (xop >= 0x14 && xop < 0x18) ||
                        xop == 0xe || xop == 0x1e) {
-                gen_movl_reg_TN(rd, cpu_val);
+                TCGv cpu_val = gen_load_gpr(dc, rd);
+
                 switch (xop) {
                 case 0x4: /* st, store word */
                     gen_address_mask(dc, cpu_addr);
@@ -4981,16 +4941,21 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                     else {
                         TCGv_i32 r_const;
+                        TCGv_i64 t64;
+                        TCGv lo;
 
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                         gen_address_mask(dc, cpu_addr);
                         r_const = tcg_const_i32(7);
                         /* XXX remove alignment check */
                         gen_helper_check_align(cpu_env, cpu_addr, r_const);
                         tcg_temp_free_i32(r_const);
-                        gen_movl_reg_TN(rd + 1, cpu_tmp0);
-                        tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, cpu_val);
-                        tcg_gen_qemu_st64(cpu_tmp64, cpu_addr, dc->mem_idx);
+                        lo = gen_load_gpr(dc, rd + 1);
+
+                        t64 = tcg_temp_new_i64();
+                        tcg_gen_concat_tl_i64(t64, lo, cpu_val);
+                        tcg_gen_qemu_st64(t64, cpu_addr, dc->mem_idx);
+                        tcg_temp_free_i64(t64);
                     }
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
@@ -5001,7 +4966,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 4);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5012,7 +4977,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 1);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5023,7 +4988,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 2);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5037,8 +5002,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (rd & 1)
                         goto illegal_insn;
                     else {
-                        save_state(dc, cpu_cond);
-                        gen_stda_asi(cpu_val, cpu_addr, insn, rd);
+                        save_state(dc);
+                        gen_stda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     }
                     break;
 #endif
@@ -5048,7 +5013,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_qemu_st64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x1e: /* V9 stxa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 8);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5057,28 +5022,34 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
             } else if (xop > 0x23 && xop < 0x28) {
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
-                save_state(dc, cpu_cond);
+                }
+                save_state(dc);
                 switch (xop) {
                 case 0x24: /* stf, store fpreg */
-                    gen_address_mask(dc, cpu_addr);
-                    cpu_src1_32 = gen_load_fpr_F(dc, rd);
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_src1_32);
-                    tcg_gen_qemu_st32(cpu_tmp0, cpu_addr, dc->mem_idx);
+                    {
+                        TCGv t = get_temp_tl(dc);
+                        gen_address_mask(dc, cpu_addr);
+                        cpu_src1_32 = gen_load_fpr_F(dc, rd);
+                        tcg_gen_ext_i32_tl(t, cpu_src1_32);
+                        tcg_gen_qemu_st32(t, cpu_addr, dc->mem_idx);
+                    }
                     break;
                 case 0x25: /* stfsr, V9 stxfsr */
+                    {
+                        TCGv t = get_temp_tl(dc);
+
+                        tcg_gen_ld_tl(t, cpu_env, offsetof(CPUSPARCState, fsr));
 #ifdef TARGET_SPARC64
-                    gen_address_mask(dc, cpu_addr);
-                    tcg_gen_ld_i64(cpu_tmp64, cpu_env, offsetof(CPUSPARCState, fsr));
-                    if (rd == 1)
-                        tcg_gen_qemu_st64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                    else
-                        tcg_gen_qemu_st32(cpu_tmp64, cpu_addr, dc->mem_idx);
-#else
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env, offsetof(CPUSPARCState, fsr));
-                    tcg_gen_qemu_st32(cpu_tmp32, cpu_addr, dc->mem_idx);
+                        gen_address_mask(dc, cpu_addr);
+                        if (rd == 1) {
+                            tcg_gen_qemu_st64(t, cpu_addr, dc->mem_idx);
+                            break;
+                        }
 #endif
+                        tcg_gen_qemu_st32(t, cpu_addr, dc->mem_idx);
+                    }
                     break;
                 case 0x26:
 #ifdef TARGET_SPARC64
@@ -5101,8 +5072,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #else
                     if (!supervisor(dc))
                         goto priv_insn;
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     goto nfq_insn;
 #endif
 #endif
@@ -5115,11 +5087,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
             } else if (xop > 0x33 && xop < 0x3f) {
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 switch (xop) {
 #ifdef TARGET_SPARC64
                 case 0x34: /* V9 stfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
                     gen_stf_asi(cpu_addr, insn, 4, rd);
@@ -5129,7 +5101,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         TCGv_i32 r_const;
 
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                        if (gen_trap_ifnofpu(dc)) {
                             goto jmp_insn;
                         }
                         r_const = tcg_const_i32(7);
@@ -5139,18 +5111,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                     break;
                 case 0x37: /* V9 stdfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
                     gen_stf_asi(cpu_addr, insn, 8, DFPREG(rd));
                     break;
                 case 0x3c: /* V9 casa */
-                    gen_cas_asi(cpu_val, cpu_addr, cpu_src2, insn, rd);
-                    gen_movl_TN_reg(rd, cpu_val);
+                    rs2 = GET_FIELD(insn, 27, 31);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    gen_cas_asi(dc, cpu_addr, cpu_src2, insn, rd);
                     break;
                 case 0x3e: /* V9 casxa */
-                    gen_casx_asi(cpu_val, cpu_addr, cpu_src2, insn, rd);
-                    gen_movl_TN_reg(rd, cpu_val);
+                    rs2 = GET_FIELD(insn, 27, 31);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    gen_casx_asi(dc, cpu_addr, cpu_src2, insn, rd);
                     break;
 #else
                 case 0x34: /* stc */
@@ -5162,8 +5136,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 default:
                     goto illegal_insn;
                 }
-            } else
+            } else {
                 goto illegal_insn;
+            }
         }
         break;
     }
@@ -5185,7 +5160,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_ILL_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5196,7 +5171,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_UNIMP_FLUSH);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5208,7 +5183,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_PRIV_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5217,13 +5192,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     goto egress;
 #endif
  nfpu_insn:
-    save_state(dc, cpu_cond);
+    save_state(dc);
     gen_op_fpexception_im(FSR_FTT_UNIMPFPOP);
     dc->is_br = 1;
     goto egress;
 #if !defined(CONFIG_USER_ONLY) && !defined(TARGET_SPARC64)
  nfq_insn:
-    save_state(dc, cpu_cond);
+    save_state(dc);
     gen_op_fpexception_im(FSR_FTT_SEQ_ERROR);
     dc->is_br = 1;
     goto egress;
@@ -5233,7 +5208,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_NCP_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free(r_const);
@@ -5242,8 +5217,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     goto egress;
 #endif
  egress:
-    tcg_temp_free(cpu_tmp1);
-    tcg_temp_free(cpu_tmp2);
     if (dc->n_t32 != 0) {
         int i;
         for (i = dc->n_t32 - 1; i >= 0; --i) {
@@ -5251,6 +5224,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
         }
         dc->n_t32 = 0;
     }
+    if (dc->n_ttl != 0) {
+        int i;
+        for (i = dc->n_ttl - 1; i >= 0; --i) {
+            tcg_temp_free(dc->ttl[i]);
+        }
+        dc->n_ttl = 0;
+    }
 }
 
 static inline void gen_intermediate_code_internal(TranslationBlock * tb,
@@ -5277,17 +5257,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
     dc->fpu_enabled = tb_fpu_enabled(tb->flags);
     dc->address_mask_32bit = tb_am_enabled(tb->flags);
     dc->singlestep = (env->singlestep_enabled || singlestep);
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
-
-    cpu_tmp0 = tcg_temp_new();
-    cpu_tmp32 = tcg_temp_new_i32();
-    cpu_tmp64 = tcg_temp_new_i64();
-
-    cpu_dst = tcg_temp_local_new();
-
-    // loads and stores
-    cpu_val = tcg_temp_local_new();
-    cpu_addr = tcg_temp_local_new();
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
@@ -5299,7 +5269,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
                     if (dc->pc != pc_start)
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                     gen_helper_debug(cpu_env);
                     tcg_gen_exit_tb(0);
                     dc->is_br = 1;
@@ -5309,21 +5279,22 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
         }
         if (spc) {
             qemu_log("Search PC...\n");
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
-                gen_opc_pc[lj] = dc->pc;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
+                tcg_ctx.gen_opc_pc[lj] = dc->pc;
                 gen_opc_npc[lj] = dc->npc;
-                gen_opc_instr_start[lj] = 1;
-                gen_opc_icount[lj] = num_insns;
+                tcg_ctx.gen_opc_instr_start[lj] = 1;
+                tcg_ctx.gen_opc_icount[lj] = num_insns;
             }
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         last_pc = dc->pc;
         insn = cpu_ldl_code(env, dc->pc);
+
         disas_sparc_insn(dc, insn);
         num_insns++;
 
@@ -5341,39 +5312,34 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
         if (dc->singlestep) {
             break;
         }
-    } while ((gen_opc_ptr < gen_opc_end) &&
+    } while ((tcg_ctx.gen_opc_ptr < gen_opc_end) &&
              (dc->pc - pc_start) < (TARGET_PAGE_SIZE - 32) &&
              num_insns < max_insns);
 
  exit_gen_loop:
-    tcg_temp_free(cpu_addr);
-    tcg_temp_free(cpu_val);
-    tcg_temp_free(cpu_dst);
-    tcg_temp_free_i64(cpu_tmp64);
-    tcg_temp_free_i32(cpu_tmp32);
-    tcg_temp_free(cpu_tmp0);
-
-    if (tb->cflags & CF_LAST_IO)
+    if (tb->cflags & CF_LAST_IO) {
         gen_io_end();
+    }
     if (!dc->is_br) {
         if (dc->pc != DYNAMIC_PC &&
             (dc->npc != DYNAMIC_PC && dc->npc != JUMP_PC)) {
             /* static PC and NPC: we can use direct chaining */
             gen_goto_tb(dc, 0, dc->pc, dc->npc);
         } else {
-            if (dc->pc != DYNAMIC_PC)
+            if (dc->pc != DYNAMIC_PC) {
                 tcg_gen_movi_tl(cpu_pc, dc->pc);
-            save_npc(dc, cpu_cond);
+            }
+            save_npc(dc);
             tcg_gen_exit_tb(0);
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (spc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
 #if 0
         log_page_dump();
 #endif
@@ -5387,7 +5353,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("--------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, last_pc + 4 - pc_start, 0);
+        log_target_disas(env, pc_start, last_pc + 4 - pc_start, 0);
         qemu_log("\n");
     }
 #endif
@@ -5512,7 +5478,7 @@ void gen_intermediate_code_init(CPUSPARCState *env)
 void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
 {
     target_ulong npc;
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
     npc = gen_opc_npc[pc_pos];
     if (npc == 1) {
         /* dynamic NPC: already stored */
@@ -5526,9 +5492,4 @@ void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
     } else {
         env->npc = npc;
     }
-
-    /* flush pending conditional evaluations before exposing cpu state */
-    if (CC_OP != CC_OP_FLAGS) {
-        helper_compute_psr(env);
-    }
 }
diff --git a/target-unicore32/Makefile.objs b/target-unicore32/Makefile.objs
index 777f01fef86..8e143da9373 100644
--- a/target-unicore32/Makefile.objs
+++ b/target-unicore32/Makefile.objs
@@ -2,5 +2,3 @@ obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += ucf64_helper.o
 
 obj-$(CONFIG_SOFTMMU) += machine.o softmmu.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-unicore32/cpu-qom.h b/target-unicore32/cpu-qom.h
index 342d85e39b6..fe40b2d6a8a 100644
--- a/target-unicore32/cpu-qom.h
+++ b/target-unicore32/cpu-qom.h
@@ -11,7 +11,7 @@
 #ifndef QEMU_UC32_CPU_QOM_H
 #define QEMU_UC32_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_UNICORE32_CPU "unicore32-cpu"
diff --git a/target-unicore32/cpu.c b/target-unicore32/cpu.c
index 3425bbeac91..c120440653f 100644
--- a/target-unicore32/cpu.c
+++ b/target-unicore32/cpu.c
@@ -12,7 +12,7 @@
  * or (at your option) any later version.
  */
 
-#include "cpu-qom.h"
+#include "cpu.h"
 #include "qemu-common.h"
 
 static inline void set_feature(CPUUniCore32State *env, int feature)
@@ -22,6 +22,22 @@ static inline void set_feature(CPUUniCore32State *env, int feature)
 
 /* CPU models */
 
+static ObjectClass *uc32_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && (!object_class_dynamic_cast(oc, TYPE_UNICORE32_CPU) ||
+                       object_class_is_abstract(oc))) {
+        oc = NULL;
+    }
+    return oc;
+}
+
 typedef struct UniCore32CPUInfo {
     const char *name;
     void (*instance_init)(Object *obj);
@@ -80,6 +96,13 @@ static void uc32_cpu_initfn(Object *obj)
     tlb_flush(env, 1);
 }
 
+static void uc32_cpu_class_init(ObjectClass *oc, void *data)
+{
+    CPUClass *cc = CPU_CLASS(oc);
+
+    cc->class_by_name = uc32_cpu_class_by_name;
+}
+
 static void uc32_register_cpu_type(const UniCore32CPUInfo *info)
 {
     TypeInfo type_info = {
@@ -88,7 +111,7 @@ static void uc32_register_cpu_type(const UniCore32CPUInfo *info)
         .instance_init = info->instance_init,
     };
 
-    type_register_static(&type_info);
+    type_register(&type_info);
 }
 
 static const TypeInfo uc32_cpu_type_info = {
@@ -98,6 +121,7 @@ static const TypeInfo uc32_cpu_type_info = {
     .instance_init = uc32_cpu_initfn,
     .abstract = true,
     .class_size = sizeof(UniCore32CPUClass),
+    .class_init = uc32_cpu_class_init,
 };
 
 static void uc32_cpu_register_types(void)
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 06508a12782..509ce7c69dd 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -23,8 +23,8 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 
 #define NB_MMU_MODES            2
 
@@ -157,9 +157,9 @@ static inline void cpu_set_tls(CPUUniCore32State *env, target_ulong newtls)
     env->regs[16] = newtls;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "cpu-qom.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUUniCore32State *env, TranslationBlock *tb)
 {
@@ -181,8 +181,10 @@ void uc32_translate_init(void);
 void do_interrupt(CPUUniCore32State *);
 void switch_mode(CPUUniCore32State *, int);
 
-static inline bool cpu_has_work(CPUUniCore32State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUUniCore32State *env = &UNICORE32_CPU(cpu)->env;
+
     return env->interrupt_request &
         (CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
 }
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index a9e226bde4f..183b5b35775 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -10,10 +10,12 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "helper.h"
-#include "host-utils.h"
-#include "console.h"
+#include "qemu/host-utils.h"
+#ifndef CONFIG_USER_ONLY
+#include "ui/console.h"
+#endif
 
 #undef DEBUG_UC32
 
@@ -27,12 +29,14 @@ CPUUniCore32State *uc32_cpu_init(const char *cpu_model)
 {
     UniCore32CPU *cpu;
     CPUUniCore32State *env;
+    ObjectClass *oc;
     static int inited = 1;
 
-    if (object_class_by_name(cpu_model) == NULL) {
+    oc = cpu_class_by_name(TYPE_UNICORE32_CPU, cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = UNICORE32_CPU(object_new(cpu_model));
+    cpu = UNICORE32_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
 
     if (inited) {
diff --git a/target-unicore32/helper.h b/target-unicore32/helper.h
index 305318ae594..e85ce6c2016 100644
--- a/target-unicore32/helper.h
+++ b/target-unicore32/helper.h
@@ -6,7 +6,7 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_4(cp0_set, void, env, i32, i32, i32)
@@ -17,26 +17,26 @@ DEF_HELPER_1(cp1_putc, void, i32)
 DEF_HELPER_1(clz, i32, i32)
 DEF_HELPER_1(clo, i32, i32)
 
-DEF_HELPER_1(exception, void, i32)
+DEF_HELPER_2(exception, void, env, i32)
 
-DEF_HELPER_2(asr_write, void, i32, i32)
-DEF_HELPER_0(asr_read, i32)
+DEF_HELPER_3(asr_write, void, env, i32, i32)
+DEF_HELPER_1(asr_read, i32, env)
 
-DEF_HELPER_1(get_user_reg, i32, i32)
-DEF_HELPER_2(set_user_reg, void, i32, i32)
+DEF_HELPER_2(get_user_reg, i32, env, i32)
+DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
-DEF_HELPER_2(add_cc, i32, i32, i32)
-DEF_HELPER_2(adc_cc, i32, i32, i32)
-DEF_HELPER_2(sub_cc, i32, i32, i32)
-DEF_HELPER_2(sbc_cc, i32, i32, i32)
+DEF_HELPER_3(add_cc, i32, env, i32, i32)
+DEF_HELPER_3(adc_cc, i32, env, i32, i32)
+DEF_HELPER_3(sub_cc, i32, env, i32, i32)
+DEF_HELPER_3(sbc_cc, i32, env, i32, i32)
 
 DEF_HELPER_2(shl, i32, i32, i32)
 DEF_HELPER_2(shr, i32, i32, i32)
 DEF_HELPER_2(sar, i32, i32, i32)
-DEF_HELPER_2(shl_cc, i32, i32, i32)
-DEF_HELPER_2(shr_cc, i32, i32, i32)
-DEF_HELPER_2(sar_cc, i32, i32, i32)
-DEF_HELPER_2(ror_cc, i32, i32, i32)
+DEF_HELPER_3(shl_cc, i32, env, i32, i32)
+DEF_HELPER_3(shr_cc, i32, env, i32, i32)
+DEF_HELPER_3(sar_cc, i32, env, i32, i32)
+DEF_HELPER_3(ror_cc, i32, env, i32, i32)
 
 DEF_HELPER_1(ucf64_get_fpscr, i32, env)
 DEF_HELPER_2(ucf64_set_fpscr, void, env, i32)
@@ -65,4 +65,4 @@ DEF_HELPER_2(ucf64_si2df, f64, f32, env)
 DEF_HELPER_2(ucf64_sf2si, f32, f32, env)
 DEF_HELPER_2(ucf64_df2si, f32, f64, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-unicore32/op_helper.c b/target-unicore32/op_helper.c
index c63789d6cb3..6443ffec1c4 100644
--- a/target-unicore32/op_helper.c
+++ b/target-unicore32/op_helper.c
@@ -9,19 +9,18 @@
  * later version. See the COPYING file in the top-level directory.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
-void HELPER(exception)(uint32_t excp)
+void HELPER(exception)(CPUUniCore32State *env, uint32_t excp)
 {
     env->exception_index = excp;
     cpu_loop_exit(env);
 }
 
-static target_ulong asr_read(void)
+static target_ulong asr_read(CPUUniCore32State *env)
 {
     int ZF;
     ZF = (env->ZF == 0);
@@ -29,24 +28,18 @@ static target_ulong asr_read(void)
         (env->CF << 29) | ((env->VF & 0x80000000) >> 3);
 }
 
-target_ulong cpu_asr_read(CPUUniCore32State *env1)
+target_ulong cpu_asr_read(CPUUniCore32State *env)
 {
-    CPUUniCore32State *saved_env;
-    target_ulong ret;
-
-    saved_env = env;
-    env = env1;
-    ret = asr_read();
-    env = saved_env;
-    return ret;
+    return asr_read(env);
 }
 
-target_ulong HELPER(asr_read)(void)
+target_ulong HELPER(asr_read)(CPUUniCore32State *env)
 {
-    return asr_read();
+    return asr_read(env);
 }
 
-static void asr_write(target_ulong val, target_ulong mask)
+static void asr_write(CPUUniCore32State *env, target_ulong val,
+                      target_ulong mask)
 {
     if (mask & ASR_NZCV) {
         env->ZF = (~val) & ASR_Z;
@@ -62,23 +55,19 @@ static void asr_write(target_ulong val, target_ulong mask)
     env->uncached_asr = (env->uncached_asr & ~mask) | (val & mask);
 }
 
-void cpu_asr_write(CPUUniCore32State *env1, target_ulong val, target_ulong mask)
+void cpu_asr_write(CPUUniCore32State *env, target_ulong val, target_ulong mask)
 {
-    CPUUniCore32State *saved_env;
-
-    saved_env = env;
-    env = env1;
-    asr_write(val, mask);
-    env = saved_env;
+    asr_write(env, val, mask);
 }
 
-void HELPER(asr_write)(target_ulong val, target_ulong mask)
+void HELPER(asr_write)(CPUUniCore32State *env, target_ulong val,
+                       target_ulong mask)
 {
-    asr_write(val, mask);
+    asr_write(env, val, mask);
 }
 
 /* Access to user mode registers from privileged modes.  */
-uint32_t HELPER(get_user_reg)(uint32_t regno)
+uint32_t HELPER(get_user_reg)(CPUUniCore32State *env, uint32_t regno)
 {
     uint32_t val;
 
@@ -92,7 +81,7 @@ uint32_t HELPER(get_user_reg)(uint32_t regno)
     return val;
 }
 
-void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
+void HELPER(set_user_reg)(CPUUniCore32State *env, uint32_t regno, uint32_t val)
 {
     if (regno == 29) {
         env->banked_r29[0] = val;
@@ -107,7 +96,7 @@ void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
    The only way to do that in TCG is a conditional branch, which clobbers
    all our temporaries.  For now implement these as helper functions.  */
 
-uint32_t HELPER(add_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     result = a + b;
@@ -117,7 +106,7 @@ uint32_t HELPER(add_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(adc_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -132,7 +121,7 @@ uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     result = a - b;
@@ -142,7 +131,7 @@ uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sbc_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -186,7 +175,7 @@ uint32_t HELPER(sar)(uint32_t x, uint32_t i)
     return (int32_t)x >> shift;
 }
 
-uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shl_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -203,7 +192,7 @@ uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shr_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -220,7 +209,7 @@ uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(sar_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -233,7 +222,7 @@ uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(ror_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift1, shift;
     shift1 = i & 0xff;
@@ -253,40 +242,29 @@ uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
-void tlb_fill(CPUUniCore32State *env1, target_ulong addr, int is_write,
-        int mmu_idx, uintptr_t retaddr)
+void tlb_fill(CPUUniCore32State *env, target_ulong addr, int is_write,
+              int mmu_idx, uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUUniCore32State *saved_env;
-    unsigned long pc;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = uc32_cpu_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            pc = (unsigned long)retaddr;
-            tb = tb_find_pc(pc);
-            if (tb) {/* the PC is inside the translated code.
-                        It means that we have a virtual CPU fault */
-                cpu_restore_state(tb, env, pc);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
diff --git a/target-unicore32/softmmu.c b/target-unicore32/softmmu.c
index 373f94b2743..fc27100f270 100644
--- a/target-unicore32/softmmu.c
+++ b/target-unicore32/softmmu.c
@@ -31,7 +31,7 @@
 
 
 /* Map CPU modes onto saved register banks.  */
-static inline int bank_number(int mode)
+static inline int bank_number(CPUUniCore32State *env, int mode)
 {
     switch (mode) {
     case ASR_MODE_USER:
@@ -46,7 +46,7 @@ static inline int bank_number(int mode)
     case ASR_MODE_INTR:
         return 4;
     }
-    cpu_abort(cpu_single_env, "Bad mode %x\n", mode);
+    cpu_abort(env, "Bad mode %x\n", mode);
     return -1;
 }
 
@@ -60,12 +60,12 @@ void switch_mode(CPUUniCore32State *env, int mode)
         return;
     }
 
-    i = bank_number(old_mode);
+    i = bank_number(env, old_mode);
     env->banked_r29[i] = env->regs[29];
     env->banked_r30[i] = env->regs[30];
     env->banked_bsr[i] = env->bsr;
 
-    i = bank_number(mode);
+    i = bank_number(env, mode);
     env->regs[29] = env->banked_r29[i];
     env->regs[30] = env->banked_r30[i];
     env->bsr = env->banked_bsr[i];
@@ -259,7 +259,7 @@ int uc32_cpu_handle_mmu_fault(CPUUniCore32State *env, target_ulong address,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUUniCore32State *env,
+hwaddr cpu_get_phys_page_debug(CPUUniCore32State *env,
         target_ulong addr)
 {
     cpu_abort(env, "%s not supported yet\n", __func__);
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 188bf8c52a8..f4498bcb146 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -15,9 +15,9 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -55,7 +55,7 @@ static TCGv_i32 cpu_R[32];
 static TCGv cpu_F0s, cpu_F1s;
 static TCGv_i64 cpu_F0d, cpu_F1d;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static const char *regnames[] = {
       "r00", "r01", "r02", "r03", "r04", "r05", "r06", "r07",
@@ -253,7 +253,7 @@ static void disas_ocd_insn(CPUUniCore32State *env, DisasContext *s,
 static inline void gen_set_asr(TCGv var, uint32_t mask)
 {
     TCGv tmp_mask = tcg_const_i32(mask);
-    gen_helper_asr_write(var, tmp_mask);
+    gen_helper_asr_write(cpu_env, var, tmp_mask);
     tcg_temp_free_i32(tmp_mask);
 }
 /* Set NZCV flags from the high 4 bits of var.  */
@@ -263,7 +263,7 @@ static void gen_exception(int excp)
 {
     TCGv tmp = new_tmp();
     tcg_gen_movi_i32(tmp, excp);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     dead_tmp(tmp);
 }
 
@@ -416,16 +416,16 @@ static inline void gen_uc32_shift_reg(TCGv var, int shiftop,
     if (flags) {
         switch (shiftop) {
         case 0:
-            gen_helper_shl_cc(var, var, shift);
+            gen_helper_shl_cc(var, cpu_env, var, shift);
             break;
         case 1:
-            gen_helper_shr_cc(var, var, shift);
+            gen_helper_shr_cc(var, cpu_env, var, shift);
             break;
         case 2:
-            gen_helper_sar_cc(var, var, shift);
+            gen_helper_sar_cc(var, cpu_env, var, shift);
             break;
         case 3:
-            gen_helper_ror_cc(var, var, shift);
+            gen_helper_ror_cc(var, cpu_env, var, shift);
             break;
         }
     } else {
@@ -1323,11 +1323,11 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
             if (IS_USER(s)) {
                 ILLEGAL;
             }
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
             gen_exception_return(s, tmp);
         } else {
             if (UCOP_SET_S) {
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 tcg_gen_sub_i32(tmp, tmp, tmp2);
             }
@@ -1336,7 +1336,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x03:
         if (UCOP_SET_S) {
-            gen_helper_sub_cc(tmp, tmp2, tmp);
+            gen_helper_sub_cc(tmp, cpu_env, tmp2, tmp);
         } else {
             tcg_gen_sub_i32(tmp, tmp2, tmp);
         }
@@ -1344,7 +1344,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x04:
         if (UCOP_SET_S) {
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_helper_add_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             tcg_gen_add_i32(tmp, tmp, tmp2);
         }
@@ -1352,7 +1352,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x05:
         if (UCOP_SET_S) {
-            gen_helper_adc_cc(tmp, tmp, tmp2);
+            gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             gen_add_carry(tmp, tmp, tmp2);
         }
@@ -1360,7 +1360,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x06:
         if (UCOP_SET_S) {
-            gen_helper_sbc_cc(tmp, tmp, tmp2);
+            gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             gen_sub_carry(tmp, tmp, tmp2);
         }
@@ -1368,7 +1368,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x07:
         if (UCOP_SET_S) {
-            gen_helper_sbc_cc(tmp, tmp2, tmp);
+            gen_helper_sbc_cc(tmp, cpu_env, tmp2, tmp);
         } else {
             gen_sub_carry(tmp, tmp2, tmp);
         }
@@ -1390,13 +1390,13 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x0a:
         if (UCOP_SET_S) {
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
         }
         dead_tmp(tmp);
         break;
     case 0x0b:
         if (UCOP_SET_S) {
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_helper_add_cc(tmp, cpu_env, tmp, tmp2);
         }
         dead_tmp(tmp);
         break;
@@ -1536,7 +1536,7 @@ static void do_misc(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
             tmp = load_cpu_field(bsr);
         } else {
             tmp = new_tmp();
-            gen_helper_asr_read(tmp);
+            gen_helper_asr_read(tmp, cpu_env);
         }
         store_reg(s, UCOP_REG_D, tmp);
         return;
@@ -1760,7 +1760,7 @@ static void do_ldst_m(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
                     gen_bx(s, tmp);
                 } else if (user) {
                     tmp2 = tcg_const_i32(reg);
-                    gen_helper_set_user_reg(tmp2, tmp);
+                    gen_helper_set_user_reg(cpu_env, tmp2, tmp);
                     tcg_temp_free_i32(tmp2);
                     dead_tmp(tmp);
                 } else if (reg == UCOP_REG_N) {
@@ -1778,7 +1778,7 @@ static void do_ldst_m(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
                 } else if (user) {
                     tmp = new_tmp();
                     tmp2 = tcg_const_i32(reg);
-                    gen_helper_get_user_reg(tmp, tmp2);
+                    gen_helper_get_user_reg(tmp, cpu_env, tmp2);
                     tcg_temp_free_i32(tmp2);
                 } else {
                     tmp = load_reg(s, reg);
@@ -1861,7 +1861,11 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
 {
     unsigned int insn;
 
-    insn = ldl_code(s->pc);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(s->pc);
+    }
+
+    insn = cpu_ldl_code(env, s->pc);
     s->pc += 4;
 
     /* UniCore instructions class:
@@ -1928,8 +1932,6 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
         }
         ILLEGAL;
     }
-
-    return;
 }
 
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
@@ -1954,7 +1956,7 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -1997,16 +1999,16 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
@@ -2029,7 +2031,7 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
         num_insns++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              dc->pc < next_page_start &&
@@ -2101,21 +2103,21 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
 
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -2201,5 +2203,5 @@ void cpu_dump_state(CPUUniCore32State *env, FILE *f,
 
 void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->regs[31] = gen_opc_pc[pc_pos];
+    env->regs[31] = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-xtensa/core-dc232b.c b/target-xtensa/core-dc232b.c
index 804fdef56a0..0bfcf2414cb 100644
--- a/target-xtensa/core-dc232b.c
+++ b/target-xtensa/core-dc232b.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 
 #include "core-dc232b/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/core-dc233c.c b/target-xtensa/core-dc233c.c
index d643f41d373..11acbf35809 100644
--- a/target-xtensa/core-dc233c.c
+++ b/target-xtensa/core-dc233c.c
@@ -26,10 +26,10 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
 #include "qemu-common.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "core-dc233c/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/core-fsf.c b/target-xtensa/core-fsf.c
index e36b0de9d52..d4660edde91 100644
--- a/target-xtensa/core-fsf.c
+++ b/target-xtensa/core-fsf.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 
 #include "core-fsf/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/cpu-qom.h b/target-xtensa/cpu-qom.h
index 1fd2f274a1d..e344a9aa79b 100644
--- a/target-xtensa/cpu-qom.h
+++ b/target-xtensa/cpu-qom.h
@@ -29,7 +29,7 @@
 #ifndef QEMU_XTENSA_CPU_QOM_H
 #define QEMU_XTENSA_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_XTENSA_CPU "xtensa-cpu"
diff --git a/target-xtensa/cpu.c b/target-xtensa/cpu.c
index 9d01983d444..035b07c1c5c 100644
--- a/target-xtensa/cpu.c
+++ b/target-xtensa/cpu.c
@@ -48,6 +48,9 @@ static void xtensa_cpu_reset(CPUState *s)
             XTENSA_OPTION_INTERRUPT) ? 0x1f : 0x10;
     env->sregs[VECBASE] = env->config->vecbase;
     env->sregs[IBREAKENABLE] = 0;
+    env->sregs[CACHEATTR] = 0x22222222;
+    env->sregs[ATOMCTL] = xtensa_option_enabled(env->config,
+            XTENSA_OPTION_ATOMCTL) ? 0x28 : 0x15;
 
     env->pending_irq_level = 0;
     reset_mmu(env);
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index 177094ae9a8..5acf78c692b 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -35,7 +35,8 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -64,6 +65,7 @@ enum {
     XTENSA_OPTION_FP_COPROCESSOR,
     XTENSA_OPTION_MP_SYNCHRO,
     XTENSA_OPTION_CONDITIONAL_STORE,
+    XTENSA_OPTION_ATOMCTL,
 
     /* Interrupts and exceptions */
     XTENSA_OPTION_EXCEPTION,
@@ -92,6 +94,7 @@ enum {
     XTENSA_OPTION_REGION_PROTECTION,
     XTENSA_OPTION_REGION_TRANSLATION,
     XTENSA_OPTION_MMU,
+    XTENSA_OPTION_CACHEATTR,
 
     /* Other */
     XTENSA_OPTION_WINDOWED_REGISTER,
@@ -127,6 +130,8 @@ enum {
     ITLBCFG = 91,
     DTLBCFG = 92,
     IBREAKENABLE = 96,
+    CACHEATTR = 98,
+    ATOMCTL = 99,
     IBREAKA = 128,
     DBREAKA = 144,
     DBREAKC = 160,
@@ -148,6 +153,7 @@ enum {
     ICOUNTLEVEL = 237,
     EXCVADDR = 238,
     CCOMPARE = 240,
+    MISC = 244,
 };
 
 #define PS_INTLEVEL 0xf
@@ -192,6 +198,14 @@ enum {
 
 #define REGION_PAGE_MASK 0xe0000000
 
+#define PAGE_CACHE_MASK    0x700
+#define PAGE_CACHE_SHIFT   8
+#define PAGE_CACHE_INVALID 0x000
+#define PAGE_CACHE_BYPASS  0x100
+#define PAGE_CACHE_WT      0x200
+#define PAGE_CACHE_WB      0x400
+#define PAGE_CACHE_ISOLATE 0x600
+
 enum {
     /* Static vectors */
     EXC_RESET,
@@ -325,6 +339,8 @@ typedef struct CPUXtensaState {
     uint32_t sregs[256];
     uint32_t uregs[256];
     uint32_t phys_regs[MAX_NAREG];
+    float32 fregs[16];
+    float_status fp_status;
 
     xtensa_tlb_entry itlb[7][MAX_TLB_WAY_SIZE];
     xtensa_tlb_entry dtlb[10][MAX_TLB_WAY_SIZE];
@@ -401,6 +417,7 @@ void debug_exception_env(CPUXtensaState *new_env, uint32_t cause);
 
 
 #define XTENSA_OPTION_BIT(opt) (((uint64_t)1) << (opt))
+#define XTENSA_OPTION_ALL (~(uint64_t)0)
 
 static inline bool xtensa_option_bits_enabled(const XtensaConfig *config,
         uint64_t opt)
@@ -465,6 +482,8 @@ static inline int cpu_mmu_index(CPUXtensaState *env)
 #define XTENSA_TBFLAG_LITBASE 0x8
 #define XTENSA_TBFLAG_DEBUG 0x10
 #define XTENSA_TBFLAG_ICOUNT 0x20
+#define XTENSA_TBFLAG_CPENABLE_MASK 0x3fc0
+#define XTENSA_TBFLAG_CPENABLE_SHIFT 6
 
 static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
         target_ulong *cs_base, int *flags)
@@ -488,13 +507,18 @@ static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
             *flags |= XTENSA_TBFLAG_ICOUNT;
         }
     }
+    if (xtensa_option_enabled(env->config, XTENSA_OPTION_COPROCESSOR)) {
+        *flags |= env->sregs[CPENABLE] << XTENSA_TBFLAG_CPENABLE_SHIFT;
+    }
 }
 
-#include "cpu-all.h"
-#include "exec-all.h"
+#include "exec/cpu-all.h"
+#include "exec/exec-all.h"
 
-static inline int cpu_has_work(CPUXtensaState *env)
+static inline int cpu_has_work(CPUState *cpu)
 {
+    CPUXtensaState *env = &XTENSA_CPU(cpu)->env;
+
     return env->pending_irq_level;
 }
 
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index d5bb171fcd9..94c03a1d3c0 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/loader.h"
 #endif
@@ -118,7 +118,7 @@ void xtensa_cpu_list(FILE *f, fprintf_function cpu_fprintf)
     }
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUXtensaState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUXtensaState *env, target_ulong addr)
 {
     uint32_t paddr;
     uint32_t page_size;
@@ -390,6 +390,7 @@ int xtensa_tlb_lookup(const CPUXtensaState *env, uint32_t addr, bool dtlb,
 static unsigned mmu_attr_to_access(uint32_t attr)
 {
     unsigned access = 0;
+
     if (attr < 12) {
         access |= PAGE_READ;
         if (attr & 0x1) {
@@ -398,8 +399,22 @@ static unsigned mmu_attr_to_access(uint32_t attr)
         if (attr & 0x2) {
             access |= PAGE_WRITE;
         }
+
+        switch (attr & 0xc) {
+        case 0:
+            access |= PAGE_CACHE_BYPASS;
+            break;
+
+        case 4:
+            access |= PAGE_CACHE_WB;
+            break;
+
+        case 8:
+            access |= PAGE_CACHE_WT;
+            break;
+        }
     } else if (attr == 13) {
-        access |= PAGE_READ | PAGE_WRITE;
+        access |= PAGE_READ | PAGE_WRITE | PAGE_CACHE_ISOLATE;
     }
     return access;
 }
@@ -410,14 +425,35 @@ static unsigned mmu_attr_to_access(uint32_t attr)
  */
 static unsigned region_attr_to_access(uint32_t attr)
 {
-    unsigned access = 0;
-    if ((attr < 6 && attr != 3) || attr == 14) {
-        access |= PAGE_READ | PAGE_WRITE;
-    }
-    if (attr > 0 && attr < 6) {
-        access |= PAGE_EXEC;
-    }
-    return access;
+    static const unsigned access[16] = {
+         [0] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_WT,
+         [1] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WT,
+         [2] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_BYPASS,
+         [3] =                          PAGE_EXEC | PAGE_CACHE_WB,
+         [4] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+         [5] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+        [14] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_ISOLATE,
+    };
+
+    return access[attr & 0xf];
+}
+
+/*!
+ * Convert cacheattr to PAGE_{READ,WRITE,EXEC} mask.
+ * See ISA, A.2.14 The Cache Attribute Register
+ */
+static unsigned cacheattr_attr_to_access(uint32_t attr)
+{
+    static const unsigned access[16] = {
+         [0] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_WT,
+         [1] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WT,
+         [2] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_BYPASS,
+         [3] =                          PAGE_EXEC | PAGE_CACHE_WB,
+         [4] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+        [14] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_ISOLATE,
+    };
+
+    return access[attr & 0xf];
 }
 
 static bool is_access_granted(unsigned access, int is_write)
@@ -486,7 +522,8 @@ static int get_physical_addr_mmu(CPUXtensaState *env, bool update_tlb,
             INST_FETCH_PRIVILEGE_CAUSE;
     }
 
-    *access = mmu_attr_to_access(entry->attr);
+    *access = mmu_attr_to_access(entry->attr) &
+        ~(dtlb ? PAGE_EXEC : PAGE_READ | PAGE_WRITE);
     if (!is_access_granted(*access, is_write)) {
         return dtlb ?
             (is_write ?
@@ -566,7 +603,8 @@ int xtensa_get_physical_addr(CPUXtensaState *env, bool update_tlb,
     } else {
         *paddr = vaddr;
         *page_size = TARGET_PAGE_SIZE;
-        *access = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+        *access = cacheattr_attr_to_access(
+                env->sregs[CACHEATTR] >> ((vaddr & 0xe0000000) >> 27));
         return 0;
     }
 }
@@ -599,24 +637,34 @@ static void dump_tlb(FILE *f, fprintf_function cpu_fprintf,
                 xtensa_tlb_get_entry(env, dtlb, wi, ei);
 
             if (entry->asid) {
+                static const char * const cache_text[8] = {
+                    [PAGE_CACHE_BYPASS >> PAGE_CACHE_SHIFT] = "Bypass",
+                    [PAGE_CACHE_WT >> PAGE_CACHE_SHIFT] = "WT",
+                    [PAGE_CACHE_WB >> PAGE_CACHE_SHIFT] = "WB",
+                    [PAGE_CACHE_ISOLATE >> PAGE_CACHE_SHIFT] = "Isolate",
+                };
                 unsigned access = attr_to_access(entry->attr);
+                unsigned cache_idx = (access & PAGE_CACHE_MASK) >>
+                    PAGE_CACHE_SHIFT;
 
                 if (print_header) {
                     print_header = false;
                     cpu_fprintf(f, "Way %u (%d %s)\n", wi, sz, sz_text);
                     cpu_fprintf(f,
-                            "\tVaddr       Paddr       ASID  Attr RWX\n"
-                            "\t----------  ----------  ----  ---- ---\n");
+                            "\tVaddr       Paddr       ASID  Attr RWX Cache\n"
+                            "\t----------  ----------  ----  ---- --- -------\n");
                 }
                 cpu_fprintf(f,
-                        "\t0x%08x  0x%08x  0x%02x  0x%02x %c%c%c\n",
+                        "\t0x%08x  0x%08x  0x%02x  0x%02x %c%c%c %-7s\n",
                         entry->vaddr,
                         entry->paddr,
                         entry->asid,
                         entry->attr,
                         (access & PAGE_READ) ? 'R' : '-',
                         (access & PAGE_WRITE) ? 'W' : '-',
-                        (access & PAGE_EXEC) ? 'X' : '-');
+                        (access & PAGE_EXEC) ? 'X' : '-',
+                        cache_text[cache_idx] ? cache_text[cache_idx] :
+                            "Invalid");
             }
         }
     }
diff --git a/target-xtensa/helper.h b/target-xtensa/helper.h
index 152fec044dd..38d7157f34b 100644
--- a/target-xtensa/helper.h
+++ b/target-xtensa/helper.h
@@ -1,12 +1,12 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_2(exception, noreturn, env, i32)
 DEF_HELPER_3(exception_cause, noreturn, env, i32, i32)
 DEF_HELPER_4(exception_cause_vaddr, noreturn, env, i32, i32, i32)
 DEF_HELPER_3(debug_exception, noreturn, env, i32, i32)
 
-DEF_HELPER_FLAGS_1(nsa, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_1(nsau, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
+DEF_HELPER_FLAGS_1(nsa, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(nsau, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_2(wsr_windowbase, void, env, i32)
 DEF_HELPER_4(entry, void, env, i32, i32, i32)
 DEF_HELPER_2(retw, i32, env, i32)
@@ -23,10 +23,11 @@ DEF_HELPER_3(waiti, void, env, i32, i32)
 DEF_HELPER_3(timer_irq, void, env, i32, i32)
 DEF_HELPER_2(advance_ccount, void, env, i32)
 DEF_HELPER_1(check_interrupts, void, env)
+DEF_HELPER_3(check_atomctl, void, env, i32, i32)
 
 DEF_HELPER_2(wsr_rasid, void, env, i32)
-DEF_HELPER_FLAGS_3(rtlb0, TCG_CALL_CONST | TCG_CALL_PURE, i32, env, i32, i32)
-DEF_HELPER_FLAGS_3(rtlb1, TCG_CALL_CONST | TCG_CALL_PURE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(rtlb0, TCG_CALL_NO_RWG_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(rtlb1, TCG_CALL_NO_RWG_SE, i32, env, i32, i32)
 DEF_HELPER_3(itlb, void, env, i32, i32)
 DEF_HELPER_3(ptlb, i32, env, i32, i32)
 DEF_HELPER_4(wtlb, void, env, i32, i32, i32)
@@ -36,4 +37,25 @@ DEF_HELPER_3(wsr_ibreaka, void, env, i32, i32)
 DEF_HELPER_3(wsr_dbreaka, void, env, i32, i32)
 DEF_HELPER_3(wsr_dbreakc, void, env, i32, i32)
 
-#include "def-helper.h"
+DEF_HELPER_2(wur_fcr, void, env, i32)
+DEF_HELPER_FLAGS_1(abs_s, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_FLAGS_1(neg_s, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_3(add_s, f32, env, f32, f32)
+DEF_HELPER_3(sub_s, f32, env, f32, f32)
+DEF_HELPER_3(mul_s, f32, env, f32, f32)
+DEF_HELPER_4(madd_s, f32, env, f32, f32, f32)
+DEF_HELPER_4(msub_s, f32, env, f32, f32, f32)
+DEF_HELPER_FLAGS_3(ftoi, TCG_CALL_NO_RWG_SE, i32, f32, i32, i32)
+DEF_HELPER_FLAGS_3(ftoui, TCG_CALL_NO_RWG_SE, i32, f32, i32, i32)
+DEF_HELPER_3(itof, f32, env, i32, i32)
+DEF_HELPER_3(uitof, f32, env, i32, i32)
+
+DEF_HELPER_4(un_s, void, env, i32, f32, f32)
+DEF_HELPER_4(oeq_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ueq_s, void, env, i32, f32, f32)
+DEF_HELPER_4(olt_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ult_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ole_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ule_s, void, env, i32, f32, f32)
+
+#include "exec/def-helper.h"
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index 2659c0e00f7..3813a72626e 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -27,7 +27,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 static void do_unaligned_access(CPUXtensaState *env,
         target_ulong addr, int is_write, int is_user, uintptr_t retaddr);
@@ -36,33 +36,23 @@ static void do_unaligned_access(CPUXtensaState *env,
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
-
-static void do_restore_state(CPUXtensaState *env, uintptr_t pc)
-{
-    TranslationBlock *tb;
-
-    tb = tb_find_pc(pc);
-    if (tb) {
-        cpu_restore_state(tb, env, pc);
-    }
-}
+#include "exec/softmmu_template.h"
 
 static void do_unaligned_access(CPUXtensaState *env,
         target_ulong addr, int is_write, int is_user, uintptr_t retaddr)
 {
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_UNALIGNED_EXCEPTION) &&
             !xtensa_option_enabled(env->config, XTENSA_OPTION_HW_ALIGNMENT)) {
-        do_restore_state(env, retaddr);
+        cpu_restore_state(env, retaddr);
         HELPER(exception_cause_vaddr)(env,
                 env->pc, LOAD_STORE_ALIGNMENT_CAUSE, addr);
     }
@@ -86,7 +76,7 @@ void tlb_fill(CPUXtensaState *env,
                 paddr & TARGET_PAGE_MASK,
                 access, mmu_idx, page_size);
     } else {
-        do_restore_state(env, retaddr);
+        cpu_restore_state(env, retaddr);
         HELPER(exception_cause_vaddr)(env, env->pc, ret, vaddr);
     }
 }
@@ -415,6 +405,63 @@ void HELPER(check_interrupts)(CPUXtensaState *env)
     check_interrupts(env);
 }
 
+/*!
+ * Check vaddr accessibility/cache attributes and raise an exception if
+ * specified by the ATOMCTL SR.
+ *
+ * Note: local memory exclusion is not implemented
+ */
+void HELPER(check_atomctl)(CPUXtensaState *env, uint32_t pc, uint32_t vaddr)
+{
+    uint32_t paddr, page_size, access;
+    uint32_t atomctl = env->sregs[ATOMCTL];
+    int rc = xtensa_get_physical_addr(env, true, vaddr, 1,
+            xtensa_get_cring(env), &paddr, &page_size, &access);
+
+    /*
+     * s32c1i never causes LOAD_PROHIBITED_CAUSE exceptions,
+     * see opcode description in the ISA
+     */
+    if (rc == 0 &&
+            (access & (PAGE_READ | PAGE_WRITE)) != (PAGE_READ | PAGE_WRITE)) {
+        rc = STORE_PROHIBITED_CAUSE;
+    }
+
+    if (rc) {
+        HELPER(exception_cause_vaddr)(env, pc, rc, vaddr);
+    }
+
+    /*
+     * When data cache is not configured use ATOMCTL bypass field.
+     * See ISA, 4.3.12.4 The Atomic Operation Control Register (ATOMCTL)
+     * under the Conditional Store Option.
+     */
+    if (!xtensa_option_enabled(env->config, XTENSA_OPTION_DCACHE)) {
+        access = PAGE_CACHE_BYPASS;
+    }
+
+    switch (access & PAGE_CACHE_MASK) {
+    case PAGE_CACHE_WB:
+        atomctl >>= 2;
+    case PAGE_CACHE_WT:
+        atomctl >>= 2;
+    case PAGE_CACHE_BYPASS:
+        if ((atomctl & 0x3) == 0) {
+            HELPER(exception_cause_vaddr)(env, pc,
+                    LOAD_STORE_ERROR_CAUSE, vaddr);
+        }
+        break;
+
+    case PAGE_CACHE_ISOLATE:
+        HELPER(exception_cause_vaddr)(env, pc,
+                LOAD_STORE_ERROR_CAUSE, vaddr);
+        break;
+
+    default:
+        break;
+    }
+}
+
 void HELPER(wsr_rasid)(CPUXtensaState *env, uint32_t v)
 {
     v = (v & 0xffffff00) | 0x1;
@@ -771,3 +818,137 @@ void HELPER(wsr_dbreakc)(CPUXtensaState *env, uint32_t i, uint32_t v)
     }
     env->sregs[DBREAKC + i] = v;
 }
+
+void HELPER(wur_fcr)(CPUXtensaState *env, uint32_t v)
+{
+    static const int rounding_mode[] = {
+        float_round_nearest_even,
+        float_round_to_zero,
+        float_round_up,
+        float_round_down,
+    };
+
+    env->uregs[FCR] = v & 0xfffff07f;
+    set_float_rounding_mode(rounding_mode[v & 3], &env->fp_status);
+}
+
+float32 HELPER(abs_s)(float32 v)
+{
+    return float32_abs(v);
+}
+
+float32 HELPER(neg_s)(float32 v)
+{
+    return float32_chs(v);
+}
+
+float32 HELPER(add_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_add(a, b, &env->fp_status);
+}
+
+float32 HELPER(sub_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_sub(a, b, &env->fp_status);
+}
+
+float32 HELPER(mul_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_mul(a, b, &env->fp_status);
+}
+
+float32 HELPER(madd_s)(CPUXtensaState *env, float32 a, float32 b, float32 c)
+{
+    return float32_muladd(b, c, a, 0,
+            &env->fp_status);
+}
+
+float32 HELPER(msub_s)(CPUXtensaState *env, float32 a, float32 b, float32 c)
+{
+    return float32_muladd(b, c, a, float_muladd_negate_product,
+            &env->fp_status);
+}
+
+uint32_t HELPER(ftoi)(float32 v, uint32_t rounding_mode, uint32_t scale)
+{
+    float_status fp_status = {0};
+
+    set_float_rounding_mode(rounding_mode, &fp_status);
+    return float32_to_int32(
+            float32_scalbn(v, scale, &fp_status), &fp_status);
+}
+
+uint32_t HELPER(ftoui)(float32 v, uint32_t rounding_mode, uint32_t scale)
+{
+    float_status fp_status = {0};
+    float32 res;
+
+    set_float_rounding_mode(rounding_mode, &fp_status);
+
+    res = float32_scalbn(v, scale, &fp_status);
+
+    if (float32_is_neg(v) && !float32_is_any_nan(v)) {
+        return float32_to_int32(res, &fp_status);
+    } else {
+        return float32_to_uint32(res, &fp_status);
+    }
+}
+
+float32 HELPER(itof)(CPUXtensaState *env, uint32_t v, uint32_t scale)
+{
+    return float32_scalbn(int32_to_float32(v, &env->fp_status),
+            (int32_t)scale, &env->fp_status);
+}
+
+float32 HELPER(uitof)(CPUXtensaState *env, uint32_t v, uint32_t scale)
+{
+    return float32_scalbn(uint32_to_float32(v, &env->fp_status),
+            (int32_t)scale, &env->fp_status);
+}
+
+static inline void set_br(CPUXtensaState *env, bool v, uint32_t br)
+{
+    if (v) {
+        env->sregs[BR] |= br;
+    } else {
+        env->sregs[BR] &= ~br;
+    }
+}
+
+void HELPER(un_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_unordered_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(oeq_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_eq_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ueq_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v == float_relation_equal || v == float_relation_unordered, br);
+}
+
+void HELPER(olt_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_lt_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ult_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v == float_relation_less || v == float_relation_unordered, br);
+}
+
+void HELPER(ole_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_le_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ule_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v != float_relation_greater, br);
+}
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index a3a5650fb0f..dd4f51a7b75 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -42,6 +42,10 @@
 #define XCHAL_VECBASE_RESET_VADDR 0
 #endif
 
+#ifndef XCHAL_HW_MIN_VERSION
+#define XCHAL_HW_MIN_VERSION 0
+#endif
+
 #define XCHAL_OPTION(xchal, qemu) ((xchal) ? XTENSA_OPTION_BIT(qemu) : 0)
 
 #define XTENSA_OPTIONS ( \
@@ -58,9 +62,12 @@
     XCHAL_OPTION(XCHAL_HAVE_SEXT, XTENSA_OPTION_MISC_OP_SEXT) | \
     XCHAL_OPTION(XCHAL_HAVE_CLAMPS, XTENSA_OPTION_MISC_OP_CLAMPS) | \
     XCHAL_OPTION(XCHAL_HAVE_CP, XTENSA_OPTION_COPROCESSOR) | \
+    XCHAL_OPTION(XCHAL_HAVE_BOOLEANS, XTENSA_OPTION_BOOLEAN) | \
     XCHAL_OPTION(XCHAL_HAVE_FP, XTENSA_OPTION_FP_COPROCESSOR) | \
     XCHAL_OPTION(XCHAL_HAVE_RELEASE_SYNC, XTENSA_OPTION_MP_SYNCHRO) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
+    XCHAL_OPTION(XCHAL_HAVE_S32C1I && XCHAL_HW_MIN_VERSION >= 230000, \
+        XTENSA_OPTION_ATOMCTL) | \
     /* Interrupts and exceptions */ \
     XCHAL_OPTION(XCHAL_HAVE_EXCEPTIONS, XTENSA_OPTION_EXCEPTION) | \
     XCHAL_OPTION(XCHAL_HAVE_VECBASE, XTENSA_OPTION_RELOCATABLE_VECTOR) | \
@@ -84,9 +91,13 @@
     XCHAL_OPTION(XCHAL_HAVE_XLT_CACHEATTR, \
             XTENSA_OPTION_REGION_TRANSLATION) | \
     XCHAL_OPTION(XCHAL_HAVE_PTP_MMU, XTENSA_OPTION_MMU) | \
+    XCHAL_OPTION(XCHAL_HAVE_CACHEATTR, XTENSA_OPTION_CACHEATTR) | \
     /* Other, TODO */ \
     XCHAL_OPTION(XCHAL_HAVE_WINDOWED, XTENSA_OPTION_WINDOWED_REGISTER) | \
-    XCHAL_OPTION(XCHAL_HAVE_DEBUG, XTENSA_OPTION_DEBUG))
+    XCHAL_OPTION(XCHAL_HAVE_DEBUG, XTENSA_OPTION_DEBUG) |\
+    XCHAL_OPTION(XCHAL_NUM_MISC_REGS > 0, XTENSA_OPTION_MISC_SR) | \
+    XCHAL_OPTION(XCHAL_HAVE_THREADPTR, XTENSA_OPTION_THREAD_POINTER) | \
+    XCHAL_OPTION(XCHAL_HAVE_PRID, XTENSA_OPTION_PROCESSOR_ID))
 
 #ifndef XCHAL_WINDOW_OF4_VECOFS
 #define XCHAL_WINDOW_OF4_VECOFS         0x00000000
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 1900bd5d449..7029ac48145 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -31,11 +31,11 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "disas.h"
+#include "exec/exec-all.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
-#include "sysemu.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -65,86 +65,130 @@ typedef struct DisasContext {
     bool debug;
     bool icount;
     TCGv_i32 next_icount;
+
+    unsigned cpenable;
 } DisasContext;
 
 static TCGv_ptr cpu_env;
 static TCGv_i32 cpu_pc;
 static TCGv_i32 cpu_R[16];
+static TCGv_i32 cpu_FR[16];
 static TCGv_i32 cpu_SR[256];
 static TCGv_i32 cpu_UR[256];
 
-#include "gen-icount.h"
-
-static const char * const sregnames[256] = {
-    [LBEG] = "LBEG",
-    [LEND] = "LEND",
-    [LCOUNT] = "LCOUNT",
-    [SAR] = "SAR",
-    [BR] = "BR",
-    [LITBASE] = "LITBASE",
-    [SCOMPARE1] = "SCOMPARE1",
-    [ACCLO] = "ACCLO",
-    [ACCHI] = "ACCHI",
-    [MR] = "MR0",
-    [MR + 1] = "MR1",
-    [MR + 2] = "MR2",
-    [MR + 3] = "MR3",
-    [WINDOW_BASE] = "WINDOW_BASE",
-    [WINDOW_START] = "WINDOW_START",
-    [PTEVADDR] = "PTEVADDR",
-    [RASID] = "RASID",
-    [ITLBCFG] = "ITLBCFG",
-    [DTLBCFG] = "DTLBCFG",
-    [IBREAKENABLE] = "IBREAKENABLE",
-    [IBREAKA] = "IBREAKA0",
-    [IBREAKA + 1] = "IBREAKA1",
-    [DBREAKA] = "DBREAKA0",
-    [DBREAKA + 1] = "DBREAKA1",
-    [DBREAKC] = "DBREAKC0",
-    [DBREAKC + 1] = "DBREAKC1",
-    [EPC1] = "EPC1",
-    [EPC1 + 1] = "EPC2",
-    [EPC1 + 2] = "EPC3",
-    [EPC1 + 3] = "EPC4",
-    [EPC1 + 4] = "EPC5",
-    [EPC1 + 5] = "EPC6",
-    [EPC1 + 6] = "EPC7",
-    [DEPC] = "DEPC",
-    [EPS2] = "EPS2",
-    [EPS2 + 1] = "EPS3",
-    [EPS2 + 2] = "EPS4",
-    [EPS2 + 3] = "EPS5",
-    [EPS2 + 4] = "EPS6",
-    [EPS2 + 5] = "EPS7",
-    [EXCSAVE1] = "EXCSAVE1",
-    [EXCSAVE1 + 1] = "EXCSAVE2",
-    [EXCSAVE1 + 2] = "EXCSAVE3",
-    [EXCSAVE1 + 3] = "EXCSAVE4",
-    [EXCSAVE1 + 4] = "EXCSAVE5",
-    [EXCSAVE1 + 5] = "EXCSAVE6",
-    [EXCSAVE1 + 6] = "EXCSAVE7",
-    [CPENABLE] = "CPENABLE",
-    [INTSET] = "INTSET",
-    [INTCLEAR] = "INTCLEAR",
-    [INTENABLE] = "INTENABLE",
-    [PS] = "PS",
-    [VECBASE] = "VECBASE",
-    [EXCCAUSE] = "EXCCAUSE",
-    [DEBUGCAUSE] = "DEBUGCAUSE",
-    [CCOUNT] = "CCOUNT",
-    [PRID] = "PRID",
-    [ICOUNT] = "ICOUNT",
-    [ICOUNTLEVEL] = "ICOUNTLEVEL",
-    [EXCVADDR] = "EXCVADDR",
-    [CCOMPARE] = "CCOMPARE0",
-    [CCOMPARE + 1] = "CCOMPARE1",
-    [CCOMPARE + 2] = "CCOMPARE2",
+#include "exec/gen-icount.h"
+
+typedef struct XtensaReg {
+    const char *name;
+    uint64_t opt_bits;
+    enum {
+        SR_R = 1,
+        SR_W = 2,
+        SR_X = 4,
+        SR_RW = 3,
+        SR_RWX = 7,
+    } access;
+} XtensaReg;
+
+#define XTENSA_REG_ACCESS(regname, opt, acc) { \
+        .name = (regname), \
+        .opt_bits = XTENSA_OPTION_BIT(opt), \
+        .access = (acc), \
+    }
+
+#define XTENSA_REG(regname, opt) XTENSA_REG_ACCESS(regname, opt, SR_RWX)
+
+#define XTENSA_REG_BITS(regname, opt) { \
+        .name = (regname), \
+        .opt_bits = (opt), \
+        .access = SR_RWX, \
+    }
+
+static const XtensaReg sregnames[256] = {
+    [LBEG] = XTENSA_REG("LBEG", XTENSA_OPTION_LOOP),
+    [LEND] = XTENSA_REG("LEND", XTENSA_OPTION_LOOP),
+    [LCOUNT] = XTENSA_REG("LCOUNT", XTENSA_OPTION_LOOP),
+    [SAR] = XTENSA_REG_BITS("SAR", XTENSA_OPTION_ALL),
+    [BR] = XTENSA_REG("BR", XTENSA_OPTION_BOOLEAN),
+    [LITBASE] = XTENSA_REG("LITBASE", XTENSA_OPTION_EXTENDED_L32R),
+    [SCOMPARE1] = XTENSA_REG("SCOMPARE1", XTENSA_OPTION_CONDITIONAL_STORE),
+    [ACCLO] = XTENSA_REG("ACCLO", XTENSA_OPTION_MAC16),
+    [ACCHI] = XTENSA_REG("ACCHI", XTENSA_OPTION_MAC16),
+    [MR] = XTENSA_REG("MR0", XTENSA_OPTION_MAC16),
+    [MR + 1] = XTENSA_REG("MR1", XTENSA_OPTION_MAC16),
+    [MR + 2] = XTENSA_REG("MR2", XTENSA_OPTION_MAC16),
+    [MR + 3] = XTENSA_REG("MR3", XTENSA_OPTION_MAC16),
+    [WINDOW_BASE] = XTENSA_REG("WINDOW_BASE", XTENSA_OPTION_WINDOWED_REGISTER),
+    [WINDOW_START] = XTENSA_REG("WINDOW_START",
+            XTENSA_OPTION_WINDOWED_REGISTER),
+    [PTEVADDR] = XTENSA_REG("PTEVADDR", XTENSA_OPTION_MMU),
+    [RASID] = XTENSA_REG("RASID", XTENSA_OPTION_MMU),
+    [ITLBCFG] = XTENSA_REG("ITLBCFG", XTENSA_OPTION_MMU),
+    [DTLBCFG] = XTENSA_REG("DTLBCFG", XTENSA_OPTION_MMU),
+    [IBREAKENABLE] = XTENSA_REG("IBREAKENABLE", XTENSA_OPTION_DEBUG),
+    [CACHEATTR] = XTENSA_REG("CACHEATTR", XTENSA_OPTION_CACHEATTR),
+    [ATOMCTL] = XTENSA_REG("ATOMCTL", XTENSA_OPTION_ATOMCTL),
+    [IBREAKA] = XTENSA_REG("IBREAKA0", XTENSA_OPTION_DEBUG),
+    [IBREAKA + 1] = XTENSA_REG("IBREAKA1", XTENSA_OPTION_DEBUG),
+    [DBREAKA] = XTENSA_REG("DBREAKA0", XTENSA_OPTION_DEBUG),
+    [DBREAKA + 1] = XTENSA_REG("DBREAKA1", XTENSA_OPTION_DEBUG),
+    [DBREAKC] = XTENSA_REG("DBREAKC0", XTENSA_OPTION_DEBUG),
+    [DBREAKC + 1] = XTENSA_REG("DBREAKC1", XTENSA_OPTION_DEBUG),
+    [EPC1] = XTENSA_REG("EPC1", XTENSA_OPTION_EXCEPTION),
+    [EPC1 + 1] = XTENSA_REG("EPC2", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 2] = XTENSA_REG("EPC3", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 3] = XTENSA_REG("EPC4", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 4] = XTENSA_REG("EPC5", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 5] = XTENSA_REG("EPC6", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 6] = XTENSA_REG("EPC7", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [DEPC] = XTENSA_REG("DEPC", XTENSA_OPTION_EXCEPTION),
+    [EPS2] = XTENSA_REG("EPS2", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 1] = XTENSA_REG("EPS3", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 2] = XTENSA_REG("EPS4", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 3] = XTENSA_REG("EPS5", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 4] = XTENSA_REG("EPS6", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 5] = XTENSA_REG("EPS7", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1] = XTENSA_REG("EXCSAVE1", XTENSA_OPTION_EXCEPTION),
+    [EXCSAVE1 + 1] = XTENSA_REG("EXCSAVE2",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 2] = XTENSA_REG("EXCSAVE3",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 3] = XTENSA_REG("EXCSAVE4",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 4] = XTENSA_REG("EXCSAVE5",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 5] = XTENSA_REG("EXCSAVE6",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 6] = XTENSA_REG("EXCSAVE7",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [CPENABLE] = XTENSA_REG("CPENABLE", XTENSA_OPTION_COPROCESSOR),
+    [INTSET] = XTENSA_REG_ACCESS("INTSET", XTENSA_OPTION_INTERRUPT, SR_RW),
+    [INTCLEAR] = XTENSA_REG_ACCESS("INTCLEAR", XTENSA_OPTION_INTERRUPT, SR_W),
+    [INTENABLE] = XTENSA_REG("INTENABLE", XTENSA_OPTION_INTERRUPT),
+    [PS] = XTENSA_REG_BITS("PS", XTENSA_OPTION_ALL),
+    [VECBASE] = XTENSA_REG("VECBASE", XTENSA_OPTION_RELOCATABLE_VECTOR),
+    [EXCCAUSE] = XTENSA_REG("EXCCAUSE", XTENSA_OPTION_EXCEPTION),
+    [DEBUGCAUSE] = XTENSA_REG_ACCESS("DEBUGCAUSE", XTENSA_OPTION_DEBUG, SR_R),
+    [CCOUNT] = XTENSA_REG("CCOUNT", XTENSA_OPTION_TIMER_INTERRUPT),
+    [PRID] = XTENSA_REG_ACCESS("PRID", XTENSA_OPTION_PROCESSOR_ID, SR_R),
+    [ICOUNT] = XTENSA_REG("ICOUNT", XTENSA_OPTION_DEBUG),
+    [ICOUNTLEVEL] = XTENSA_REG("ICOUNTLEVEL", XTENSA_OPTION_DEBUG),
+    [EXCVADDR] = XTENSA_REG("EXCVADDR", XTENSA_OPTION_EXCEPTION),
+    [CCOMPARE] = XTENSA_REG("CCOMPARE0", XTENSA_OPTION_TIMER_INTERRUPT),
+    [CCOMPARE + 1] = XTENSA_REG("CCOMPARE1",
+            XTENSA_OPTION_TIMER_INTERRUPT),
+    [CCOMPARE + 2] = XTENSA_REG("CCOMPARE2",
+            XTENSA_OPTION_TIMER_INTERRUPT),
+    [MISC] = XTENSA_REG("MISC0", XTENSA_OPTION_MISC_SR),
+    [MISC + 1] = XTENSA_REG("MISC1", XTENSA_OPTION_MISC_SR),
+    [MISC + 2] = XTENSA_REG("MISC2", XTENSA_OPTION_MISC_SR),
+    [MISC + 3] = XTENSA_REG("MISC3", XTENSA_OPTION_MISC_SR),
 };
 
-static const char * const uregnames[256] = {
-    [THREADPTR] = "THREADPTR",
-    [FCR] = "FCR",
-    [FSR] = "FSR",
+static const XtensaReg uregnames[256] = {
+    [THREADPTR] = XTENSA_REG("THREADPTR", XTENSA_OPTION_THREAD_POINTER),
+    [FCR] = XTENSA_REG("FCR", XTENSA_OPTION_FP_COPROCESSOR),
+    [FSR] = XTENSA_REG("FSR", XTENSA_OPTION_FP_COPROCESSOR),
 };
 
 void xtensa_translate_init(void)
@@ -155,6 +199,12 @@ void xtensa_translate_init(void)
         "ar8", "ar9", "ar10", "ar11",
         "ar12", "ar13", "ar14", "ar15",
     };
+    static const char * const fregnames[] = {
+        "f0", "f1", "f2", "f3",
+        "f4", "f5", "f6", "f7",
+        "f8", "f9", "f10", "f11",
+        "f12", "f13", "f14", "f15",
+    };
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
@@ -167,19 +217,25 @@ void xtensa_translate_init(void)
                 regnames[i]);
     }
 
+    for (i = 0; i < 16; i++) {
+        cpu_FR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+                offsetof(CPUXtensaState, fregs[i]),
+                fregnames[i]);
+    }
+
     for (i = 0; i < 256; ++i) {
-        if (sregnames[i]) {
+        if (sregnames[i].name) {
             cpu_SR[i] = tcg_global_mem_new_i32(TCG_AREG0,
                     offsetof(CPUXtensaState, sregs[i]),
-                    sregnames[i]);
+                    sregnames[i].name);
         }
     }
 
     for (i = 0; i < 256; ++i) {
-        if (uregnames[i]) {
+        if (uregnames[i].name) {
             cpu_UR[i] = tcg_global_mem_new_i32(TCG_AREG0,
                     offsetof(CPUXtensaState, uregs[i]),
-                    uregnames[i]);
+                    uregnames[i].name);
         }
     }
 #define GEN_HELPER 2
@@ -318,6 +374,15 @@ static void gen_check_privilege(DisasContext *dc)
     }
 }
 
+static void gen_check_cpenable(DisasContext *dc, unsigned cp)
+{
+    if (option_enabled(dc, XTENSA_OPTION_COPROCESSOR) &&
+            !(dc->cpenable & (1 << cp))) {
+        gen_exception_cause(dc, COPROCESSOR0_DISABLED + cp);
+        dc->is_jmp = DISAS_UPDATE;
+    }
+}
+
 static void gen_jump_slot(DisasContext *dc, TCGv dest, int slot)
 {
     tcg_gen_mov_i32(cpu_pc, dest);
@@ -426,6 +491,28 @@ static void gen_brcondi(DisasContext *dc, TCGCond cond,
     tcg_temp_free(tmp);
 }
 
+static void gen_check_sr(DisasContext *dc, uint32_t sr, unsigned access)
+{
+    if (!xtensa_option_bits_enabled(dc->config, sregnames[sr].opt_bits)) {
+        if (sregnames[sr].name) {
+            qemu_log("SR %s is not configured\n", sregnames[sr].name);
+        } else {
+            qemu_log("SR %d is not implemented\n", sr);
+        }
+        gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
+    } else if (!(sregnames[sr].access & access)) {
+        static const char * const access_text[] = {
+            [SR_R] = "rsr",
+            [SR_W] = "wsr",
+            [SR_X] = "xsr",
+        };
+        assert(access < ARRAY_SIZE(access_text) && access_text[access]);
+        qemu_log("SR %s is not available for %s\n", sregnames[sr].name,
+                access_text[access]);
+        gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
+    }
+}
+
 static void gen_rsr_ccount(DisasContext *dc, TCGv_i32 d, uint32_t sr)
 {
     gen_advance_ccount(dc);
@@ -447,14 +534,10 @@ static void gen_rsr(DisasContext *dc, TCGv_i32 d, uint32_t sr)
         [PTEVADDR] = gen_rsr_ptevaddr,
     };
 
-    if (sregnames[sr]) {
-        if (rsr_handler[sr]) {
-            rsr_handler[sr](dc, d, sr);
-        } else {
-            tcg_gen_mov_i32(d, cpu_SR[sr]);
-        }
+    if (rsr_handler[sr]) {
+        rsr_handler[sr](dc, d, sr);
     } else {
-        qemu_log("RSR %d not implemented, ", sr);
+        tcg_gen_mov_i32(d, cpu_SR[sr]);
     }
 }
 
@@ -532,6 +615,11 @@ static void gen_wsr_ibreakenable(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     gen_jumpi_check_loop_end(dc, 0);
 }
 
+static void gen_wsr_atomctl(DisasContext *dc, uint32_t sr, TCGv_i32 v)
+{
+    tcg_gen_andi_i32(cpu_SR[sr], v, 0x3f);
+}
+
 static void gen_wsr_ibreaka(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     unsigned id = sr - IBREAKA;
@@ -566,6 +654,13 @@ static void gen_wsr_dbreakc(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     }
 }
 
+static void gen_wsr_cpenable(DisasContext *dc, uint32_t sr, TCGv_i32 v)
+{
+    tcg_gen_andi_i32(cpu_SR[sr], v, 0xff);
+    /* This can change tb->flags, so exit tb */
+    gen_jumpi_check_loop_end(dc, -1);
+}
+
 static void gen_wsr_intset(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     tcg_gen_andi_i32(cpu_SR[sr], v,
@@ -609,14 +704,6 @@ static void gen_wsr_ps(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     gen_jumpi_check_loop_end(dc, -1);
 }
 
-static void gen_wsr_debugcause(DisasContext *dc, uint32_t sr, TCGv_i32 v)
-{
-}
-
-static void gen_wsr_prid(DisasContext *dc, uint32_t sr, TCGv_i32 v)
-{
-}
-
 static void gen_wsr_icount(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     if (dc->icount) {
@@ -662,18 +749,18 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [ITLBCFG] = gen_wsr_tlbcfg,
         [DTLBCFG] = gen_wsr_tlbcfg,
         [IBREAKENABLE] = gen_wsr_ibreakenable,
+        [ATOMCTL] = gen_wsr_atomctl,
         [IBREAKA] = gen_wsr_ibreaka,
         [IBREAKA + 1] = gen_wsr_ibreaka,
         [DBREAKA] = gen_wsr_dbreaka,
         [DBREAKA + 1] = gen_wsr_dbreaka,
         [DBREAKC] = gen_wsr_dbreakc,
         [DBREAKC + 1] = gen_wsr_dbreakc,
+        [CPENABLE] = gen_wsr_cpenable,
         [INTSET] = gen_wsr_intset,
         [INTCLEAR] = gen_wsr_intclear,
         [INTENABLE] = gen_wsr_intenable,
         [PS] = gen_wsr_ps,
-        [DEBUGCAUSE] = gen_wsr_debugcause,
-        [PRID] = gen_wsr_prid,
         [ICOUNT] = gen_wsr_icount,
         [ICOUNTLEVEL] = gen_wsr_icountlevel,
         [CCOMPARE] = gen_wsr_ccompare,
@@ -681,14 +768,27 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [CCOMPARE + 2] = gen_wsr_ccompare,
     };
 
-    if (sregnames[sr]) {
-        if (wsr_handler[sr]) {
-            wsr_handler[sr](dc, sr, s);
-        } else {
-            tcg_gen_mov_i32(cpu_SR[sr], s);
-        }
+    if (wsr_handler[sr]) {
+        wsr_handler[sr](dc, sr, s);
     } else {
-        qemu_log("WSR %d not implemented, ", sr);
+        tcg_gen_mov_i32(cpu_SR[sr], s);
+    }
+}
+
+static void gen_wur(uint32_t ur, TCGv_i32 s)
+{
+    switch (ur) {
+    case FCR:
+        gen_helper_wur_fcr(cpu_env, s);
+        break;
+
+    case FSR:
+        tcg_gen_andi_i32(cpu_UR[ur], s, 0xffffff80);
+        break;
+
+    default:
+        tcg_gen_mov_i32(cpu_UR[ur], s);
+        break;
     }
 }
 
@@ -761,7 +861,7 @@ static TCGv_i32 gen_mac16_m(TCGv_i32 v, bool hi, bool is_unsigned)
     return m;
 }
 
-static void disas_xtensa_insn(DisasContext *dc)
+static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 {
 #define HAS_OPTION_BITS(opt) do { \
         if (!option_bits_enabled(dc, opt)) { \
@@ -851,8 +951,8 @@ static void disas_xtensa_insn(DisasContext *dc)
 
 #define RSR_SR (b1)
 
-    uint8_t b0 = cpu_ldub_code(cpu_single_env, dc->pc);
-    uint8_t b1 = cpu_ldub_code(cpu_single_env, dc->pc + 1);
+    uint8_t b0 = cpu_ldub_code(env, dc->pc);
+    uint8_t b1 = cpu_ldub_code(env, dc->pc + 1);
     uint8_t b2 = 0;
 
     static const uint32_t B4CONST[] = {
@@ -868,7 +968,7 @@ static void disas_xtensa_insn(DisasContext *dc)
         HAS_OPTION(XTENSA_OPTION_CODE_DENSITY);
     } else {
         dc->next_pc = dc->pc + 3;
-        b2 = cpu_ldub_code(cpu_single_env, dc->pc + 2);
+        b2 = cpu_ldub_code(env, dc->pc + 2);
     }
 
     switch (OP0) {
@@ -1303,12 +1403,14 @@ static void disas_xtensa_insn(DisasContext *dc)
 
                 case 1: /*ABS*/
                     {
-                        int label = gen_new_label();
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                        tcg_gen_brcondi_i32(
-                                TCG_COND_GE, cpu_R[RRR_R], 0, label);
-                        tcg_gen_neg_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                        gen_set_label(label);
+                        TCGv_i32 zero = tcg_const_i32(0);
+                        TCGv_i32 neg = tcg_temp_new_i32();
+
+                        tcg_gen_neg_i32(neg, cpu_R[RRR_T]);
+                        tcg_gen_movcond_i32(TCG_COND_GE, cpu_R[RRR_R],
+                                cpu_R[RRR_T], zero, cpu_R[RRR_T], neg);
+                        tcg_temp_free(neg);
+                        tcg_temp_free(zero);
                     }
                     break;
 
@@ -1382,6 +1484,7 @@ static void disas_xtensa_insn(DisasContext *dc)
             case 6: /*XSR*/
                 {
                     TCGv_i32 tmp = tcg_temp_new_i32();
+                    gen_check_sr(dc, RSR_SR, SR_X);
                     if (RSR_SR >= 64) {
                         gen_check_privilege(dc);
                     }
@@ -1390,9 +1493,6 @@ static void disas_xtensa_insn(DisasContext *dc)
                     gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
                     gen_wsr(dc, RSR_SR, tmp);
                     tcg_temp_free(tmp);
-                    if (!sregnames[RSR_SR]) {
-                        TBD();
-                    }
                 }
                 break;
 
@@ -1615,25 +1715,21 @@ static void disas_xtensa_insn(DisasContext *dc)
         case 3: /*RST3*/
             switch (OP2) {
             case 0: /*RSR*/
+                gen_check_sr(dc, RSR_SR, SR_R);
                 if (RSR_SR >= 64) {
                     gen_check_privilege(dc);
                 }
                 gen_window_check1(dc, RRR_T);
                 gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
-                if (!sregnames[RSR_SR]) {
-                    TBD();
-                }
                 break;
 
             case 1: /*WSR*/
+                gen_check_sr(dc, RSR_SR, SR_W);
                 if (RSR_SR >= 64) {
                     gen_check_privilege(dc);
                 }
                 gen_window_check1(dc, RRR_T);
                 gen_wsr(dc, RSR_SR, cpu_R[RRR_T]);
-                if (!sregnames[RSR_SR]) {
-                    TBD();
-                }
                 break;
 
             case 2: /*SEXTu*/
@@ -1661,22 +1757,20 @@ static void disas_xtensa_insn(DisasContext *dc)
                 {
                     TCGv_i32 tmp1 = tcg_temp_new_i32();
                     TCGv_i32 tmp2 = tcg_temp_new_i32();
-                    int label = gen_new_label();
+                    TCGv_i32 zero = tcg_const_i32(0);
 
                     tcg_gen_sari_i32(tmp1, cpu_R[RRR_S], 24 - RRR_T);
                     tcg_gen_xor_i32(tmp2, tmp1, cpu_R[RRR_S]);
                     tcg_gen_andi_i32(tmp2, tmp2, 0xffffffff << (RRR_T + 7));
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    tcg_gen_brcondi_i32(TCG_COND_EQ, tmp2, 0, label);
 
                     tcg_gen_sari_i32(tmp1, cpu_R[RRR_S], 31);
-                    tcg_gen_xori_i32(cpu_R[RRR_R], tmp1,
-                            0xffffffff >> (25 - RRR_T));
-
-                    gen_set_label(label);
+                    tcg_gen_xori_i32(tmp1, tmp1, 0xffffffff >> (25 - RRR_T));
 
+                    tcg_gen_movcond_i32(TCG_COND_EQ, cpu_R[RRR_R], tmp2, zero,
+                            cpu_R[RRR_S], tmp1);
                     tcg_temp_free(tmp1);
                     tcg_temp_free(tmp2);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1693,19 +1787,9 @@ static void disas_xtensa_insn(DisasContext *dc)
                         TCG_COND_LEU,
                         TCG_COND_GEU
                     };
-                    int label = gen_new_label();
-
-                    if (RRR_R != RRR_T) {
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                        tcg_gen_brcond_i32(cond[OP2 - 4],
-                                cpu_R[RRR_S], cpu_R[RRR_T], label);
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                    } else {
-                        tcg_gen_brcond_i32(cond[OP2 - 4],
-                                cpu_R[RRR_T], cpu_R[RRR_S], label);
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    }
-                    gen_set_label(label);
+                    tcg_gen_movcond_i32(cond[OP2 - 4], cpu_R[RRR_R],
+                            cpu_R[RRR_S], cpu_R[RRR_T],
+                            cpu_R[RRR_S], cpu_R[RRR_T]);
                 }
                 break;
 
@@ -1716,15 +1800,16 @@ static void disas_xtensa_insn(DisasContext *dc)
                 gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
                 {
                     static const TCGCond cond[] = {
-                        TCG_COND_NE,
                         TCG_COND_EQ,
+                        TCG_COND_NE,
+                        TCG_COND_LT,
                         TCG_COND_GE,
-                        TCG_COND_LT
                     };
-                    int label = gen_new_label();
-                    tcg_gen_brcondi_i32(cond[OP2 - 8], cpu_R[RRR_T], 0, label);
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    gen_set_label(label);
+                    TCGv_i32 zero = tcg_const_i32(0);
+
+                    tcg_gen_movcond_i32(cond[OP2 - 8], cpu_R[RRR_R],
+                            cpu_R[RRR_T], zero, cpu_R[RRR_S], cpu_R[RRR_R]);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1733,16 +1818,16 @@ static void disas_xtensa_insn(DisasContext *dc)
                 HAS_OPTION(XTENSA_OPTION_BOOLEAN);
                 gen_window_check2(dc, RRR_R, RRR_S);
                 {
-                    int label = gen_new_label();
+                    TCGv_i32 zero = tcg_const_i32(0);
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
                     tcg_gen_andi_i32(tmp, cpu_SR[BR], 1 << RRR_T);
-                    tcg_gen_brcondi_i32(
-                            OP2 & 1 ? TCG_COND_EQ : TCG_COND_NE,
-                            tmp, 0, label);
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    gen_set_label(label);
+                    tcg_gen_movcond_i32(OP2 & 1 ? TCG_COND_NE : TCG_COND_EQ,
+                            cpu_R[RRR_R], tmp, zero,
+                            cpu_R[RRR_S], cpu_R[RRR_R]);
+
                     tcg_temp_free(tmp);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1750,7 +1835,7 @@ static void disas_xtensa_insn(DisasContext *dc)
                 gen_window_check1(dc, RRR_R);
                 {
                     int st = (RRR_S << 4) + RRR_T;
-                    if (uregnames[st]) {
+                    if (uregnames[st].name) {
                         tcg_gen_mov_i32(cpu_R[RRR_R], cpu_UR[st]);
                     } else {
                         qemu_log("RUR %d not implemented, ", st);
@@ -1761,13 +1846,11 @@ static void disas_xtensa_insn(DisasContext *dc)
 
             case 15: /*WUR*/
                 gen_window_check1(dc, RRR_T);
-                {
-                    if (uregnames[RSR_SR]) {
-                        tcg_gen_mov_i32(cpu_UR[RSR_SR], cpu_R[RRR_T]);
-                    } else {
-                        qemu_log("WUR %d not implemented, ", RSR_SR);
-                        TBD();
-                    }
+                if (uregnames[RSR_SR].name) {
+                    gen_wur(RSR_SR, cpu_R[RRR_T]);
+                } else {
+                    qemu_log("WUR %d not implemented, ", RSR_SR);
+                    TBD();
                 }
                 break;
 
@@ -1778,7 +1861,7 @@ static void disas_xtensa_insn(DisasContext *dc)
         case 5:
             gen_window_check2(dc, RRR_R, RRR_T);
             {
-                int shiftimm = RRR_S | (OP1 << 4);
+                int shiftimm = RRR_S | ((OP1 & 1) << 4);
                 int maskimm = (1 << (OP2 + 1)) - 1;
 
                 TCGv_i32 tmp = tcg_temp_new_i32();
@@ -1797,8 +1880,34 @@ static void disas_xtensa_insn(DisasContext *dc)
             break;
 
         case 8: /*LSCXp*/
-            HAS_OPTION(XTENSA_OPTION_COPROCESSOR);
-            TBD();
+            switch (OP2) {
+            case 0: /*LSXf*/
+            case 1: /*LSXUf*/
+            case 4: /*SSXf*/
+            case 5: /*SSXUf*/
+                HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
+                gen_window_check2(dc, RRR_S, RRR_T);
+                gen_check_cpenable(dc, 0);
+                {
+                    TCGv_i32 addr = tcg_temp_new_i32();
+                    tcg_gen_add_i32(addr, cpu_R[RRR_S], cpu_R[RRR_T]);
+                    gen_load_store_alignment(dc, 2, addr, false);
+                    if (OP2 & 0x4) {
+                        tcg_gen_qemu_st32(cpu_FR[RRR_R], addr, dc->cring);
+                    } else {
+                        tcg_gen_qemu_ld32u(cpu_FR[RRR_R], addr, dc->cring);
+                    }
+                    if (OP2 & 0x1) {
+                        tcg_gen_mov_i32(cpu_R[RRR_S], addr);
+                    }
+                    tcg_temp_free(addr);
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         case 9: /*LSC4*/
@@ -1836,12 +1945,214 @@ static void disas_xtensa_insn(DisasContext *dc)
 
         case 10: /*FP0*/
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-            TBD();
+            switch (OP2) {
+            case 0: /*ADD.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_add_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 1: /*SUB.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_sub_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 2: /*MUL.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_mul_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 4: /*MADD.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_madd_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 5: /*MSUB.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_msub_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 8: /*ROUND.Sf*/
+            case 9: /*TRUNC.Sf*/
+            case 10: /*FLOOR.Sf*/
+            case 11: /*CEIL.Sf*/
+            case 14: /*UTRUNC.Sf*/
+                gen_window_check1(dc, RRR_R);
+                gen_check_cpenable(dc, 0);
+                {
+                    static const unsigned rounding_mode_const[] = {
+                        float_round_nearest_even,
+                        float_round_to_zero,
+                        float_round_down,
+                        float_round_up,
+                        [6] = float_round_to_zero,
+                    };
+                    TCGv_i32 rounding_mode = tcg_const_i32(
+                            rounding_mode_const[OP2 & 7]);
+                    TCGv_i32 scale = tcg_const_i32(RRR_T);
+
+                    if (OP2 == 14) {
+                        gen_helper_ftoui(cpu_R[RRR_R], cpu_FR[RRR_S],
+                                rounding_mode, scale);
+                    } else {
+                        gen_helper_ftoi(cpu_R[RRR_R], cpu_FR[RRR_S],
+                                rounding_mode, scale);
+                    }
+
+                    tcg_temp_free(rounding_mode);
+                    tcg_temp_free(scale);
+                }
+                break;
+
+            case 12: /*FLOAT.Sf*/
+            case 13: /*UFLOAT.Sf*/
+                gen_window_check1(dc, RRR_S);
+                gen_check_cpenable(dc, 0);
+                {
+                    TCGv_i32 scale = tcg_const_i32(-RRR_T);
+
+                    if (OP2 == 13) {
+                        gen_helper_uitof(cpu_FR[RRR_R], cpu_env,
+                                cpu_R[RRR_S], scale);
+                    } else {
+                        gen_helper_itof(cpu_FR[RRR_R], cpu_env,
+                                cpu_R[RRR_S], scale);
+                    }
+                    tcg_temp_free(scale);
+                }
+                break;
+
+            case 15: /*FP1OP*/
+                switch (RRR_T) {
+                case 0: /*MOV.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 1: /*ABS.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    gen_helper_abs_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 4: /*RFRf*/
+                    gen_window_check1(dc, RRR_R);
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 5: /*WFRf*/
+                    gen_window_check1(dc, RRR_S);
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_R[RRR_S]);
+                    break;
+
+                case 6: /*NEG.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    gen_helper_neg_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                default: /*reserved*/
+                    RESERVED();
+                    break;
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         case 11: /*FP1*/
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-            TBD();
+
+#define gen_compare(rel, br, a, b) \
+    do { \
+        TCGv_i32 bit = tcg_const_i32(1 << br); \
+        \
+        gen_check_cpenable(dc, 0); \
+        gen_helper_##rel(cpu_env, bit, cpu_FR[a], cpu_FR[b]); \
+        tcg_temp_free(bit); \
+    } while (0)
+
+            switch (OP2) {
+            case 1: /*UN.Sf*/
+                gen_compare(un_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 2: /*OEQ.Sf*/
+                gen_compare(oeq_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 3: /*UEQ.Sf*/
+                gen_compare(ueq_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 4: /*OLT.Sf*/
+                gen_compare(olt_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 5: /*ULT.Sf*/
+                gen_compare(ult_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 6: /*OLE.Sf*/
+                gen_compare(ole_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 7: /*ULE.Sf*/
+                gen_compare(ule_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+#undef gen_compare
+
+            case 8: /*MOVEQZ.Sf*/
+            case 9: /*MOVNEZ.Sf*/
+            case 10: /*MOVLTZ.Sf*/
+            case 11: /*MOVGEZ.Sf*/
+                gen_window_check1(dc, RRR_T);
+                gen_check_cpenable(dc, 0);
+                {
+                    static const TCGCond cond[] = {
+                        TCG_COND_EQ,
+                        TCG_COND_NE,
+                        TCG_COND_LT,
+                        TCG_COND_GE,
+                    };
+                    TCGv_i32 zero = tcg_const_i32(0);
+
+                    tcg_gen_movcond_i32(cond[OP2 - 8], cpu_FR[RRR_R],
+                            cpu_R[RRR_T], zero, cpu_FR[RRR_S], cpu_FR[RRR_R]);
+                    tcg_temp_free(zero);
+                }
+                break;
+
+            case 12: /*MOVF.Sf*/
+            case 13: /*MOVT.Sf*/
+                HAS_OPTION(XTENSA_OPTION_BOOLEAN);
+                gen_check_cpenable(dc, 0);
+                {
+                    TCGv_i32 zero = tcg_const_i32(0);
+                    TCGv_i32 tmp = tcg_temp_new_i32();
+
+                    tcg_gen_andi_i32(tmp, cpu_SR[BR], 1 << RRR_T);
+                    tcg_gen_movcond_i32(OP2 & 1 ? TCG_COND_NE : TCG_COND_EQ,
+                            cpu_FR[RRR_R], tmp, zero,
+                            cpu_FR[RRR_S], cpu_FR[RRR_R]);
+
+                    tcg_temp_free(tmp);
+                    tcg_temp_free(zero);
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         default: /*reserved*/
@@ -2043,10 +2354,15 @@ static void disas_xtensa_insn(DisasContext *dc)
                 int label = gen_new_label();
                 TCGv_i32 tmp = tcg_temp_local_new_i32();
                 TCGv_i32 addr = tcg_temp_local_new_i32();
+                TCGv_i32 tpc;
 
                 tcg_gen_mov_i32(tmp, cpu_R[RRI8_T]);
                 tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2);
                 gen_load_store_alignment(dc, 2, addr, true);
+
+                gen_advance_ccount(dc);
+                tpc = tcg_const_i32(dc->pc);
+                gen_helper_check_atomctl(cpu_env, tpc, addr);
                 tcg_gen_qemu_ld32u(cpu_R[RRI8_T], addr, dc->cring);
                 tcg_gen_brcond_i32(TCG_COND_NE, cpu_R[RRI8_T],
                         cpu_SR[SCOMPARE1], label);
@@ -2054,6 +2370,7 @@ static void disas_xtensa_insn(DisasContext *dc)
                 tcg_gen_qemu_st32(tmp, addr, dc->cring);
 
                 gen_set_label(label);
+                tcg_temp_free(tpc);
                 tcg_temp_free(addr);
                 tcg_temp_free(tmp);
             }
@@ -2072,8 +2389,34 @@ static void disas_xtensa_insn(DisasContext *dc)
         break;
 
     case 3: /*LSCIp*/
-        HAS_OPTION(XTENSA_OPTION_COPROCESSOR);
-        TBD();
+        switch (RRI8_R) {
+        case 0: /*LSIf*/
+        case 4: /*SSIf*/
+        case 8: /*LSIUf*/
+        case 12: /*SSIUf*/
+            HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
+            gen_window_check1(dc, RRI8_S);
+            gen_check_cpenable(dc, 0);
+            {
+                TCGv_i32 addr = tcg_temp_new_i32();
+                tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2);
+                gen_load_store_alignment(dc, 2, addr, false);
+                if (RRI8_R & 0x4) {
+                    tcg_gen_qemu_st32(cpu_FR[RRI8_T], addr, dc->cring);
+                } else {
+                    tcg_gen_qemu_ld32u(cpu_FR[RRI8_T], addr, dc->cring);
+                }
+                if (RRI8_R & 0x8) {
+                    tcg_gen_mov_i32(cpu_R[RRI8_S], addr);
+                }
+                tcg_temp_free(addr);
+            }
+            break;
+
+        default: /*reserved*/
+            RESERVED();
+            break;
+        }
         break;
 
     case 4: /*MAC16d*/
@@ -2502,7 +2845,9 @@ static void disas_xtensa_insn(DisasContext *dc)
         break;
     }
 
-    gen_check_loop_end(dc, 0);
+    if (dc->is_jmp == DISAS_NEXT) {
+        gen_check_loop_end(dc, 0);
+    }
     dc->pc = dc->next_pc;
 
     return;
@@ -2547,7 +2892,7 @@ static void gen_intermediate_code_internal(
     DisasContext dc;
     int insn_count = 0;
     int j, lj = -1;
-    uint16_t *gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    uint16_t *gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     int max_insns = tb->cflags & CF_COUNT_MASK;
     uint32_t pc_start = tb->pc;
     uint32_t next_page_start =
@@ -2569,6 +2914,8 @@ static void gen_intermediate_code_internal(
     dc.ccount_delta = 0;
     dc.debug = tb->flags & XTENSA_TBFLAG_DEBUG;
     dc.icount = tb->flags & XTENSA_TBFLAG_ICOUNT;
+    dc.cpenable = (tb->flags & XTENSA_TBFLAG_CPENABLE_MASK) >>
+        XTENSA_TBFLAG_CPENABLE_SHIFT;
 
     init_litbase(&dc);
     init_sar_tracker(&dc);
@@ -2589,19 +2936,19 @@ static void gen_intermediate_code_internal(
         check_breakpoint(env, &dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc.pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = insn_count;
+            tcg_ctx.gen_opc_pc[lj] = dc.pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = insn_count;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc.pc);
         }
 
@@ -2627,7 +2974,7 @@ static void gen_intermediate_code_internal(
             gen_ibreak_check(env, &dc);
         }
 
-        disas_xtensa_insn(&dc);
+        disas_xtensa_insn(env, &dc);
         ++insn_count;
         if (dc.icount) {
             tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount);
@@ -2640,7 +2987,7 @@ static void gen_intermediate_code_internal(
     } while (dc.is_jmp == DISAS_NEXT &&
             insn_count < max_insns &&
             dc.pc < next_page_start &&
-            gen_opc_ptr < gen_opc_end);
+            tcg_ctx.gen_opc_ptr < gen_opc_end);
 
     reset_litbase(&dc);
     reset_sar_tracker(&dc);
@@ -2656,9 +3003,13 @@ static void gen_intermediate_code_internal(
         gen_jumpi(&dc, dc.pc, 0);
     }
     gen_icount_end(tb, insn_count);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
-    if (!search_pc) {
+    if (search_pc) {
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+        memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
+                (j - lj) * sizeof(tcg_ctx.gen_opc_instr_start[0]));
+    } else {
         tb->size = dc.pc - pc_start;
         tb->icount = insn_count;
     }
@@ -2682,8 +3033,8 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "PC=%08x\n\n", env->pc);
 
     for (i = j = 0; i < 256; ++i) {
-        if (sregnames[i]) {
-            cpu_fprintf(f, "%s=%08x%c", sregnames[i], env->sregs[i],
+        if (xtensa_option_bits_enabled(env->config, sregnames[i].opt_bits)) {
+            cpu_fprintf(f, "%12s=%08x%c", sregnames[i].name, env->sregs[i],
                     (j++ % 4) == 3 ? '\n' : ' ');
         }
     }
@@ -2691,8 +3042,8 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, (j % 4) == 0 ? "\n" : "\n\n");
 
     for (i = j = 0; i < 256; ++i) {
-        if (uregnames[i]) {
-            cpu_fprintf(f, "%s=%08x%c", uregnames[i], env->uregs[i],
+        if (xtensa_option_bits_enabled(env->config, uregnames[i].opt_bits)) {
+            cpu_fprintf(f, "%s=%08x%c", uregnames[i].name, env->uregs[i],
                     (j++ % 4) == 3 ? '\n' : ' ');
         }
     }
@@ -2700,7 +3051,7 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, (j % 4) == 0 ? "\n" : "\n\n");
 
     for (i = 0; i < 16; ++i) {
-        cpu_fprintf(f, "A%02d=%08x%c", i, env->regs[i],
+        cpu_fprintf(f, " A%02d=%08x%c", i, env->regs[i],
                 (i % 4) == 3 ? '\n' : ' ');
     }
 
@@ -2710,9 +3061,19 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
         cpu_fprintf(f, "AR%02d=%08x%c", i, env->phys_regs[i],
                 (i % 4) == 3 ? '\n' : ' ');
     }
+
+    if (xtensa_option_enabled(env->config, XTENSA_OPTION_FP_COPROCESSOR)) {
+        cpu_fprintf(f, "\n");
+
+        for (i = 0; i < 16; ++i) {
+            cpu_fprintf(f, "F%02d=%08x (%+10.8e)%c", i,
+                    float32_val(env->fregs[i]),
+                    *(float *)&env->fregs[i], (i % 2) == 1 ? '\n' : ' ');
+        }
+    }
 }
 
 void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-xtensa/xtensa-semi.c b/target-xtensa/xtensa-semi.c
index e745bef0eea..5fe0361c02e 100644
--- a/target-xtensa/xtensa-semi.c
+++ b/target-xtensa/xtensa-semi.c
@@ -31,7 +31,7 @@
 #include <stddef.h>
 #include "cpu.h"
 #include "helper.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 enum {
     TARGET_SYS_exit = 1,
@@ -110,7 +110,9 @@ static uint32_t errno_h2g(int host_errno)
         [ENOMEM]        = TARGET_ENOMEM,
         [EACCES]        = TARGET_EACCES,
         [EFAULT]        = TARGET_EFAULT,
+#ifdef ENOTBLK
         [ENOTBLK]       = TARGET_ENOTBLK,
+#endif
         [EBUSY]         = TARGET_EBUSY,
         [EEXIST]        = TARGET_EEXIST,
         [EXDEV]         = TARGET_EXDEV,
@@ -121,7 +123,9 @@ static uint32_t errno_h2g(int host_errno)
         [ENFILE]        = TARGET_ENFILE,
         [EMFILE]        = TARGET_EMFILE,
         [ENOTTY]        = TARGET_ENOTTY,
+#ifdef ETXTBSY
         [ETXTBSY]       = TARGET_ETXTBSY,
+#endif
         [EFBIG]         = TARGET_EFBIG,
         [ENOSPC]        = TARGET_ENOSPC,
         [ESPIPE]        = TARGET_ESPIPE,
@@ -131,7 +135,9 @@ static uint32_t errno_h2g(int host_errno)
         [EDOM]          = TARGET_EDOM,
         [ERANGE]        = TARGET_ERANGE,
         [ENOSYS]        = TARGET_ENOSYS,
+#ifdef ELOOP
         [ELOOP]         = TARGET_ELOOP,
+#endif
     };
 
     if (host_errno == 0) {
@@ -163,12 +169,12 @@ void HELPER(simcall)(CPUXtensaState *env)
             uint32_t len = regs[5];
 
             while (len > 0) {
-                target_phys_addr_t paddr =
+                hwaddr paddr =
                     cpu_get_phys_page_debug(env, vaddr);
                 uint32_t page_left =
                     TARGET_PAGE_SIZE - (vaddr & (TARGET_PAGE_SIZE - 1));
                 uint32_t io_sz = page_left < len ? page_left : len;
-                target_phys_addr_t sz = io_sz;
+                hwaddr sz = io_sz;
                 void *buf = cpu_physical_memory_map(paddr, &sz, is_write);
 
                 if (buf) {
@@ -289,7 +295,7 @@ void HELPER(simcall)(CPUXtensaState *env)
             uint32_t sz = regs[5];
 
             while (sz) {
-                target_phys_addr_t len = sz;
+                hwaddr len = sz;
                 void *buf = cpu_physical_memory_map(base, &len, 1);
 
                 if (buf && len) {
diff --git a/targphys.h b/targphys.h
deleted file mode 100644
index bd4938fc02d..00000000000
--- a/targphys.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Define target_phys_addr_t if it exists.  */
-
-#ifndef TARGPHYS_H
-#define TARGPHYS_H
-
-#ifdef TARGET_PHYS_ADDR_BITS
-/* target_phys_addr_t is the type of a physical address (its size can
-   be different from 'target_ulong').  */
-
-#if TARGET_PHYS_ADDR_BITS == 32
-typedef uint32_t target_phys_addr_t;
-#define TARGET_PHYS_ADDR_MAX UINT32_MAX
-#define TARGET_FMT_plx "%08x"
-/* Format strings for printing target_phys_addr_t types.
- * These are recommended over the less flexible TARGET_FMT_plx,
- * which is retained for the benefit of existing code.
- */
-#define TARGET_PRIdPHYS PRId32
-#define TARGET_PRIiPHYS PRIi32
-#define TARGET_PRIoPHYS PRIo32
-#define TARGET_PRIuPHYS PRIu32
-#define TARGET_PRIxPHYS PRIx32
-#define TARGET_PRIXPHYS PRIX32
-#elif TARGET_PHYS_ADDR_BITS == 64
-typedef uint64_t target_phys_addr_t;
-#define TARGET_PHYS_ADDR_MAX UINT64_MAX
-#define TARGET_FMT_plx "%016" PRIx64
-#define TARGET_PRIdPHYS PRId64
-#define TARGET_PRIiPHYS PRIi64
-#define TARGET_PRIoPHYS PRIo64
-#define TARGET_PRIuPHYS PRIu64
-#define TARGET_PRIxPHYS PRIx64
-#define TARGET_PRIXPHYS PRIX64
-#endif
-#endif
-
-#endif
diff --git a/tcg/README b/tcg/README
index cfdfd96d095..ec1ac79375a 100644
--- a/tcg/README
+++ b/tcg/README
@@ -77,19 +77,27 @@ destroyed, but local temporaries and globals are preserved.
 Using the tcg_gen_helper_x_y it is possible to call any function
 taking i32, i64 or pointer types. By default, before calling a helper,
 all globals are stored at their canonical location and it is assumed
-that the function can modify them. This can be overridden by the
-TCG_CALL_CONST function modifier. By default, the helper is allowed to
-modify the CPU state or raise an exception. This can be overridden by
-the TCG_CALL_PURE function modifier, in which case the call to the
-function is removed if the return value is not used.
+that the function can modify them. By default, the helper is allowed to
+modify the CPU state or raise an exception.
+
+This can be overridden using the following function modifiers:
+- TCG_CALL_NO_READ_GLOBALS means that the helper does not read globals,
+  either directly or via an exception. They will not be saved to their
+  canonical locations before calling the helper.
+- TCG_CALL_NO_WRITE_GLOBALS means that the helper does not modify any globals.
+  They will only be saved to their canonical location before calling helpers,
+  but they won't be reloaded afterwise.
+- TCG_CALL_NO_SIDE_EFFECTS means that the call to the function is removed if
+  the return value is not used.
+
+Note that TCG_CALL_NO_READ_GLOBALS implies TCG_CALL_NO_WRITE_GLOBALS.
 
 On some TCG targets (e.g. x86), several calling conventions are
 supported.
 
 * Branches:
 
-Use the instruction 'br' to jump to a label. Use 'jmp' to jump to an
-explicit address. Conditional branches can only jump to labels.
+Use the instruction 'br' to jump to a label.
 
 3.3) Code Optimizations
 
@@ -129,10 +137,6 @@ call function 'ptr' (pointer type)
 
 ********* Jumps/Labels
 
-* jmp t0
-
-Absolute jump to address t0 (pointer type).
-
 * set_label $label
 
 Define label 'label' at the current program point.
@@ -141,7 +145,7 @@ Define label 'label' at the current program point.
 
 Jump to label.
 
-* brcond_i32/i64 cond, t0, t1, label
+* brcond_i32/i64 t0, t1, cond, label
 
 Conditional jump if t0 cond t1 is true. cond can be:
     TCG_COND_EQ
@@ -301,12 +305,18 @@ This operation would be equivalent to
 
 ********* Conditional moves
 
-* setcond_i32/i64 cond, dest, t1, t2
+* setcond_i32/i64 dest, t1, t2, cond
 
 dest = (t1 cond t2)
 
 Set DEST to 1 if (T1 cond T2) is true, otherwise set to 0.
 
+* movcond_i32/i64 dest, c1, c2, v1, v2, cond
+
+dest = (c1 cond c2 ? v1 : v2)
+
+Set DEST to V1 if (C1 cond C2) is true, otherwise set to V2.
+
 ********* Type conversions
 
 * ext_i32_i64 t0, t1
@@ -348,13 +358,16 @@ st32_i64 t0, t1, offset
 write(t0, t1 + offset)
 Write 8, 16, 32 or 64 bits to host memory.
 
+All this opcodes assume that the pointed host memory doesn't correspond
+to a global. In the latter case the behaviour is unpredictable.
+
 ********* 64-bit target on 32-bit host support
 
 The following opcodes are internal to TCG.  Thus they are to be implemented by
 32-bit host code generators, but are not to be emitted by guest translators.
 They are emitted as needed by inline functions within "tcg-op.h".
 
-* brcond2_i32 cond, t0_low, t0_high, t1_low, t1_high, label
+* brcond2_i32 t0_low, t0_high, t1_low, t1_high, cond, label
 
 Similar to brcond, except that the 64-bit values T0 and T1
 are formed from two 32-bit arguments.
@@ -371,7 +384,7 @@ is returned in two 32-bit outputs.
 Similar to mul, except two 32-bit (unsigned) inputs T1 and T2 yielding
 the full 64-bit product T0.  The later is returned in two 32-bit outputs.
 
-* setcond2_i32 cond, dest, t1_low, t1_high, t2_low, t2_high
+* setcond2_i32 dest, t1_low, t1_high, t2_low, t2_high, cond
 
 Similar to setcond, except that the 64-bit values T1 and T2 are
 formed from two 32-bit arguments.  The result is a 32-bit value.
@@ -386,7 +399,8 @@ Exit the current TB and return the value t0 (word type).
 
 Exit the current TB and jump to the TB index 'index' (constant) if the
 current TB was linked to this TB. Otherwise execute the next
-instructions.
+instructions. Only indices 0 and 1 are valid and tcg_gen_goto_tb may be issued
+at most once with each slot index per TB.
 
 * qemu_ld8u t0, t1, flags
 qemu_ld8s t0, t1, flags
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index cf0ca3d0b3a..d9c33d850f5 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -145,12 +145,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -176,7 +170,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
            so don't use these. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-#if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if TARGET_LONG_BITS == 64
         /* If we're passing env to the helper as r0 and need a regpair
          * for the address then r2 will be overwritten as we're setting
          * up the args to the helper.
@@ -204,8 +198,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
            use these. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-#if defined(CONFIG_SOFTMMU) && \
-    defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if defined(CONFIG_SOFTMMU) && (TARGET_LONG_BITS == 64)
         /* Avoid clashes with registers being used for helper args */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
@@ -223,7 +216,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 #ifdef CONFIG_SOFTMMU
         /* r2 is still needed to load data_reg, so don't use it. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
-#if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if TARGET_LONG_BITS == 64
         /* Avoid clashes with registers being used for helper args */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
 #endif
@@ -342,7 +335,7 @@ enum arm_cond_code_e {
     COND_AL = 0xe,
 };
 
-static const uint8_t tcg_cond_to_arm_cond[10] = {
+static const uint8_t tcg_cond_to_arm_cond[] = {
     [TCG_COND_EQ] = COND_EQ,
     [TCG_COND_NE] = COND_NE,
     [TCG_COND_LT] = COND_LT,
@@ -474,6 +467,21 @@ static inline void tcg_out_movi32(TCGContext *s,
     }
 }
 
+static inline void tcg_out_dat_rI(TCGContext *s, int cond, int opc, TCGArg dst,
+                                  TCGArg lhs, TCGArg rhs, int rhs_is_const)
+{
+    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
+     * rhs must satisfy the "rI" constraint.
+     */
+    if (rhs_is_const) {
+        int rot = encode_imm(rhs);
+        assert(rot >= 0);
+        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
+    } else {
+        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
+    }
+}
+
 static inline void tcg_out_mul32(TCGContext *s,
                 int cond, int rd, int rs, int rm)
 {
@@ -603,6 +611,22 @@ static inline void tcg_out_bswap16(TCGContext *s, int cond, int rd, int rn)
     }
 }
 
+/* swap the two low bytes assuming that the two high input bytes and the
+   two high output bit can hold any value. */
+static inline void tcg_out_bswap16st(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* rev16 */
+        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_R8, 0, rn, SHIFT_IMM_LSR(8));
+        tcg_out_dat_imm(s, cond, ARITH_AND, TCG_REG_R8, TCG_REG_R8, 0xff);
+        tcg_out_dat_reg(s, cond, ARITH_ORR,
+                        rd, TCG_REG_R8, rn, SHIFT_IMM_LSL(8));
+    }
+}
+
 static inline void tcg_out_bswap32(TCGContext *s, int cond, int rd, int rn)
 {
     if (use_armv6_instructions) {
@@ -631,6 +655,22 @@ static inline void tcg_out_ld32_12(TCGContext *s, int cond,
                         (rn << 16) | (rd << 12) | ((-im) & 0xfff));
 }
 
+/* Offset pre-increment with base writeback.  */
+static inline void tcg_out_ld32_12wb(TCGContext *s, int cond,
+                                     int rd, int rn, tcg_target_long im)
+{
+    /* ldr with writeback and both register equals is UNPREDICTABLE */
+    assert(rd != rn);
+
+    if (im >= 0) {
+        tcg_out32(s, (cond << 28) | 0x05b00000 |
+                        (rn << 16) | (rd << 12) | (im & 0xfff));
+    } else {
+        tcg_out32(s, (cond << 28) | 0x05300000 |
+                        (rn << 16) | (rd << 12) | ((-im) & 0xfff));
+    }
+}
+
 static inline void tcg_out_st32_12(TCGContext *s, int cond,
                 int rd, int rn, tcg_target_long im)
 {
@@ -952,9 +992,8 @@ static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
 
 #ifdef CONFIG_SOFTMMU
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -972,25 +1011,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 /* Helper routines for marshalling helper function arguments into
  * the correct registers and stack.
@@ -1083,7 +1103,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, data_reg, data_reg2, bswap;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
+    int mem_index, s_bits, tlb_offset;
     TCGReg argreg;
 # if TARGET_LONG_BITS == 64
     int addr_reg2;
@@ -1123,19 +1143,15 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                     TCG_REG_R0, TCG_REG_R8, CPU_TLB_SIZE - 1);
     tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_AREG0,
                     TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
-    /* In the
-     *  ldr r1 [r0, #(offsetof(CPUArchState, tlb_table[mem_index][0].addr_read))]
-     * below, the offset is likely to exceed 12 bits if mem_index != 0 and
-     * not exceed otherwise, so use an
-     *  add r0, r0, #(mem_index * sizeof *CPUArchState.tlb_table)
-     * before.
-     */
-    if (mem_index)
+    /* We assume that the offset is contained within 20 bits.  */
+    tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
+    assert((tlb_offset & ~0xfffff) == 0);
+    if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
-                        (mem_index << (TLB_SHIFT & 1)) |
-                        ((16 - (TLB_SHIFT >> 1)) << 8));
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_read));
+                        0xa00 | (tlb_offset >> 12));
+        tlb_offset &= 0xfff;
+    }
+    tcg_out_ld32_12wb(s, COND_AL, TCG_REG_R1, TCG_REG_R0, tlb_offset);
     tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R1,
                     TCG_REG_R8, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
     /* Check alignment.  */
@@ -1143,15 +1159,14 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_dat_imm(s, COND_EQ, ARITH_TST,
                         0, addr_reg, (1 << s_bits) - 1);
 #  if TARGET_LONG_BITS == 64
-    /* XXX: possibly we could use a block data load or writeback in
-     * the first access.  */
-    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_read) + 4);
+    /* XXX: possibly we could use a block data load in the first access.  */
+    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0, 4);
     tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
                     TCG_REG_R1, addr_reg2, SHIFT_IMM_LSL(0));
 #  endif
     tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addend));
+                    offsetof(CPUTLBEntry, addend)
+                    - offsetof(CPUTLBEntry, addr_read));
 
     switch (opc) {
     case 0:
@@ -1203,9 +1218,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
      * trash by moving the earlier arguments into them.
      */
     argreg = TCG_REG_R0;
-#ifdef CONFIG_TCG_PASS_AREG0
     argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
-#endif
 #if TARGET_LONG_BITS == 64
     argreg = tcg_out_arg_reg64(s, argreg, addr_reg, addr_reg2);
 #else
@@ -1226,20 +1239,11 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     case 1:
     case 2:
     default:
-        if (data_reg != TCG_REG_R0) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg, 0, TCG_REG_R0, SHIFT_IMM_LSL(0));
-        }
+        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
         break;
     case 3:
-        if (data_reg != TCG_REG_R0) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg, 0, TCG_REG_R0, SHIFT_IMM_LSL(0));
-        }
-        if (data_reg2 != TCG_REG_R1) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg2, 0, TCG_REG_R1, SHIFT_IMM_LSL(0));
-        }
+        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
+        tcg_out_mov_reg(s, COND_AL, data_reg2, TCG_REG_R1);
         break;
     }
 
@@ -1311,7 +1315,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, data_reg, data_reg2, bswap;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
+    int mem_index, s_bits, tlb_offset;
     TCGReg argreg;
 # if TARGET_LONG_BITS == 64
     int addr_reg2;
@@ -1348,19 +1352,15 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
                     TCG_REG_R0, TCG_REG_R8, CPU_TLB_SIZE - 1);
     tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R0,
                     TCG_AREG0, TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
-    /* In the
-     *  ldr r1 [r0, #(offsetof(CPUArchState, tlb_table[mem_index][0].addr_write))]
-     * below, the offset is likely to exceed 12 bits if mem_index != 0 and
-     * not exceed otherwise, so use an
-     *  add r0, r0, #(mem_index * sizeof *CPUArchState.tlb_table)
-     * before.
-     */
-    if (mem_index)
+    /* We assume that the offset is contained within 20 bits.  */
+    tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    assert((tlb_offset & ~0xfffff) == 0);
+    if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
-                        (mem_index << (TLB_SHIFT & 1)) |
-                        ((16 - (TLB_SHIFT >> 1)) << 8));
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_write));
+                        0xa00 | (tlb_offset >> 12));
+        tlb_offset &= 0xfff;
+    }
+    tcg_out_ld32_12wb(s, COND_AL, TCG_REG_R1, TCG_REG_R0, tlb_offset);
     tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R1,
                     TCG_REG_R8, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
     /* Check alignment.  */
@@ -1368,15 +1368,14 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_dat_imm(s, COND_EQ, ARITH_TST,
                         0, addr_reg, (1 << s_bits) - 1);
 #  if TARGET_LONG_BITS == 64
-    /* XXX: possibly we could use a block data load or writeback in
-     * the first access.  */
-    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_write) + 4);
+    /* XXX: possibly we could use a block data load in the first access.  */
+    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0, 4);
     tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
                     TCG_REG_R1, addr_reg2, SHIFT_IMM_LSL(0));
 #  endif
     tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addend));
+                    offsetof(CPUTLBEntry, addend)
+                    - offsetof(CPUTLBEntry, addr_write));
 
     switch (opc) {
     case 0:
@@ -1384,7 +1383,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         break;
     case 1:
         if (bswap) {
-            tcg_out_bswap16(s, COND_EQ, TCG_REG_R0, data_reg);
+            tcg_out_bswap16st(s, COND_EQ, TCG_REG_R0, data_reg);
             tcg_out_st16_r(s, COND_EQ, TCG_REG_R0, addr_reg, TCG_REG_R1);
         } else {
             tcg_out_st16_r(s, COND_EQ, data_reg, addr_reg, TCG_REG_R1);
@@ -1421,9 +1420,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
      * trash by moving the earlier arguments into them.
      */
     argreg = TCG_REG_R0;
-#ifdef CONFIG_TCG_PASS_AREG0
     argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
-#endif
 #if TARGET_LONG_BITS == 64
     argreg = tcg_out_arg_reg64(s, argreg, addr_reg, addr_reg2);
 #else
@@ -1472,7 +1469,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         break;
     case 1:
         if (bswap) {
-            tcg_out_bswap16(s, COND_AL, TCG_REG_R0, data_reg);
+            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, data_reg);
             tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addr_reg, 0);
         } else {
             tcg_out_st16_8(s, COND_AL, data_reg, addr_reg, 0);
@@ -1561,12 +1558,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         else
             tcg_out_callr(s, COND_AL, args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0])
-            tcg_out_goto(s, COND_AL, args[0]);
-        else
-            tcg_out_bx(s, COND_AL, args[0]);
-        break;
     case INDEX_op_br:
         tcg_out_goto_label(s, COND_AL, args[0]);
         break;
@@ -1603,6 +1594,15 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_movi_i32:
         tcg_out_movi32(s, COND_AL, args[0], args[1]);
         break;
+    case INDEX_op_movcond_i32:
+        /* Constraints mean that v2 is always in the same register as dest,
+         * so we only need to do "if condition passed, move v1 to dest".
+         */
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[1], args[2], const_args[2]);
+        tcg_out_dat_rI(s, tcg_cond_to_arm_cond[args[5]],
+                       ARITH_MOV, args[0], 0, args[3], const_args[3]);
+        break;
     case INDEX_op_add_i32:
         c = ARITH_ADD;
         goto gen_arith;
@@ -1622,14 +1622,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         c = ARITH_EOR;
         /* Fall through.  */
     gen_arith:
-        if (const_args[2]) {
-            int rot;
-            rot = encode_imm(args[2]);
-            tcg_out_dat_imm(s, COND_AL, c,
-                            args[0], args[1], rotl(args[2], rot) | (rot << 7));
-        } else
-            tcg_out_dat_reg(s, COND_AL, c,
-                            args[0], args[1], args[2], SHIFT_IMM_LSL(0));
+        tcg_out_dat_rI(s, COND_AL, c, args[0], args[1], args[2], const_args[2]);
         break;
     case INDEX_op_add2_i32:
         tcg_out_dat_reg2(s, COND_AL, ARITH_ADD, ARITH_ADC,
@@ -1689,15 +1682,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_brcond_i32:
-        if (const_args[1]) {
-            int rot;
-            rot = encode_imm(args[1]);
-            tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0,
-                            args[0], rotl(args[1], rot) | (rot << 7));
-        } else {
-            tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0,
-                            args[0], args[1], SHIFT_IMM_LSL(0));
-        }
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[0], args[1], const_args[1]);
         tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[2]], args[3]);
         break;
     case INDEX_op_brcond2_i32:
@@ -1716,15 +1702,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[4]], args[5]);
         break;
     case INDEX_op_setcond_i32:
-        if (const_args[2]) {
-            int rot;
-            rot = encode_imm(args[2]);
-            tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0,
-                            args[1], rotl(args[2], rot) | (rot << 7));
-        } else {
-            tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0,
-                            args[1], args[2], SHIFT_IMM_LSL(0));
-        }
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[1], args[2], const_args[2]);
         tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]],
                         ARITH_MOV, args[0], 0, 1);
         tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])],
@@ -1800,7 +1779,6 @@ static const TCGTargetOpDef arm_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1835,6 +1813,7 @@ static const TCGTargetOpDef arm_op_defs[] = {
 
     { INDEX_op_brcond_i32, { "r", "rI" } },
     { INDEX_op_setcond_i32, { "r", "r", "rI" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rI", "rI", "0" } },
 
     /* TODO: "r", "r", "r", "r", "ri", "ri" */
     { INDEX_op_add2_i32, { "r", "r", "r", "r", "r", "r" } },
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index f90b834795b..7083f3a700d 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_ARM 
 #define TCG_TARGET_ARM 1
 
 #undef TCG_TARGET_WORDS_BIGENDIAN
@@ -73,11 +74,9 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
-
-#define TCG_TARGET_HAS_GUEST_BASE
+#define TCG_TARGET_HAS_movcond_i32      1
 
 enum {
-    /* Note: must be synced with dyngen-exec.h */
     TCG_AREG0 = TCG_REG_R6,
 };
 
@@ -93,3 +92,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
+
+#endif
diff --git a/tcg/hppa/tcg-target.c b/tcg/hppa/tcg-target.c
index 2885212e3a1..656e73621a7 100644
--- a/tcg/hppa/tcg-target.c
+++ b/tcg/hppa/tcg-target.c
@@ -175,12 +175,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     *insn_ptr = insn;
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -738,7 +732,7 @@ static void tcg_out_branch(TCGContext *s, int label_index, int nul)
     }
 }
 
-static const uint8_t tcg_cond_to_cmp_cond[10] =
+static const uint8_t tcg_cond_to_cmp_cond[] =
 {
     [TCG_COND_EQ] = COND_EQ,
     [TCG_COND_NE] = COND_EQ | COND_FALSE,
@@ -826,13 +820,15 @@ static void tcg_out_brcond2(TCGContext *s, int cond, TCGArg al, TCGArg ah,
 {
     switch (cond) {
     case TCG_COND_EQ:
+        tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, al, bl, blconst);
+        tcg_out_brcond(s, TCG_COND_EQ, ah, bh, bhconst, label_index);
+        break;
     case TCG_COND_NE:
-        tcg_out_comclr(s, tcg_invert_cond(cond), TCG_REG_R0, al, bl, blconst);
-        tcg_out_brcond(s, cond, ah, bh, bhconst, label_index);
+        tcg_out_brcond(s, TCG_COND_NE, al, bl, blconst, label_index);
+        tcg_out_brcond(s, TCG_COND_NE, ah, bh, bhconst, label_index);
         break;
-
     default:
-        tcg_out_brcond(s, cond, ah, bh, bhconst, label_index);
+        tcg_out_brcond(s, tcg_high_cond(cond), ah, bh, bhconst, label_index);
         tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, ah, bh, bhconst);
         tcg_out_brcond(s, tcg_unsigned_cond(cond),
                        al, bl, blconst, label_index);
@@ -853,9 +849,8 @@ static void tcg_out_setcond2(TCGContext *s, int cond, TCGArg ret,
 {
     int scratch = TCG_REG_R20;
 
-    if (ret != al && ret != ah
-        && (blconst || ret != bl)
-        && (bhconst || ret != bh)) {
+    /* Note that the low parts are fully consumed before scratch is set.  */
+    if (ret != ah && (bhconst || ret != bh)) {
         scratch = ret;
     }
 
@@ -867,22 +862,52 @@ static void tcg_out_setcond2(TCGContext *s, int cond, TCGArg ret,
         tcg_out_movi(s, TCG_TYPE_I32, scratch, cond == TCG_COND_NE);
         break;
 
-    default:
+    case TCG_COND_GE:
+    case TCG_COND_GEU:
+    case TCG_COND_LT:
+    case TCG_COND_LTU:
+        /* Optimize compares with low part zero.  */
+        if (bl == 0) {
+            tcg_out_setcond(s, cond, ret, ah, bh, bhconst);
+            return;
+        }
+        /* FALLTHRU */
+
+    case TCG_COND_LE:
+    case TCG_COND_LEU:
+    case TCG_COND_GT:
+    case TCG_COND_GTU:
+        /* <= : ah < bh | (ah == bh && al <= bl) */
         tcg_out_setcond(s, tcg_unsigned_cond(cond), scratch, al, bl, blconst);
         tcg_out_comclr(s, TCG_COND_EQ, TCG_REG_R0, ah, bh, bhconst);
         tcg_out_movi(s, TCG_TYPE_I32, scratch, 0);
-        tcg_out_comclr(s, cond, TCG_REG_R0, ah, bh, bhconst);
+        tcg_out_comclr(s, tcg_invert_cond(tcg_high_cond(cond)),
+                       TCG_REG_R0, ah, bh, bhconst);
         tcg_out_movi(s, TCG_TYPE_I32, scratch, 1);
         break;
+
+    default:
+        tcg_abort();
     }
 
     tcg_out_mov(s, TCG_TYPE_I32, ret, scratch);
 }
 
+static void tcg_out_movcond(TCGContext *s, int cond, TCGArg ret,
+                            TCGArg c1, TCGArg c2, int c2const,
+                            TCGArg v1, int v1const)
+{
+    tcg_out_comclr(s, tcg_invert_cond(cond), TCG_REG_R0, c1, c2, c2const);
+    if (v1const) {
+        tcg_out_movi(s, TCG_TYPE_I32, ret, v1);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_I32, ret, v1);
+    }
+}
+
 #if defined(CONFIG_SOFTMMU)
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -900,25 +925,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 /* Load and compare a TLB entry, and branch if TLB miss.  OFFSET is set to
    the offset of the first ADDR_READ or ADDR_WRITE member of the appropriate
@@ -963,10 +969,11 @@ static int tcg_out_tlb_read(TCGContext *s, int r0, int r1, int addrlo,
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, r1, offset);
     }
 
-    /* Compute the value that ought to appear in the TLB for a hit, namely, the page
-       of the address.  We include the low N bits of the address to catch unaligned
-       accesses and force them onto the slow path.  Do this computation after having
-       issued the load from the TLB slot to give the load time to complete.  */
+    /* Compute the value that ought to appear in the TLB for a hit, namely,
+       the page of the address.  We include the low N bits of the address
+       to catch unaligned accesses and force them onto the slow path.  Do
+       this computation after having issued the load from the TLB slot to
+       give the load time to complete.  */
     tcg_out_andi(s, r0, addrlo, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
 
     /* If not equal, jump to lab_miss. */
@@ -979,6 +986,36 @@ static int tcg_out_tlb_read(TCGContext *s, int r0, int r1, int addrlo,
 
     return ret;
 }
+
+static int tcg_out_arg_reg32(TCGContext *s, int argno, TCGArg v, bool vconst)
+{
+    if (argno < 4) {
+        if (vconst) {
+            tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
+        }
+    } else {
+        if (vconst && v != 0) {
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R20, v);
+            v = TCG_REG_R20;
+        }
+        tcg_out_st(s, TCG_TYPE_I32, v, TCG_REG_CALL_STACK,
+                   TCG_TARGET_CALL_STACK_OFFSET - ((argno - 3) * 4));
+    }
+    return argno + 1;
+}
+
+static int tcg_out_arg_reg64(TCGContext *s, int argno, TCGArg vl, TCGArg vh)
+{
+    /* 64-bit arguments must go in even reg pairs and stack slots.  */
+    if (argno & 1) {
+        argno++;
+    }
+    argno = tcg_out_arg_reg32(s, argno, vl, false);
+    argno = tcg_out_arg_reg32(s, argno, vh, false);
+    return argno;
+}
 #endif
 
 static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo_reg, int datahi_reg,
@@ -1059,41 +1096,36 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     /* Note that addrhi_reg is only used for 64-bit guests.  */
     int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
     int mem_index = *args;
-    int lab1, lab2, argreg, offset;
+    int lab1, lab2, argno, offset;
 
     lab1 = gen_new_label();
     lab2 = gen_new_label();
 
     offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg, addrhi_reg,
-                              opc & 3, lab1, offset);
+    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
+                              addrhi_reg, opc & 3, lab1, offset);
 
     /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, (offset ? TCG_REG_R1 : TCG_REG_R25),
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
+               (offset ? TCG_REG_R1 : TCG_REG_R25),
                offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
-    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg, TCG_REG_R20, opc);
+    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg,
+                           TCG_REG_R20, opc);
     tcg_out_branch(s, lab2, 1);
 
     /* TLB Miss.  */
     /* label1: */
     tcg_out_label(s, lab1, s->code_ptr);
 
-    argreg = TCG_REG_R26;
-    tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrlo_reg);
+    argno = 0;
+    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrhi_reg);
+        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
+    } else {
+        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
     }
-    tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
-
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
+    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
+
     tcg_out_call(s, qemu_ld_helpers[opc & 3]);
 
     switch (opc) {
@@ -1129,8 +1161,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 #endif
 }
 
-static void tcg_out_qemu_st_direct(TCGContext *s, int datalo_reg, int datahi_reg,
-                                   int addr_reg, int opc)
+static void tcg_out_qemu_st_direct(TCGContext *s, int datalo_reg,
+                                   int datahi_reg, int addr_reg, int opc)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 0;
@@ -1183,17 +1215,18 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
     /* Note that addrhi_reg is only used for 64-bit guests.  */
     int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
     int mem_index = *args;
-    int lab1, lab2, argreg, offset;
+    int lab1, lab2, argno, next, offset;
 
     lab1 = gen_new_label();
     lab2 = gen_new_label();
 
     offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg, addrhi_reg,
-                              opc, lab1, offset);
+    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
+                              addrhi_reg, opc, lab1, offset);
 
     /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, (offset ? TCG_REG_R1 : TCG_REG_R25),
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
+               (offset ? TCG_REG_R1 : TCG_REG_R25),
                offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
 
     /* There are no indexed stores, so we must do this addition explitly.
@@ -1206,65 +1239,46 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
     /* label1: */
     tcg_out_label(s, lab1, s->code_ptr);
 
-    argreg = TCG_REG_R26;
-    tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrlo_reg);
+    argno = 0;
+    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrhi_reg);
+        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
+    } else {
+        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
     }
 
+    next = (argno < 4 ? tcg_target_call_iarg_regs[argno] : TCG_REG_R20);
     switch(opc) {
     case 0:
-        tcg_out_andi(s, argreg--, datalo_reg, 0xff);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        tcg_out_andi(s, next, datalo_reg, 0xff);
+        argno = tcg_out_arg_reg32(s, argno, next, false);
         break;
     case 1:
-        tcg_out_andi(s, argreg--, datalo_reg, 0xffff);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        tcg_out_andi(s, next, datalo_reg, 0xffff);
+        argno = tcg_out_arg_reg32(s, argno, next, false);
         break;
     case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, datalo_reg);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        argno = tcg_out_arg_reg32(s, argno, datalo_reg, false);
         break;
     case 3:
-        /* Because of the alignment required by the 64-bit data argument,
-           we will always use R23/R24.  Also, we will always run out of
-           argument registers for storing mem_index, so that will have
-           to go on the stack.  */
-        if (mem_index == 0) {
-            argreg = TCG_REG_R0;
-        } else {
-            argreg = TCG_REG_R20;
-            tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
-        }
-        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R23, datahi_reg);
-        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R24, datalo_reg);
-        tcg_out_st(s, TCG_TYPE_I32, argreg, TCG_REG_CALL_STACK,
-                   TCG_TARGET_CALL_STACK_OFFSET - 4);
+        argno = tcg_out_arg_reg64(s, argno, datalo_reg, datahi_reg);
         break;
     default:
         tcg_abort();
     }
+    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
 
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
     tcg_out_call(s, qemu_st_helpers[opc]);
 
     /* label2: */
     tcg_out_label(s, lab2, s->code_ptr);
 #else
-    /* There are no indexed stores, so if GUEST_BASE is set we must do the add
-       explicitly.  Careful to avoid R20, which is used for the bswaps to follow.  */
+    /* There are no indexed stores, so if GUEST_BASE is set we must do
+       the add explicitly.  Careful to avoid R20, which is used for the
+       bswaps to follow.  */
     if (GUEST_BASE != 0) {
-        tcg_out_arith(s, TCG_REG_R31, addrlo_reg, TCG_GUEST_BASE_REG, INSN_ADDL);
+        tcg_out_arith(s, TCG_REG_R31, addrlo_reg,
+                      TCG_GUEST_BASE_REG, INSN_ADDL);
         addrlo_reg = TCG_REG_R31;
     }
     tcg_out_qemu_st_direct(s, datalo_reg, datahi_reg, addrlo_reg, opc);
@@ -1326,11 +1340,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         }
         break;
 
-    case INDEX_op_jmp:
-        fprintf(stderr, "unimplemented jmp\n");
-        tcg_abort();
-        break;
-
     case INDEX_op_br:
         tcg_out_branch(s, args[0], 1);
         break;
@@ -1499,6 +1508,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
                          args[3], const_args[3], args[4], const_args[4]);
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2], const_args[2],
+                        args[3], const_args[3]);
+        break;
+
     case INDEX_op_add2_i32:
         tcg_out_add2(s, args[0], args[1], args[2], args[3],
                      args[4], args[5], const_args[4]);
@@ -1560,7 +1574,6 @@ static const TCGTargetOpDef hppa_op_defs[] = {
     { INDEX_op_goto_tb, { } },
 
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "r" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1607,6 +1620,10 @@ static const TCGTargetOpDef hppa_op_defs[] = {
     { INDEX_op_setcond_i32, { "r", "rZ", "rI" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rI", "rI" } },
 
+    /* ??? We can actually support a signed 14-bit arg3, but we
+       only have existing constraints for a signed 11-bit.  */
+    { INDEX_op_movcond_i32, { "r", "rZ", "rI", "rI", "0" } },
+
     { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rI", "rZ" } },
     { INDEX_op_sub2_i32, { "r", "r", "rI", "rZ", "rK", "rZ" } },
 
diff --git a/tcg/hppa/tcg-target.h b/tcg/hppa/tcg-target.h
index d4bf6fe756c..e2754fe970b 100644
--- a/tcg/hppa/tcg-target.h
+++ b/tcg/hppa/tcg-target.h
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#ifndef TCG_TARGET_HPPA
 #define TCG_TARGET_HPPA 1
 
 #if TCG_TARGET_REG_BITS != 32
@@ -96,15 +97,13 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_movcond_i32      1
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub rd, 0, rs */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* and rd, rs, 0xff */
 #define TCG_TARGET_HAS_ext16u_i32       0 /* and rd, rs, 0xffff */
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_R17
 
 
@@ -121,3 +120,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
         start += 32;
     }
 }
+
+#endif
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index da17bba39df..7aec3043e35 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -75,9 +75,7 @@ static const int tcg_target_call_iarg_regs[] = {
     TCG_REG_R8,
     TCG_REG_R9,
 #else
-    TCG_REG_EAX,
-    TCG_REG_EDX,
-    TCG_REG_ECX
+    /* 32 bit mode uses stack based calling convention (GCC default). */
 #endif
 };
 
@@ -88,6 +86,29 @@ static const int tcg_target_call_oarg_regs[] = {
 #endif
 };
 
+/* Registers used with L constraint, which are the first argument 
+   registers on x86_64, and two random call clobbered registers on
+   i386. */
+#if TCG_TARGET_REG_BITS == 64
+# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
+# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
+#else
+# define TCG_REG_L0 TCG_REG_EAX
+# define TCG_REG_L1 TCG_REG_EDX
+#endif
+
+/* For 32-bit, we are going to attempt to determine at runtime whether cmov
+   is available.  However, the host compiler must supply <cpuid.h>, as we're
+   not going to go so far as our own inline assembly.  */
+#if TCG_TARGET_REG_BITS == 64
+# define have_cmov 1
+#elif defined(CONFIG_CPUID_H)
+#include <cpuid.h>
+static bool have_cmov;
+#else
+# define have_cmov 0
+#endif
+
 static uint8_t *tb_ret_addr;
 
 static void patch_reloc(uint8_t *code_ptr, int type,
@@ -114,16 +135,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    if (TCG_TARGET_REG_BITS == 64) {
-        return 6;
-    }
-
-    return 0;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -179,18 +190,13 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         /* qemu_ld/st address constraint */
     case 'L':
         ct->ct |= TCG_CT_REG;
-        if (TCG_TARGET_REG_BITS == 64) {
+#if TCG_TARGET_REG_BITS == 64
             tcg_regset_set32(ct->u.regs, 0, 0xffff);
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[0]);
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[1]);
-#ifdef CONFIG_TCG_PASS_AREG0
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[2]);
-#endif
-        } else {
+#else
             tcg_regset_set32(ct->u.regs, 0, 0xff);
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EAX);
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EDX);
-        }
+#endif
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L1);
         break;
 
     case 'e':
@@ -238,11 +244,13 @@ static inline int tcg_target_const_match(tcg_target_long val,
 # define P_REXW		0x800		/* Set REX.W = 1 */
 # define P_REXB_R	0x1000		/* REG field as byte register */
 # define P_REXB_RM	0x2000		/* R/M field as byte register */
+# define P_GS           0x4000          /* gs segment override */
 #else
 # define P_ADDR32	0
 # define P_REXW		0
 # define P_REXB_R	0
 # define P_REXB_RM	0
+# define P_GS           0
 #endif
 
 #define OPC_ARITH_EvIz	(0x81)
@@ -251,6 +259,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define OPC_ADD_GvEv	(OPC_ARITH_GvEv | (ARITH_ADD << 3))
 #define OPC_BSWAP	(0xc8 | P_EXT)
 #define OPC_CALL_Jz	(0xe8)
+#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */
 #define OPC_CMP_GvEv	(OPC_ARITH_GvEv | (ARITH_CMP << 3))
 #define OPC_DEC_r32	(0x48)
 #define OPC_IMUL_GvEv	(0xaf | P_EXT)
@@ -265,6 +274,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define OPC_MOVB_EvGv	(0x88)		/* stores, more or less */
 #define OPC_MOVL_EvGv	(0x89)		/* stores, more or less */
 #define OPC_MOVL_GvEv	(0x8b)		/* loads, more or less */
+#define OPC_MOVB_EvIz   (0xc6)
 #define OPC_MOVL_EvIz	(0xc7)
 #define OPC_MOVL_Iv     (0xb8)
 #define OPC_MOVSBL	(0xbe | P_EXT)
@@ -338,7 +348,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define JCC_JLE 0xe
 #define JCC_JG  0xf
 
-static const uint8_t tcg_cond_to_jcc[10] = {
+static const uint8_t tcg_cond_to_jcc[] = {
     [TCG_COND_EQ] = JCC_JE,
     [TCG_COND_NE] = JCC_JNE,
     [TCG_COND_LT] = JCC_JL,
@@ -356,6 +366,9 @@ static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
 {
     int rex;
 
+    if (opc & P_GS) {
+        tcg_out8(s, 0x65);
+    }
     if (opc & P_DATA16) {
         /* We should never be asking for both 16 and 64-bit operation.  */
         assert((opc & P_REXW) == 0);
@@ -937,6 +950,31 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
 }
 #endif
 
+static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, 0);
+    if (have_cmov) {
+        tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
+    } else {
+        int over = gen_new_label();
+        tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
+        tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
+        tcg_out_label(s, over, s->code_ptr);
+    }
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
+    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | P_REXW, dest, v1);
+}
+#endif
+
 static void tcg_out_branch(TCGContext *s, int call, tcg_target_long dest)
 {
     tcg_target_long disp = dest - (tcg_target_long)s->code_ptr - 5;
@@ -963,9 +1001,8 @@ static void tcg_out_jmp(TCGContext *s, tcg_target_long dest)
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void *qemu_ld_helpers[4] = {
@@ -983,25 +1020,17 @@ static const void *qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
+static void add_qemu_ldst_label(TCGContext *s,
+                                int is_ld,
+                                int opc,
+                                int data_reg,
+                                int data_reg2,
+                                int addrlo_reg,
+                                int addrhi_reg,
+                                int mem_index,
+                                uint8_t *raddr,
+                                uint8_t **label_ptr);
 
 /* Perform the TLB load and compare.
 
@@ -1018,12 +1047,12 @@ static void *qemu_st_helpers[4] = {
    LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
    positions of the displacements of forward jumps to the TLB miss case.
 
-   First argument register is loaded with the low part of the address.
+   Second argument register is loaded with the low part of the address.
    In the TLB hit case, it has been adjusted as indicated by the TLB
    and so is a host address.  In the TLB miss case, it continues to
    hold a guest address.
 
-   Second argument register is clobbered.  */
+   First argument register is clobbered.  */
 
 static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
                                     int mem_index, int s_bits,
@@ -1031,8 +1060,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
                                     uint8_t **label_ptr, int which)
 {
     const int addrlo = args[addrlo_idx];
-    const int r0 = tcg_target_call_iarg_regs[0];
-    const int r1 = tcg_target_call_iarg_regs[1];
+    const int r0 = TCG_REG_L0;
+    const int r1 = TCG_REG_L1;
     TCGType type = TCG_TYPE_I32;
     int rexw = 0;
 
@@ -1041,51 +1070,68 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
         rexw = P_REXW;
     }
 
-    tcg_out_mov(s, type, r1, addrlo);
     tcg_out_mov(s, type, r0, addrlo);
+    tcg_out_mov(s, type, r1, addrlo);
 
-    tcg_out_shifti(s, SHIFT_SHR + rexw, r1,
+    tcg_out_shifti(s, SHIFT_SHR + rexw, r0,
                    TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
 
-    tgen_arithi(s, ARITH_AND + rexw, r0,
-                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
     tgen_arithi(s, ARITH_AND + rexw, r1,
+                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
+    tgen_arithi(s, ARITH_AND + rexw, r0,
                 (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
 
-    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r1, TCG_AREG0, r1, 0,
+    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r0, TCG_AREG0, r0, 0,
                              offsetof(CPUArchState, tlb_table[mem_index][0])
                              + which);
 
-    /* cmp 0(r1), r0 */
-    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r0, r1, 0);
+    /* cmp 0(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r1, r0, 0);
 
-    tcg_out_mov(s, type, r0, addrlo);
+    tcg_out_mov(s, type, r1, addrlo);
 
-    /* jne label1 */
-    tcg_out8(s, OPC_JCC_short + JCC_JNE);
+    /* jne slow_path */
+    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
     label_ptr[0] = s->code_ptr;
-    s->code_ptr++;
+    s->code_ptr += 4;
 
     if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        /* cmp 4(r1), addrhi */
-        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r1, 4);
+        /* cmp 4(r0), addrhi */
+        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r0, 4);
 
-        /* jne label1 */
-        tcg_out8(s, OPC_JCC_short + JCC_JNE);
+        /* jne slow_path */
+        tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
         label_ptr[1] = s->code_ptr;
-        s->code_ptr++;
+        s->code_ptr += 4;
     }
 
     /* TLB Hit.  */
 
-    /* add addend(r1), r0 */
-    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r0, r1,
+    /* add addend(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r1, r0,
                          offsetof(CPUTLBEntry, addend) - which);
 }
-#endif
+#elif defined(__x86_64__) && defined(__linux__)
+# include <asm/prctl.h>
+# include <sys/prctl.h>
+
+int arch_prctl(int code, unsigned long addr);
+
+static int guest_base_flags;
+static inline void setup_guest_base_seg(void)
+{
+    if (arch_prctl(ARCH_SET_GS, GUEST_BASE) == 0) {
+        guest_base_flags = P_GS;
+    }
+}
+#else
+# define guest_base_flags 0
+static inline void setup_guest_base_seg(void) { }
+#endif /* SOFTMMU */
 
 static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int sizeop)
+                                   int base, tcg_target_long ofs, int seg,
+                                   int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 1;
@@ -1094,28 +1140,29 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
 #endif
     switch (sizeop) {
     case 0:
-        tcg_out_modrm_offset(s, OPC_MOVZBL, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVZBL + seg, datalo, base, ofs);
         break;
     case 0 | 4:
-        tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW + seg, datalo, base, ofs);
         break;
     case 1:
-        tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_rolw_8(s, datalo);
         }
         break;
     case 1 | 4:
         if (bswap) {
-            tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
             tcg_out_rolw_8(s, datalo);
             tcg_out_modrm(s, OPC_MOVSWL + P_REXW, datalo, datalo);
         } else {
-            tcg_out_modrm_offset(s, OPC_MOVSWL + P_REXW, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVSWL + P_REXW + seg,
+                                 datalo, base, ofs);
         }
         break;
     case 2:
-        tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_bswap32(s, datalo);
         }
@@ -1123,17 +1170,18 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
 #if TCG_TARGET_REG_BITS == 64
     case 2 | 4:
         if (bswap) {
-            tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
             tcg_out_bswap32(s, datalo);
             tcg_out_ext32s(s, datalo, datalo);
         } else {
-            tcg_out_modrm_offset(s, OPC_MOVSLQ, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVSLQ + seg, datalo, base, ofs);
         }
         break;
 #endif
     case 3:
         if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_ld(s, TCG_TYPE_I64, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_GvEv + P_REXW + seg,
+                                 datalo, base, ofs);
             if (bswap) {
                 tcg_out_bswap64(s, datalo);
             }
@@ -1144,11 +1192,15 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
                 datahi = t;
             }
             if (base != datalo) {
-                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
-                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datalo, base, ofs);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datahi, base, ofs + 4);
             } else {
-                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
-                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datahi, base, ofs + 4);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datalo, base, ofs);
             }
             if (bswap) {
                 tcg_out_bswap32(s, datalo);
@@ -1171,12 +1223,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
     int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
     int mem_index, s_bits;
-#if TCG_TARGET_REG_BITS == 64
-    int arg_idx;
-#else
-    int stack_adjust;
-#endif
-    uint8_t *label_ptr[3];
+    uint8_t *label_ptr[2];
 #endif
 
     data_reg = args[0];
@@ -1194,133 +1241,48 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
                      label_ptr, offsetof(CPUTLBEntry, addr_read));
 
     /* TLB Hit.  */
-    tcg_out_qemu_ld_direct(s, data_reg, data_reg2,
-                           tcg_target_call_iarg_regs[0], 0, opc);
-
-    /* jmp label2 */
-    tcg_out8(s, OPC_JMP_short);
-    label_ptr[2] = s->code_ptr;
-    s->code_ptr++;
-
-    /* TLB Miss.  */
-
-    /* label1: */
-    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
-    }
-
-    /* XXX: move that code at the end of the TB */
-#if TCG_TARGET_REG_BITS == 32
-    tcg_out_pushi(s, mem_index);
-    stack_adjust = 4;
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_push(s, args[addrlo_idx + 1]);
-        stack_adjust += 4;
-    }
-    tcg_out_push(s, args[addrlo_idx]);
-    stack_adjust += 4;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_push(s, TCG_AREG0);
-    stack_adjust += 4;
-#endif
-#else
-    /* The first argument is already loaded with addrlo.  */
-    arg_idx = 1;
-    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[arg_idx],
-                 mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
-#endif
-
-    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
-
-#if TCG_TARGET_REG_BITS == 32
-    if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
-        /* Pop and discard.  This is 2 bytes smaller than the add.  */
-        tcg_out_pop(s, TCG_REG_ECX);
-    } else if (stack_adjust != 0) {
-        tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
-    }
-#endif
-
-    switch(opc) {
-    case 0 | 4:
-        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case 1 | 4:
-        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case 0:
-        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 1:
-        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case 2 | 4:
-        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
-        break;
-#endif
-    case 3:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
-        } else if (data_reg == TCG_REG_EDX) {
-            /* xchg %edx, %eax */
-            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EAX);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EDX);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-
-    /* label2: */
-    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
+    tcg_out_qemu_ld_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+
+    /* Record the current context of a load into ldst label */
+    add_qemu_ldst_label(s,
+                        1,
+                        opc,
+                        data_reg,
+                        data_reg2,
+                        args[addrlo_idx],
+                        args[addrlo_idx + 1],
+                        mem_index,
+                        s->code_ptr,
+                        label_ptr);
 #else
     {
         int32_t offset = GUEST_BASE;
         int base = args[addrlo_idx];
-
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* ??? We assume all operations have left us with register
-               contents that are zero extended.  So far this appears to
-               be true.  If we want to enforce this, we can either do
-               an explicit zero-extension here, or (if GUEST_BASE == 0)
-               use the ADDR32 prefix.  For now, do nothing.  */
-
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64,
-                             tcg_target_call_iarg_regs[0], GUEST_BASE);
-                tgen_arithr(s, ARITH_ADD + P_REXW,
-                            tcg_target_call_iarg_regs[0], base);
-                base = tcg_target_call_iarg_regs[0];
-                offset = 0;
-            }
+        int seg = 0;
+
+        /* ??? We assume all operations have left us with register contents
+           that are zero extended.  So far this appears to be true.  If we
+           want to enforce this, we can either do an explicit zero-extension
+           here, or (if GUEST_BASE == 0, or a segment register is in use)
+           use the ADDR32 prefix.  For now, do nothing.  */
+        if (GUEST_BASE && guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+        } else if (TCG_TARGET_REG_BITS == 64 && offset != GUEST_BASE) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
+            base = TCG_REG_L1;
+            offset = 0;
         }
 
-        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, opc);
+        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, seg, opc);
     }
 #endif
 }
 
 static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int sizeop)
+                                   int base, tcg_target_long ofs, int seg,
+                                   int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 1;
@@ -1330,12 +1292,13 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
     /* ??? Ideally we wouldn't need a scratch register.  For user-only,
        we could perform the bswap twice to restore the original value
        instead of moving to the scratch.  But as it is, the L constraint
-       means that the second argument reg is definitely free here.  */
-    int scratch = tcg_target_call_iarg_regs[1];
+       means that TCG_REG_L0 is definitely free here.  */
+    const int scratch = TCG_REG_L0;
 
     switch (sizeop) {
     case 0:
-        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
+                             datalo, base, ofs);
         break;
     case 1:
         if (bswap) {
@@ -1343,7 +1306,8 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
             tcg_out_rolw_8(s, scratch);
             datalo = scratch;
         }
-        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16 + seg,
+                             datalo, base, ofs);
         break;
     case 2:
         if (bswap) {
@@ -1351,7 +1315,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
             tcg_out_bswap32(s, scratch);
             datalo = scratch;
         }
-        tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datalo, base, ofs);
         break;
     case 3:
         if (TCG_TARGET_REG_BITS == 64) {
@@ -1360,17 +1324,18 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
                 tcg_out_bswap64(s, scratch);
                 datalo = scratch;
             }
-            tcg_out_st(s, TCG_TYPE_I64, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_REXW + seg,
+                                 datalo, base, ofs);
         } else if (bswap) {
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
             tcg_out_bswap32(s, scratch);
-            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs);
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
             tcg_out_bswap32(s, scratch);
-            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs + 4);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs+4);
         } else {
-            tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
-            tcg_out_st(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datahi, base, ofs+4);
         }
         break;
     default:
@@ -1385,8 +1350,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
     int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
     int mem_index, s_bits;
-    int stack_adjust;
-    uint8_t *label_ptr[3];
+    uint8_t *label_ptr[2];
 #endif
 
     data_reg = args[0];
@@ -1404,23 +1368,223 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
                      label_ptr, offsetof(CPUTLBEntry, addr_write));
 
     /* TLB Hit.  */
-    tcg_out_qemu_st_direct(s, data_reg, data_reg2,
-                           tcg_target_call_iarg_regs[0], 0, opc);
+    tcg_out_qemu_st_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+
+    /* Record the current context of a store into ldst label */
+    add_qemu_ldst_label(s,
+                        0,
+                        opc,
+                        data_reg,
+                        data_reg2,
+                        args[addrlo_idx],
+                        args[addrlo_idx + 1],
+                        mem_index,
+                        s->code_ptr,
+                        label_ptr);
+#else
+    {
+        int32_t offset = GUEST_BASE;
+        int base = args[addrlo_idx];
+        int seg = 0;
+
+        /* ??? We assume all operations have left us with register contents
+           that are zero extended.  So far this appears to be true.  If we
+           want to enforce this, we can either do an explicit zero-extension
+           here, or (if GUEST_BASE == 0, or a segment register is in use)
+           use the ADDR32 prefix.  For now, do nothing.  */
+        if (GUEST_BASE && guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+        } else if (TCG_TARGET_REG_BITS == 64 && offset != GUEST_BASE) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
+            base = TCG_REG_L1;
+            offset = 0;
+        }
+
+        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, seg, opc);
+    }
+#endif
+}
+
+#if defined(CONFIG_SOFTMMU)
+/*
+ * Record the context of a call to the out of line helper code for the slow path
+ * for a load or store, so that we can later generate the correct helper code
+ */
+static void add_qemu_ldst_label(TCGContext *s,
+                                int is_ld,
+                                int opc,
+                                int data_reg,
+                                int data_reg2,
+                                int addrlo_reg,
+                                int addrhi_reg,
+                                int mem_index,
+                                uint8_t *raddr,
+                                uint8_t **label_ptr)
+{
+    int idx;
+    TCGLabelQemuLdst *label;
+
+    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
+        tcg_abort();
+    }
 
-    /* jmp label2 */
+    idx = s->nb_qemu_ldst_labels++;
+    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = data_reg;
+    label->datahi_reg = data_reg2;
+    label->addrlo_reg = addrlo_reg;
+    label->addrhi_reg = addrhi_reg;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+/*
+ * Generate code for the slow path for a load at the end of block
+ */
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+#if TCG_TARGET_REG_BITS == 32
+    int stack_adjust;
+    int addrlo_reg = label->addrlo_reg;
+    int addrhi_reg = label->addrhi_reg;
+#endif
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
+    }
+
+#if TCG_TARGET_REG_BITS == 32
+    tcg_out_pushi(s, mem_index);
+    stack_adjust = 4;
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_push(s, addrhi_reg);
+        stack_adjust += 4;
+    }
+    tcg_out_push(s, addrlo_reg);
+    stack_adjust += 4;
+    tcg_out_push(s, TCG_AREG0);
+    stack_adjust += 4;
+#else
+    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0], TCG_AREG0);
+    /* The second argument is already loaded with addrlo.  */
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], mem_index);
+#endif
+
+    /* Code generation of qemu_ld/st's slow path calling MMU helper
+
+       PRE_PROC ...
+       call MMU helper
+       jmp POST_PROC (2b) : short forward jump <- GETRA()
+       jmp next_code (5b) : dummy long backward jump which is never executed
+       POST_PROC ... : do post-processing <- GETRA() + 7
+       jmp next_code : jump to the code corresponding to next IR of qemu_ld/st
+    */
+
+    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
+
+    /* Jump to post-processing code */
     tcg_out8(s, OPC_JMP_short);
-    label_ptr[2] = s->code_ptr;
-    s->code_ptr++;
+    tcg_out8(s, 5);
+    /* Dummy backward jump having information of fast path'pc for MMU helpers */
+    tcg_out8(s, OPC_JMP_long);
+    *(int32_t *)s->code_ptr = (int32_t)(raddr - s->code_ptr - 4);
+    s->code_ptr += 4;
 
-    /* TLB Miss.  */
+#if TCG_TARGET_REG_BITS == 32
+    if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
+        /* Pop and discard.  This is 2 bytes smaller than the add.  */
+        tcg_out_pop(s, TCG_REG_ECX);
+    } else if (stack_adjust != 0) {
+        tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
+    }
+#endif
 
-    /* label1: */
-    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
+    switch(opc) {
+    case 0 | 4:
+        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case 1 | 4:
+        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case 0:
+        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
+        break;
+    case 1:
+        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
+        break;
+    case 2:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case 2 | 4:
+        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
+        break;
+#endif
+    case 3:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
+        } else if (data_reg == TCG_REG_EDX) {
+            /* xchg %edx, %eax */
+            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EAX);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EDX);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, (tcg_target_long)raddr);
+}
+
+/*
+ * Generate code for the slow path for a store at the end of block
+ */
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int stack_adjust;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+#if TCG_TARGET_REG_BITS == 32
+    int data_reg2 = label->datahi_reg;
+    int addrlo_reg = label->addrlo_reg;
+    int addrhi_reg = label->addrhi_reg;
+#endif
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
     if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
     }
 
-    /* XXX: move that code at the end of the TB */
 #if TCG_TARGET_REG_BITS == 32
     tcg_out_pushi(s, mem_index);
     stack_adjust = 4;
@@ -1431,35 +1595,42 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
     tcg_out_push(s, data_reg);
     stack_adjust += 4;
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_push(s, args[addrlo_idx + 1]);
+        tcg_out_push(s, addrhi_reg);
         stack_adjust += 4;
     }
-    tcg_out_push(s, args[addrlo_idx]);
+    tcg_out_push(s, addrlo_reg);
     stack_adjust += 4;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_push(s, TCG_AREG0);
     stack_adjust += 4;
-#endif
 #else
+    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0], TCG_AREG0);
+    /* The second argument is already loaded with addrlo.  */
     tcg_out_mov(s, (opc == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32),
-                tcg_target_call_iarg_regs[1], data_reg);
-    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], mem_index);
+                tcg_target_call_iarg_regs[2], data_reg);
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], mem_index);
     stack_adjust = 0;
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
 #endif
 
+    /* Code generation of qemu_ld/st's slow path calling MMU helper
+
+       PRE_PROC ...
+       call MMU helper
+       jmp POST_PROC (2b) : short forward jump <- GETRA()
+       jmp next_code (5b) : dummy long backward jump which is never executed
+       POST_PROC ... : do post-processing <- GETRA() + 7
+       jmp next_code : jump to the code corresponding to next IR of qemu_ld/st
+    */
+
     tcg_out_calli(s, (tcg_target_long)qemu_st_helpers[s_bits]);
 
+    /* Jump to post-processing code */
+    tcg_out8(s, OPC_JMP_short);
+    tcg_out8(s, 5);
+    /* Dummy backward jump having information of fast path'pc for MMU helpers */
+    tcg_out8(s, OPC_JMP_long);
+    *(int32_t *)s->code_ptr = (int32_t)(raddr - s->code_ptr - 4);
+    s->code_ptr += 4;
+
     if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
         /* Pop and discard.  This is 2 bytes smaller than the add.  */
         tcg_out_pop(s, TCG_REG_ECX);
@@ -1467,34 +1638,29 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
         tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
     }
 
-    /* label2: */
-    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
-#else
-    {
-        int32_t offset = GUEST_BASE;
-        int base = args[addrlo_idx];
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, (tcg_target_long)raddr);
+}
 
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* ??? We assume all operations have left us with register
-               contents that are zero extended.  So far this appears to
-               be true.  If we want to enforce this, we can either do
-               an explicit zero-extension here, or (if GUEST_BASE == 0)
-               use the ADDR32 prefix.  For now, do nothing.  */
-
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64,
-                             tcg_target_call_iarg_regs[0], GUEST_BASE);
-                tgen_arithr(s, ARITH_ADD + P_REXW,
-                            tcg_target_call_iarg_regs[0], base);
-                base = tcg_target_call_iarg_regs[0];
-                offset = 0;
-            }
+/*
+ * Generate TB finalization at the end of block
+ */
+void tcg_out_tb_finalize(TCGContext *s)
+{
+    int i;
+    TCGLabelQemuLdst *label;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
+        label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[i];
+        if (label->is_ld) {
+            tcg_out_qemu_ld_slow_path(s, label);
+        } else {
+            tcg_out_qemu_st_slow_path(s, label);
         }
-
-        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, opc);
     }
-#endif
 }
+#endif  /* CONFIG_SOFTMMU */
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg *args, const int *const_args)
@@ -1537,14 +1703,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_modrm(s, OPC_GRP5, EXT5_CALLN_Ev, args[0]);
         }
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_jmp(s, args[0]);
-        } else {
-            /* jmp *reg */
-            tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, args[0]);
-        }
-        break;
     case INDEX_op_br:
         tcg_out_jxx(s, JCC_JMP, args[0], 0);
         break;
@@ -1573,18 +1731,35 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     OP_32_64(st8):
-        tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
-                             args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvIz,
+                                 0, args[1], args[2]);
+            tcg_out8(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
+                                 args[0], args[1], args[2]);
+        }
         break;
     OP_32_64(st16):
-        tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
-                             args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16,
+                                 0, args[1], args[2]);
+            tcg_out16(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
+                                 args[0], args[1], args[2]);
+        }
         break;
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_st32_i64:
 #endif
     case INDEX_op_st_i32:
-        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        }
         break;
 
     OP_32_64(add):
@@ -1680,6 +1855,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_setcond32(s, args[3], args[0], args[1],
                           args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond32(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
 
     OP_32_64(bswap16):
         tcg_out_rolw_8(s, args[0]);
@@ -1788,7 +1967,13 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
         break;
     case INDEX_op_st_i64:
-        tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW,
+                                 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        }
         break;
     case INDEX_op_qemu_ld32s:
         tcg_out_qemu_ld(s, args, 2 | 4);
@@ -1802,6 +1987,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_setcond64(s, args[3], args[0], args[1],
                           args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond64(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
 
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, args[0]);
@@ -1841,7 +2030,6 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
     { INDEX_op_mov_i32, { "r", "r" } },
     { INDEX_op_movi_i32, { "r" } },
@@ -1850,9 +2038,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ld16u_i32, { "r", "r" } },
     { INDEX_op_ld16s_i32, { "r", "r" } },
     { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "q", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "qi", "r" } },
+    { INDEX_op_st16_i32, { "ri", "r" } },
+    { INDEX_op_st_i32, { "ri", "r" } },
 
     { INDEX_op_add_i32, { "r", "r", "ri" } },
     { INDEX_op_sub_i32, { "r", "0", "ri" } },
@@ -1886,6 +2074,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_setcond_i32, { "q", "r", "ri" } },
 
     { INDEX_op_deposit_i32, { "Q", "0", "Q" } },
+#if TCG_TARGET_HAS_movcond_i32
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "0" } },
+#endif
 
 #if TCG_TARGET_REG_BITS == 32
     { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
@@ -1903,12 +2094,12 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ld32u_i64, { "r", "r" } },
     { INDEX_op_ld32s_i64, { "r", "r" } },
     { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
+    { INDEX_op_st8_i64, { "ri", "r" } },
+    { INDEX_op_st16_i64, { "ri", "r" } },
+    { INDEX_op_st32_i64, { "ri", "r" } },
+    { INDEX_op_st_i64, { "re", "r" } },
 
-    { INDEX_op_add_i64, { "r", "0", "re" } },
+    { INDEX_op_add_i64, { "r", "r", "re" } },
     { INDEX_op_mul_i64, { "r", "0", "re" } },
     { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
     { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
@@ -1940,6 +2131,7 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
     { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
+    { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
 #endif
 
 #if TCG_TARGET_REG_BITS == 64
@@ -2038,15 +2230,17 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 #if TCG_TARGET_REG_BITS == 32
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
                (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
-    tcg_out_ld(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[1], TCG_REG_ESP,
-               (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4);
+    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
+    /* jmp *tb.  */
+    tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
+		         (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
+			 + stack_addend);
 #else
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-#endif
     tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
-
     /* jmp *tb.  */
     tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
+#endif
 
     /* TB epilogue */
     tb_ret_addr = s->code_ptr;
@@ -2057,10 +2251,27 @@ static void tcg_target_qemu_prologue(TCGContext *s)
         tcg_out_pop(s, tcg_target_callee_save_regs[i]);
     }
     tcg_out_opc(s, OPC_RET, 0, 0, 0);
+
+#if !defined(CONFIG_SOFTMMU)
+    /* Try to set up a segment register to point to GUEST_BASE.  */
+    if (GUEST_BASE) {
+        setup_guest_base_seg();
+    }
+#endif
 }
 
 static void tcg_target_init(TCGContext *s)
 {
+    /* For 32-bit, 99% certainty that we're running on hardware that supports
+       cmov, but we still need to check.  In case cmov is not available, we'll
+       use a small forward branch.  */
+#ifndef have_cmov
+    {
+        unsigned a, b, c, d;
+        have_cmov = (__get_cpuid(1, &a, &b, &c, &d) && (d & bit_CMOV));
+    }
+#endif
+
 #if !defined(CONFIG_USER_ONLY)
     /* fail safe */
     if ((1 << CPU_TLB_ENTRY_BITS) != sizeof(CPUTLBEntry))
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index c3cfe05f62b..e63db9cfe9b 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_I386 
 #define TCG_TARGET_I386 1
 
 //#define TCG_TARGET_WORDS_BIGENDIAN
@@ -67,7 +68,11 @@ typedef enum {
 /* used for function call generation */
 #define TCG_REG_CALL_STACK TCG_REG_ESP 
 #define TCG_TARGET_STACK_ALIGN 16
+#if defined(_WIN64)
+#define TCG_TARGET_CALL_STACK_OFFSET 32
+#else
 #define TCG_TARGET_CALL_STACK_OFFSET 0
+#endif
 
 /* optional instructions */
 #define TCG_TARGET_HAS_div2_i32         1
@@ -86,6 +91,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_movcond_i32      1
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -107,6 +113,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      1
+#define TCG_TARGET_HAS_movcond_i64      1
 #endif
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) \
@@ -114,9 +121,6 @@ typedef enum {
      ((ofs) == 0 && (len) == 16))
 #define TCG_TARGET_deposit_i64_valid    TCG_TARGET_deposit_i32_valid
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
-/* Note: must be synced with dyngen-exec.h */
 #if TCG_TARGET_REG_BITS == 64
 # define TCG_AREG0 TCG_REG_R14
 #else
@@ -127,3 +131,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
 }
+
+#endif
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index dc588dbea52..2373d9ef79a 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -176,12 +176,6 @@ static const int tcg_target_call_oarg_regs[] = {
     TCG_REG_R8
 };
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 8;
-}
-
 /*
  * opcode formation
  */
@@ -236,6 +230,8 @@ enum {
     OPC_CMP4_LT_A6            = 0x18400000000ull,
     OPC_CMP4_LTU_A6           = 0x1a400000000ull,
     OPC_CMP4_EQ_A6            = 0x1c400000000ull,
+    OPC_DEP_I14               = 0x0ae00000000ull,
+    OPC_DEP_I15               = 0x08000000000ull,
     OPC_DEP_Z_I12             = 0x0a600000000ull,
     OPC_EXTR_I11              = 0x0a400002000ull,
     OPC_EXTR_U_I11            = 0x0a400000000ull,
@@ -507,6 +503,30 @@ static inline uint64_t tcg_opc_i12(int qp, uint64_t opc, int r1,
            | (qp & 0x3f);
 }
 
+static inline uint64_t tcg_opc_i14(int qp, uint64_t opc, int r1, uint64_t imm,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((imm & 0x01) << 36)
+           | ((len & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((pos & 0x3f) << 14)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i15(int qp, uint64_t opc, int r1, int r2,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((pos & 0x3f) << 31)
+           | ((len & 0x0f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
 static inline uint64_t tcg_opc_i18(int qp, uint64_t opc, uint64_t imm)
 {
     return opc
@@ -1318,6 +1338,37 @@ static inline void tcg_out_bswap64(TCGContext *s, TCGArg ret, TCGArg arg)
                    tcg_opc_i3 (TCG_REG_P0, OPC_MUX1_I3, ret, arg, 0xb));
 }
 
+static inline void tcg_out_deposit(TCGContext *s, TCGArg ret, TCGArg a1,
+                                   TCGArg a2, int const_a2, int pos, int len)
+{
+    uint64_t i1 = 0, i2 = 0;
+    int cpos = 63 - pos, lm1 = len - 1;
+
+    if (const_a2) {
+        /* Truncate the value of a constant a2 to the width of the field.  */
+        int mask = (1u << len) - 1;
+        a2 &= mask;
+
+        if (a2 == 0 || a2 == mask) {
+            /* 1-bit signed constant inserted into register.  */
+            i2 = tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, ret, a2, a1, cpos, lm1);
+        } else {
+            /* Otherwise, load any constant into a temporary.  Do this into
+               the first I slot to help out with cross-unit delays.  */
+            i1 = tcg_opc_a5(TCG_REG_P0, OPC_ADDL_A5,
+                            TCG_REG_R2, a2, TCG_REG_R0);
+            a2 = TCG_REG_R2;
+        }
+    }
+    if (i2 == 0) {
+        i2 = tcg_opc_i15(TCG_REG_P0, OPC_DEP_I15, ret, a2, a1, cpos, lm1);
+    }
+    tcg_out_bundle(s, (i1 ? mII : miI),
+                   tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                   i1 ? i1 : tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                   i2);
+}
+
 static inline uint64_t tcg_opc_cmp_a(int qp, TCGCond cond, TCGArg arg1,
                                      TCGArg arg2, int cmp4)
 {
@@ -1410,21 +1461,47 @@ static inline void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGArg ret,
                    tcg_opc_a5(TCG_REG_P7, OPC_ADDL_A5, ret, 0, TCG_REG_R0));
 }
 
+static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
+                                   TCGArg c1, TCGArg c2,
+                                   TCGArg v1, int const_v1,
+                                   TCGArg v2, int const_v2, int cmp4)
+{
+    uint64_t opc1, opc2;
+
+    if (const_v1) {
+        opc1 = tcg_opc_a5(TCG_REG_P6, OPC_ADDL_A5, ret, v1, TCG_REG_R0);
+    } else if (ret == v1) {
+        opc1 = tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0);
+    } else {
+        opc1 = tcg_opc_a4(TCG_REG_P6, OPC_ADDS_A4, ret, 0, v1);
+    }
+    if (const_v2) {
+        opc2 = tcg_opc_a5(TCG_REG_P7, OPC_ADDL_A5, ret, v2, TCG_REG_R0);
+    } else if (ret == v2) {
+        opc2 = tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0);
+    } else {
+        opc2 = tcg_opc_a4(TCG_REG_P7, OPC_ADDS_A4, ret, 0, v2);
+    }
+
+    tcg_out_bundle(s, MmI,
+                   tcg_opc_cmp_a(TCG_REG_P0, cond, c1, c2, cmp4),
+                   opc1,
+                   opc2);
+}
+
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* Load and compare a TLB entry, and return the result in (p6, p7).
    R2 is loaded with the address of the addend TLB entry.
-   R56 is loaded with the address, zero extented on 32-bit targets. */
+   R57 is loaded with the address, zero extented on 32-bit targets. */
 static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
                                     int s_bits, uint64_t offset_rw,
                                     uint64_t offset_addend)
 {
     tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P0, OPC_ADDL_A5, TCG_REG_R3,
-                               TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                               TCG_REG_R0),
+                   tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
                    tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, TCG_REG_R2,
                                addr_reg, TARGET_PAGE_BITS, CPU_TLB_BITS - 1),
                    tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, TCG_REG_R2,
@@ -1434,9 +1511,9 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
                    tcg_opc_a5 (TCG_REG_P0, OPC_ADDL_A5, TCG_REG_R2,
                                offset_rw, TCG_REG_R2),
 #if TARGET_LONG_BITS == 32
-                   tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R56, addr_reg),
+                   tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R57, addr_reg),
 #else
-                   tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, TCG_REG_R56,
+                   tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, TCG_REG_R57,
                               0, addr_reg),
 #endif
                    tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
@@ -1444,15 +1521,15 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
     tcg_out_bundle(s, mII,
                    tcg_opc_m3 (TCG_REG_P0,
                                (TARGET_LONG_BITS == 32
-                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R57,
+                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R56,
                                TCG_REG_R2, offset_addend - offset_rw),
-                   tcg_opc_a1 (TCG_REG_P0, OPC_AND_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                   tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, TCG_REG_R3, 0,
+                               TCG_REG_R57, 63 - s_bits,
+                               TARGET_PAGE_BITS - s_bits - 1),
                    tcg_opc_a6 (TCG_REG_P0, OPC_CMP_EQ_A6, TCG_REG_P6,
-                               TCG_REG_P7, TCG_REG_R3, TCG_REG_R57));
+                               TCG_REG_P7, TCG_REG_R3, TCG_REG_R56));
 }
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -1461,16 +1538,6 @@ static const void * const qemu_ld_helpers[4] = {
     helper_ldl_mmu,
     helper_ldq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-#endif
 
 static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 {
@@ -1497,8 +1564,8 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 
     /* P6 is the fast path, and P7 the slow path */
     tcg_out_bundle(s, mLX,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R57,
-                               mem_index, TCG_REG_R0),
+                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
+                               TCG_REG_R56, 0, TCG_AREG0),
                    tcg_opc_l2 ((tcg_target_long) qemu_ld_helpers[s_bits]),
                    tcg_opc_x2 (TCG_REG_P7, OPC_MOVL_X2, TCG_REG_R2,
                                (tcg_target_long) qemu_ld_helpers[s_bits]));
@@ -1506,7 +1573,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                    tcg_opc_m3 (TCG_REG_P0, OPC_LD8_M3, TCG_REG_R3,
                                TCG_REG_R2, 8),
                    tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                               TCG_REG_R3, TCG_REG_R57),
                    tcg_opc_i21(TCG_REG_P7, OPC_MOV_I21, TCG_REG_B6,
                                TCG_REG_R3, 0));
     if (bswap && s_bits == 1) {
@@ -1530,25 +1597,17 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1, TCG_REG_R1, TCG_REG_R2),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0));
     }
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
-                               mem_index, TCG_REG_R0),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R57, 0, TCG_REG_R56),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R56, 0, TCG_AREG0));
-#endif
     if (!bswap || s_bits == 0) {
         tcg_out_bundle(s, miB,
-                       tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                       tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                                   mem_index, TCG_REG_R0),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
                                    TCG_REG_B0, TCG_REG_B6));
     } else {
         tcg_out_bundle(s, miB,
-                       tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                       tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                                   mem_index, TCG_REG_R0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R8, TCG_REG_R8, 0xb),
                        tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
@@ -1570,7 +1629,6 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     }
 }
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
    uintxx_t val, int mmu_idx) */
 static const void * const qemu_st_helpers[4] = {
@@ -1579,16 +1637,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 {
@@ -1611,8 +1659,8 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 
     /* P6 is the fast path, and P7 the slow path */
     tcg_out_bundle(s, mLX,
-                   tcg_opc_a4(TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R57,
-                              0, data_reg),
+                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
+                               TCG_REG_R56, 0, TCG_AREG0),
                    tcg_opc_l2 ((tcg_target_long) qemu_st_helpers[opc]),
                    tcg_opc_x2 (TCG_REG_P7, OPC_MOVL_X2, TCG_REG_R2,
                                (tcg_target_long) qemu_st_helpers[opc]));
@@ -1620,31 +1668,42 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
                    tcg_opc_m3 (TCG_REG_P0, OPC_LD8_M3, TCG_REG_R3,
                                TCG_REG_R2, 8),
                    tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                               TCG_REG_R3, TCG_REG_R57),
                    tcg_opc_i21(TCG_REG_P7, OPC_MOV_I21, TCG_REG_B6,
                                TCG_REG_R3, 0));
 
     if (!bswap || opc == 0) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, mii,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
-                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0));
     } else if (opc == 1) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i12(TCG_REG_P6, OPC_DEP_Z_I12,
-                                   TCG_REG_R2, data_reg, 15, 15),
+                                   TCG_REG_R2, data_reg, 15, 15));
+        tcg_out_bundle(s, miI,
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, TCG_REG_R2, 0xb));
         data_reg = TCG_REG_R2;
     } else if (opc == 2) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i12(TCG_REG_P6, OPC_DEP_Z_I12,
-                                   TCG_REG_R2, data_reg, 31, 31),
+                                   TCG_REG_R2, data_reg, 31, 31));
+        tcg_out_bundle(s, miI,
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, TCG_REG_R2, 0xb));
         data_reg = TCG_REG_R2;
@@ -1652,37 +1711,20 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
-                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, data_reg, 0xb));
         data_reg = TCG_REG_R2;
     }
 
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R59,
-                               mem_index, TCG_REG_R0),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R58, 0, TCG_REG_R57),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R57, 0, TCG_REG_R56));
     tcg_out_bundle(s, miB,
                    tcg_opc_m4 (TCG_REG_P6, opc_st_m4[opc],
                                data_reg, TCG_REG_R3),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R56, 0, TCG_AREG0),
-                   tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
-                               TCG_REG_B0, TCG_REG_B6));
-#else
-    tcg_out_bundle(s, miB,
-                   tcg_opc_m4 (TCG_REG_P6, opc_st_m4[opc],
-                               data_reg, TCG_REG_R3),
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R59,
                                mem_index, TCG_REG_R0),
                    tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
                                TCG_REG_B0, TCG_REG_B6));
-#endif
 }
 
 #else /* !CONFIG_SOFTMMU */
@@ -1956,9 +1998,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:
         tcg_out_goto_tb(s, args[0]);
         break;
-    case INDEX_op_jmp:
-        tcg_out_jmp(s, args[0]);
-        break;
 
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
@@ -2135,6 +2174,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap64(s, args[0], args[1]);
         break;
 
+    case INDEX_op_deposit_i32:
+    case INDEX_op_deposit_i64:
+        tcg_out_deposit(s, args[0], args[1], args[2], const_args[2],
+                        args[3], args[4]);
+        break;
+
     case INDEX_op_brcond_i32:
         tcg_out_brcond(s, args[2], args[0], const_args[0],
                        args[1], const_args[1], args[3], 1);
@@ -2149,6 +2194,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_setcond_i64:
         tcg_out_setcond(s, args[3], args[0], args[1], args[2], 0);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 1);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 0);
+        break;
 
     case INDEX_op_qemu_ld8u:
         tcg_out_qemu_ld(s, args, 0);
@@ -2196,7 +2249,6 @@ static const TCGTargetOpDef ia64_op_defs[] = {
     { INDEX_op_call, { "r" } },
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
-    { INDEX_op_jmp, { "r" } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
     { INDEX_op_movi_i32, { "r" } },
@@ -2240,6 +2292,7 @@ static const TCGTargetOpDef ia64_op_defs[] = {
 
     { INDEX_op_brcond_i32, { "rI", "rI" } },
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rI", "rI" } },
 
     { INDEX_op_mov_i64, { "r", "r" } },
     { INDEX_op_movi_i64, { "r" } },
@@ -2289,6 +2342,10 @@ static const TCGTargetOpDef ia64_op_defs[] = {
 
     { INDEX_op_brcond_i64, { "rI", "rI" } },
     { INDEX_op_setcond_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "rZ", "rZ", "rI", "rI" } },
+
+    { INDEX_op_deposit_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_deposit_i64, { "r", "rZ", "ri" } },
 
     { INDEX_op_qemu_ld8u, { "r", "r" } },
     { INDEX_op_qemu_ld8s, { "r", "r" } },
@@ -2313,9 +2370,12 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     int frame_size;
 
     /* reserve some stack space */
-    frame_size = TCG_STATIC_CALL_ARGS_SIZE;
+    frame_size = TCG_STATIC_CALL_ARGS_SIZE +
+                 CPU_TEMP_BUF_NLONGS * sizeof(long);
     frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
                  ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 
     /* First emit adhoc function descriptor */
     *(uint64_t *)(s->code_ptr) = (uint64_t)s->code_ptr + 16; /* entry point */
@@ -2422,6 +2482,4 @@ static void tcg_target_init(TCGContext *s)
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_R6);
 
     tcg_add_target_add_op_defs(ia64_op_defs);
-    tcg_set_frame(s, TCG_AREG0, offsetof(CPUArchState, temp_buf),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 }
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index 0631b9f1805..7f3401ecdd3 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_IA64 
 #define TCG_TARGET_IA64 1
 
 /* We only map the first 64 registers */
@@ -131,8 +132,13 @@ typedef enum {
 #define TCG_TARGET_HAS_orc_i64          1
 #define TCG_TARGET_HAS_rot_i32          1
 #define TCG_TARGET_HAS_rot_i64          1
-#define TCG_TARGET_HAS_deposit_i32      0
-#define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i32      1
+#define TCG_TARGET_HAS_movcond_i64      1
+#define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_deposit_i64      1
+
+#define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
+#define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub r1, r0, r3 */
@@ -140,12 +146,8 @@ typedef enum {
 #define TCG_TARGET_HAS_not_i32          0 /* xor r1, -1, r3 */
 #define TCG_TARGET_HAS_not_i64          0 /* xor r1, -1, r3 */
 
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_R7
 
-/* Guest base is supported */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
@@ -157,3 +159,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     }
     asm volatile (";;sync.i;;srlz.i;;");
 }
+
+#endif
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 1006e28000c..bd8c8589891 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -68,7 +68,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #endif
 
 /* check if we really need so many registers :P */
-static const int tcg_target_reg_alloc_order[] = {
+static const TCGReg tcg_target_reg_alloc_order[] = {
     TCG_REG_S0,
     TCG_REG_S1,
     TCG_REG_S2,
@@ -94,14 +94,14 @@ static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_V1
 };
 
-static const int tcg_target_call_iarg_regs[4] = {
+static const TCGReg tcg_target_call_iarg_regs[4] = {
     TCG_REG_A0,
     TCG_REG_A1,
     TCG_REG_A2,
     TCG_REG_A3
 };
 
-static const int tcg_target_call_oarg_regs[2] = {
+static const TCGReg tcg_target_call_oarg_regs[2] = {
     TCG_REG_V0,
     TCG_REG_V1
 };
@@ -185,12 +185,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -217,7 +211,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set(ct->u.regs, 0xffffffff);
 #if defined(CONFIG_SOFTMMU)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
-# if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+# if (TARGET_LONG_BITS == 64)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
 # endif
 #endif
@@ -227,12 +221,11 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set(ct->u.regs, 0xffffffff);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
 #if defined(CONFIG_SOFTMMU)
-# if (defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 32) || \
-     (!defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 64)
+# if (TARGET_LONG_BITS == 32)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
 # endif
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-# if defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 64
+# if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
 # endif
 #endif
@@ -279,6 +272,8 @@ static inline int tcg_target_const_match(tcg_target_long val,
 enum {
     OPC_BEQ      = 0x04 << 26,
     OPC_BNE      = 0x05 << 26,
+    OPC_BLEZ     = 0x06 << 26,
+    OPC_BGTZ     = 0x07 << 26,
     OPC_ADDIU    = 0x09 << 26,
     OPC_SLTI     = 0x0A << 26,
     OPC_SLTIU    = 0x0B << 26,
@@ -299,12 +294,16 @@ enum {
     OPC_SPECIAL  = 0x00 << 26,
     OPC_SLL      = OPC_SPECIAL | 0x00,
     OPC_SRL      = OPC_SPECIAL | 0x02,
+    OPC_ROTR     = OPC_SPECIAL | (0x01 << 21) | 0x02,
     OPC_SRA      = OPC_SPECIAL | 0x03,
     OPC_SLLV     = OPC_SPECIAL | 0x04,
     OPC_SRLV     = OPC_SPECIAL | 0x06,
+    OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
     OPC_SRAV     = OPC_SPECIAL | 0x07,
     OPC_JR       = OPC_SPECIAL | 0x08,
     OPC_JALR     = OPC_SPECIAL | 0x09,
+    OPC_MOVZ     = OPC_SPECIAL | 0x0A,
+    OPC_MOVN     = OPC_SPECIAL | 0x0B,
     OPC_MFHI     = OPC_SPECIAL | 0x10,
     OPC_MFLO     = OPC_SPECIAL | 0x12,
     OPC_MULT     = OPC_SPECIAL | 0x18,
@@ -320,7 +319,16 @@ enum {
     OPC_SLT      = OPC_SPECIAL | 0x2A,
     OPC_SLTU     = OPC_SPECIAL | 0x2B,
 
+    OPC_REGIMM   = 0x01 << 26,
+    OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
+    OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
+
+    OPC_SPECIAL2 = 0x1c << 26,
+    OPC_MUL      = OPC_SPECIAL2 | 0x002,
+
     OPC_SPECIAL3 = 0x1f << 26,
+    OPC_INS      = OPC_SPECIAL3 | 0x004,
+    OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
     OPC_SEB      = OPC_SPECIAL3 | 0x420,
     OPC_SEH      = OPC_SPECIAL3 | 0x620,
 };
@@ -328,7 +336,8 @@ enum {
 /*
  * Type reg
  */
-static inline void tcg_out_opc_reg(TCGContext *s, int opc, int rd, int rs, int rt)
+static inline void tcg_out_opc_reg(TCGContext *s, int opc,
+                                   TCGReg rd, TCGReg rs, TCGReg rt)
 {
     int32_t inst;
 
@@ -342,7 +351,8 @@ static inline void tcg_out_opc_reg(TCGContext *s, int opc, int rd, int rs, int r
 /*
  * Type immediate
  */
-static inline void tcg_out_opc_imm(TCGContext *s, int opc, int rt, int rs, int imm)
+static inline void tcg_out_opc_imm(TCGContext *s, int opc,
+                                   TCGReg rt, TCGReg rs, TCGArg imm)
 {
     int32_t inst;
 
@@ -356,7 +366,8 @@ static inline void tcg_out_opc_imm(TCGContext *s, int opc, int rt, int rs, int i
 /*
  * Type branch
  */
-static inline void tcg_out_opc_br(TCGContext *s, int opc, int rt, int rs)
+static inline void tcg_out_opc_br(TCGContext *s, int opc,
+                                  TCGReg rt, TCGReg rs)
 {
     /* We pay attention here to not modify the branch target by reading
        the existing value and using it again. This ensure that caches and
@@ -369,7 +380,8 @@ static inline void tcg_out_opc_br(TCGContext *s, int opc, int rt, int rs)
 /*
  * Type sa
  */
-static inline void tcg_out_opc_sa(TCGContext *s, int opc, int rd, int rt, int sa)
+static inline void tcg_out_opc_sa(TCGContext *s, int opc,
+                                  TCGReg rd, TCGReg rt, TCGArg sa)
 {
     int32_t inst;
 
@@ -408,38 +420,47 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
     }
 }
 
-static inline void tcg_out_bswap16(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+#else
     /* ret and arg can't be register at */
     if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
     }
 
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0x00ff);
-
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
     tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_bswap16s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
+#else
     /* ret and arg can't be register at */
     if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
     }
 
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff);
-
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
     tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_bswap32(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
+#else
     /* ret and arg must be different and can't be register at */
     if (ret == arg || ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
@@ -457,11 +478,12 @@ static inline void tcg_out_bswap32(TCGContext *s, int ret, int arg)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
     tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff00);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_ext8s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#ifdef _MIPS_ARCH_MIPS32R2
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
     tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
 #else
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
@@ -469,9 +491,9 @@ static inline void tcg_out_ext8s(TCGContext *s, int ret, int arg)
 #endif
 }
 
-static inline void tcg_out_ext16s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#ifdef _MIPS_ARCH_MIPS32R2
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
     tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
 #else
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
@@ -479,8 +501,8 @@ static inline void tcg_out_ext16s(TCGContext *s, int ret, int arg)
 #endif
 }
 
-static inline void tcg_out_ldst(TCGContext *s, int opc, int arg,
-                              int arg1, tcg_target_long arg2)
+static inline void tcg_out_ldst(TCGContext *s, int opc, TCGArg arg,
+                                TCGReg arg1, TCGArg arg2)
 {
     if (arg2 == (int16_t) arg2) {
         tcg_out_opc_imm(s, opc, arg, arg1, arg2);
@@ -503,7 +525,7 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
     tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
 }
 
-static inline void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
+static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
 {
     if (val == (int16_t)val) {
         tcg_out_opc_imm(s, OPC_ADDIU, reg, reg, val);
@@ -544,7 +566,7 @@ DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_reg16, TCGReg arg)
 #undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
 #define DEFINE_TCG_OUT_CALL_IARG_GET_ARG(A) \
     tcg_out_movi(s, TCG_TYPE_I32, A, arg);
-DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_imm32, uint32_t arg)
+DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_imm32, TCGArg arg)
 #undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
 
 /* We don't use the macro for this one to avoid an unnecessary reg-reg
@@ -574,8 +596,8 @@ static inline void tcg_out_call_iarg_reg64(TCGContext *s, int *arg_num,
 #endif
 }
 
-static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
-                           int arg2, int label_index)
+static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGArg arg1,
+                           TCGArg arg2, int label_index)
 {
     TCGLabel *l = &s->labels[label_index];
 
@@ -587,32 +609,48 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
         tcg_out_opc_br(s, OPC_BNE, arg1, arg2);
         break;
     case TCG_COND_LT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BLTZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
+            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_LTU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
         tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_GE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BGEZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
+            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_GEU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
         tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_LE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BLEZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
+            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_LEU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
         tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_GT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BGTZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
+            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_GTU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
@@ -632,8 +670,9 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
 
 /* XXX: we implement it at the target level to avoid having to
    handle cross basic blocks temporaries */
-static void tcg_out_brcond2(TCGContext *s, TCGCond cond, int arg1,
-                            int arg2, int arg3, int arg4, int label_index)
+static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGArg arg1,
+                            TCGArg arg2, TCGArg arg3, TCGArg arg4,
+                            int label_index)
 {
     void *label_ptr;
 
@@ -695,8 +734,70 @@ static void tcg_out_brcond2(TCGContext *s, TCGCond cond, int arg1,
     reloc_pc16(label_ptr, (tcg_target_long) s->code_ptr);
 }
 
-static void tcg_out_setcond(TCGContext *s, TCGCond cond, int ret,
-                            int arg1, int arg2)
+static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGArg c1, TCGArg c2, TCGArg v)
+{
+    switch (cond) {
+    case TCG_COND_EQ:
+        if (c1 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c2);
+        } else if (c2 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c1);
+        } else {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        }
+        break;
+    case TCG_COND_NE:
+        if (c1 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c2);
+        } else if (c2 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c1);
+        } else {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        }
+        break;
+    case TCG_COND_LT:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LTU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GE:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GEU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LE:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LEU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GT:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GTU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    default:
+        tcg_abort();
+        break;
+    }
+}
+
+static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGArg arg1, TCGArg arg2)
 {
     switch (cond) {
     case TCG_COND_EQ:
@@ -755,8 +856,8 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, int ret,
 
 /* XXX: we implement it at the target level to avoid having to
    handle cross basic blocks temporaries */
-static void tcg_out_setcond2(TCGContext *s, TCGCond cond, int ret,
-                             int arg1, int arg2, int arg3, int arg4)
+static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
+                             TCGArg arg1, TCGArg arg2, TCGArg arg3, TCGArg arg4)
 {
     switch (cond) {
     case TCG_COND_EQ:
@@ -819,9 +920,8 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, int ret,
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -839,42 +939,22 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 #endif
 
 static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
                             int opc)
 {
-    int addr_regl, addr_meml;
-    int data_regl, data_regh, data_reg1, data_reg2;
-    int mem_index, s_bits;
+    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
 #if defined(CONFIG_SOFTMMU)
     void *label1_ptr, *label2_ptr;
     int arg_num;
-#endif
-#if TARGET_LONG_BITS == 64
-# if defined(CONFIG_SOFTMMU)
+    int mem_index, s_bits;
+    int addr_meml;
+# if TARGET_LONG_BITS == 64
     uint8_t *label3_ptr;
+    TCGReg addr_regh;
+    int addr_memh;
 # endif
-    int addr_regh, addr_memh;
 #endif
     data_regl = *args++;
     if (opc == 3)
@@ -882,11 +962,22 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
     else
         data_regh = 0;
     addr_regl = *args++;
-#if TARGET_LONG_BITS == 64
+#if defined(CONFIG_SOFTMMU)
+# if TARGET_LONG_BITS == 64
     addr_regh = *args++;
-#endif
+#  if defined(TCG_TARGET_WORDS_BIGENDIAN)
+    addr_memh = 0;
+    addr_meml = 4;
+#  else
+    addr_memh = 4;
+    addr_meml = 0;
+#  endif
+# else
+    addr_meml = 0;
+# endif
     mem_index = *args;
     s_bits = opc & 3;
+#endif
 
     if (opc == 3) {
 #if defined(TCG_TARGET_WORDS_BIGENDIAN)
@@ -900,18 +991,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
         data_reg1 = data_regl;
         data_reg2 = 0;
     }
-#if TARGET_LONG_BITS == 64
-# if defined(TCG_TARGET_WORDS_BIGENDIAN)
-    addr_memh = 0;
-    addr_meml = 4;
-# else
-    addr_memh = 4;
-    addr_meml = 0;
-# endif
-#else
-    addr_meml = 0;
-#endif
-
 #if defined(CONFIG_SOFTMMU)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
     tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
@@ -942,9 +1021,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
 
     /* slow path */
     arg_num = 0;
-# ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# endif
 # if TARGET_LONG_BITS == 64
     tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
 # else
@@ -1052,50 +1129,56 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
                             int opc)
 {
-    int addr_regl, addr_meml;
-    int data_regl, data_regh, data_reg1, data_reg2;
-    int mem_index, s_bits;
+    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
 #if defined(CONFIG_SOFTMMU)
     uint8_t *label1_ptr, *label2_ptr;
     int arg_num;
+    int mem_index, s_bits;
+    int addr_meml;
 #endif
 #if TARGET_LONG_BITS == 64
 # if defined(CONFIG_SOFTMMU)
     uint8_t *label3_ptr;
+    TCGReg addr_regh;
+    int addr_memh;
 # endif
-    int addr_regh, addr_memh;
 #endif
-
     data_regl = *args++;
     if (opc == 3) {
         data_regh = *args++;
-#if defined(TCG_TARGET_WORDS_BIGENDIAN)
-        data_reg1 = data_regh;
-        data_reg2 = data_regl;
-#else
-        data_reg1 = data_regl;
-        data_reg2 = data_regh;
-#endif
     } else {
-        data_reg1 = data_regl;
-        data_reg2 = 0;
         data_regh = 0;
     }
     addr_regl = *args++;
-#if TARGET_LONG_BITS == 64
+#if defined(CONFIG_SOFTMMU)
+# if TARGET_LONG_BITS == 64
     addr_regh = *args++;
-# if defined(TCG_TARGET_WORDS_BIGENDIAN)
+#  if defined(TCG_TARGET_WORDS_BIGENDIAN)
     addr_memh = 0;
     addr_meml = 4;
-# else
+#  else
     addr_memh = 4;
     addr_meml = 0;
-# endif
-#else
+#  endif
+# else
     addr_meml = 0;
-#endif
+# endif
     mem_index = *args;
     s_bits = opc;
+#endif
+
+    if (opc == 3) {
+#if defined(TCG_TARGET_WORDS_BIGENDIAN)
+        data_reg1 = data_regh;
+        data_reg2 = data_regl;
+#else
+        data_reg1 = data_regl;
+        data_reg2 = data_regh;
+#endif
+    } else {
+        data_reg1 = data_regl;
+        data_reg2 = 0;
+    }
 
 #if defined(CONFIG_SOFTMMU)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
@@ -1127,9 +1210,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
 
     /* slow path */
     arg_num = 0;
-# ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# endif
 # if TARGET_LONG_BITS == 64
     tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
 # else
@@ -1182,7 +1263,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
         break;
     case 1:
         if (TCG_NEED_BSWAP) {
-            tcg_out_bswap16(s, TCG_REG_T0, data_reg1);
+            tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_T0, data_reg1, 0xffff);
+            tcg_out_bswap16(s, TCG_REG_T0, TCG_REG_T0);
             tcg_out_opc_imm(s, OPC_SH, TCG_REG_T0, TCG_REG_A0, 0);
         } else {
             tcg_out_opc_imm(s, OPC_SH, data_reg1, TCG_REG_A0, 0);
@@ -1243,10 +1325,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, args[0], 0);
         tcg_out_nop(s);
         break;
-    case INDEX_op_jmp:
-        tcg_out_opc_reg(s, OPC_JR, 0, args[0], 0);
-        tcg_out_nop(s);
-        break;
     case INDEX_op_br:
         tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, args[0]);
         break;
@@ -1328,8 +1406,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_mov(s, TCG_TYPE_I32, args[0], TCG_REG_AT);
         break;
     case INDEX_op_mul_i32:
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 1)
+        tcg_out_opc_reg(s, OPC_MUL, args[0], args[1], args[2]);
+#else
         tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
         tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
+#endif
         break;
     case INDEX_op_mulu2_i32:
         tcg_out_opc_reg(s, OPC_MULTU, 0, args[2], args[3]);
@@ -1402,6 +1484,31 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_opc_reg(s, OPC_SRLV, args[0], args[2], args[1]);
         }
         break;
+    case INDEX_op_rotl_i32:
+        if (const_args[2]) {
+            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], 0x20 - args[2]);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_AT, 32);
+            tcg_out_opc_reg(s, OPC_SUBU, TCG_REG_AT, TCG_REG_AT, args[2]);
+            tcg_out_opc_reg(s, OPC_ROTRV, args[0], TCG_REG_AT, args[1]);
+        }
+        break;
+    case INDEX_op_rotr_i32:
+        if (const_args[2]) {
+            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], args[2]);
+        } else {
+            tcg_out_opc_reg(s, OPC_ROTRV, args[0], args[2], args[1]);
+        }
+        break;
+
+    /* The bswap routines do not work on non-R2 CPU. In that case
+       we let TCG generating the corresponding code. */
+    case INDEX_op_bswap16_i32:
+        tcg_out_bswap16(s, args[0], args[1]);
+        break;
+    case INDEX_op_bswap32_i32:
+        tcg_out_bswap32(s, args[0], args[1]);
+        break;
 
     case INDEX_op_ext8s_i32:
         tcg_out_ext8s(s, args[0], args[1]);
@@ -1410,6 +1517,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext16s(s, args[0], args[1]);
         break;
 
+    case INDEX_op_deposit_i32:
+        tcg_out_opc_imm(s, OPC_INS, args[0], args[2],
+                        ((args[3] + args[4] - 1) << 11) | (args[3] << 6));
+        break;
+
     case INDEX_op_brcond_i32:
         tcg_out_brcond(s, args[2], args[0], args[1], args[3]);
         break;
@@ -1417,6 +1529,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond2(s, args[4], args[0], args[1], args[2], args[3], args[5]);
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2], args[3]);
+        break;
+
     case INDEX_op_setcond_i32:
         tcg_out_setcond(s, args[3], args[0], args[1], args[2]);
         break;
@@ -1464,7 +1580,6 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "C" } },
-    { INDEX_op_jmp, { "r" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1478,34 +1593,42 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_st16_i32, { "rZ", "r" } },
     { INDEX_op_st_i32, { "rZ", "r" } },
 
-    { INDEX_op_add_i32, { "r", "rZ", "rJZ" } },
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
     { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
     { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_remu_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rJZ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
 
-    { INDEX_op_and_i32, { "r", "rZ", "rIZ" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rI" } },
     { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_not_i32, { "r", "rZ" } },
     { INDEX_op_or_i32, { "r", "rZ", "rIZ" } },
     { INDEX_op_xor_i32, { "r", "rZ", "rIZ" } },
 
-    { INDEX_op_shl_i32, { "r", "rZ", "riZ" } },
-    { INDEX_op_shr_i32, { "r", "rZ", "riZ" } },
-    { INDEX_op_sar_i32, { "r", "rZ", "riZ" } },
+    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
 
     { INDEX_op_ext8s_i32, { "r", "rZ" } },
     { INDEX_op_ext16s_i32, { "r", "rZ" } },
 
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+
     { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJZ", "rJZ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJZ", "rJZ" } },
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
     { INDEX_op_brcond2_i32, { "rZ", "rZ", "rZ", "rZ" } },
 
 #if TARGET_LONG_BITS == 32
@@ -1545,7 +1668,6 @@ static int tcg_target_callee_save_regs[] = {
     TCG_REG_S5,
     TCG_REG_S6,
     TCG_REG_S7,
-    TCG_REG_GP,
     TCG_REG_FP,
     TCG_REG_RA,       /* should be last for ABI compliance */
 };
@@ -1555,11 +1677,15 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 {
     int i, frame_size;
 
-    /* reserve some stack space */
+    /* reserve some stack space, also for TCG temps. */
     frame_size = ARRAY_SIZE(tcg_target_callee_save_regs) * 4
-                 + TCG_STATIC_CALL_ARGS_SIZE;
+                 + TCG_STATIC_CALL_ARGS_SIZE
+                 + CPU_TEMP_BUF_NLONGS * sizeof(long);
     frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
                  ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_SP, ARRAY_SIZE(tcg_target_callee_save_regs) * 4
+                  + TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 
     /* TB prologue */
     tcg_out_addi(s, TCG_REG_SP, -frame_size);
@@ -1611,8 +1737,7 @@ static void tcg_target_init(TCGContext *s)
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_T0);   /* internal use */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);   /* return address */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);   /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
 
     tcg_add_target_add_op_defs(mips_op_defs);
-    tcg_set_frame(s, TCG_AREG0, offsetof(CPUArchState, temp_buf),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 }
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index d3c804d9a4f..78af664ccab 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_MIPS 
 #define TCG_TARGET_MIPS 1
 
 #ifdef __MIPSEB__
@@ -80,28 +81,42 @@ typedef enum {
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_nor_i32          1
-#define TCG_TARGET_HAS_rot_i32          0
 #define TCG_TARGET_HAS_ext8s_i32        1
 #define TCG_TARGET_HAS_ext16s_i32       1
-#define TCG_TARGET_HAS_bswap32_i32      0
-#define TCG_TARGET_HAS_bswap16_i32      0
 #define TCG_TARGET_HAS_andc_i32         0
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
+
+/* optional instructions only implemented on MIPS4, MIPS32 and Loongson 2 */
+#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \
+    defined(_MIPS_ARCH_LOONGSON2E) || defined(_MIPS_ARCH_LOONGSON2F) || \
+    defined(_MIPS_ARCH_MIPS4)
+#define TCG_TARGET_HAS_movcond_i32      1
+#else
+#define TCG_TARGET_HAS_movcond_i32      0
+#endif
+
+/* optional instructions only implemented on MIPS32R2 */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+#define TCG_TARGET_HAS_bswap16_i32      1
+#define TCG_TARGET_HAS_bswap32_i32      1
+#define TCG_TARGET_HAS_rot_i32          1
+#define TCG_TARGET_HAS_deposit_i32      1
+#else
+#define TCG_TARGET_HAS_bswap16_i32      0
+#define TCG_TARGET_HAS_bswap32_i32      0
+#define TCG_TARGET_HAS_rot_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#endif
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub  rd, zero, rt   */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* andi rt, rs, 0xff   */
 #define TCG_TARGET_HAS_ext16u_i32       0 /* andi rt, rs, 0xffff */
 
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_S0
 
-/* guest base is supported */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 #ifdef __OpenBSD__
 #include <machine/sysarch.h>
 #else
@@ -113,3 +128,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
 {
     cacheflush ((void *)start, stop-start, ICACHE);
 }
+
+#endif
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 9c65474a8c7..973d2d679fb 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -39,8 +39,6 @@ typedef enum {
     TCG_TEMP_UNDEF = 0,
     TCG_TEMP_CONST,
     TCG_TEMP_COPY,
-    TCG_TEMP_HAS_COPY,
-    TCG_TEMP_ANY
 } tcg_temp_state;
 
 struct tcg_temp_info {
@@ -48,42 +46,34 @@ struct tcg_temp_info {
     uint16_t prev_copy;
     uint16_t next_copy;
     tcg_target_ulong val;
+    tcg_target_ulong mask;
 };
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
 
-/* Reset TEMP's state to TCG_TEMP_ANY.  If TEMP was a representative of some
-   class of equivalent temp's, a new representative should be chosen in this
-   class. */
-static void reset_temp(TCGArg temp, int nb_temps, int nb_globals)
+/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
+   the copy flag from the left temp.  */
+static void reset_temp(TCGArg temp)
 {
-    int i;
-    TCGArg new_base = (TCGArg)-1;
-    if (temps[temp].state == TCG_TEMP_HAS_COPY) {
-        for (i = temps[temp].next_copy; i != temp; i = temps[i].next_copy) {
-            if (i >= nb_globals) {
-                temps[i].state = TCG_TEMP_HAS_COPY;
-                new_base = i;
-                break;
-            }
+    if (temps[temp].state == TCG_TEMP_COPY) {
+        if (temps[temp].prev_copy == temps[temp].next_copy) {
+            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
+        } else {
+            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
+            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
         }
-        for (i = temps[temp].next_copy; i != temp; i = temps[i].next_copy) {
-            if (new_base == (TCGArg)-1) {
-                temps[i].state = TCG_TEMP_ANY;
-            } else {
-                temps[i].val = new_base;
-            }
-        }
-        temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-        temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-    } else if (temps[temp].state == TCG_TEMP_COPY) {
-        temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-        temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        new_base = temps[temp].val;
     }
-    temps[temp].state = TCG_TEMP_ANY;
-    if (new_base != (TCGArg)-1 && temps[new_base].next_copy == new_base) {
-        temps[new_base].state = TCG_TEMP_ANY;
+    temps[temp].state = TCG_TEMP_UNDEF;
+    temps[temp].mask = -1;
+}
+
+/* Reset all temporaries, given that there are NB_TEMPS of them.  */
+static void reset_all_temps(int nb_temps)
+{
+    int i;
+    for (i = 0; i < nb_temps; i++) {
+        temps[i].state = TCG_TEMP_UNDEF;
+        temps[i].mask = -1;
     }
 }
 
@@ -107,40 +97,89 @@ static TCGOpcode op_to_movi(TCGOpcode op)
     }
 }
 
-static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args, TCGArg dst,
-                            TCGArg src, int nb_temps, int nb_globals)
+static TCGArg find_better_copy(TCGContext *s, TCGArg temp)
+{
+    TCGArg i;
+
+    /* If this is already a global, we can't do better. */
+    if (temp < s->nb_globals) {
+        return temp;
+    }
+
+    /* Search for a global first. */
+    for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
+        if (i < s->nb_globals) {
+            return i;
+        }
+    }
+
+    /* If it is a temp, search for a temp local. */
+    if (!s->temps[temp].temp_local) {
+        for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
+            if (s->temps[i].temp_local) {
+                return i;
+            }
+        }
+    }
+
+    /* Failure to find a better representation, return the same temp. */
+    return temp;
+}
+
+static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
 {
-        reset_temp(dst, nb_temps, nb_globals);
-        assert(temps[src].state != TCG_TEMP_COPY);
-        /* Don't try to copy if one of temps is a global or either one
-           is local and another is register */
-        if (src >= nb_globals && dst >= nb_globals &&
-            tcg_arg_is_local(s, src) == tcg_arg_is_local(s, dst)) {
-            assert(temps[src].state != TCG_TEMP_CONST);
-            if (temps[src].state != TCG_TEMP_HAS_COPY) {
-                temps[src].state = TCG_TEMP_HAS_COPY;
-                temps[src].next_copy = src;
-                temps[src].prev_copy = src;
-            }
-            temps[dst].state = TCG_TEMP_COPY;
-            temps[dst].val = src;
-            temps[dst].next_copy = temps[src].next_copy;
-            temps[dst].prev_copy = src;
-            temps[temps[dst].next_copy].prev_copy = dst;
-            temps[src].next_copy = dst;
+    TCGArg i;
+
+    if (arg1 == arg2) {
+        return true;
+    }
+
+    if (temps[arg1].state != TCG_TEMP_COPY
+        || temps[arg2].state != TCG_TEMP_COPY) {
+        return false;
+    }
+
+    for (i = temps[arg1].next_copy ; i != arg1 ; i = temps[i].next_copy) {
+        if (i == arg2) {
+            return true;
         }
-        gen_args[0] = dst;
-        gen_args[1] = src;
+    }
+
+    return false;
 }
 
-static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val,
-                             int nb_temps, int nb_globals)
+static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args,
+                            TCGArg dst, TCGArg src)
 {
-        reset_temp(dst, nb_temps, nb_globals);
-        temps[dst].state = TCG_TEMP_CONST;
-        temps[dst].val = val;
-        gen_args[0] = dst;
-        gen_args[1] = val;
+    reset_temp(dst);
+    temps[dst].mask = temps[src].mask;
+    assert(temps[src].state != TCG_TEMP_CONST);
+
+    if (s->temps[src].type == s->temps[dst].type) {
+        if (temps[src].state != TCG_TEMP_COPY) {
+            temps[src].state = TCG_TEMP_COPY;
+            temps[src].next_copy = src;
+            temps[src].prev_copy = src;
+        }
+        temps[dst].state = TCG_TEMP_COPY;
+        temps[dst].next_copy = temps[src].next_copy;
+        temps[dst].prev_copy = src;
+        temps[temps[dst].next_copy].prev_copy = dst;
+        temps[src].next_copy = dst;
+    }
+
+    gen_args[0] = dst;
+    gen_args[1] = src;
+}
+
+static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val)
+{
+    reset_temp(dst);
+    temps[dst].state = TCG_TEMP_CONST;
+    temps[dst].val = val;
+    temps[dst].mask = val;
+    gen_args[0] = dst;
+    gen_args[1] = val;
 }
 
 static TCGOpcode op_to_mov(TCGOpcode op)
@@ -267,37 +306,217 @@ static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y)
     return res;
 }
 
+static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_EQ:
+        return x == y;
+    case TCG_COND_NE:
+        return x != y;
+    case TCG_COND_LT:
+        return (int32_t)x < (int32_t)y;
+    case TCG_COND_GE:
+        return (int32_t)x >= (int32_t)y;
+    case TCG_COND_LE:
+        return (int32_t)x <= (int32_t)y;
+    case TCG_COND_GT:
+        return (int32_t)x > (int32_t)y;
+    case TCG_COND_LTU:
+        return x < y;
+    case TCG_COND_GEU:
+        return x >= y;
+    case TCG_COND_LEU:
+        return x <= y;
+    case TCG_COND_GTU:
+        return x > y;
+    default:
+        tcg_abort();
+    }
+}
+
+static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_EQ:
+        return x == y;
+    case TCG_COND_NE:
+        return x != y;
+    case TCG_COND_LT:
+        return (int64_t)x < (int64_t)y;
+    case TCG_COND_GE:
+        return (int64_t)x >= (int64_t)y;
+    case TCG_COND_LE:
+        return (int64_t)x <= (int64_t)y;
+    case TCG_COND_GT:
+        return (int64_t)x > (int64_t)y;
+    case TCG_COND_LTU:
+        return x < y;
+    case TCG_COND_GEU:
+        return x >= y;
+    case TCG_COND_LEU:
+        return x <= y;
+    case TCG_COND_GTU:
+        return x > y;
+    default:
+        tcg_abort();
+    }
+}
+
+static bool do_constant_folding_cond_eq(TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_GT:
+    case TCG_COND_LTU:
+    case TCG_COND_LT:
+    case TCG_COND_GTU:
+    case TCG_COND_NE:
+        return 0;
+    case TCG_COND_GE:
+    case TCG_COND_GEU:
+    case TCG_COND_LE:
+    case TCG_COND_LEU:
+    case TCG_COND_EQ:
+        return 1;
+    default:
+        tcg_abort();
+    }
+}
+
+/* Return 2 if the condition can't be simplified, and the result
+   of the condition (0 or 1) if it can */
+static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
+                                       TCGArg y, TCGCond c)
+{
+    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
+        switch (op_bits(op)) {
+        case 32:
+            return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
+        case 64:
+            return do_constant_folding_cond_64(temps[x].val, temps[y].val, c);
+        default:
+            tcg_abort();
+        }
+    } else if (temps_are_copies(x, y)) {
+        return do_constant_folding_cond_eq(c);
+    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+        switch (c) {
+        case TCG_COND_LTU:
+            return 0;
+        case TCG_COND_GEU:
+            return 1;
+        default:
+            return 2;
+        }
+    } else {
+        return 2;
+    }
+}
+
+/* Return 2 if the condition can't be simplified, and the result
+   of the condition (0 or 1) if it can */
+static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
+{
+    TCGArg al = p1[0], ah = p1[1];
+    TCGArg bl = p2[0], bh = p2[1];
+
+    if (temps[bl].state == TCG_TEMP_CONST
+        && temps[bh].state == TCG_TEMP_CONST) {
+        uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
+
+        if (temps[al].state == TCG_TEMP_CONST
+            && temps[ah].state == TCG_TEMP_CONST) {
+            uint64_t a;
+            a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
+            return do_constant_folding_cond_64(a, b, c);
+        }
+        if (b == 0) {
+            switch (c) {
+            case TCG_COND_LTU:
+                return 0;
+            case TCG_COND_GEU:
+                return 1;
+            default:
+                break;
+            }
+        }
+    }
+    if (temps_are_copies(al, bl) && temps_are_copies(ah, bh)) {
+        return do_constant_folding_cond_eq(c);
+    }
+    return 2;
+}
+
+static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
+{
+    TCGArg a1 = *p1, a2 = *p2;
+    int sum = 0;
+    sum += temps[a1].state == TCG_TEMP_CONST;
+    sum -= temps[a2].state == TCG_TEMP_CONST;
+
+    /* Prefer the constant in second argument, and then the form
+       op a, a, b, which is better handled on non-RISC hosts. */
+    if (sum > 0 || (sum == 0 && dest == a2)) {
+        *p1 = a2;
+        *p2 = a1;
+        return true;
+    }
+    return false;
+}
+
+static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
+{
+    int sum = 0;
+    sum += temps[p1[0]].state == TCG_TEMP_CONST;
+    sum += temps[p1[1]].state == TCG_TEMP_CONST;
+    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
+    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    if (sum > 0) {
+        TCGArg t;
+        t = p1[0], p1[0] = p2[0], p2[0] = t;
+        t = p1[1], p1[1] = p2[1], p2[1] = t;
+        return true;
+    }
+    return false;
+}
+
 /* Propagate constants and copies, fold constant expressions. */
 static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                                     TCGArg *args, TCGOpDef *tcg_op_defs)
 {
     int i, nb_ops, op_index, nb_temps, nb_globals, nb_call_args;
+    tcg_target_ulong mask, affected;
     TCGOpcode op;
     const TCGOpDef *def;
     TCGArg *gen_args;
     TCGArg tmp;
+
     /* Array VALS has an element for each temp.
        If this temp holds a constant then its value is kept in VALS' element.
-       If this temp is a copy of other ones then this equivalence class'
-       representative is kept in VALS' element.
-       If this temp is neither copy nor constant then corresponding VALS'
-       element is unused. */
+       If this temp is a copy of other ones then the other copies are
+       available through the doubly linked circular list. */
 
     nb_temps = s->nb_temps;
     nb_globals = s->nb_globals;
-    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+    reset_all_temps(nb_temps);
 
-    nb_ops = tcg_opc_ptr - gen_opc_buf;
+    nb_ops = tcg_opc_ptr - s->gen_opc_buf;
     gen_args = args;
     for (op_index = 0; op_index < nb_ops; op_index++) {
-        op = gen_opc_buf[op_index];
+        op = s->gen_opc_buf[op_index];
         def = &tcg_op_defs[op];
         /* Do copy propagation */
-        if (!(def->flags & (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS))) {
-            assert(op != INDEX_op_call);
+        if (op == INDEX_op_call) {
+            int nb_oargs = args[0] >> 16;
+            int nb_iargs = args[0] & 0xffff;
+            for (i = nb_oargs + 1; i < nb_oargs + nb_iargs + 1; i++) {
+                if (temps[args[i]].state == TCG_TEMP_COPY) {
+                    args[i] = find_better_copy(s, args[i]);
+                }
+            }
+        } else {
             for (i = def->nb_oargs; i < def->nb_oargs + def->nb_iargs; i++) {
                 if (temps[args[i]].state == TCG_TEMP_COPY) {
-                    args[i] = temps[args[i]].val;
+                    args[i] = find_better_copy(s, args[i]);
                 }
             }
         }
@@ -312,17 +531,71 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(eqv):
         CASE_OP_32_64(nand):
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = args[1];
-                args[1] = args[2];
-                args[2] = tmp;
+            swap_commutative(args[0], &args[1], &args[2]);
+            break;
+        CASE_OP_32_64(brcond):
+            if (swap_commutative(-1, &args[0], &args[1])) {
+                args[2] = tcg_swap_cond(args[2]);
+            }
+            break;
+        CASE_OP_32_64(setcond):
+            if (swap_commutative(args[0], &args[1], &args[2])) {
+                args[3] = tcg_swap_cond(args[3]);
+            }
+            break;
+        CASE_OP_32_64(movcond):
+            if (swap_commutative(-1, &args[1], &args[2])) {
+                args[5] = tcg_swap_cond(args[5]);
+            }
+            /* For movcond, we canonicalize the "false" input reg to match
+               the destination reg so that the tcg backend can implement
+               a "move if true" operation.  */
+            if (swap_commutative(args[0], &args[4], &args[3])) {
+                args[5] = tcg_invert_cond(args[5]);
+            }
+            break;
+        case INDEX_op_add2_i32:
+            swap_commutative(args[0], &args[2], &args[4]);
+            swap_commutative(args[1], &args[3], &args[5]);
+            break;
+        case INDEX_op_mulu2_i32:
+            swap_commutative(args[0], &args[2], &args[3]);
+            break;
+        case INDEX_op_brcond2_i32:
+            if (swap_commutative2(&args[0], &args[2])) {
+                args[4] = tcg_swap_cond(args[4]);
+            }
+            break;
+        case INDEX_op_setcond2_i32:
+            if (swap_commutative2(&args[1], &args[3])) {
+                args[5] = tcg_swap_cond(args[5]);
             }
             break;
         default:
             break;
         }
 
-        /* Simplify expression if possible. */
+        /* Simplify expressions for "shift/rot r, 0, a => movi r, 0" */
+        switch (op) {
+        CASE_OP_32_64(shl):
+        CASE_OP_32_64(shr):
+        CASE_OP_32_64(sar):
+        CASE_OP_32_64(rotl):
+        CASE_OP_32_64(rotr):
+            if (temps[args[1]].state == TCG_TEMP_CONST
+                && temps[args[1]].val == 0) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
+                args += 3;
+                gen_args += 2;
+                continue;
+            }
+            break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, 0 => mov r, a" cases */
         switch (op) {
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
@@ -331,50 +604,182 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
+        CASE_OP_32_64(or):
+        CASE_OP_32_64(xor):
             if (temps[args[1]].state == TCG_TEMP_CONST) {
                 /* Proceed with possible constant folding. */
                 break;
             }
             if (temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0) {
-                if ((temps[args[0]].state == TCG_TEMP_COPY
-                    && temps[args[0]].val == args[1])
-                    || args[0] == args[1]) {
-                    args += 3;
-                    gen_opc_buf[op_index] = INDEX_op_nop;
+                if (temps_are_copies(args[0], args[1])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else {
-                    gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[1],
-                                    nb_temps, nb_globals);
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                     gen_args += 2;
-                    args += 3;
                 }
+                args += 3;
                 continue;
             }
             break;
+        default:
+            break;
+        }
+
+        /* Simplify using known-zero bits */
+        mask = -1;
+        affected = -1;
+        switch (op) {
+        CASE_OP_32_64(ext8s):
+            if ((temps[args[1]].mask & 0x80) != 0) {
+                break;
+            }
+        CASE_OP_32_64(ext8u):
+            mask = 0xff;
+            goto and_const;
+        CASE_OP_32_64(ext16s):
+            if ((temps[args[1]].mask & 0x8000) != 0) {
+                break;
+            }
+        CASE_OP_32_64(ext16u):
+            mask = 0xffff;
+            goto and_const;
+        case INDEX_op_ext32s_i64:
+            if ((temps[args[1]].mask & 0x80000000) != 0) {
+                break;
+            }
+        case INDEX_op_ext32u_i64:
+            mask = 0xffffffffU;
+            goto and_const;
+
+        CASE_OP_32_64(and):
+            mask = temps[args[2]].mask;
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+        and_const:
+                affected = temps[args[1]].mask & ~mask;
+            }
+            mask = temps[args[1]].mask & mask;
+            break;
+
+        CASE_OP_32_64(sar):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = ((tcg_target_long)temps[args[1]].mask
+                        >> temps[args[2]].val);
+            }
+            break;
+
+        CASE_OP_32_64(shr):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = temps[args[1]].mask >> temps[args[2]].val;
+            }
+            break;
+
+        CASE_OP_32_64(shl):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = temps[args[1]].mask << temps[args[2]].val;
+            }
+            break;
+
+        CASE_OP_32_64(neg):
+            /* Set to 1 all bits to the left of the rightmost.  */
+            mask = -(temps[args[1]].mask & -temps[args[1]].mask);
+            break;
+
+        CASE_OP_32_64(deposit):
+            tmp = ((1ull << args[4]) - 1);
+            mask = ((temps[args[1]].mask & ~(tmp << args[3]))
+                    | ((temps[args[2]].mask & tmp) << args[3]));
+            break;
+
+        CASE_OP_32_64(or):
+        CASE_OP_32_64(xor):
+            mask = temps[args[1]].mask | temps[args[2]].mask;
+            break;
+
+        CASE_OP_32_64(setcond):
+            mask = 1;
+            break;
+
+        CASE_OP_32_64(movcond):
+            mask = temps[args[3]].mask | temps[args[4]].mask;
+            break;
+
+        default:
+            break;
+        }
+
+        if (mask == 0) {
+            assert(def->nb_oargs == 1);
+            s->gen_opc_buf[op_index] = op_to_movi(op);
+            tcg_opt_gen_movi(gen_args, args[0], 0);
+            args += def->nb_oargs + def->nb_iargs + def->nb_cargs;
+            gen_args += 2;
+            continue;
+        }
+        if (affected == 0) {
+            assert(def->nb_oargs == 1);
+            if (temps_are_copies(args[0], args[1])) {
+                s->gen_opc_buf[op_index] = INDEX_op_nop;
+            } else if (temps[args[1]].state != TCG_TEMP_CONST) {
+                s->gen_opc_buf[op_index] = op_to_mov(op);
+                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                gen_args += 2;
+            } else {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], temps[args[1]].val);
+                gen_args += 2;
+            }
+            args += def->nb_iargs + 1;
+            continue;
+        }
+
+        /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
+        switch (op) {
+        CASE_OP_32_64(and):
         CASE_OP_32_64(mul):
             if ((temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0)) {
-                gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], 0, nb_temps, nb_globals);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
                 args += 3;
                 gen_args += 2;
                 continue;
             }
             break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, a => mov r, a" cases */
+        switch (op) {
         CASE_OP_32_64(or):
         CASE_OP_32_64(and):
-            if (args[1] == args[2]) {
-                if (args[1] == args[0]) {
-                    args += 3;
-                    gen_opc_buf[op_index] = INDEX_op_nop;
+            if (temps_are_copies(args[1], args[2])) {
+                if (temps_are_copies(args[0], args[1])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else {
-                    gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[1], nb_temps,
-                                    nb_globals);
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                     gen_args += 2;
-                    args += 3;
                 }
+                args += 3;
+                continue;
+            }
+            break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, a => movi r, 0" cases */
+        switch (op) {
+        CASE_OP_32_64(sub):
+        CASE_OP_32_64(xor):
+            if (temps_are_copies(args[1], args[2])) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
+                gen_args += 2;
+                args += 3;
                 continue;
             }
             break;
@@ -387,16 +792,13 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
            allocator where needed and possible.  Also detect copies. */
         switch (op) {
         CASE_OP_32_64(mov):
-            if ((temps[args[1]].state == TCG_TEMP_COPY
-                && temps[args[1]].val == args[0])
-                || args[0] == args[1]) {
+            if (temps_are_copies(args[0], args[1])) {
                 args += 2;
-                gen_opc_buf[op_index] = INDEX_op_nop;
+                s->gen_opc_buf[op_index] = INDEX_op_nop;
                 break;
             }
             if (temps[args[1]].state != TCG_TEMP_CONST) {
-                tcg_opt_gen_mov(s, gen_args, args[0], args[1],
-                                nb_temps, nb_globals);
+                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                 gen_args += 2;
                 args += 2;
                 break;
@@ -404,14 +806,15 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             /* Source argument is constant.  Rewrite the operation and
                let movi case handle it. */
             op = op_to_movi(op);
-            gen_opc_buf[op_index] = op;
+            s->gen_opc_buf[op_index] = op;
             args[1] = temps[args[1]].val;
             /* fallthrough */
         CASE_OP_32_64(movi):
-            tcg_opt_gen_movi(gen_args, args[0], args[1], nb_temps, nb_globals);
+            tcg_opt_gen_movi(gen_args, args[0], args[1]);
             gen_args += 2;
             args += 2;
             break;
+
         CASE_OP_32_64(not):
         CASE_OP_32_64(neg):
         CASE_OP_32_64(ext8s):
@@ -421,20 +824,15 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
             if (temps[args[1]].state == TCG_TEMP_CONST) {
-                gen_opc_buf[op_index] = op_to_movi(op);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val, 0);
-                tcg_opt_gen_movi(gen_args, args[0], tmp, nb_temps, nb_globals);
-                gen_args += 2;
-                args += 2;
-                break;
-            } else {
-                reset_temp(args[0], nb_temps, nb_globals);
-                gen_args[0] = args[0];
-                gen_args[1] = args[1];
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
                 gen_args += 2;
                 args += 2;
                 break;
             }
+            goto do_default;
+
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
         CASE_OP_32_64(mul):
@@ -453,31 +851,200 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(nor):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[2]].state == TCG_TEMP_CONST) {
-                gen_opc_buf[op_index] = op_to_movi(op);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val,
                                           temps[args[2]].val);
-                tcg_opt_gen_movi(gen_args, args[0], tmp, nb_temps, nb_globals);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
                 gen_args += 2;
                 args += 3;
                 break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(deposit):
+            if (temps[args[1]].state == TCG_TEMP_CONST
+                && temps[args[2]].state == TCG_TEMP_CONST) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tmp = ((1ull << args[4]) - 1);
+                tmp = (temps[args[1]].val & ~(tmp << args[3]))
+                      | ((temps[args[2]].val & tmp) << args[3]);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+                args += 5;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(setcond):
+            tmp = do_constant_folding_cond(op, args[1], args[2], args[3]);
+            if (tmp != 2) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(brcond):
+            tmp = do_constant_folding_cond(op, args[0], args[1], args[2]);
+            if (tmp != 2) {
+                if (tmp) {
+                    reset_all_temps(nb_temps);
+                    s->gen_opc_buf[op_index] = INDEX_op_br;
+                    gen_args[0] = args[3];
+                    gen_args += 1;
+                } else {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                }
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(movcond):
+            tmp = do_constant_folding_cond(op, args[1], args[2], args[5]);
+            if (tmp != 2) {
+                if (temps_are_copies(args[0], args[4-tmp])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                } else if (temps[args[4-tmp]].state == TCG_TEMP_CONST) {
+                    s->gen_opc_buf[op_index] = op_to_movi(op);
+                    tcg_opt_gen_movi(gen_args, args[0], temps[args[4-tmp]].val);
+                    gen_args += 2;
+                } else {
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[4-tmp]);
+                    gen_args += 2;
+                }
+                args += 6;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_add2_i32:
+        case INDEX_op_sub2_i32:
+            if (temps[args[2]].state == TCG_TEMP_CONST
+                && temps[args[3]].state == TCG_TEMP_CONST
+                && temps[args[4]].state == TCG_TEMP_CONST
+                && temps[args[5]].state == TCG_TEMP_CONST) {
+                uint32_t al = temps[args[2]].val;
+                uint32_t ah = temps[args[3]].val;
+                uint32_t bl = temps[args[4]].val;
+                uint32_t bh = temps[args[5]].val;
+                uint64_t a = ((uint64_t)ah << 32) | al;
+                uint64_t b = ((uint64_t)bh << 32) | bl;
+                TCGArg rl, rh;
+
+                if (op == INDEX_op_add2_i32) {
+                    a += b;
+                } else {
+                    a -= b;
+                }
+
+                /* We emit the extra nop when we emit the add2/sub2.  */
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+
+                rl = args[0];
+                rh = args[1];
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)a);
+                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(a >> 32));
+                gen_args += 4;
+                args += 6;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_mulu2_i32:
+            if (temps[args[2]].state == TCG_TEMP_CONST
+                && temps[args[3]].state == TCG_TEMP_CONST) {
+                uint32_t a = temps[args[2]].val;
+                uint32_t b = temps[args[3]].val;
+                uint64_t r = (uint64_t)a * b;
+                TCGArg rl, rh;
+
+                /* We emit the extra nop when we emit the mulu2.  */
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+
+                rl = args[0];
+                rh = args[1];
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)r);
+                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(r >> 32));
+                gen_args += 4;
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_brcond2_i32:
+            tmp = do_constant_folding_cond2(&args[0], &args[2], args[4]);
+            if (tmp != 2) {
+                if (tmp) {
+                    reset_all_temps(nb_temps);
+                    s->gen_opc_buf[op_index] = INDEX_op_br;
+                    gen_args[0] = args[5];
+                    gen_args += 1;
+                } else {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                }
+            } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
+                       && temps[args[2]].state == TCG_TEMP_CONST
+                       && temps[args[3]].state == TCG_TEMP_CONST
+                       && temps[args[2]].val == 0
+                       && temps[args[3]].val == 0) {
+                /* Simplify LT/GE comparisons vs zero to a single compare
+                   vs the high word of the input.  */
+                reset_all_temps(nb_temps);
+                s->gen_opc_buf[op_index] = INDEX_op_brcond_i32;
+                gen_args[0] = args[1];
+                gen_args[1] = args[3];
+                gen_args[2] = args[4];
+                gen_args[3] = args[5];
+                gen_args += 4;
             } else {
-                reset_temp(args[0], nb_temps, nb_globals);
+                goto do_default;
+            }
+            args += 6;
+            break;
+
+        case INDEX_op_setcond2_i32:
+            tmp = do_constant_folding_cond2(&args[1], &args[3], args[5]);
+            if (tmp != 2) {
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+            } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
+                       && temps[args[3]].state == TCG_TEMP_CONST
+                       && temps[args[4]].state == TCG_TEMP_CONST
+                       && temps[args[3]].val == 0
+                       && temps[args[4]].val == 0) {
+                /* Simplify LT/GE comparisons vs zero to a single compare
+                   vs the high word of the input.  */
+                s->gen_opc_buf[op_index] = INDEX_op_setcond_i32;
                 gen_args[0] = args[0];
-                gen_args[1] = args[1];
-                gen_args[2] = args[2];
-                gen_args += 3;
-                args += 3;
-                break;
+                gen_args[1] = args[2];
+                gen_args[2] = args[4];
+                gen_args[3] = args[5];
+                gen_args += 4;
+            } else {
+                goto do_default;
             }
+            args += 6;
+            break;
+
         case INDEX_op_call:
             nb_call_args = (args[0] >> 16) + (args[0] & 0xffff);
-            if (!(args[nb_call_args + 1] & (TCG_CALL_CONST | TCG_CALL_PURE))) {
+            if (!(args[nb_call_args + 1] & (TCG_CALL_NO_READ_GLOBALS |
+                                            TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i, nb_temps, nb_globals);
+                    reset_temp(i);
                 }
             }
             for (i = 0; i < (args[0] >> 16); i++) {
-                reset_temp(args[i + 1], nb_temps, nb_globals);
+                reset_temp(args[i + 1]);
             }
             i = nb_call_args + 3;
             while (i) {
@@ -487,22 +1054,20 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                 i--;
             }
             break;
-        case INDEX_op_set_label:
-        case INDEX_op_jmp:
-        case INDEX_op_br:
-        CASE_OP_32_64(brcond):
-            memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
-            for (i = 0; i < def->nb_args; i++) {
-                *gen_args = *args;
-                args++;
-                gen_args++;
-            }
-            break;
+
         default:
-            /* Default case: we do know nothing about operation so no
-               propagation is done.  We only trash output args.  */
-            for (i = 0; i < def->nb_oargs; i++) {
-                reset_temp(args[i], nb_temps, nb_globals);
+        do_default:
+            /* Default case: we know nothing about operation (or were unable
+               to compute the operation result) so no propagation is done.
+               We trash everything if the operation is the end of a basic
+               block, otherwise we only trash the output args.  "mask" is
+               the non-zero bits mask for the first output arg.  */
+            if (def->flags & TCG_OPF_BB_END) {
+                reset_all_temps(nb_temps);
+            } else {
+                for (i = 0; i < def->nb_oargs; i++) {
+                    reset_temp(args[i]);
+                }
             }
             for (i = 0; i < def->nb_args; i++) {
                 gen_args[i] = args[i];
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 0cff1812574..29ca934e66e 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -39,8 +39,6 @@ static uint8_t *tb_ret_addr;
 #define LR_OFFSET 4
 #endif
 
-#define FAST_PATH
-
 #ifndef GUEST_BASE
 #define GUEST_BASE 0
 #endif
@@ -221,12 +219,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return ARRAY_SIZE (tcg_target_call_iarg_regs);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -248,7 +240,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
 #if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
@@ -256,11 +247,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
 #endif
 #endif
-#else /* !AREG0 */
-#if TARGET_LONG_BITS == 64
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-#endif
-#endif
         break;
     case 'K':                   /* qemu_st[8..32] constraint */
         ct->ct |= TCG_CT_REG;
@@ -268,7 +254,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
 #if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
@@ -276,11 +261,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R8);
 #endif
 #endif
-#else /* !AREG0 */
-#if TARGET_LONG_BITS == 64
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
-#endif
-#endif
         break;
     case 'M':                   /* qemu_st64 constraint */
         ct->ct |= TCG_CT_REG;
@@ -290,12 +270,10 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
-#if defined(CONFIG_TCG_PASS_AREG0)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R8);
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R9);
 #endif
-#endif
         break;
 #else
     case 'L':
@@ -404,6 +382,7 @@ static int tcg_target_const_match(tcg_target_long val,
 #define ORC    XO31(412)
 #define EQV    XO31(284)
 #define NAND   XO31(476)
+#define ISEL   XO31( 15)
 
 #define LBZX   XO31( 87)
 #define LHZX   XO31(279)
@@ -456,7 +435,7 @@ enum {
     CR_SO
 };
 
-static const uint32_t tcg_to_bc[10] = {
+static const uint32_t tcg_to_bc[] = {
     [TCG_COND_EQ]  = BC | BI (7, CR_EQ) | BO_COND_TRUE,
     [TCG_COND_NE]  = BC | BI (7, CR_EQ) | BO_COND_FALSE,
     [TCG_COND_LT]  = BC | BI (7, CR_LT) | BO_COND_TRUE,
@@ -539,9 +518,39 @@ static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+static void add_qemu_ldst_label (TCGContext *s,
+                                 int is_ld,
+                                 int opc,
+                                 int data_reg,
+                                 int data_reg2,
+                                 int addrlo_reg,
+                                 int addrhi_reg,
+                                 int mem_index,
+                                 uint8_t *raddr,
+                                 uint8_t *label_ptr)
+{
+    int idx;
+    TCGLabelQemuLdst *label;
+
+    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
+        tcg_abort();
+    }
+
+    idx = s->nb_qemu_ldst_labels++;
+    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = data_reg;
+    label->datahi_reg = data_reg2;
+    label->addrlo_reg = addrlo_reg;
+    label->addrhi_reg = addrhi_reg;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
+
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -559,55 +568,15 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
-#endif
+static void *ld_trampolines[4];
+static void *st_trampolines[4];
 
-static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_tlb_check (TCGContext *s, int r0, int r1, int r2,
+                               int addr_reg, int addr_reg2, int s_bits,
+                               int offset1, int offset2, uint8_t **label_ptr)
 {
-    int addr_reg, data_reg, data_reg2, r0, r1, rbase, bswap;
-#ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits, r2, ir;
-    void *label1_ptr, *label2_ptr;
-#if TARGET_LONG_BITS == 64
-    int addr_reg2;
-#endif
-#endif
-
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
-
-#ifdef CONFIG_SOFTMMU
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#endif
-    mem_index = *args;
-    s_bits = opc & 3;
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
+    uint16_t retranst;
 
     tcg_out32 (s, (RLWINM
                    | RA (r0)
@@ -621,7 +590,7 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
     tcg_out32 (s, (LWZU
                    | RT (r1)
                    | RA (r0)
-                   | offsetof (CPUArchState, tlb_table[mem_index][0].addr_read)
+                   | offset1
                    )
         );
     tcg_out32 (s, (RLWINM
@@ -639,79 +608,58 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
     tcg_out32 (s, CMP | BF (6) | RA (addr_reg2) | RB (r1));
     tcg_out32 (s, CRAND | BT (7, CR_EQ) | BA (6, CR_EQ) | BB (7, CR_EQ));
 #endif
+    *label_ptr = s->code_ptr;
+    retranst = ((uint16_t *) s->code_ptr)[1] & ~3;
+    tcg_out32 (s, BC | BI (7, CR_EQ) | retranst | BO_COND_FALSE);
 
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-#endif
-
-    /* slow path */
-    ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, TCG_AREG0);
-#endif
-#if TARGET_LONG_BITS == 32
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#else
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-    ir |= 1;
-#endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg2);
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#endif
-    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-
-    tcg_out_call (s, (tcg_target_long) qemu_ld_helpers[s_bits], 1);
-    switch (opc) {
-    case 0|4:
-        tcg_out32 (s, EXTSB | RA (data_reg) | RS (3));
-        break;
-    case 1|4:
-        tcg_out32 (s, EXTSH | RA (data_reg) | RS (3));
-        break;
-    case 0:
-    case 1:
-    case 2:
-        if (data_reg != 3)
-            tcg_out_mov (s, TCG_TYPE_I32, data_reg, 3);
-        break;
-    case 3:
-        if (data_reg == 3) {
-            if (data_reg2 == 4) {
-                tcg_out_mov (s, TCG_TYPE_I32, 0, 4);
-                tcg_out_mov (s, TCG_TYPE_I32, 4, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 0);
-            }
-            else {
-                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 4);
-            }
-        }
-        else {
-            if (data_reg != 4) tcg_out_mov (s, TCG_TYPE_I32, data_reg, 4);
-            if (data_reg2 != 3) tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
-        }
-        break;
-    }
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
-
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
-#endif
-
-    /* r0 now contains &env->tlb_table[mem_index][index].addr_read */
+    /* r0 now contains &env->tlb_table[mem_index][index].addr_x */
     tcg_out32 (s, (LWZ
                    | RT (r0)
                    | RA (r0)
-                   | (offsetof (CPUTLBEntry, addend)
-                      - offsetof (CPUTLBEntry, addr_read))
-                   ));
+                   | offset2
+                   )
+        );
     /* r0 = env->tlb_table[mem_index][index].addend */
     tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (addr_reg));
     /* r0 = env->tlb_table[mem_index][index].addend + addr */
 
+}
+#endif
+
+static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+{
+    int addr_reg, data_reg, data_reg2, r0, r1, rbase, bswap;
+#ifdef CONFIG_SOFTMMU
+    int mem_index, s_bits, r2, addr_reg2;
+    uint8_t *label_ptr;
+#endif
+
+    data_reg = *args++;
+    if (opc == 3)
+        data_reg2 = *args++;
+    else
+        data_reg2 = 0;
+    addr_reg = *args++;
+
+#ifdef CONFIG_SOFTMMU
+#if TARGET_LONG_BITS == 64
+    addr_reg2 = *args++;
+#else
+    addr_reg2 = 0;
+#endif
+    mem_index = *args;
+    s_bits = opc & 3;
+    r0 = 3;
+    r1 = 4;
+    r2 = 0;
+    rbase = 0;
+
+    tcg_out_tlb_check (
+        s, r0, r1, r2, addr_reg, addr_reg2, s_bits,
+        offsetof (CPUArchState, tlb_table[mem_index][0].addr_read),
+        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_read),
+        &label_ptr
+        );
 #else  /* !CONFIG_SOFTMMU */
     r0 = addr_reg;
     r1 = 3;
@@ -777,9 +725,17 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
         }
         break;
     }
-
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
+    add_qemu_ldst_label (s,
+                         1,
+                         opc,
+                         data_reg,
+                         data_reg2,
+                         addr_reg,
+                         addr_reg2,
+                         mem_index,
+                         s->code_ptr,
+                         label_ptr);
 #endif
 }
 
@@ -787,11 +743,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, r0, r1, data_reg, data_reg2, bswap, rbase;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, r2, ir;
-    void *label1_ptr, *label2_ptr;
-#if TARGET_LONG_BITS == 64
-    int addr_reg2;
-#endif
+    int mem_index, r2, addr_reg2;
+    uint8_t *label_ptr;
 #endif
 
     data_reg = *args++;
@@ -804,6 +757,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 #ifdef CONFIG_SOFTMMU
 #if TARGET_LONG_BITS == 64
     addr_reg2 = *args++;
+#else
+    addr_reg2 = 0;
 #endif
     mem_index = *args;
     r0 = 3;
@@ -811,54 +766,162 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
     r2 = 0;
     rbase = 0;
 
-    tcg_out32 (s, (RLWINM
-                   | RA (r0)
-                   | RS (addr_reg)
-                   | SH (32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS))
-                   | MB (32 - (CPU_TLB_ENTRY_BITS + CPU_TLB_BITS))
-                   | ME (31 - CPU_TLB_ENTRY_BITS)
-                   )
-        );
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (TCG_AREG0));
-    tcg_out32 (s, (LWZU
-                   | RT (r1)
-                   | RA (r0)
-                   | offsetof (CPUArchState, tlb_table[mem_index][0].addr_write)
-                   )
-        );
-    tcg_out32 (s, (RLWINM
-                   | RA (r2)
-                   | RS (addr_reg)
-                   | SH (0)
-                   | MB ((32 - opc) & 31)
-                   | ME (31 - TARGET_PAGE_BITS)
-                   )
+    tcg_out_tlb_check (
+        s, r0, r1, r2, addr_reg, addr_reg2, opc & 3,
+        offsetof (CPUArchState, tlb_table[mem_index][0].addr_write),
+        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_write),
+        &label_ptr
         );
+#else  /* !CONFIG_SOFTMMU */
+    r0 = addr_reg;
+    r1 = 3;
+    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+#endif
 
-    tcg_out32 (s, CMP | (7 << 23) | RA (r2) | RB (r1));
-#if TARGET_LONG_BITS == 64
-    tcg_out32 (s, LWZ | RT (r1) | RA (r0) | 4);
-    tcg_out32 (s, CMP | BF (6) | RA (addr_reg2) | RB (r1));
-    tcg_out32 (s, CRAND | BT (7, CR_EQ) | BA (6, CR_EQ) | BB (7, CR_EQ));
+#ifdef TARGET_WORDS_BIGENDIAN
+    bswap = 0;
+#else
+    bswap = 1;
+#endif
+    switch (opc) {
+    case 0:
+        tcg_out32 (s, STBX | SAB (data_reg, rbase, r0));
+        break;
+    case 1:
+        if (bswap)
+            tcg_out32 (s, STHBRX | SAB (data_reg, rbase, r0));
+        else
+            tcg_out32 (s, STHX | SAB (data_reg, rbase, r0));
+        break;
+    case 2:
+        if (bswap)
+            tcg_out32 (s, STWBRX | SAB (data_reg, rbase, r0));
+        else
+            tcg_out32 (s, STWX | SAB (data_reg, rbase, r0));
+        break;
+    case 3:
+        if (bswap) {
+            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
+            tcg_out32 (s, STWBRX | SAB (data_reg,  rbase, r0));
+            tcg_out32 (s, STWBRX | SAB (data_reg2, rbase, r1));
+        }
+        else {
+#ifdef CONFIG_USE_GUEST_BASE
+            tcg_out32 (s, STWX | SAB (data_reg2, rbase, r0));
+            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
+            tcg_out32 (s, STWX | SAB (data_reg,  rbase, r1));
+#else
+            tcg_out32 (s, STW | RS (data_reg2) | RA (r0));
+            tcg_out32 (s, STW | RS (data_reg) | RA (r0) | 4);
 #endif
+        }
+        break;
+    }
 
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
+#ifdef CONFIG_SOFTMMU
+    add_qemu_ldst_label (s,
+                         0,
+                         opc,
+                         data_reg,
+                         data_reg2,
+                         addr_reg,
+                         addr_reg2,
+                         mem_index,
+                         s->code_ptr,
+                         label_ptr);
 #endif
+}
+
+#if defined(CONFIG_SOFTMMU)
+static void tcg_out_qemu_ld_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int ir;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    int addr_reg = label->addrlo_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
 
     /* slow path */
-    ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, TCG_AREG0);
+    ir = 4;
+#if TARGET_LONG_BITS == 32
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
+#else
+#ifdef TCG_TARGET_CALL_ALIGN_ARGS
+    ir |= 1;
+#endif
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #endif
+    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
+    tcg_out_call (s, (tcg_target_long) ld_trampolines[s_bits], 1);
+    tcg_out32 (s, (tcg_target_long) raddr);
+    switch (opc) {
+    case 0|4:
+        tcg_out32 (s, EXTSB | RA (data_reg) | RS (3));
+        break;
+    case 1|4:
+        tcg_out32 (s, EXTSH | RA (data_reg) | RS (3));
+        break;
+    case 0:
+    case 1:
+    case 2:
+        if (data_reg != 3)
+            tcg_out_mov (s, TCG_TYPE_I32, data_reg, 3);
+        break;
+    case 3:
+        if (data_reg == 3) {
+            if (data_reg2 == 4) {
+                tcg_out_mov (s, TCG_TYPE_I32, 0, 4);
+                tcg_out_mov (s, TCG_TYPE_I32, 4, 3);
+                tcg_out_mov (s, TCG_TYPE_I32, 3, 0);
+            }
+            else {
+                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
+                tcg_out_mov (s, TCG_TYPE_I32, 3, 4);
+            }
+        }
+        else {
+            if (data_reg != 4) tcg_out_mov (s, TCG_TYPE_I32, data_reg, 4);
+            if (data_reg2 != 3) tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
+        }
+        break;
+    }
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_b (s, 0, (tcg_target_long) raddr);
+}
+
+static void tcg_out_qemu_st_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int ir;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    int addr_reg = label->addrlo_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    /* resolve label address */
+    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
+
+    /* slow path */
+    ir = 4;
 #if TARGET_LONG_BITS == 32
     tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #else
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
     ir |= 1;
 #endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg2);
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
     tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #endif
 
@@ -893,75 +956,39 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
     ir++;
 
     tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-    tcg_out_call (s, (tcg_target_long) qemu_st_helpers[opc], 1);
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
-
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
-#endif
-
-    tcg_out32 (s, (LWZ
-                   | RT (r0)
-                   | RA (r0)
-                   | (offsetof (CPUTLBEntry, addend)
-                      - offsetof (CPUTLBEntry, addr_write))
-                   ));
-    /* r0 = env->tlb_table[mem_index][index].addend */
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (addr_reg));
-    /* r0 = env->tlb_table[mem_index][index].addend + addr */
-
-#else  /* !CONFIG_SOFTMMU */
-    r0 = addr_reg;
-    r1 = 3;
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
-#endif
+    tcg_out_call (s, (tcg_target_long) st_trampolines[opc], 1);
+    tcg_out32 (s, (tcg_target_long) raddr);
+    tcg_out_b (s, 0, (tcg_target_long) raddr);
+}
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 0;
-#else
-    bswap = 1;
-#endif
-    switch (opc) {
-    case 0:
-        tcg_out32 (s, STBX | SAB (data_reg, rbase, r0));
-        break;
-    case 1:
-        if (bswap)
-            tcg_out32 (s, STHBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STHX | SAB (data_reg, rbase, r0));
-        break;
-    case 2:
-        if (bswap)
-            tcg_out32 (s, STWBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STWX | SAB (data_reg, rbase, r0));
-        break;
-    case 3:
-        if (bswap) {
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWBRX | SAB (data_reg,  rbase, r0));
-            tcg_out32 (s, STWBRX | SAB (data_reg2, rbase, r1));
+void tcg_out_tb_finalize(TCGContext *s)
+{
+    int i;
+    TCGLabelQemuLdst *label;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
+        label = (TCGLabelQemuLdst *) &s->qemu_ldst_labels[i];
+        if (label->is_ld) {
+            tcg_out_qemu_ld_slow_path (s, label);
         }
         else {
-#ifdef CONFIG_USE_GUEST_BASE
-            tcg_out32 (s, STWX | SAB (data_reg2, rbase, r0));
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWX | SAB (data_reg,  rbase, r1));
-#else
-            tcg_out32 (s, STW | RS (data_reg2) | RA (r0));
-            tcg_out32 (s, STW | RS (data_reg) | RA (r0) | 4);
-#endif
+            tcg_out_qemu_st_slow_path (s, label);
         }
-        break;
     }
+}
+#endif
 
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
-#endif
+static void emit_ldst_trampoline (TCGContext *s, const void *ptr)
+{
+    tcg_out32 (s, MFSPR | RT (3) | LR);
+    tcg_out32 (s, ADDI | RT (3) | RA (3) | 4);
+    tcg_out32 (s, MTSPR | RS (3) | LR);
+    tcg_out_mov (s, TCG_TYPE_I32, 3, TCG_AREG0);
+    tcg_out_b (s, 0, (tcg_target_long) ptr);
 }
+#endif
 
 static void tcg_target_qemu_prologue (TCGContext *s)
 {
@@ -1023,6 +1050,16 @@ static void tcg_target_qemu_prologue (TCGContext *s)
     tcg_out32 (s, MTSPR | RS (0) | LR);
     tcg_out32 (s, ADDI | RT (1) | RA (1) | frame_size);
     tcg_out32 (s, BCLR | BO_ALWAYS);
+
+#ifdef CONFIG_SOFTMMU
+    for (i = 0; i < 4; ++i) {
+        ld_trampolines[i] = s->code_ptr;
+        emit_ldst_trampoline (s, qemu_ld_helpers[i]);
+
+        st_trampolines[i] = s->code_ptr;
+        emit_ldst_trampoline (s, qemu_st_helpers[i]);
+    }
+#endif
 }
 
 static void tcg_out_ld (TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
@@ -1307,6 +1344,72 @@ static void tcg_out_setcond2 (TCGContext *s, const TCGArg *args,
         );
 }
 
+static void tcg_out_movcond (TCGContext *s, TCGCond cond,
+                             TCGArg dest,
+                             TCGArg c1, TCGArg c2,
+                             TCGArg v1, TCGArg v2,
+                             int const_c2)
+{
+    tcg_out_cmp (s, cond, c1, c2, const_c2, 7);
+
+    if (1) {
+        /* At least here on 7747A bit twiddling hacks are outperformed
+           by jumpy code (the testing was not scientific) */
+        if (dest == v2) {
+            cond = tcg_invert_cond (cond);
+            v2 = v1;
+        }
+        else {
+            if (dest != v1) {
+                tcg_out_mov (s, TCG_TYPE_I32, dest, v1);
+            }
+        }
+        /* Branch forward over one insn */
+        tcg_out32 (s, tcg_to_bc[cond] | 8);
+        tcg_out_mov (s, TCG_TYPE_I32, dest, v2);
+    }
+    else {
+        /* isel version, "if (1)" above should be replaced once a way
+           to figure out availability of isel on the underlying
+           hardware is found */
+        int tab, bc;
+
+        switch (cond) {
+        case TCG_COND_EQ:
+            tab = TAB (dest, v1, v2);
+            bc = CR_EQ;
+            break;
+        case TCG_COND_NE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_EQ;
+            break;
+        case TCG_COND_LTU:
+        case TCG_COND_LT:
+            tab = TAB (dest, v1, v2);
+            bc = CR_LT;
+            break;
+        case TCG_COND_GEU:
+        case TCG_COND_GE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_LT;
+            break;
+        case TCG_COND_LEU:
+        case TCG_COND_LE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_GT;
+            break;
+        case TCG_COND_GTU:
+        case TCG_COND_GT:
+            tab = TAB (dest, v1, v2);
+            bc = CR_GT;
+            break;
+        default:
+            tcg_abort ();
+        }
+        tcg_out32 (s, ISEL | tab | ((bc + 28) << 6));
+    }
+}
+
 static void tcg_out_brcond (TCGContext *s, TCGCond cond,
                             TCGArg arg1, TCGArg arg2, int const_arg2,
                             int label_index)
@@ -1394,15 +1497,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_call (s, args[0], const_args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_b (s, 0, args[0]);
-        }
-        else {
-            tcg_out32 (s, MTSPR | RS (args[0]) | CTR);
-            tcg_out32 (s, BCCTR | BO_ALWAYS);
-        }
-        break;
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -1864,6 +1958,13 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
             );
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond (s, args[5], args[0],
+                         args[1], args[2],
+                         args[3], args[4],
+                         const_args[2]);
+        break;
+
     default:
         tcg_dump_ops (s);
         tcg_abort ();
@@ -1874,7 +1975,6 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1960,6 +2060,7 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_ext16u_i32, { "r", "r" } },
 
     { INDEX_op_deposit_i32, { "r", "0", "r" } },
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "r" } },
 
     { -1 },
 };
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index 2f37fd289bc..ea267695814 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_PPC 
 #define TCG_TARGET_PPC 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -92,11 +93,12 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         1
 #define TCG_TARGET_HAS_nor_i32          1
 #define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_movcond_i32      1
 
 #define TCG_AREG0 TCG_REG_R27
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
 #define tcg_qemu_tb_exec(env, tb_ptr) \
     ((long __attribute__ ((longcall)) \
       (*)(void *, void *))code_gen_prologue)(env, tb_ptr)
+
+#endif
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 27a0ae88ec4..833fe0c10b9 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -208,12 +208,6 @@ static void patch_reloc (uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static int tcg_target_get_call_iarg_regs_count (int flags)
-{
-    return ARRAY_SIZE (tcg_target_call_iarg_regs);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
 {
@@ -235,10 +229,8 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
 #ifdef CONFIG_SOFTMMU
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
 #endif
-#endif
         break;
     case 'S':                   /* qemu_st constraint */
         ct->ct |= TCG_CT_REG;
@@ -247,10 +239,8 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
 #ifdef CONFIG_SOFTMMU
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R6);
 #endif
-#endif
         break;
     case 'Z':
         ct->ct |= TCG_CT_CONST_U32;
@@ -428,7 +418,7 @@ enum {
     CR_SO
 };
 
-static const uint32_t tcg_to_bc[10] = {
+static const uint32_t tcg_to_bc[] = {
     [TCG_COND_EQ]  = BC | BI (7, CR_EQ) | BO_COND_TRUE,
     [TCG_COND_NE]  = BC | BI (7, CR_EQ) | BO_COND_FALSE,
     [TCG_COND_LT]  = BC | BI (7, CR_LT) | BO_COND_TRUE,
@@ -556,9 +546,8 @@ static void tcg_out_ldsta (TCGContext *s, int ret, int addr,
 
 #if defined (CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -576,25 +565,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 static void tcg_out_tlb_read (TCGContext *s, int r0, int r1, int r2,
                               int addr_reg, int s_bits, int offset)
@@ -676,9 +646,7 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
 
     /* slow path */
     ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-#endif
     tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
     tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
 
@@ -827,9 +795,7 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 
     /* slow path */
     ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-#endif
     tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
     tcg_out_rld (s, RLDICL, ir++, data_reg, 0, 64 - (1 << (3 + opc)));
     tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
@@ -1279,15 +1245,6 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_call (s, args[0], const_args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_b (s, 0, args[0]);
-        }
-        else {
-            tcg_out32 (s, MTSPR | RS (args[0]) | CTR);
-            tcg_out32 (s, BCCTR | BO_ALWAYS);
-        }
-        break;
     case INDEX_op_movi_i32:
         tcg_out_movi (s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -1622,7 +1579,6 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index 97eec0843de..9b8e9a07b83 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_PPC64 
 #define TCG_TARGET_PPC64 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -83,6 +84,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rot_i64          0
@@ -103,8 +105,10 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 
 #define TCG_AREG0 TCG_REG_R27
 
-#define TCG_TARGET_HAS_GUEST_BASE
 #define TCG_TARGET_EXTEND_ARGS 1
+
+#endif
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index 04662c15fd7..e12a15221a1 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -268,7 +268,7 @@ static const int tcg_target_call_oarg_regs[] = {
 #define S390_CC_ALWAYS  15
 
 /* Condition codes that result from a COMPARE and COMPARE LOGICAL.  */
-static const uint8_t tcg_cond_to_s390_cond[10] = {
+static const uint8_t tcg_cond_to_s390_cond[] = {
     [TCG_COND_EQ]  = S390_CC_EQ,
     [TCG_COND_NE]  = S390_CC_NE,
     [TCG_COND_LT]  = S390_CC_LT,
@@ -284,7 +284,7 @@ static const uint8_t tcg_cond_to_s390_cond[10] = {
 /* Condition codes that result from a LOAD AND TEST.  Here, we have no
    unsigned instruction variation, however since the test is vs zero we
    can re-map the outcomes appropriately.  */
-static const uint8_t tcg_cond_to_ltr_cond[10] = {
+static const uint8_t tcg_cond_to_ltr_cond[] = {
     [TCG_COND_EQ]  = S390_CC_EQ,
     [TCG_COND_NE]  = S390_CC_NE,
     [TCG_COND_LT]  = S390_CC_LT,
@@ -299,9 +299,8 @@ static const uint8_t tcg_cond_to_ltr_cond[10] = {
 
 #ifdef CONFIG_SOFTMMU
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -319,25 +318,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 #endif
 
 static uint8_t *tb_ret_addr;
@@ -376,11 +356,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return sizeof(tcg_target_call_iarg_regs) / sizeof(int);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -1138,7 +1113,7 @@ static void tgen64_xori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
 static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
                     TCGArg c2, int c2const)
 {
-    bool is_unsigned = (c > TCG_COND_GT);
+    bool is_unsigned = is_unsigned_cond(c);
     if (c2const) {
         if (c2 == 0) {
             if (type == TCG_TYPE_I32) {
@@ -1507,29 +1482,25 @@ static void tcg_prepare_qemu_ldst(TCGContext* s, TCGReg data_reg,
             tcg_abort();
         }
         tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
         /* XXX/FIXME: suboptimal */
-        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
+                    tcg_target_call_iarg_regs[2]);
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
                     tcg_target_call_iarg_regs[1]);
-        tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
                     tcg_target_call_iarg_regs[0]);
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
                     TCG_AREG0);
-#endif
         tgen_calli(s, (tcg_target_ulong)qemu_st_helpers[s_bits]);
     } else {
         tcg_out_movi(s, TCG_TYPE_I32, arg1, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
         /* XXX/FIXME: suboptimal */
-        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                    tcg_target_call_iarg_regs[2]);
         tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
                     tcg_target_call_iarg_regs[1]);
-        tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
                     tcg_target_call_iarg_regs[0]);
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
                     TCG_AREG0);
-#endif
         tgen_calli(s, (tcg_target_ulong)qemu_ld_helpers[s_bits]);
 
         /* sign extension */
@@ -2066,11 +2037,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-    case INDEX_op_jmp:
-        /* This one is obsolete and never emitted.  */
-        tcg_abort();
-        break;
-
     default:
         fprintf(stderr,"unimplemented opc 0x%x\n",opc);
         tcg_abort();
@@ -2081,7 +2047,6 @@ static const TCGTargetOpDef s390_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index d12f90b7055..c87b4138b5d 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_S390 
 #define TCG_TARGET_S390 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -63,6 +64,7 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -84,10 +86,9 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
 /* used for function call generation */
 #define TCG_REG_CALL_STACK		TCG_REG_R15
 #define TCG_TARGET_STACK_ALIGN		8
@@ -96,7 +97,6 @@ typedef enum TCGReg {
 #define TCG_TARGET_EXTEND_ARGS 1
 
 enum {
-    /* Note: must be synced with dyngen-exec.h */
     TCG_AREG0 = TCG_REG_R10,
 };
 
@@ -104,3 +104,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
 }
+
+#endif
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 247a27814ab..03db514a1de 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -59,10 +59,14 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 };
 #endif
 
-#ifdef CONFIG_TCG_PASS_AREG0
-#define ARG_OFFSET 1
+/* Define some temporary registers.  T2 is used for constant generation.  */
+#define TCG_REG_T1  TCG_REG_G1
+#define TCG_REG_T2  TCG_REG_O7
+
+#ifdef CONFIG_USE_GUEST_BASE
+# define TCG_GUEST_BASE_REG TCG_REG_I5
 #else
-#define ARG_OFFSET 0
+# define TCG_GUEST_BASE_REG TCG_REG_G0
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
@@ -74,11 +78,25 @@ static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_L5,
     TCG_REG_L6,
     TCG_REG_L7,
+
     TCG_REG_I0,
     TCG_REG_I1,
     TCG_REG_I2,
     TCG_REG_I3,
     TCG_REG_I4,
+    TCG_REG_I5,
+
+    TCG_REG_G2,
+    TCG_REG_G3,
+    TCG_REG_G4,
+    TCG_REG_G5,
+
+    TCG_REG_O0,
+    TCG_REG_O1,
+    TCG_REG_O2,
+    TCG_REG_O3,
+    TCG_REG_O4,
+    TCG_REG_O5,
 };
 
 static const int tcg_target_call_iarg_regs[6] = {
@@ -97,105 +115,6 @@ static const int tcg_target_call_oarg_regs[] = {
     TCG_REG_O3,
 };
 
-static inline int check_fit_tl(tcg_target_long val, unsigned int bits)
-{
-    return (val << ((sizeof(tcg_target_long) * 8 - bits))
-            >> (sizeof(tcg_target_long) * 8 - bits)) == val;
-}
-
-static inline int check_fit_i32(uint32_t val, unsigned int bits)
-{
-    return ((val << (32 - bits)) >> (32 - bits)) == val;
-}
-
-static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
-{
-    value += addend;
-    switch (type) {
-    case R_SPARC_32:
-        if (value != (uint32_t)value)
-            tcg_abort();
-        *(uint32_t *)code_ptr = value;
-        break;
-    case R_SPARC_WDISP22:
-        value -= (long)code_ptr;
-        value >>= 2;
-        if (!check_fit_tl(value, 22))
-            tcg_abort();
-        *(uint32_t *)code_ptr = ((*(uint32_t *)code_ptr) & ~0x3fffff) | value;
-        break;
-    case R_SPARC_WDISP19:
-        value -= (long)code_ptr;
-        value >>= 2;
-        if (!check_fit_tl(value, 19))
-            tcg_abort();
-        *(uint32_t *)code_ptr = ((*(uint32_t *)code_ptr) & ~0x7ffff) | value;
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 6;
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        break;
-    case 'L': /* qemu_ld/st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        // Helper args
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O3);
-#endif
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S11;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_S13;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct;
-
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST)
-        return 1;
-    else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11))
-        return 1;
-    else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13))
-        return 1;
-    else
-        return 0;
-}
-
 #define INSN_OP(x)  ((x) << 30)
 #define INSN_OP2(x) ((x) << 22)
 #define INSN_OP3(x) ((x) << 19)
@@ -205,12 +124,13 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define INSN_RS2(x) (x)
 #define INSN_ASI(x) ((x) << 5)
 
+#define INSN_IMM10(x) ((1 << 13) | ((x) & 0x3ff))
 #define INSN_IMM11(x) ((1 << 13) | ((x) & 0x7ff))
 #define INSN_IMM13(x) ((1 << 13) | ((x) & 0x1fff))
+#define INSN_OFF16(x) ((((x) >> 2) & 0x3fff) | ((((x) >> 16) & 3) << 20))
 #define INSN_OFF19(x) (((x) >> 2) & 0x07ffff)
-#define INSN_OFF22(x) (((x) >> 2) & 0x3fffff)
+#define INSN_COND(x) ((x) << 25)
 
-#define INSN_COND(x, a) (((x) << 25) | ((a) << 29))
 #define COND_N     0x0
 #define COND_E     0x1
 #define COND_LE    0x2
@@ -227,11 +147,26 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define COND_CC    0xd
 #define COND_POS   0xe
 #define COND_VC    0xf
-#define BA         (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2))
+#define BA         (INSN_OP(0) | INSN_COND(COND_A) | INSN_OP2(0x2))
+
+#define RCOND_Z    1
+#define RCOND_LEZ  2
+#define RCOND_LZ   3
+#define RCOND_NZ   5
+#define RCOND_GZ   6
+#define RCOND_GEZ  7
 
 #define MOVCC_ICC  (1 << 18)
 #define MOVCC_XCC  (1 << 18 | 1 << 12)
 
+#define BPCC_ICC   0
+#define BPCC_XCC   (2 << 20)
+#define BPCC_PT    (1 << 19)
+#define BPCC_PN    0
+#define BPCC_A     (1 << 29)
+
+#define BPR_PT     BPCC_PT
+
 #define ARITH_ADD  (INSN_OP(2) | INSN_OP3(0x00))
 #define ARITH_ADDCC (INSN_OP(2) | INSN_OP3(0x10))
 #define ARITH_AND  (INSN_OP(2) | INSN_OP3(0x01))
@@ -242,7 +177,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ARITH_XOR  (INSN_OP(2) | INSN_OP3(0x03))
 #define ARITH_SUB  (INSN_OP(2) | INSN_OP3(0x04))
 #define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14))
-#define ARITH_ADDX (INSN_OP(2) | INSN_OP3(0x10))
+#define ARITH_ADDX (INSN_OP(2) | INSN_OP3(0x08))
 #define ARITH_SUBX (INSN_OP(2) | INSN_OP3(0x0c))
 #define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a))
 #define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e))
@@ -251,6 +186,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ARITH_UDIVX (INSN_OP(2) | INSN_OP3(0x0d))
 #define ARITH_SDIVX (INSN_OP(2) | INSN_OP3(0x2d))
 #define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c))
+#define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f))
 
 #define SHIFT_SLL  (INSN_OP(2) | INSN_OP3(0x25))
 #define SHIFT_SRL  (INSN_OP(2) | INSN_OP3(0x26))
@@ -294,6 +230,119 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ASI_PRIMARY_LITTLE 0x88
 #endif
 
+#define LDUH_LE    (LDUHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSH_LE    (LDSHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDUW_LE    (LDUWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSW_LE    (LDSWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDX_LE     (LDXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+#define STH_LE     (STHA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STW_LE     (STWA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STX_LE     (STXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+static inline int check_fit_tl(tcg_target_long val, unsigned int bits)
+{
+    return (val << ((sizeof(tcg_target_long) * 8 - bits))
+            >> (sizeof(tcg_target_long) * 8 - bits)) == val;
+}
+
+static inline int check_fit_i32(uint32_t val, unsigned int bits)
+{
+    return ((val << (32 - bits)) >> (32 - bits)) == val;
+}
+
+static void patch_reloc(uint8_t *code_ptr, int type,
+                        tcg_target_long value, tcg_target_long addend)
+{
+    uint32_t insn;
+    value += addend;
+    switch (type) {
+    case R_SPARC_32:
+        if (value != (uint32_t)value) {
+            tcg_abort();
+        }
+        *(uint32_t *)code_ptr = value;
+        break;
+    case R_SPARC_WDISP16:
+        value -= (long)code_ptr;
+        if (!check_fit_tl(value >> 2, 16)) {
+            tcg_abort();
+        }
+        insn = *(uint32_t *)code_ptr;
+        insn &= ~INSN_OFF16(-1);
+        insn |= INSN_OFF16(value);
+        *(uint32_t *)code_ptr = insn;
+        break;
+    case R_SPARC_WDISP19:
+        value -= (long)code_ptr;
+        if (!check_fit_tl(value >> 2, 19)) {
+            tcg_abort();
+        }
+        insn = *(uint32_t *)code_ptr;
+        insn &= ~INSN_OFF19(-1);
+        insn |= INSN_OFF19(value);
+        *(uint32_t *)code_ptr = insn;
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        break;
+    case 'L': /* qemu_ld/st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        // Helper args
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_S11;
+        break;
+    case 'J':
+        ct->ct |= TCG_CT_CONST_S13;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
 static inline void tcg_out_arith(TCGContext *s, int rd, int rs1, int rs2,
                                  int op)
 {
@@ -318,7 +367,9 @@ static void tcg_out_arithc(TCGContext *s, int rd, int rs1,
 static inline void tcg_out_mov(TCGContext *s, TCGType type,
                                TCGReg ret, TCGReg arg)
 {
-    tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
+    if (ret != arg) {
+        tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
+    }
 }
 
 static inline void tcg_out_sethi(TCGContext *s, int ret, uint32_t arg)
@@ -359,71 +410,50 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
         tcg_out_sethi(s, ret, ~arg);
         tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR);
     } else {
-        tcg_out_movi_imm32(s, TCG_REG_I4, arg >> (TCG_TARGET_REG_BITS / 2));
-        tcg_out_arithi(s, TCG_REG_I4, TCG_REG_I4, 32, SHIFT_SLLX);
-        tcg_out_movi_imm32(s, ret, arg);
-        tcg_out_arith(s, ret, ret, TCG_REG_I4, ARITH_OR);
+        tcg_out_movi_imm32(s, ret, arg >> (TCG_TARGET_REG_BITS / 2));
+        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
+        tcg_out_movi_imm32(s, TCG_REG_T2, arg);
+        tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR);
     }
 }
 
-static inline void tcg_out_ld_raw(TCGContext *s, int ret,
-                                  tcg_target_long arg)
-{
-    tcg_out_sethi(s, ret, arg);
-    tcg_out32(s, LDUW | INSN_RD(ret) | INSN_RS1(ret) |
-              INSN_IMM13(arg & 0x3ff));
-}
-
-static inline void tcg_out_ld_ptr(TCGContext *s, int ret,
-                                  tcg_target_long arg)
+static inline void tcg_out_ldst_rr(TCGContext *s, int data, int a1,
+                                   int a2, int op)
 {
-    if (!check_fit_tl(arg, 10))
-        tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ffULL);
-    if (TCG_TARGET_REG_BITS == 64) {
-        tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(ret) |
-                  INSN_IMM13(arg & 0x3ff));
-    } else {
-        tcg_out32(s, LDUW | INSN_RD(ret) | INSN_RS1(ret) |
-                  INSN_IMM13(arg & 0x3ff));
-    }
+    tcg_out32(s, op | INSN_RD(data) | INSN_RS1(a1) | INSN_RS2(a2));
 }
 
-static inline void tcg_out_ldst(TCGContext *s, int ret, int addr, int offset, int op)
+static inline void tcg_out_ldst(TCGContext *s, int ret, int addr,
+                                int offset, int op)
 {
-    if (check_fit_tl(offset, 13))
+    if (check_fit_tl(offset, 13)) {
         tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(addr) |
                   INSN_IMM13(offset));
-    else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, offset);
-        tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(TCG_REG_I5) |
-                  INSN_RS2(addr));
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, offset);
+        tcg_out_ldst_rr(s, ret, addr, TCG_REG_T1, op);
     }
 }
 
-static inline void tcg_out_ldst_asi(TCGContext *s, int ret, int addr,
-                                    int offset, int op, int asi)
-{
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, offset);
-    tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(TCG_REG_I5) |
-              INSN_ASI(asi) | INSN_RS2(addr));
-}
-
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
                               TCGReg arg1, tcg_target_long arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst(s, ret, arg1, arg2, LDUW);
-    else
-        tcg_out_ldst(s, ret, arg1, arg2, LDX);
+    tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
                               TCGReg arg1, tcg_target_long arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst(s, arg, arg1, arg2, STW);
-    else
-        tcg_out_ldst(s, arg, arg1, arg2, STX);
+    tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
+}
+
+static inline void tcg_out_ld_ptr(TCGContext *s, int ret,
+                                  tcg_target_long arg)
+{
+    if (!check_fit_tl(arg, 10)) {
+        tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
+    }
+    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff);
 }
 
 static inline void tcg_out_sety(TCGContext *s, int rs)
@@ -442,20 +472,21 @@ static inline void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
         if (check_fit_tl(val, 13))
             tcg_out_arithi(s, reg, reg, val, ARITH_ADD);
         else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, val);
-            tcg_out_arith(s, reg, reg, TCG_REG_I5, ARITH_ADD);
+            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, val);
+            tcg_out_arith(s, reg, reg, TCG_REG_T1, ARITH_ADD);
         }
     }
 }
 
-static inline void tcg_out_andi(TCGContext *s, int reg, tcg_target_long val)
+static inline void tcg_out_andi(TCGContext *s, int rd, int rs,
+                                tcg_target_long val)
 {
     if (val != 0) {
         if (check_fit_tl(val, 13))
-            tcg_out_arithi(s, reg, reg, val, ARITH_AND);
+            tcg_out_arithi(s, rd, rs, val, ARITH_AND);
         else {
-            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, val);
-            tcg_out_arith(s, reg, reg, TCG_REG_I5, ARITH_AND);
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T1, val);
+            tcg_out_arith(s, rd, rs, TCG_REG_T1, ARITH_AND);
         }
     }
 }
@@ -467,8 +498,8 @@ static void tcg_out_div32(TCGContext *s, int rd, int rs1,
     if (uns) {
         tcg_out_sety(s, TCG_REG_G0);
     } else {
-        tcg_out_arithi(s, TCG_REG_I5, rs1, 31, SHIFT_SRA);
-        tcg_out_sety(s, TCG_REG_I5);
+        tcg_out_arithi(s, TCG_REG_T1, rs1, 31, SHIFT_SRA);
+        tcg_out_sety(s, TCG_REG_T1);
     }
 
     tcg_out_arithc(s, rd, rs1, val2, val2const,
@@ -480,37 +511,7 @@ static inline void tcg_out_nop(TCGContext *s)
     tcg_out_sethi(s, TCG_REG_G0, 0);
 }
 
-static void tcg_out_branch_i32(TCGContext *s, int opc, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x2)
-                      | INSN_OFF22(l->u.value - (unsigned long)s->code_ptr)));
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP22, label_index, 0);
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x2) | 0));
-    }
-}
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_branch_i64(TCGContext *s, int opc, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x1) |
-                      (0x5 << 19) |
-                      INSN_OFF19(l->u.value - (unsigned long)s->code_ptr)));
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, label_index, 0);
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x1) |
-                      (0x5 << 19) | 0));
-    }
-}
-#endif
-
-static const uint8_t tcg_cond_to_bcond[10] = {
+static const uint8_t tcg_cond_to_bcond[] = {
     [TCG_COND_EQ] = COND_E,
     [TCG_COND_NE] = COND_NE,
     [TCG_COND_LT] = COND_L,
@@ -523,70 +524,144 @@ static const uint8_t tcg_cond_to_bcond[10] = {
     [TCG_COND_GTU] = COND_GU,
 };
 
+static const uint8_t tcg_cond_to_rcond[] = {
+    [TCG_COND_EQ] = RCOND_Z,
+    [TCG_COND_NE] = RCOND_NZ,
+    [TCG_COND_LT] = RCOND_LZ,
+    [TCG_COND_GT] = RCOND_GZ,
+    [TCG_COND_LE] = RCOND_LEZ,
+    [TCG_COND_GE] = RCOND_GEZ
+};
+
+static void tcg_out_bpcc0(TCGContext *s, int scond, int flags, int off19)
+{
+    tcg_out32(s, INSN_OP(0) | INSN_OP2(1) | INSN_COND(scond) | flags | off19);
+}
+
+static void tcg_out_bpcc(TCGContext *s, int scond, int flags, int label)
+{
+    TCGLabel *l = &s->labels[label];
+    int off19;
+
+    if (l->has_value) {
+        off19 = INSN_OFF19(l->u.value - (unsigned long)s->code_ptr);
+    } else {
+        /* Make sure to preserve destinations during retranslation.  */
+        off19 = *(uint32_t *)s->code_ptr & INSN_OFF19(-1);
+        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, label, 0);
+    }
+    tcg_out_bpcc0(s, scond, flags, off19);
+}
+
 static void tcg_out_cmp(TCGContext *s, TCGArg c1, TCGArg c2, int c2const)
 {
     tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, ARITH_SUBCC);
 }
 
-static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond,
-                               TCGArg arg1, TCGArg arg2, int const_arg2,
-                               int label_index)
+static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGArg arg1,
+                               TCGArg arg2, int const_arg2, int label)
 {
     tcg_out_cmp(s, arg1, arg2, const_arg2);
-    tcg_out_branch_i32(s, tcg_cond_to_bcond[cond], label_index);
+    tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC | BPCC_PT, label);
     tcg_out_nop(s);
 }
 
+static void tcg_out_movcc(TCGContext *s, TCGCond cond, int cc, TCGArg ret,
+                          TCGArg v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret)
+              | INSN_RS1(tcg_cond_to_bcond[cond])
+              | (v1const ? INSN_IMM11(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
+                                TCGArg c1, TCGArg c2, int c2const,
+                                TCGArg v1, int v1const)
+{
+    tcg_out_cmp(s, c1, c2, c2const);
+    tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const);
+}
+
 #if TCG_TARGET_REG_BITS == 64
-static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond,
-                               TCGArg arg1, TCGArg arg2, int const_arg2,
-                               int label_index)
+static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGArg arg1,
+                               TCGArg arg2, int const_arg2, int label)
 {
-    tcg_out_cmp(s, arg1, arg2, const_arg2);
-    tcg_out_branch_i64(s, tcg_cond_to_bcond[cond], label_index);
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.  */
+    if (arg2 == 0 && !is_unsigned_cond(cond)) {
+        TCGLabel *l = &s->labels[label];
+        int off16;
+
+        if (l->has_value) {
+            off16 = INSN_OFF16(l->u.value - (unsigned long)s->code_ptr);
+        } else {
+            /* Make sure to preserve destinations during retranslation.  */
+            off16 = *(uint32_t *)s->code_ptr & INSN_OFF16(-1);
+            tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, label, 0);
+        }
+        tcg_out32(s, INSN_OP(0) | INSN_OP2(3) | BPR_PT | INSN_RS1(arg1)
+                  | INSN_COND(tcg_cond_to_rcond[cond]) | off16);
+    } else {
+        tcg_out_cmp(s, arg1, arg2, const_arg2);
+        tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC | BPCC_PT, label);
+    }
     tcg_out_nop(s);
 }
+
+static void tcg_out_movr(TCGContext *s, TCGCond cond, TCGArg ret, TCGArg c1,
+                         TCGArg v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1)
+              | (tcg_cond_to_rcond[cond] << 10)
+              | (v1const ? INSN_IMM10(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGArg ret,
+                                TCGArg c1, TCGArg c2, int c2const,
+                                TCGArg v1, int v1const)
+{
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.
+       Note that the immediate range is one bit smaller, so we must check
+       for that as well.  */
+    if (c2 == 0 && !is_unsigned_cond(cond)
+        && (!v1const || check_fit_tl(v1, 10))) {
+        tcg_out_movr(s, cond, ret, c1, v1, v1const);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const);
+    }
+}
 #else
 static void tcg_out_brcond2_i32(TCGContext *s, TCGCond cond,
                                 TCGArg al, TCGArg ah,
                                 TCGArg bl, int blconst,
                                 TCGArg bh, int bhconst, int label_dest)
 {
-    int cc, label_next = gen_new_label();
+    int scond, label_next = gen_new_label();
 
     tcg_out_cmp(s, ah, bh, bhconst);
 
     /* Note that we fill one of the delay slots with the second compare.  */
     switch (cond) {
     case TCG_COND_EQ:
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_next);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_next);
         tcg_out_cmp(s, al, bl, blconst);
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_EQ], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_E, BPCC_ICC | BPCC_PT, label_dest);
         break;
 
     case TCG_COND_NE:
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_dest);
         tcg_out_cmp(s, al, bl, blconst);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_dest);
         break;
 
     default:
-        /* ??? One could fairly easily special-case 64-bit unsigned
-           compares against 32-bit zero-extended constants.  For instance,
-           we know that (unsigned)AH < 0 is false and need not emit it.
-           Similarly, (unsigned)AH > 0 being true implies AH != 0, so the
-           second branch will never be taken.  */
-        cc = INSN_COND(tcg_cond_to_bcond[cond], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        scond = tcg_cond_to_bcond[tcg_high_cond(cond)];
+        tcg_out_bpcc(s, scond, BPCC_ICC | BPCC_PT, label_dest);
         tcg_out_nop(s);
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_next);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_next);
         tcg_out_cmp(s, al, bl, blconst);
-        cc = INSN_COND(tcg_cond_to_bcond[tcg_unsigned_cond(cond)], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        scond = tcg_cond_to_bcond[tcg_unsigned_cond(cond)];
+        tcg_out_bpcc(s, scond, BPCC_ICC | BPCC_PT, label_dest);
         break;
     }
     tcg_out_nop(s);
@@ -598,47 +673,42 @@ static void tcg_out_brcond2_i32(TCGContext *s, TCGCond cond,
 static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
                                 TCGArg c1, TCGArg c2, int c2const)
 {
-    TCGArg t;
-
     /* For 32-bit comparisons, we can play games with ADDX/SUBX.  */
     switch (cond) {
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+        /* The result of the comparison is in the carry bit.  */
+        break;
+
     case TCG_COND_EQ:
     case TCG_COND_NE:
+        /* For equality, we can transform to inequality vs zero.  */
         if (c2 != 0) {
             tcg_out_arithc(s, ret, c1, c2, c2const, ARITH_XOR);
         }
         c1 = TCG_REG_G0, c2 = ret, c2const = 0;
-        cond = (cond == TCG_COND_EQ ? TCG_COND_LEU : TCG_COND_LTU);
+        cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
 	break;
 
     case TCG_COND_GTU:
-    case TCG_COND_GEU:
-        if (c2const && c2 != 0) {
-            tcg_out_movi_imm13(s, TCG_REG_I5, c2);
-            c2 = TCG_REG_I5;
-        }
-        t = c1, c1 = c2, c2 = t, c2const = 0;
-        cond = tcg_swap_cond(cond);
-        break;
-
-    case TCG_COND_LTU:
     case TCG_COND_LEU:
-        break;
+        /* If we don't need to load a constant into a register, we can
+           swap the operands on GTU/LEU.  There's no benefit to loading
+           the constant into a temporary register.  */
+        if (!c2const || c2 == 0) {
+            TCGArg t = c1;
+            c1 = c2;
+            c2 = t;
+            c2const = 0;
+            cond = tcg_swap_cond(cond);
+            break;
+        }
+        /* FALLTHRU */
 
     default:
         tcg_out_cmp(s, c1, c2, c2const);
-#if defined(__sparc_v9__) || defined(__sparc_v8plus__)
         tcg_out_movi_imm13(s, ret, 0);
-        tcg_out32 (s, ARITH_MOVCC | INSN_RD(ret)
-                   | INSN_RS1(tcg_cond_to_bcond[cond])
-                   | MOVCC_ICC | INSN_IMM11(1));
-#else
-        t = gen_new_label();
-        tcg_out_branch_i32(s, INSN_COND(tcg_cond_to_bcond[cond], 1), t);
-        tcg_out_movi_imm13(s, ret, 1);
-        tcg_out_movi_imm13(s, ret, 0);
-        tcg_out_label(s, t, s->code_ptr);
-#endif
+        tcg_out_movcc(s, cond, MOVCC_ICC, ret, 1, 1);
         return;
     }
 
@@ -654,11 +724,16 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
 static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGArg ret,
                                 TCGArg c1, TCGArg c2, int c2const)
 {
-    tcg_out_cmp(s, c1, c2, c2const);
-    tcg_out_movi_imm13(s, ret, 0);
-    tcg_out32 (s, ARITH_MOVCC | INSN_RD(ret)
-               | INSN_RS1(tcg_cond_to_bcond[cond])
-               | MOVCC_XCC | INSN_IMM11(1));
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare
+       if the input does not overlap the output.  */
+    if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) {
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movr(s, cond, ret, c1, 1, 1);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, 1, 1);
+    }
 }
 #else
 static void tcg_out_setcond2_i32(TCGContext *s, TCGCond cond, TCGArg ret,
@@ -666,56 +741,98 @@ static void tcg_out_setcond2_i32(TCGContext *s, TCGCond cond, TCGArg ret,
                                  TCGArg bl, int blconst,
                                  TCGArg bh, int bhconst)
 {
-    int lab;
+    int tmp = TCG_REG_T1;
+
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (ret != ah && (bhconst || ret != bh)) {
+        tmp = ret;
+    }
 
     switch (cond) {
     case TCG_COND_EQ:
-        tcg_out_setcond_i32(s, TCG_COND_EQ, TCG_REG_I5, al, bl, blconst);
-        tcg_out_setcond_i32(s, TCG_COND_EQ, ret, ah, bh, bhconst);
-        tcg_out_arith(s, ret, ret, TCG_REG_I5, ARITH_AND);
-        break;
-
     case TCG_COND_NE:
-        tcg_out_setcond_i32(s, TCG_COND_NE, TCG_REG_I5, al, al, blconst);
-        tcg_out_setcond_i32(s, TCG_COND_NE, ret, ah, bh, bhconst);
-        tcg_out_arith(s, ret, ret, TCG_REG_I5, ARITH_OR);
+        if (bl == 0 && bh == 0) {
+            if (cond == TCG_COND_EQ) {
+                tcg_out_arith(s, TCG_REG_G0, al, ah, ARITH_ORCC);
+                tcg_out_movi(s, TCG_TYPE_I32, ret, 1);
+            } else {
+                tcg_out_arith(s, ret, al, ah, ARITH_ORCC);
+            }
+        } else {
+            tcg_out_setcond_i32(s, cond, tmp, al, bl, blconst);
+            tcg_out_cmp(s, ah, bh, bhconst);
+            tcg_out_mov(s, TCG_TYPE_I32, ret, tmp);
+        }
+        tcg_out_movcc(s, TCG_COND_NE, MOVCC_ICC, ret, cond == TCG_COND_NE, 1);
         break;
 
     default:
-        lab = gen_new_label();
-
+        /* <= : ah < bh | (ah == bh && al <= bl) */
+        tcg_out_setcond_i32(s, tcg_unsigned_cond(cond), tmp, al, bl, blconst);
         tcg_out_cmp(s, ah, bh, bhconst);
-        tcg_out_branch_i32(s, INSN_COND(tcg_cond_to_bcond[cond], 1), lab);
-        tcg_out_movi_imm13(s, ret, 1);
-        tcg_out_branch_i32(s, INSN_COND(COND_NE, 1), lab);
-        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_mov(s, TCG_TYPE_I32, ret, tmp);
+        tcg_out_movcc(s, TCG_COND_NE, MOVCC_ICC, ret, 0, 1);
+        tcg_out_movcc(s, tcg_high_cond(cond), MOVCC_ICC, ret, 1, 1);
+        break;
+    }
+}
 
-        tcg_out_setcond_i32(s, tcg_unsigned_cond(cond), ret, al, bl, blconst);
+static void tcg_out_addsub2(TCGContext *s, TCGArg rl, TCGArg rh,
+                            TCGArg al, TCGArg ah, TCGArg bl, int blconst,
+                            TCGArg bh, int bhconst, int opl, int oph)
+{
+    TCGArg tmp = TCG_REG_T1;
 
-        tcg_out_label(s, lab, s->code_ptr);
-        break;
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (rl != ah && (bhconst || rl != bh)) {
+        tmp = rl;
     }
+
+    tcg_out_arithc(s, tmp, al, bl, blconst, opl);
+    tcg_out_arithc(s, rh, ah, bh, bhconst, oph);
+    tcg_out_mov(s, TCG_TYPE_I32, rl, tmp);
 }
 #endif
 
 /* Generate global QEMU prologue and epilogue code */
 static void tcg_target_qemu_prologue(TCGContext *s)
 {
-    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_CALL_STACK_OFFSET,
-                  CPU_TEMP_BUF_NLONGS * (int)sizeof(long));
+    int tmp_buf_size, frame_size;
+
+    /* The TCG temp buffer is at the top of the frame, immediately
+       below the frame pointer.  */
+    tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long);
+    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_STACK_BIAS - tmp_buf_size,
+                  tmp_buf_size);
+
+    /* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is
+       otherwise the minimal frame usable by callees.  */
+    frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS;
+    frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size;
+    frame_size += TCG_TARGET_STACK_ALIGN - 1;
+    frame_size &= -TCG_TARGET_STACK_ALIGN;
     tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
-              INSN_IMM13(-(TCG_TARGET_STACK_MINFRAME +
-                           CPU_TEMP_BUF_NLONGS * (int)sizeof(long))));
+              INSN_IMM13(-frame_size));
+
+#ifdef CONFIG_USE_GUEST_BASE
+    if (GUEST_BASE != 0) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+#endif
+
     tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I1) |
               INSN_RS2(TCG_REG_G0));
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_I0);
+    /* delay slot */
+    tcg_out_nop(s);
+
+    /* No epilogue required.  We issue ret + restore directly in the TB.  */
 }
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -733,441 +850,307 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
-#endif
+/* Perform the TLB load and compare.
 
-#if TARGET_LONG_BITS == 32
-#define TARGET_LD_OP LDUW
-#else
-#define TARGET_LD_OP LDX
-#endif
+   Inputs:
+   ADDRLO_IDX contains the index into ARGS of the low part of the
+   address; the high part of the address is at ADDR_LOW_IDX+1.
 
-#if defined(CONFIG_SOFTMMU)
-#if HOST_LONG_BITS == 32
-#define TARGET_ADDEND_LD_OP LDUW
+   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
+
+   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
+   This should be offsetof addr_read or addr_write.
+
+   The result of the TLB comparison is in %[ix]cc.  The sanitized address
+   is in the returned register, maybe %o0.  The TLB addend is in %o1.  */
+
+static int tcg_out_tlb_load(TCGContext *s, int addrlo_idx, int mem_index,
+                            int s_bits, const TCGArg *args, int which)
+{
+    const int addrlo = args[addrlo_idx];
+    const int r0 = TCG_REG_O0;
+    const int r1 = TCG_REG_O1;
+    const int r2 = TCG_REG_O2;
+    int addr = addrlo;
+    int tlb_ofs;
+
+    if (TCG_TARGET_REG_BITS == 32 && TARGET_LONG_BITS == 64) {
+        /* Assemble the 64-bit address in R0.  */
+        tcg_out_arithi(s, r0, addrlo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, r1, args[addrlo_idx + 1], 32, SHIFT_SLLX);
+        tcg_out_arith(s, r0, r0, r1, ARITH_OR);
+    }
+
+    /* Shift the page number down to tlb-entry.  */
+    tcg_out_arithi(s, r1, addrlo,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS, SHIFT_SRL);
+
+    /* Mask out the page offset, except for the required alignment.  */
+    tcg_out_andi(s, r0, addr, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+
+    /* Compute tlb index, modulo tlb size.  */
+    tcg_out_andi(s, r1, r1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
+
+    /* Relative to the current ENV.  */
+    tcg_out_arith(s, r1, TCG_AREG0, r1, ARITH_ADD);
+
+    /* Find a base address that can load both tlb comparator and addend.  */
+    tlb_ofs = offsetof(CPUArchState, tlb_table[mem_index][0]);
+    if (!check_fit_tl(tlb_ofs + sizeof(CPUTLBEntry), 13)) {
+        tcg_out_addi(s, r1, tlb_ofs);
+        tlb_ofs = 0;
+    }
+
+    /* Load the tlb comparator and the addend.  */
+    tcg_out_ld(s, TCG_TYPE_TL, r2, r1, tlb_ofs + which);
+    tcg_out_ld(s, TCG_TYPE_PTR, r1, r1, tlb_ofs+offsetof(CPUTLBEntry, addend));
+
+    /* subcc arg0, arg2, %g0 */
+    tcg_out_cmp(s, r0, r2, 0);
+
+    /* If the guest address must be zero-extended, do so now.  */
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, r0, addrlo, 0, SHIFT_SRL);
+        return r0;
+    }
+    return addrlo;
+}
+#endif /* CONFIG_SOFTMMU */
+
+static const int qemu_ld_opc[8] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+    LDUB, LDUH, LDUW, LDX, LDSB, LDSH, LDSW, LDX
 #else
-#define TARGET_ADDEND_LD_OP LDX
-#endif
+    LDUB, LDUH_LE, LDUW_LE, LDX_LE, LDSB, LDSH_LE, LDSW_LE, LDX_LE
 #endif
+};
 
-#ifdef __arch64__
-#define HOST_LD_OP LDX
-#define HOST_ST_OP STX
-#define HOST_SLL_OP SHIFT_SLLX
-#define HOST_SRA_OP SHIFT_SRAX
+static const int qemu_st_opc[4] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+    STB, STH, STW, STX
 #else
-#define HOST_LD_OP LDUW
-#define HOST_ST_OP STW
-#define HOST_SLL_OP SHIFT_SLL
-#define HOST_SRA_OP SHIFT_SRA
+    STB, STH_LE, STW_LE, STX_LE
 #endif
+};
 
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int sizeop)
 {
-    int addr_reg, data_reg, arg0, arg1, arg2, mem_index, s_bits;
+    int addrlo_idx = 1, datalo, datahi, addr_reg;
 #if defined(CONFIG_SOFTMMU)
-    uint32_t *label1_ptr, *label2_ptr;
+    int memi_idx, memi, s_bits, n;
+    uint32_t *label_ptr[2];
 #endif
 
-    data_reg = *args++;
-    addr_reg = *args++;
-    mem_index = *args;
-    s_bits = opc & 3;
-
-    arg0 = TCG_REG_O0;
-    arg1 = TCG_REG_O1;
-    arg2 = TCG_REG_O2;
+    datahi = datalo = args[0];
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        datahi = args[1];
+        addrlo_idx = 2;
+    }
 
 #if defined(CONFIG_SOFTMMU)
-    /* srl addr_reg, x, arg1 */
-    tcg_out_arithi(s, arg1, addr_reg, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS,
-                   SHIFT_SRL);
-    /* and addr_reg, x, arg0 */
-    tcg_out_arithi(s, arg0, addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                   ARITH_AND);
-
-    /* and arg1, x, arg1 */
-    tcg_out_andi(s, arg1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    /* add arg1, x, arg1 */
-    tcg_out_addi(s, arg1, offsetof(CPUArchState,
-                                   tlb_table[mem_index][0].addr_read));
+    memi_idx = addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS);
+    memi = args[memi_idx];
+    s_bits = sizeop & 3;
+
+    addr_reg = tcg_out_tlb_load(s, addrlo_idx, memi, s_bits, args,
+                                offsetof(CPUTLBEntry, addr_read));
+
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        int reg64;
+
+        /* bne,pn %[xi]cc, label0 */
+        label_ptr[0] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_NE, BPCC_PN
+                      | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+
+        /* TLB Hit.  */
+        /* Load all 64-bits into an O/G register.  */
+        reg64 = (datalo < 16 ? datalo : TCG_REG_O0);
+        tcg_out_ldst_rr(s, reg64, addr_reg, TCG_REG_O1, qemu_ld_opc[sizeop]);
+
+        /* Move the two 32-bit pieces into the destination registers.  */
+        tcg_out_arithi(s, datahi, reg64, 32, SHIFT_SRLX);
+        if (reg64 != datalo) {
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, reg64);
+        }
 
-    /* add env, arg1, arg1 */
-    tcg_out_arith(s, arg1, TCG_AREG0, arg1, ARITH_ADD);
+        /* b,a,pt label1 */
+        label_ptr[1] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_A, BPCC_A | BPCC_PT, 0);
+    } else {
+        /* The fast path is exactly one insn.  Thus we can perform the
+           entire TLB Hit in the (annulled) delay slot of the branch
+           over the TLB Miss case.  */
+
+        /* beq,a,pt %[xi]cc, label0 */
+        label_ptr[0] = NULL;
+        label_ptr[1] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                      | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+        /* delay slot */
+        tcg_out_ldst_rr(s, datalo, addr_reg, TCG_REG_O1, qemu_ld_opc[sizeop]);
+    }
 
-    /* ld [arg1], arg2 */
-    tcg_out32(s, TARGET_LD_OP | INSN_RD(arg2) | INSN_RS1(arg1) |
-              INSN_RS2(TCG_REG_G0));
+    /* TLB Miss.  */
 
-    /* subcc arg0, arg2, %g0 */
-    tcg_out_arith(s, TCG_REG_G0, arg0, arg2, ARITH_SUBCC);
-
-    /* will become:
-       be label1
-        or
-       be,pt %xcc label1 */
-    label1_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* mov (delay slot) */
-    tcg_out_mov(s, TCG_TYPE_PTR, arg0, addr_reg);
-
-    /* mov */
-    tcg_out_movi(s, TCG_TYPE_I32, arg1, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
+    if (label_ptr[0]) {
+        *label_ptr[0] |= INSN_OFF19((unsigned long)s->code_ptr -
+                                    (unsigned long)label_ptr[0]);
+    }
+    n = 0;
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[n++], TCG_AREG0);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                    args[addrlo_idx + 1]);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                args[addrlo_idx]);
 
-    /* XXX: move that code at the end of the TB */
     /* qemu_ld_helper[s_bits](arg0, arg1) */
     tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_ld_helpers[s_bits]
                            - (tcg_target_ulong)s->code_ptr) >> 2)
                          & 0x3fffffff));
-    /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-       global registers */
-    // delay slot
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_ST_OP);
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_LD_OP);
-
-    /* data_reg = sign_extend(arg0) */
-    switch(opc) {
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[n], memi);
+
+    n = tcg_target_call_oarg_regs[0];
+    /* datalo = sign_extend(arg0) */
+    switch (sizeop) {
     case 0 | 4:
-        /* sll arg0, 24/56, data_reg */
-        tcg_out_arithi(s, data_reg, arg0, (int)sizeof(tcg_target_long) * 8 - 8,
-                       HOST_SLL_OP);
-        /* sra data_reg, 24/56, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg,
-                       (int)sizeof(tcg_target_long) * 8 - 8, HOST_SRA_OP);
+        /* Recall that SRA sign extends from bit 31 through bit 63.  */
+        tcg_out_arithi(s, datalo, n, 24, SHIFT_SLL);
+        tcg_out_arithi(s, datalo, datalo, 24, SHIFT_SRA);
         break;
     case 1 | 4:
-        /* sll arg0, 16/48, data_reg */
-        tcg_out_arithi(s, data_reg, arg0,
-                       (int)sizeof(tcg_target_long) * 8 - 16, HOST_SLL_OP);
-        /* sra data_reg, 16/48, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg,
-                       (int)sizeof(tcg_target_long) * 8 - 16, HOST_SRA_OP);
+        tcg_out_arithi(s, datalo, n, 16, SHIFT_SLL);
+        tcg_out_arithi(s, datalo, datalo, 16, SHIFT_SRA);
         break;
     case 2 | 4:
-        /* sll arg0, 32, data_reg */
-        tcg_out_arithi(s, data_reg, arg0, 32, HOST_SLL_OP);
-        /* sra data_reg, 32, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg, 32, HOST_SRA_OP);
+        tcg_out_arithi(s, datalo, n, 0, SHIFT_SRA);
         break;
+    case 3:
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_mov(s, TCG_TYPE_REG, datahi, n);
+            tcg_out_mov(s, TCG_TYPE_REG, datalo, n + 1);
+            break;
+        }
+        /* FALLTHRU */
     case 0:
     case 1:
     case 2:
-    case 3:
     default:
         /* mov */
-        tcg_out_mov(s, TCG_TYPE_REG, data_reg, arg0);
+        tcg_out_mov(s, TCG_TYPE_REG, datalo, n);
         break;
     }
 
-    /* will become:
-       ba label2 */
-    label2_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* nop (delay slot */
-    tcg_out_nop(s);
-
-    /* label1: */
-#if TARGET_LONG_BITS == 32
-    /* be label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#else
-    /* be,pt %xcc label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x1) |
-                   (0x5 << 19) | INSN_OFF19((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#endif
-
-    /* ld [arg1 + x], arg1 */
-    tcg_out_ldst(s, arg1, arg1, offsetof(CPUTLBEntry, addend) -
-                 offsetof(CPUTLBEntry, addr_read), TARGET_ADDEND_LD_OP);
-
-#if TARGET_LONG_BITS == 32
-    /* and addr_reg, x, arg0 */
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, 0xffffffff);
-    tcg_out_arith(s, arg0, addr_reg, TCG_REG_I5, ARITH_AND);
-    /* add arg0, arg1, arg0 */
-    tcg_out_arith(s, arg0, arg0, arg1, ARITH_ADD);
+    *label_ptr[1] |= INSN_OFF19((unsigned long)s->code_ptr -
+                                (unsigned long)label_ptr[1]);
 #else
-    /* add addr_reg, arg1, arg0 */
-    tcg_out_arith(s, arg0, addr_reg, arg1, ARITH_ADD);
-#endif
+    addr_reg = args[addrlo_idx];
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr_reg, 0, SHIFT_SRL);
+        addr_reg = TCG_REG_T1;
+    }
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        int reg64 = (datalo < 16 ? datalo : TCG_REG_O0);
 
-#else
-    arg0 = addr_reg;
-#endif
+        tcg_out_ldst_rr(s, reg64, addr_reg,
+                        (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                        qemu_ld_opc[sizeop]);
 
-    switch(opc) {
-    case 0:
-        /* ldub [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUB);
-        break;
-    case 0 | 4:
-        /* ldsb [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSB);
-        break;
-    case 1:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* lduh [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUH);
-#else
-        /* lduha [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDUHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 1 | 4:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldsh [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSH);
-#else
-        /* ldsha [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDSHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* lduw [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUW);
-#else
-        /* lduwa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDUWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2 | 4:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldsw [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSW);
-#else
-        /* ldswa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDSWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 3:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldx [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDX);
-#else
-        /* ldxa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDXA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    default:
-        tcg_abort();
+        tcg_out_arithi(s, datahi, reg64, 32, SHIFT_SRLX);
+        if (reg64 != datalo) {
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, reg64);
+        }
+    } else {
+        tcg_out_ldst_rr(s, datalo, addr_reg,
+                        (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                        qemu_ld_opc[sizeop]);
     }
-
-#if defined(CONFIG_SOFTMMU)
-    /* label2: */
-    *label2_ptr = (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label2_ptr));
-#endif
+#endif /* CONFIG_SOFTMMU */
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int sizeop)
 {
-    int addr_reg, data_reg, arg0, arg1, arg2, mem_index, s_bits;
+    int addrlo_idx = 1, datalo, datahi, addr_reg;
 #if defined(CONFIG_SOFTMMU)
-    uint32_t *label1_ptr, *label2_ptr;
+    int memi_idx, memi, n, datafull;
+    uint32_t *label_ptr;
 #endif
 
-    data_reg = *args++;
-    addr_reg = *args++;
-    mem_index = *args;
-
-    s_bits = opc;
-
-    arg0 = TCG_REG_O0;
-    arg1 = TCG_REG_O1;
-    arg2 = TCG_REG_O2;
+    datahi = datalo = args[0];
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        datahi = args[1];
+        addrlo_idx = 2;
+    }
 
 #if defined(CONFIG_SOFTMMU)
-    /* srl addr_reg, x, arg1 */
-    tcg_out_arithi(s, arg1, addr_reg, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS,
-                   SHIFT_SRL);
-
-    /* and addr_reg, x, arg0 */
-    tcg_out_arithi(s, arg0, addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                   ARITH_AND);
-
-    /* and arg1, x, arg1 */
-    tcg_out_andi(s, arg1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    /* add arg1, x, arg1 */
-    tcg_out_addi(s, arg1, offsetof(CPUArchState,
-                                   tlb_table[mem_index][0].addr_write));
-
-    /* add env, arg1, arg1 */
-    tcg_out_arith(s, arg1, TCG_AREG0, arg1, ARITH_ADD);
+    memi_idx = addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS);
+    memi = args[memi_idx];
+
+    addr_reg = tcg_out_tlb_load(s, addrlo_idx, memi, sizeop, args,
+                                offsetof(CPUTLBEntry, addr_write));
+
+    datafull = datalo;
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        /* Reconstruct the full 64-bit value.  */
+        tcg_out_arithi(s, TCG_REG_T1, datalo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, TCG_REG_O2, datahi, 32, SHIFT_SLLX);
+        tcg_out_arith(s, TCG_REG_O2, TCG_REG_T1, TCG_REG_O2, ARITH_OR);
+        datafull = TCG_REG_O2;
+    }
 
-    /* ld [arg1], arg2 */
-    tcg_out32(s, TARGET_LD_OP | INSN_RD(arg2) | INSN_RS1(arg1) |
-              INSN_RS2(TCG_REG_G0));
+    /* The fast path is exactly one insn.  Thus we can perform the entire
+       TLB Hit in the (annulled) delay slot of the branch over TLB Miss.  */
+    /* beq,a,pt %[xi]cc, label0 */
+    label_ptr = (uint32_t *)s->code_ptr;
+    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+    /* delay slot */
+    tcg_out_ldst_rr(s, datafull, addr_reg, TCG_REG_O1, qemu_st_opc[sizeop]);
+
+    /* TLB Miss.  */
+
+    n = 0;
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[n++], TCG_AREG0);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                    args[addrlo_idx + 1]);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                args[addrlo_idx]);
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++], datahi);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++], datalo);
 
-    /* subcc arg0, arg2, %g0 */
-    tcg_out_arith(s, TCG_REG_G0, arg0, arg2, ARITH_SUBCC);
-
-    /* will become:
-       be label1
-        or
-       be,pt %xcc label1 */
-    label1_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* mov (delay slot) */
-    tcg_out_mov(s, TCG_TYPE_PTR, arg0, addr_reg);
-
-    /* mov */
-    tcg_out_mov(s, TCG_TYPE_REG, arg1, data_reg);
-
-    /* mov */
-    tcg_out_movi(s, TCG_TYPE_I32, arg2, mem_index);
-
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
-    /* XXX: move that code at the end of the TB */
     /* qemu_st_helper[s_bits](arg0, arg1, arg2) */
-    tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_st_helpers[s_bits]
+    tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_st_helpers[sizeop]
                            - (tcg_target_ulong)s->code_ptr) >> 2)
                          & 0x3fffffff));
-    /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-       global registers */
-    // delay slot
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_ST_OP);
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_LD_OP);
-
-    /* will become:
-       ba label2 */
-    label2_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* nop (delay slot) */
-    tcg_out_nop(s);
-
-#if TARGET_LONG_BITS == 32
-    /* be label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#else
-    /* be,pt %xcc label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x1) |
-                   (0x5 << 19) | INSN_OFF19((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#endif
-
-    /* ld [arg1 + x], arg1 */
-    tcg_out_ldst(s, arg1, arg1, offsetof(CPUTLBEntry, addend) -
-                 offsetof(CPUTLBEntry, addr_write), TARGET_ADDEND_LD_OP);
-
-#if TARGET_LONG_BITS == 32
-    /* and addr_reg, x, arg0 */
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, 0xffffffff);
-    tcg_out_arith(s, arg0, addr_reg, TCG_REG_I5, ARITH_AND);
-    /* add arg0, arg1, arg0 */
-    tcg_out_arith(s, arg0, arg0, arg1, ARITH_ADD);
-#else
-    /* add addr_reg, arg1, arg0 */
-    tcg_out_arith(s, arg0, addr_reg, arg1, ARITH_ADD);
-#endif
-
-#else
-    arg0 = addr_reg;
-#endif
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n], memi);
 
-    switch(opc) {
-    case 0:
-        /* stb data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STB);
-        break;
-    case 1:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* sth data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STH);
-#else
-        /* stha data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* stw data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STW);
-#else
-        /* stwa data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 3:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* stx data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STX);
+    *label_ptr |= INSN_OFF19((unsigned long)s->code_ptr -
+                             (unsigned long)label_ptr);
 #else
-        /* stxa data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STXA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    default:
-        tcg_abort();
+    addr_reg = args[addrlo_idx];
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr_reg, 0, SHIFT_SRL);
+        addr_reg = TCG_REG_T1;
     }
-
-#if defined(CONFIG_SOFTMMU)
-    /* label2: */
-    *label2_ptr = (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label2_ptr));
-#endif
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        tcg_out_arithi(s, TCG_REG_T1, datalo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, TCG_REG_O2, datahi, 32, SHIFT_SLLX);
+        tcg_out_arith(s, TCG_REG_O2, TCG_REG_T1, TCG_REG_O2, ARITH_OR);
+        datalo = TCG_REG_O2;
+    }
+    tcg_out_ldst_rr(s, datalo, addr_reg,
+                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    qemu_st_opc[sizeop]);
+#endif /* CONFIG_SOFTMMU */
 }
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
@@ -1186,43 +1169,36 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
             /* direct jump method */
-            tcg_out_sethi(s, TCG_REG_I5, args[0] & 0xffffe000);
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I5) |
-                      INSN_IMM13((args[0] & 0x1fff)));
+            uint32_t old_insn = *(uint32_t *)s->code_ptr;
             s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
+            /* Make sure to preserve links during retranslation.  */
+            tcg_out32(s, CALL | (old_insn & ~INSN_OP(-1)));
         } else {
             /* indirect jump method */
-            tcg_out_ld_ptr(s, TCG_REG_I5, (tcg_target_long)(s->tb_next + args[0]));
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I5) |
+            tcg_out_ld_ptr(s, TCG_REG_T1,
+                           (tcg_target_long)(s->tb_next + args[0]));
+            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_T1) |
                       INSN_RS2(TCG_REG_G0));
         }
         tcg_out_nop(s);
         s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
         break;
     case INDEX_op_call:
-        if (const_args[0])
+        if (const_args[0]) {
             tcg_out32(s, CALL | ((((tcg_target_ulong)args[0]
                                    - (tcg_target_ulong)s->code_ptr) >> 2)
                                  & 0x3fffffff));
-        else {
-            tcg_out_ld_ptr(s, TCG_REG_I5,
+        } else {
+            tcg_out_ld_ptr(s, TCG_REG_T1,
                            (tcg_target_long)(s->tb_next + args[0]));
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_O7) | INSN_RS1(TCG_REG_I5) |
+            tcg_out32(s, JMPL | INSN_RD(TCG_REG_O7) | INSN_RS1(TCG_REG_T1) |
                       INSN_RS2(TCG_REG_G0));
         }
-        /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-           global registers */
-        // delay slot
-        tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                     TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                     sizeof(long), HOST_ST_OP);
-        tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                     TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                     sizeof(long), HOST_LD_OP);
-        break;
-    case INDEX_op_jmp:
+        /* delay slot */
+        tcg_out_nop(s);
+        break;
     case INDEX_op_br:
-        tcg_out_branch_i32(s, COND_A, args[0]);
+        tcg_out_bpcc(s, COND_A, BPCC_PT, args[0]);
         tcg_out_nop(s);
         break;
     case INDEX_op_movi_i32:
@@ -1290,13 +1266,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         goto gen_arith;
     case INDEX_op_shl_i32:
         c = SHIFT_SLL;
-        goto gen_arith;
+    do_shift32:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, args[0], args[1], args[2] & 31, const_args[2], c);
+        break;
     case INDEX_op_shr_i32:
         c = SHIFT_SRL;
-        goto gen_arith;
+        goto do_shift32;
     case INDEX_op_sar_i32:
         c = SHIFT_SRA;
-        goto gen_arith;
+        goto do_shift32;
     case INDEX_op_mul_i32:
         c = ARITH_UMUL;
         goto gen_arith;
@@ -1317,11 +1296,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
 
     case INDEX_op_rem_i32:
     case INDEX_op_remu_i32:
-        tcg_out_div32(s, TCG_REG_I5, args[1], args[2], const_args[2],
+        tcg_out_div32(s, TCG_REG_T1, args[1], args[2], const_args[2],
                       opc == INDEX_op_remu_i32);
-        tcg_out_arithc(s, TCG_REG_I5, TCG_REG_I5, args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, TCG_REG_T1, args[2], const_args[2],
                        ARITH_UMUL);
-        tcg_out_arith(s, args[0], args[1], TCG_REG_I5, ARITH_SUB);
+        tcg_out_arith(s, args[0], args[1], TCG_REG_T1, ARITH_SUB);
         break;
 
     case INDEX_op_brcond_i32:
@@ -1332,6 +1311,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_setcond_i32(s, args[3], args[0], args[1],
                             args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond_i32(s, args[5], args[0], args[1],
+                            args[2], const_args[2], args[3], const_args[3]);
+        break;
 
 #if TCG_TARGET_REG_BITS == 32
     case INDEX_op_brcond2_i32:
@@ -1345,16 +1328,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
                              args[4], const_args[4]);
         break;
     case INDEX_op_add2_i32:
-        tcg_out_arithc(s, args[0], args[2], args[4], const_args[4],
-                       ARITH_ADDCC);
-        tcg_out_arithc(s, args[1], args[3], args[5], const_args[5],
-                       ARITH_ADDX);
+        tcg_out_addsub2(s, args[0], args[1], args[2], args[3],
+                        args[4], const_args[4], args[5], const_args[5],
+                        ARITH_ADDCC, ARITH_ADDX);
         break;
     case INDEX_op_sub2_i32:
-        tcg_out_arithc(s, args[0], args[2], args[4], const_args[4],
-                       ARITH_SUBCC);
-        tcg_out_arithc(s, args[1], args[3], args[5], const_args[5],
-                       ARITH_SUBX);
+        tcg_out_addsub2(s, args[0], args[1], args[2], args[3],
+                        args[4], const_args[4], args[5], const_args[5],
+                        ARITH_SUBCC, ARITH_SUBX);
         break;
     case INDEX_op_mulu2_i32:
         tcg_out_arithc(s, args[0], args[2], args[3], const_args[3],
@@ -1386,6 +1367,9 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_qemu_ld(s, args, 2 | 4);
         break;
 #endif
+    case INDEX_op_qemu_ld64:
+        tcg_out_qemu_ld(s, args, 3);
+        break;
     case INDEX_op_qemu_st8:
         tcg_out_qemu_st(s, args, 0);
         break;
@@ -1395,6 +1379,9 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_qemu_st32:
         tcg_out_qemu_st(s, args, 2);
         break;
+    case INDEX_op_qemu_st64:
+        tcg_out_qemu_st(s, args, 3);
+        break;
 
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_movi_i64:
@@ -1411,13 +1398,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
     case INDEX_op_shl_i64:
         c = SHIFT_SLLX;
-        goto gen_arith;
+    do_shift64:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, args[0], args[1], args[2] & 63, const_args[2], c);
+        break;
     case INDEX_op_shr_i64:
         c = SHIFT_SRLX;
-        goto gen_arith;
+        goto do_shift64;
     case INDEX_op_sar_i64:
         c = SHIFT_SRAX;
-        goto gen_arith;
+        goto do_shift64;
     case INDEX_op_mul_i64:
         c = ARITH_MULX;
         goto gen_arith;
@@ -1429,11 +1419,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         goto gen_arith;
     case INDEX_op_rem_i64:
     case INDEX_op_remu_i64:
-        tcg_out_arithc(s, TCG_REG_I5, args[1], args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, args[1], args[2], const_args[2],
                        opc == INDEX_op_rem_i64 ? ARITH_SDIVX : ARITH_UDIVX);
-        tcg_out_arithc(s, TCG_REG_I5, TCG_REG_I5, args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, TCG_REG_T1, args[2], const_args[2],
                        ARITH_MULX);
-        tcg_out_arith(s, args[0], args[1], TCG_REG_I5, ARITH_SUB);
+        tcg_out_arith(s, args[0], args[1], TCG_REG_T1, ARITH_SUB);
         break;
     case INDEX_op_ext32s_i64:
         if (const_args[1]) {
@@ -1458,14 +1448,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_setcond_i64(s, args[3], args[0], args[1],
                             args[2], const_args[2]);
         break;
-
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond_i64(s, args[5], args[0], args[1],
+                            args[2], const_args[2], args[3], const_args[3]);
         break;
-
 #endif
     gen_arith:
         tcg_out_arithc(s, args[0], args[1], args[2], const_args[2], c);
@@ -1485,7 +1471,6 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1495,55 +1480,42 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ld16u_i32, { "r", "r" } },
     { INDEX_op_ld16s_i32, { "r", "r" } },
     { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "r", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "rJ" } },
-    { INDEX_op_mul_i32, { "r", "r", "rJ" } },
-    { INDEX_op_div_i32, { "r", "r", "rJ" } },
-    { INDEX_op_divu_i32, { "r", "r", "rJ" } },
-    { INDEX_op_rem_i32, { "r", "r", "rJ" } },
-    { INDEX_op_remu_i32, { "r", "r", "rJ" } },
-    { INDEX_op_sub_i32, { "r", "r", "rJ" } },
-    { INDEX_op_and_i32, { "r", "r", "rJ" } },
-    { INDEX_op_andc_i32, { "r", "r", "rJ" } },
-    { INDEX_op_or_i32, { "r", "r", "rJ" } },
-    { INDEX_op_orc_i32, { "r", "r", "rJ" } },
-    { INDEX_op_xor_i32, { "r", "r", "rJ" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "rJ" } },
-    { INDEX_op_shr_i32, { "r", "r", "rJ" } },
-    { INDEX_op_sar_i32, { "r", "r", "rJ" } },
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_div_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_divu_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_rem_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_remu_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_andc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_or_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_orc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_xor_i32, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_shl_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "rJ" } },
 
     { INDEX_op_neg_i32, { "r", "rJ" } },
     { INDEX_op_not_i32, { "r", "rJ" } },
 
-    { INDEX_op_brcond_i32, { "r", "rJ" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rJ" } },
+    { INDEX_op_brcond_i32, { "rZ", "rJ" } },
+    { INDEX_op_setcond_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rJ", "rI", "0" } },
 
 #if TCG_TARGET_REG_BITS == 32
-    { INDEX_op_brcond2_i32, { "r", "r", "rJ", "rJ" } },
-    { INDEX_op_setcond2_i32, { "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "r", "rJ" } },
-#endif
-
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld32u, { "r", "L" } },
-    { INDEX_op_qemu_ld32s, { "r", "L" } },
+    { INDEX_op_brcond2_i32, { "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rJ" } },
 #endif
 
-    { INDEX_op_qemu_st8, { "L", "L" } },
-    { INDEX_op_qemu_st16, { "L", "L" } },
-    { INDEX_op_qemu_st32, { "L", "L" } },
-
 #if TCG_TARGET_REG_BITS == 64
     { INDEX_op_mov_i64, { "r", "r" } },
     { INDEX_op_movi_i64, { "r" } },
@@ -1554,29 +1526,27 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ld32u_i64, { "r", "r" } },
     { INDEX_op_ld32s_i64, { "r", "r" } },
     { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
-    { INDEX_op_qemu_ld64, { "L", "L" } },
-    { INDEX_op_qemu_st64, { "L", "L" } },
-
-    { INDEX_op_add_i64, { "r", "r", "rJ" } },
-    { INDEX_op_mul_i64, { "r", "r", "rJ" } },
-    { INDEX_op_div_i64, { "r", "r", "rJ" } },
-    { INDEX_op_divu_i64, { "r", "r", "rJ" } },
-    { INDEX_op_rem_i64, { "r", "r", "rJ" } },
-    { INDEX_op_remu_i64, { "r", "r", "rJ" } },
-    { INDEX_op_sub_i64, { "r", "r", "rJ" } },
-    { INDEX_op_and_i64, { "r", "r", "rJ" } },
-    { INDEX_op_andc_i64, { "r", "r", "rJ" } },
-    { INDEX_op_or_i64, { "r", "r", "rJ" } },
-    { INDEX_op_orc_i64, { "r", "r", "rJ" } },
-    { INDEX_op_xor_i64, { "r", "r", "rJ" } },
-
-    { INDEX_op_shl_i64, { "r", "r", "rJ" } },
-    { INDEX_op_shr_i64, { "r", "r", "rJ" } },
-    { INDEX_op_sar_i64, { "r", "r", "rJ" } },
+    { INDEX_op_st8_i64, { "rZ", "r" } },
+    { INDEX_op_st16_i64, { "rZ", "r" } },
+    { INDEX_op_st32_i64, { "rZ", "r" } },
+    { INDEX_op_st_i64, { "rZ", "r" } },
+
+    { INDEX_op_add_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_div_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_divu_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_rem_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_remu_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_and_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_andc_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_or_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_orc_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_xor_i64, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_shl_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_shr_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_sar_i64, { "r", "rZ", "rJ" } },
 
     { INDEX_op_neg_i64, { "r", "rJ" } },
     { INDEX_op_not_i64, { "r", "rJ" } },
@@ -1584,9 +1554,51 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ext32s_i64, { "r", "ri" } },
     { INDEX_op_ext32u_i64, { "r", "ri" } },
 
-    { INDEX_op_brcond_i64, { "r", "rJ" } },
-    { INDEX_op_setcond_i64, { "r", "r", "rJ" } },
+    { INDEX_op_brcond_i64, { "rZ", "rJ" } },
+    { INDEX_op_setcond_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_movcond_i64, { "r", "rZ", "rJ", "rI", "0" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_qemu_ld8u, { "r", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
+    { INDEX_op_qemu_ld32u, { "r", "L" } },
+    { INDEX_op_qemu_ld32s, { "r", "L" } },
+    { INDEX_op_qemu_ld64, { "r", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L" } },
+#elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+    { INDEX_op_qemu_ld8u, { "r", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
+    { INDEX_op_qemu_ld64, { "r", "r", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L", "L" } },
+#else
+    { INDEX_op_qemu_ld8u, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld64, { "L", "L", "L", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L", "L", "L" } },
 #endif
+
     { -1 },
 };
 
@@ -1613,25 +1625,23 @@ static void tcg_target_init(TCGContext *s)
                      (1 << TCG_REG_O7));
 
     tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0);
-#if TCG_TARGET_REG_BITS == 64
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I4); // for internal use
-#endif
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I5); // for internal use
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O7);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0); /* zero */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G6); /* reserved for os */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G7); /* thread pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6); /* frame pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7); /* return address */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */
+
     tcg_add_target_add_op_defs(sparc_op_defs);
 }
 
 #if TCG_TARGET_REG_BITS == 64
 # define ELF_HOST_MACHINE  EM_SPARCV9
-#elif defined(__sparc_v8plus__)
+#else
 # define ELF_HOST_MACHINE  EM_SPARC32PLUS
 # define ELF_HOST_FLAGS    EF_SPARC_32PLUS
-#else
-# define ELF_HOST_MACHINE  EM_SPARC
 #endif
 
 typedef struct {
@@ -1687,3 +1697,18 @@ void tcg_register_jit(void *buf, size_t buf_size)
 
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
+
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
+{
+    uint32_t *ptr = (uint32_t *)jmp_addr;
+    tcg_target_long disp = (tcg_target_long)(addr - jmp_addr) >> 2;
+
+    /* We can reach the entire address space for 32-bit.  For 64-bit
+       the code_gen_buffer can't be larger than 2GB.  */
+    if (TCG_TARGET_REG_BITS == 64 && !check_fit_tl(disp, 30)) {
+        tcg_abort();
+    }
+
+    *ptr = CALL | (disp & 0x3fffffff);
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index ee2274d5954..256f973c6d3 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_SPARC 
 #define TCG_TARGET_SPARC 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -62,26 +63,24 @@ typedef enum {
     TCG_REG_I7,
 } TCGReg;
 
-#define TCG_CT_CONST_S11 0x100
-#define TCG_CT_CONST_S13 0x200
+#define TCG_CT_CONST_S11  0x100
+#define TCG_CT_CONST_S13  0x200
+#define TCG_CT_CONST_ZERO 0x400
 
 /* used for function call generation */
-#define TCG_REG_CALL_STACK TCG_REG_I6
-#ifdef __arch64__
-// Reserve space for AREG0
-#define TCG_TARGET_STACK_MINFRAME (176 + 4 * (int)sizeof(long) + \
-                                   TCG_STATIC_CALL_ARGS_SIZE)
-#define TCG_TARGET_CALL_STACK_OFFSET (2047 - 16)
-#define TCG_TARGET_STACK_ALIGN 16
+#define TCG_REG_CALL_STACK TCG_REG_O6
+
+#if TCG_TARGET_REG_BITS == 64
+#define TCG_TARGET_STACK_BIAS           2047
+#define TCG_TARGET_STACK_ALIGN          16
+#define TCG_TARGET_CALL_STACK_OFFSET    (128 + 6*8 + TCG_TARGET_STACK_BIAS)
 #else
-// AREG0 + one word for alignment
-#define TCG_TARGET_STACK_MINFRAME (92 + (2 + 1) * (int)sizeof(long) + \
-                                   TCG_STATIC_CALL_ARGS_SIZE)
-#define TCG_TARGET_CALL_STACK_OFFSET TCG_TARGET_STACK_MINFRAME
-#define TCG_TARGET_STACK_ALIGN 8
+#define TCG_TARGET_STACK_BIAS           0
+#define TCG_TARGET_STACK_ALIGN          8
+#define TCG_TARGET_CALL_STACK_OFFSET    (64 + 4 + 6*4)
 #endif
 
-#ifdef __arch64__
+#if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_EXTEND_ARGS 1
 #endif
 
@@ -102,6 +101,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      1
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div_i64          1
@@ -123,16 +123,10 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      1
 #endif
 
-/* Note: must be synced with dyngen-exec.h */
-#ifdef CONFIG_SOLARIS
-#define TCG_AREG0 TCG_REG_G2
-#elif defined(__sparc_v9__)
-#define TCG_AREG0 TCG_REG_G5
-#else
-#define TCG_AREG0 TCG_REG_G6
-#endif
+#define TCG_AREG0 TCG_REG_I0
 
 static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
@@ -145,3 +139,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     for (; p < stop; p += 8)
         __asm__ __volatile__("flush\t%0" : : "r" (p));
 }
+
+#endif
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index 169d3b2b0d6..91c9d80dd58 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -25,335 +25,340 @@
 
 int gen_new_label(void);
 
+static inline void tcg_gen_op0(TCGOpcode opc)
+{
+    *tcg_ctx.gen_opc_ptr++ = opc;
+}
+
 static inline void tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
 }
 
 static inline void tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
 }
 
 static inline void tcg_gen_op1i(TCGOpcode opc, TCGArg arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = arg1;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = arg1;
 }
 
 static inline void tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
 }
 
 static inline void tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
 }
 
 static inline void tcg_gen_op2i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op2i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op2ii(TCGOpcode opc, TCGArg arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = arg1;
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = arg1;
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
 }
 
 static inline void tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
 }
 
 static inline void tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 arg1,
                                     TCGv_i32 arg2, TCGArg arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
 }
 
 static inline void tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 arg1,
                                     TCGv_i64 arg2, TCGArg arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
 }
 
 static inline void tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val,
                                        TCGv_ptr base, TCGArg offset)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(val);
-    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
-    *gen_opparam_ptr++ = offset;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *tcg_ctx.gen_opparam_ptr++ = offset;
 }
 
 static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
                                        TCGv_ptr base, TCGArg offset)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
-    *gen_opparam_ptr++ = offset;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *tcg_ctx.gen_opparam_ptr++ = offset;
 }
 
 static inline void tcg_gen_qemu_ldst_op_i64_i32(TCGOpcode opc, TCGv_i64 val,
                                                 TCGv_i32 addr, TCGArg mem_index)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_I32(addr);
-    *gen_opparam_ptr++ = mem_index;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(addr);
+    *tcg_ctx.gen_opparam_ptr++ = mem_index;
 }
 
 static inline void tcg_gen_qemu_ldst_op_i64_i64(TCGOpcode opc, TCGv_i64 val,
                                                 TCGv_i64 addr, TCGArg mem_index)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_I64(addr);
-    *gen_opparam_ptr++ = mem_index;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(addr);
+    *tcg_ctx.gen_opparam_ptr++ = mem_index;
 }
 
 static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
 }
 
 static inline void tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
 }
 
 static inline void tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = arg3;
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = arg3;
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
 }
 
 static inline void tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
 }
 
 static inline void tcg_gen_op5i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 arg1,
                                      TCGv_i32 arg2, TCGv_i32 arg3,
                                      TCGArg arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = arg4;
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 arg1,
                                      TCGv_i64 arg2, TCGv_i64 arg3,
                                      TCGArg arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = arg4;
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5,
                                    TCGv_i32 arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg6);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg6);
 }
 
 static inline void tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5,
                                    TCGv_i64 arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg6);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg6);
 }
 
 static inline void tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGv_i32 arg4,
                                     TCGv_i32 arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGv_i64 arg4,
                                     TCGv_i64 arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 arg1,
                                      TCGv_i32 arg2, TCGv_i32 arg3,
                                      TCGv_i32 arg4, TCGArg arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = arg5;
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 arg1,
                                      TCGv_i64 arg2, TCGv_i64 arg3,
                                      TCGv_i64 arg4, TCGArg arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = arg5;
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void gen_set_label(int n)
@@ -396,10 +401,10 @@ static inline void tcg_gen_helperN(void *func, int flags, int sizemask,
 }
 
 /* Note: Both tcg_gen_helper32() and tcg_gen_helper64() are currently
-   reserved for helpers in tcg-runtime.c. These helpers are all const
-   and pure, hence the call to tcg_gen_callN() with TCG_CALL_CONST |
-   TCG_CALL_PURE. This may need to be adjusted if these functions
-   start to be used with other helpers. */
+   reserved for helpers in tcg-runtime.c. These helpers all do not read
+   globals and do not have side effects, hence the call to tcg_gen_callN()
+   with TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS. This may need
+   to be adjusted if these functions start to be used with other helpers. */
 static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
                                     TCGv_i32 a, TCGv_i32 b)
 {
@@ -408,8 +413,9 @@ static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
     fn = tcg_const_ptr(func);
     args[0] = GET_TCGV_I32(a);
     args[1] = GET_TCGV_I32(b);
-    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
-                  GET_TCGV_I32(ret), 2, args);
+    tcg_gen_callN(&tcg_ctx, fn,
+                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
+                  sizemask, GET_TCGV_I32(ret), 2, args);
     tcg_temp_free_ptr(fn);
 }
 
@@ -421,8 +427,9 @@ static inline void tcg_gen_helper64(void *func, int sizemask, TCGv_i64 ret,
     fn = tcg_const_ptr(func);
     args[0] = GET_TCGV_I64(a);
     args[1] = GET_TCGV_I64(b);
-    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
-                  GET_TCGV_I64(ret), 2, args);
+    tcg_gen_callN(&tcg_ctx, fn,
+                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
+                  sizemask, GET_TCGV_I64(ret), 2, args);
     tcg_temp_free_ptr(fn);
 }
 
@@ -518,18 +525,34 @@ static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
     }
 }
 
-static inline void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
+static inline void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 {
-    /* some cases can be optimized here */
-    if (arg2 == 0) {
+    TCGv_i32 t0;
+    /* Some cases can be optimized here.  */
+    switch (arg2) {
+    case 0:
         tcg_gen_movi_i32(ret, 0);
-    } else if (arg2 == 0xffffffff) {
+        return;
+    case 0xffffffffu:
         tcg_gen_mov_i32(ret, arg1);
-    } else {
-        TCGv_i32 t0 = tcg_const_i32(arg2);
-        tcg_gen_and_i32(ret, arg1, t0);
-        tcg_temp_free_i32(t0);
-    }
+        return;
+    case 0xffu:
+        /* Don't recurse with tcg_gen_ext8u_i32.  */
+        if (TCG_TARGET_HAS_ext8u_i32) {
+            tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffu:
+        if (TCG_TARGET_HAS_ext16u_i32) {
+            tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
+            return;
+        }
+        break;
+    }
+    t0 = tcg_const_i32(arg2);
+    tcg_gen_and_i32(ret, arg1, t0);
+    tcg_temp_free_i32(t0);
 }
 
 static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
@@ -543,9 +566,9 @@ static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 
 static inline void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
-    /* some cases can be optimized here */
-    if (arg2 == 0xffffffff) {
-        tcg_gen_movi_i32(ret, 0xffffffff);
+    /* Some cases can be optimized here.  */
+    if (arg2 == -1) {
+        tcg_gen_movi_i32(ret, -1);
     } else if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
     } else {
@@ -566,9 +589,12 @@ static inline void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 
 static inline void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
-    /* some cases can be optimized here */
+    /* Some cases can be optimized here.  */
     if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
+    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
+        /* Don't recurse with tcg_gen_not_i32.  */
+        tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
     } else {
         TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_xor_i32(ret, arg1, t0);
@@ -627,29 +653,49 @@ static inline void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 static inline void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1,
                                       TCGv_i32 arg2, int label_index)
 {
-    tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
+    }
 }
 
 static inline void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1,
                                        int32_t arg2, int label_index)
 {
-    TCGv_i32 t0 = tcg_const_i32(arg2);
-    tcg_gen_brcond_i32(cond, arg1, t0, label_index);
-    tcg_temp_free_i32(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        TCGv_i32 t0 = tcg_const_i32(arg2);
+        tcg_gen_brcond_i32(cond, arg1, t0, label_index);
+        tcg_temp_free_i32(t0);
+    }
 }
 
 static inline void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
                                        TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(ret, 0);
+    } else {
+        tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
+    }
 }
 
 static inline void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
                                         TCGv_i32 arg1, int32_t arg2)
 {
-    TCGv_i32 t0 = tcg_const_i32(arg2);
-    tcg_gen_setcond_i32(cond, ret, arg1, t0);
-    tcg_temp_free_i32(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(ret, 0);
+    } else {
+        TCGv_i32 t0 = tcg_const_i32(arg2);
+        tcg_gen_setcond_i32(cond, ret, arg1, t0);
+        tcg_temp_free_i32(t0);
+    }
 }
 
 static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
@@ -847,6 +893,8 @@ static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     tcg_gen_op6_i32(INDEX_op_add2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
                     TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
                     TCGV_HIGH(arg2));
+    /* Allow the optimizer room to replace add2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 }
 
 static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -854,6 +902,8 @@ static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     tcg_gen_op6_i32(INDEX_op_sub2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
                     TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
                     TCGV_HIGH(arg2));
+    /* Allow the optimizer room to replace sub2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 }
 
 static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -945,17 +995,27 @@ static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
                                       TCGv_i64 arg2, int label_index)
 {
-    tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
-                      TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
-                      TCGV_HIGH(arg2), cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
+                          TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
+                          TCGV_HIGH(arg2), cond, label_index);
+    }
 }
 
 static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
                                        TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
-                     TCGV_LOW(arg1), TCGV_HIGH(arg1),
-                     TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(TCGV_LOW(ret), 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(TCGV_LOW(ret), 0);
+    } else {
+        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
+                         TCGV_LOW(arg1), TCGV_HIGH(arg1),
+                         TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
+    }
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
@@ -969,6 +1029,8 @@ static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
     tcg_gen_op4_i32(INDEX_op_mulu2_i32, TCGV_LOW(t0), TCGV_HIGH(t0),
                     TCGV_LOW(arg1), TCGV_LOW(arg2));
+    /* Allow the optimizer room to replace mulu2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 
     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
@@ -1120,9 +1182,38 @@ static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     }
 }
 
-static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
+    TCGv_i64 t0;
+    /* Some cases can be optimized here.  */
+    switch (arg2) {
+    case 0:
+        tcg_gen_movi_i64(ret, 0);
+        return;
+    case 0xffffffffffffffffull:
+        tcg_gen_mov_i64(ret, arg1);
+        return;
+    case 0xffull:
+        /* Don't recurse with tcg_gen_ext8u_i32.  */
+        if (TCG_TARGET_HAS_ext8u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffu:
+        if (TCG_TARGET_HAS_ext16u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffffffull:
+        if (TCG_TARGET_HAS_ext32u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
+            return;
+        }
+        break;
+    }
+    t0 = tcg_const_i64(arg2);
     tcg_gen_and_i64(ret, arg1, t0);
     tcg_temp_free_i64(t0);
 }
@@ -1138,9 +1229,16 @@ static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
 static inline void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_or_i64(ret, arg1, t0);
-    tcg_temp_free_i64(t0);
+    /* Some cases can be optimized here.  */
+    if (arg2 == -1) {
+        tcg_gen_movi_i64(ret, -1);
+    } else if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_or_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1154,9 +1252,17 @@ static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
 static inline void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_xor_i64(ret, arg1, t0);
-    tcg_temp_free_i64(t0);
+    /* Some cases can be optimized here.  */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
+        /* Don't recurse with tcg_gen_not_i64.  */
+        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_xor_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1210,13 +1316,23 @@ static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
                                       TCGv_i64 arg2, int label_index)
 {
-    tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+    }
 }
 
 static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
                                        TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i64(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i64(ret, 0);
+    } else {
+        tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
+    }
 }
 
 static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1334,9 +1450,13 @@ static inline void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1,
                                        int64_t arg2, int label_index)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_brcond_i64(cond, arg1, t0, label_index);
-    tcg_temp_free_i64(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_brcond_i64(cond, arg1, t0, label_index);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
@@ -1746,36 +1866,6 @@ static inline void tcg_gen_discard_i64(TCGv_i64 arg)
 #endif
 }
 
-static inline void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
-{
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(dest), low);
-    tcg_gen_mov_i32(TCGV_HIGH(dest), high);
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    /* This extension is only needed for type correctness.
-       We may be able to do better given target specific information.  */
-    tcg_gen_extu_i32_i64(tmp, high);
-    tcg_gen_shli_i64(tmp, tmp, 32);
-    tcg_gen_extu_i32_i64(dest, low);
-    tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free_i64(tmp);
-#endif
-}
-
-static inline void tcg_gen_concat32_i64(TCGv_i64 dest, TCGv_i64 low, TCGv_i64 high)
-{
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_concat_i32_i64(dest, TCGV_LOW(low), TCGV_LOW(high));
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_ext32u_i64(dest, low);
-    tcg_gen_shli_i64(tmp, high, 32);
-    tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free_i64(tmp);
-#endif
-}
-
 static inline void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
     if (TCG_TARGET_HAS_andc_i32) {
@@ -2048,6 +2138,10 @@ static inline void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1,
     uint32_t mask;
     TCGv_i32 t1;
 
+    tcg_debug_assert(ofs < 32);
+    tcg_debug_assert(len <= 32);
+    tcg_debug_assert(ofs + len <= 32);
+
     if (ofs == 0 && len == 32) {
         tcg_gen_mov_i32(ret, arg2);
         return;
@@ -2079,6 +2173,10 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
     uint64_t mask;
     TCGv_i64 t1;
 
+    tcg_debug_assert(ofs < 64);
+    tcg_debug_assert(len <= 64);
+    tcg_debug_assert(ofs + len <= 64);
+
     if (ofs == 0 && len == 64) {
         tcg_gen_mov_i64(ret, arg2);
         return;
@@ -2118,6 +2216,102 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
     tcg_temp_free_i64(t1);
 }
 
+static inline void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low,
+                                          TCGv_i32 high)
+{
+#if TCG_TARGET_REG_BITS == 32
+    tcg_gen_mov_i32(TCGV_LOW(dest), low);
+    tcg_gen_mov_i32(TCGV_HIGH(dest), high);
+#else
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    /* These extensions are only needed for type correctness.
+       We may be able to do better given target specific information.  */
+    tcg_gen_extu_i32_i64(tmp, high);
+    tcg_gen_extu_i32_i64(dest, low);
+    /* If deposit is available, use it.  Otherwise use the extra
+       knowledge that we have of the zero-extensions above.  */
+    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
+        tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
+    } else {
+        tcg_gen_shli_i64(tmp, tmp, 32);
+        tcg_gen_or_i64(dest, dest, tmp);
+    }
+    tcg_temp_free_i64(tmp);
+#endif
+}
+
+static inline void tcg_gen_concat32_i64(TCGv_i64 dest, TCGv_i64 low,
+                                        TCGv_i64 high)
+{
+    tcg_gen_deposit_i64(dest, low, high, 32, 32);
+}
+
+static inline void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret,
+                                       TCGv_i32 c1, TCGv_i32 c2,
+                                       TCGv_i32 v1, TCGv_i32 v2)
+{
+    if (TCG_TARGET_HAS_movcond_i32) {
+        tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
+    } else {
+        TCGv_i32 t0 = tcg_temp_new_i32();
+        TCGv_i32 t1 = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cond, t0, c1, c2);
+        tcg_gen_neg_i32(t0, t0);
+        tcg_gen_and_i32(t1, v1, t0);
+        tcg_gen_andc_i32(ret, v2, t0);
+        tcg_gen_or_i32(ret, ret, t1);
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
+    }
+}
+
+static inline void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret,
+                                       TCGv_i64 c1, TCGv_i64 c2,
+                                       TCGv_i64 v1, TCGv_i64 v2)
+{
+#if TCG_TARGET_REG_BITS == 32
+    TCGv_i32 t0 = tcg_temp_new_i32();
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
+                     TCGV_LOW(c1), TCGV_HIGH(c1),
+                     TCGV_LOW(c2), TCGV_HIGH(c2), cond);
+
+    if (TCG_TARGET_HAS_movcond_i32) {
+        tcg_gen_movi_i32(t1, 0);
+        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
+                            TCGV_LOW(v1), TCGV_LOW(v2));
+        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
+                            TCGV_HIGH(v1), TCGV_HIGH(v2));
+    } else {
+        tcg_gen_neg_i32(t0, t0);
+
+        tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
+        tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
+        tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
+
+        tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
+        tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
+        tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
+    }
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+#else
+    if (TCG_TARGET_HAS_movcond_i64) {
+        tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
+    } else {
+        TCGv_i64 t0 = tcg_temp_new_i64();
+        TCGv_i64 t1 = tcg_temp_new_i64();
+        tcg_gen_setcond_i64(cond, t0, c1, c2);
+        tcg_gen_neg_i64(t0, t0);
+        tcg_gen_and_i64(t1, v1, t0);
+        tcg_gen_andc_i64(ret, v2, t0);
+        tcg_gen_or_i64(ret, ret, t1);
+        tcg_temp_free_i64(t0);
+        tcg_temp_free_i64(t1);
+    }
+#endif
+}
+
 /***************************************/
 /* QEMU specific operations. Their type depend on the QEMU CPU
    type. */
@@ -2135,6 +2329,7 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
 #define tcg_gen_qemu_ldst_op tcg_gen_op3i_i32
 #define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i32
 #define TCGV_UNUSED(x) TCGV_UNUSED_I32(x)
+#define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I32(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I32(a, b)
 #else
 #define TCGv TCGv_i64
@@ -2146,6 +2341,7 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
 #define tcg_gen_qemu_ldst_op tcg_gen_op3i_i64
 #define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i64
 #define TCGV_UNUSED(x) TCGV_UNUSED_I64(x)
+#define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I64(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I64(a, b)
 #endif
 
@@ -2166,8 +2362,15 @@ static inline void tcg_gen_exit_tb(tcg_target_long val)
     tcg_gen_op1i(INDEX_op_exit_tb, val);
 }
 
-static inline void tcg_gen_goto_tb(int idx)
+static inline void tcg_gen_goto_tb(unsigned idx)
 {
+    /* We only support two chained exits.  */
+    tcg_debug_assert(idx <= 1);
+#ifdef CONFIG_DEBUG_TCG
+    /* Verify that we havn't seen this numbered exit before.  */
+    tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
+    tcg_ctx.goto_tb_issue_mask |= 1 << idx;
+#endif
     tcg_gen_op1i(INDEX_op_goto_tb, idx);
 }
 
@@ -2434,6 +2637,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_deposit_tl tcg_gen_deposit_i64
 #define tcg_const_tl tcg_const_i64
 #define tcg_const_local_tl tcg_const_local_i64
+#define tcg_gen_movcond_tl tcg_gen_movcond_i64
 #else
 #define tcg_gen_movi_tl tcg_gen_movi_i32
 #define tcg_gen_mov_tl tcg_gen_mov_i32
@@ -2505,6 +2709,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_deposit_tl tcg_gen_deposit_i32
 #define tcg_const_tl tcg_const_i32
 #define tcg_const_local_tl tcg_const_local_i32
+#define tcg_gen_movcond_tl tcg_gen_movcond_i32
 #endif
 
 #if TCG_TARGET_REG_BITS == 32
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index 8e06d03b17e..96510634147 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -36,10 +36,9 @@ DEF(nopn, 0, 0, 1, 0) /* variable number of parameters */
 
 DEF(discard, 1, 0, 0, 0)
 
-DEF(set_label, 0, 0, 1, 0)
-DEF(call, 0, 1, 2, TCG_OPF_SIDE_EFFECTS) /* variable number of parameters */
-DEF(jmp, 0, 1, 0, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(set_label, 0, 0, 1, TCG_OPF_BB_END)
+DEF(call, 0, 1, 2, TCG_OPF_CALL_CLOBBER) /* variable number of parameters */
+DEF(br, 0, 0, 1, TCG_OPF_BB_END)
 
 #define IMPL(X) (X ? 0 : TCG_OPF_NOT_PRESENT)
 #if TCG_TARGET_REG_BITS == 32
@@ -51,15 +50,16 @@ DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
 DEF(mov_i32, 1, 1, 0, 0)
 DEF(movi_i32, 1, 0, 1, 0)
 DEF(setcond_i32, 1, 2, 1, 0)
+DEF(movcond_i32, 1, 4, 1, IMPL(TCG_TARGET_HAS_movcond_i32))
 /* load/store */
 DEF(ld8u_i32, 1, 1, 1, 0)
 DEF(ld8s_i32, 1, 1, 1, 0)
 DEF(ld16u_i32, 1, 1, 1, 0)
 DEF(ld16s_i32, 1, 1, 1, 0)
 DEF(ld_i32, 1, 1, 1, 0)
-DEF(st8_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF(st16_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF(st_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st8_i32, 0, 2, 1, 0)
+DEF(st16_i32, 0, 2, 1, 0)
+DEF(st_i32, 0, 2, 1, 0)
 /* arith */
 DEF(add_i32, 1, 2, 0, 0)
 DEF(sub_i32, 1, 2, 0, 0)
@@ -81,12 +81,11 @@ DEF(rotl_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_rot_i32))
 DEF(rotr_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_rot_i32))
 DEF(deposit_i32, 1, 2, 2, IMPL(TCG_TARGET_HAS_deposit_i32))
 
-DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END)
 
 DEF(add2_i32, 2, 4, 0, IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(sub2_i32, 2, 4, 0, IMPL(TCG_TARGET_REG_BITS == 32))
-DEF(brcond2_i32, 0, 4, 2,
-    TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS | IMPL(TCG_TARGET_REG_BITS == 32))
+DEF(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(mulu2_i32, 2, 2, 0, IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(setcond2_i32, 1, 4, 1, IMPL(TCG_TARGET_REG_BITS == 32))
 
@@ -107,6 +106,7 @@ DEF(nor_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_nor_i32))
 DEF(mov_i64, 1, 1, 0, IMPL64)
 DEF(movi_i64, 1, 0, 1, IMPL64)
 DEF(setcond_i64, 1, 2, 1, IMPL64)
+DEF(movcond_i64, 1, 4, 1, IMPL64 | IMPL(TCG_TARGET_HAS_movcond_i64))
 /* load/store */
 DEF(ld8u_i64, 1, 1, 1, IMPL64)
 DEF(ld8s_i64, 1, 1, 1, IMPL64)
@@ -115,10 +115,10 @@ DEF(ld16s_i64, 1, 1, 1, IMPL64)
 DEF(ld32u_i64, 1, 1, 1, IMPL64)
 DEF(ld32s_i64, 1, 1, 1, IMPL64)
 DEF(ld_i64, 1, 1, 1, IMPL64)
-DEF(st8_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st16_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st32_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
+DEF(st8_i64, 0, 2, 1, IMPL64)
+DEF(st16_i64, 0, 2, 1, IMPL64)
+DEF(st32_i64, 0, 2, 1, IMPL64)
+DEF(st_i64, 0, 2, 1, IMPL64)
 /* arith */
 DEF(add_i64, 1, 2, 0, IMPL64)
 DEF(sub_i64, 1, 2, 0, IMPL64)
@@ -140,7 +140,7 @@ DEF(rotl_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(rotr_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(deposit_i64, 1, 2, 2, IMPL64 | IMPL(TCG_TARGET_HAS_deposit_i64))
 
-DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS | IMPL64)
+DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | IMPL64)
 DEF(ext8s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext8s_i64))
 DEF(ext16s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext16s_i64))
 DEF(ext32s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext32s_i64))
@@ -164,8 +164,8 @@ DEF(debug_insn_start, 0, 0, 2, 0)
 #else
 DEF(debug_insn_start, 0, 0, 1, 0)
 #endif
-DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
+DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 /* Note: even if TARGET_LONG_BITS is not defined, the INDEX_op
    constants must be defined */
 #if TCG_TARGET_REG_BITS == 32
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 8386b70abd2..9275e372ff5 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -37,9 +37,9 @@
 #endif
 
 #include "qemu-common.h"
-#include "cache-utils.h"
-#include "host-utils.h"
-#include "qemu-timer.h"
+#include "qemu/cache-utils.h"
+#include "qemu/host-utils.h"
+#include "qemu/timer.h"
 
 /* Note: the long term plan is to reduce the dependancies on the QEMU
    CPU definitions. Currently they are used for qemu_ld/st
@@ -62,10 +62,6 @@
 
 #include "elf.h"
 
-#if defined(CONFIG_USE_GUEST_BASE) && !defined(TCG_TARGET_HAS_GUEST_BASE)
-#error GUEST_BASE not supported on this host.
-#endif
-
 /* Forward declarations for functions declared in tcg-target.c and used here. */
 static void tcg_target_init(TCGContext *s);
 static void tcg_target_qemu_prologue(TCGContext *s);
@@ -89,7 +85,6 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                        tcg_target_long arg2);
 static int tcg_target_const_match(tcg_target_long val,
                                   const TCGArgConstraint *arg_ct);
-static int tcg_target_get_call_iarg_regs_count(int flags);
 
 TCGOpDef tcg_op_defs[] = {
 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
@@ -101,10 +96,6 @@ const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
 static TCGRegSet tcg_target_available_regs[2];
 static TCGRegSet tcg_target_call_clobber_regs;
 
-/* XXX: move that inside the context */
-uint16_t *gen_opc_ptr;
-TCGArg *gen_opparam_ptr;
-
 static inline void tcg_out8(TCGContext *s, uint8_t v)
 {
     *s->code_ptr++ = v;
@@ -243,7 +234,6 @@ void tcg_context_init(TCGContext *s)
     int *sorted_args;
 
     memset(s, 0, sizeof(*s));
-    s->temps = s->static_temps;
     s->nb_globals = 0;
     
     /* Count total number of arguments and allocate the corresponding
@@ -299,8 +289,20 @@ void tcg_func_start(TCGContext *s)
     s->nb_labels = 0;
     s->current_frame_offset = s->frame_start;
 
-    gen_opc_ptr = gen_opc_buf;
-    gen_opparam_ptr = gen_opparam_buf;
+#ifdef CONFIG_DEBUG_TCG
+    s->goto_tb_issue_mask = 0;
+#endif
+
+    s->gen_opc_ptr = s->gen_opc_buf;
+    s->gen_opparam_ptr = s->gen_opparam_buf;
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* Initialize qemu_ld/st labels to assist code generation at the end of TB
+       for TLB miss cases at the end of TB */
+    s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) *
+                                     TCG_MAX_QEMU_LDST);
+    s->nb_qemu_ldst_labels = 0;
+#endif
 }
 
 static inline void tcg_temp_alloc(TCGContext *s, int n)
@@ -635,23 +637,23 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
     }
 #endif /* TCG_TARGET_EXTEND_ARGS */
 
-    *gen_opc_ptr++ = INDEX_op_call;
-    nparam = gen_opparam_ptr++;
+    *s->gen_opc_ptr++ = INDEX_op_call;
+    nparam = s->gen_opparam_ptr++;
     if (ret != TCG_CALL_DUMMY_ARG) {
 #if TCG_TARGET_REG_BITS < 64
         if (sizemask & 1) {
 #ifdef TCG_TARGET_WORDS_BIGENDIAN
-            *gen_opparam_ptr++ = ret + 1;
-            *gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret + 1;
+            *s->gen_opparam_ptr++ = ret;
 #else
-            *gen_opparam_ptr++ = ret;
-            *gen_opparam_ptr++ = ret + 1;
+            *s->gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret + 1;
 #endif
             nb_rets = 2;
         } else
 #endif
         {
-            *gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret;
             nb_rets = 1;
         }
     } else {
@@ -665,7 +667,7 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
             /* some targets want aligned 64 bit args */
             if (real_args & 1) {
-                *gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
+                *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
                 real_args++;
             }
 #endif
@@ -680,28 +682,28 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
 	       have to get more complicated to differentiate between
 	       stack arguments and register arguments.  */
 #if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
-            *gen_opparam_ptr++ = args[i] + 1;
-            *gen_opparam_ptr++ = args[i];
+            *s->gen_opparam_ptr++ = args[i] + 1;
+            *s->gen_opparam_ptr++ = args[i];
 #else
-            *gen_opparam_ptr++ = args[i];
-            *gen_opparam_ptr++ = args[i] + 1;
+            *s->gen_opparam_ptr++ = args[i];
+            *s->gen_opparam_ptr++ = args[i] + 1;
 #endif
             real_args += 2;
             continue;
         }
 #endif /* TCG_TARGET_REG_BITS < 64 */
 
-        *gen_opparam_ptr++ = args[i];
+        *s->gen_opparam_ptr++ = args[i];
         real_args++;
     }
-    *gen_opparam_ptr++ = GET_TCGV_PTR(func);
+    *s->gen_opparam_ptr++ = GET_TCGV_PTR(func);
 
-    *gen_opparam_ptr++ = flags;
+    *s->gen_opparam_ptr++ = flags;
 
     *nparam = (nb_rets << 16) | (real_args + 1);
 
     /* total parameters, needed to go backward in the instruction stream */
-    *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
+    *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
 
 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
     for (i = 0; i < nargs; ++i) {
@@ -778,7 +780,11 @@ static void tcg_reg_alloc_start(TCGContext *s)
     }
     for(i = s->nb_globals; i < s->nb_temps; i++) {
         ts = &s->temps[i];
-        ts->val_type = TEMP_VAL_DEAD;
+        if (ts->temp_local) {
+            ts->val_type = TEMP_VAL_MEM;
+        } else {
+            ts->val_type = TEMP_VAL_DEAD;
+        }
         ts->mem_allocated = 0;
         ts->fixed_reg = 0;
     }
@@ -794,7 +800,6 @@ static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,
 
     assert(idx >= 0 && idx < s->nb_temps);
     ts = &s->temps[idx];
-    assert(ts);
     if (idx < s->nb_globals) {
         pstrcpy(buf, buf_size, ts->name);
     } else {
@@ -861,6 +866,8 @@ static TCGHelperInfo *tcg_find_helper(TCGContext *s, tcg_target_ulong val)
 
 static const char * const cond_name[] =
 {
+    [TCG_COND_NEVER] = "never",
+    [TCG_COND_ALWAYS] = "always",
     [TCG_COND_EQ] = "eq",
     [TCG_COND_NE] = "ne",
     [TCG_COND_LT] = "lt",
@@ -884,9 +891,9 @@ void tcg_dump_ops(TCGContext *s)
     char buf[128];
 
     first_insn = 1;
-    opc_ptr = gen_opc_buf;
-    args = gen_opparam_buf;
-    while (opc_ptr < gen_opc_ptr) {
+    opc_ptr = s->gen_opc_buf;
+    args = s->gen_opparam_buf;
+    while (opc_ptr < s->gen_opc_ptr) {
         c = *opc_ptr++;
         def = &tcg_op_defs[c];
         if (c == INDEX_op_debug_insn_start) {
@@ -937,11 +944,7 @@ void tcg_dump_ops(TCGContext *s)
                                                        args[nb_oargs + i]));
                 }
             }
-        } else if (c == INDEX_op_movi_i32 
-#if TCG_TARGET_REG_BITS == 64
-                   || c == INDEX_op_movi_i64
-#endif
-                   ) {
+        } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) {
             tcg_target_ulong val;
             TCGHelperInfo *th;
 
@@ -991,17 +994,13 @@ void tcg_dump_ops(TCGContext *s)
             }
             switch (c) {
             case INDEX_op_brcond_i32:
-#if TCG_TARGET_REG_BITS == 32
-            case INDEX_op_brcond2_i32:
-#elif TCG_TARGET_REG_BITS == 64
-            case INDEX_op_brcond_i64:
-#endif
             case INDEX_op_setcond_i32:
-#if TCG_TARGET_REG_BITS == 32
+            case INDEX_op_movcond_i32:
+            case INDEX_op_brcond2_i32:
             case INDEX_op_setcond2_i32:
-#elif TCG_TARGET_REG_BITS == 64
+            case INDEX_op_brcond_i64:
             case INDEX_op_setcond_i64:
-#endif
+            case INDEX_op_movcond_i64:
                 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
                     qemu_log(",%s", cond_name[args[k++]]);
                 } else {
@@ -1188,31 +1187,27 @@ static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr,
     }
 }
 
-/* liveness analysis: end of function: globals are live, temps are
-   dead. */
-/* XXX: at this stage, not used as there would be little gains because
-   most TBs end with a conditional jump. */
-static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps)
+/* liveness analysis: end of function: all temps are dead, and globals
+   should be in memory. */
+static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
+                                   uint8_t *mem_temps)
 {
-    memset(dead_temps, 0, s->nb_globals);
-    memset(dead_temps + s->nb_globals, 1, s->nb_temps - s->nb_globals);
+    memset(dead_temps, 1, s->nb_temps);
+    memset(mem_temps, 1, s->nb_globals);
+    memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
 }
 
-/* liveness analysis: end of basic block: globals are live, temps are
-   dead, local temps are live. */
-static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps)
+/* liveness analysis: end of basic block: all temps are dead, globals
+   and local temps should be in memory. */
+static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
+                                 uint8_t *mem_temps)
 {
     int i;
-    TCGTemp *ts;
 
-    memset(dead_temps, 0, s->nb_globals);
-    ts = &s->temps[s->nb_globals];
+    memset(dead_temps, 1, s->nb_temps);
+    memset(mem_temps, 1, s->nb_globals);
     for(i = s->nb_globals; i < s->nb_temps; i++) {
-        if (ts->temp_local)
-            dead_temps[i] = 0;
-        else
-            dead_temps[i] = 1;
-        ts++;
+        mem_temps[i] = s->temps[i].temp_local;
     }
 }
 
@@ -1225,22 +1220,25 @@ static void tcg_liveness_analysis(TCGContext *s)
     TCGOpcode op;
     TCGArg *args;
     const TCGOpDef *def;
-    uint8_t *dead_temps;
-    unsigned int dead_args;
+    uint8_t *dead_temps, *mem_temps;
+    uint16_t dead_args;
+    uint8_t sync_args;
     
-    gen_opc_ptr++; /* skip end */
+    s->gen_opc_ptr++; /* skip end */
 
-    nb_ops = gen_opc_ptr - gen_opc_buf;
+    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
 
     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
+    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
     
     dead_temps = tcg_malloc(s->nb_temps);
-    memset(dead_temps, 1, s->nb_temps);
+    mem_temps = tcg_malloc(s->nb_temps);
+    tcg_la_func_end(s, dead_temps, mem_temps);
 
-    args = gen_opparam_ptr;
+    args = s->gen_opparam_ptr;
     op_index = nb_ops - 1;
     while (op_index >= 0) {
-        op = gen_opc_buf[op_index];
+        op = s->gen_opc_buf[op_index];
         def = &tcg_op_defs[op];
         switch(op) {
         case INDEX_op_call:
@@ -1256,30 +1254,41 @@ static void tcg_liveness_analysis(TCGContext *s)
 
                 /* pure functions can be removed if their result is not
                    used */
-                if (call_flags & TCG_CALL_PURE) {
+                if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
                     for(i = 0; i < nb_oargs; i++) {
                         arg = args[i];
-                        if (!dead_temps[arg])
+                        if (!dead_temps[arg] || mem_temps[arg]) {
                             goto do_not_remove_call;
+                        }
                     }
-                    tcg_set_nop(s, gen_opc_buf + op_index, 
+                    tcg_set_nop(s, s->gen_opc_buf + op_index,
                                 args - 1, nb_args);
                 } else {
                 do_not_remove_call:
 
                     /* output args are dead */
                     dead_args = 0;
+                    sync_args = 0;
                     for(i = 0; i < nb_oargs; i++) {
                         arg = args[i];
                         if (dead_temps[arg]) {
                             dead_args |= (1 << i);
                         }
+                        if (mem_temps[arg]) {
+                            sync_args |= (1 << i);
+                        }
                         dead_temps[arg] = 1;
+                        mem_temps[arg] = 0;
                     }
-                    
-                    if (!(call_flags & TCG_CALL_CONST)) {
-                        /* globals are live (they may be used by the call) */
-                        memset(dead_temps, 0, s->nb_globals);
+
+                    if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
+                        /* globals should be synced to memory */
+                        memset(mem_temps, 1, s->nb_globals);
+                    }
+                    if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
+                                        TCG_CALL_NO_READ_GLOBALS))) {
+                        /* globals should go back to memory */
+                        memset(dead_temps, 1, s->nb_globals);
                     }
 
                     /* input args are live */
@@ -1293,15 +1302,11 @@ static void tcg_liveness_analysis(TCGContext *s)
                         }
                     }
                     s->op_dead_args[op_index] = dead_args;
+                    s->op_sync_args[op_index] = sync_args;
                 }
                 args--;
             }
             break;
-        case INDEX_op_set_label:
-            args--;
-            /* mark end of basic block */
-            tcg_la_bb_end(s, dead_temps);
-            break;
         case INDEX_op_debug_insn_start:
             args -= def->nb_args;
             break;
@@ -1313,11 +1318,62 @@ static void tcg_liveness_analysis(TCGContext *s)
             args--;
             /* mark the temporary as dead */
             dead_temps[args[0]] = 1;
+            mem_temps[args[0]] = 0;
             break;
         case INDEX_op_end:
             break;
-            /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
+
+        case INDEX_op_add2_i32:
+        case INDEX_op_sub2_i32:
+            args -= 6;
+            nb_iargs = 4;
+            nb_oargs = 2;
+            /* Test if the high part of the operation is dead, but not
+               the low part.  The result can be optimized to a simple
+               add or sub.  This happens often for x86_64 guest when the
+               cpu mode is set to 32 bit.  */
+            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+                    goto do_remove;
+                }
+                /* Create the single operation plus nop.  */
+                if (op == INDEX_op_add2_i32) {
+                    op = INDEX_op_add_i32;
+                } else {
+                    op = INDEX_op_sub_i32;
+                }
+                s->gen_opc_buf[op_index] = op;
+                args[1] = args[2];
+                args[2] = args[4];
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+                tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3);
+                /* Fall through and mark the single-word operation live.  */
+                nb_iargs = 2;
+                nb_oargs = 1;
+            }
+            goto do_not_remove;
+
+        case INDEX_op_mulu2_i32:
+            args -= 4;
+            nb_iargs = 2;
+            nb_oargs = 2;
+            /* Likewise, test for the high part of the operation dead.  */
+            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+                    goto do_remove;
+                }
+                s->gen_opc_buf[op_index] = op = INDEX_op_mul_i32;
+                args[1] = args[2];
+                args[2] = args[3];
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+                tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
+                /* Fall through and mark the single-word operation live.  */
+                nb_oargs = 1;
+            }
+            goto do_not_remove;
+
         default:
+            /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
             args -= def->nb_args;
             nb_iargs = def->nb_iargs;
             nb_oargs = def->nb_oargs;
@@ -1328,10 +1384,12 @@ static void tcg_liveness_analysis(TCGContext *s)
             if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
                 for(i = 0; i < nb_oargs; i++) {
                     arg = args[i];
-                    if (!dead_temps[arg])
+                    if (!dead_temps[arg] || mem_temps[arg]) {
                         goto do_not_remove;
+                    }
                 }
-                tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args);
+            do_remove:
+                tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args);
 #ifdef CONFIG_PROFILER
                 s->del_op_count++;
 #endif
@@ -1340,20 +1398,25 @@ static void tcg_liveness_analysis(TCGContext *s)
 
                 /* output args are dead */
                 dead_args = 0;
+                sync_args = 0;
                 for(i = 0; i < nb_oargs; i++) {
                     arg = args[i];
                     if (dead_temps[arg]) {
                         dead_args |= (1 << i);
                     }
+                    if (mem_temps[arg]) {
+                        sync_args |= (1 << i);
+                    }
                     dead_temps[arg] = 1;
+                    mem_temps[arg] = 0;
                 }
 
                 /* if end of basic block, update */
                 if (def->flags & TCG_OPF_BB_END) {
-                    tcg_la_bb_end(s, dead_temps);
-                } else if (def->flags & TCG_OPF_CALL_CLOBBER) {
-                    /* globals are live */
-                    memset(dead_temps, 0, s->nb_globals);
+                    tcg_la_bb_end(s, dead_temps, mem_temps);
+                } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
+                    /* globals should be synced to memory */
+                    memset(mem_temps, 1, s->nb_globals);
                 }
 
                 /* input args are live */
@@ -1365,24 +1428,28 @@ static void tcg_liveness_analysis(TCGContext *s)
                     dead_temps[arg] = 0;
                 }
                 s->op_dead_args[op_index] = dead_args;
+                s->op_sync_args[op_index] = sync_args;
             }
             break;
         }
         op_index--;
     }
 
-    if (args != gen_opparam_buf)
+    if (args != s->gen_opparam_buf) {
         tcg_abort();
+    }
 }
 #else
 /* dummy liveness analysis */
 static void tcg_liveness_analysis(TCGContext *s)
 {
     int nb_ops;
-    nb_ops = gen_opc_ptr - gen_opc_buf;
+    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
 
     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
     memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
+    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
+    memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
 }
 #endif
 
@@ -1463,7 +1530,8 @@ static void temp_allocate_frame(TCGContext *s, int temp)
 {
     TCGTemp *ts;
     ts = &s->temps[temp];
-#ifndef __sparc_v9__ /* Sparc64 stack is accessed with offset of 2047 */
+#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
+    /* Sparc64 stack is accessed with offset of 2047 */
     s->current_frame_offset = (s->current_frame_offset +
                                (tcg_target_long)sizeof(tcg_target_long) - 1) &
         ~(sizeof(tcg_target_long) - 1);
@@ -1478,22 +1546,33 @@ static void temp_allocate_frame(TCGContext *s, int temp)
     s->current_frame_offset += (tcg_target_long)sizeof(tcg_target_long);
 }
 
+/* sync register 'reg' by saving it to the corresponding temporary */
+static inline void tcg_reg_sync(TCGContext *s, int reg)
+{
+    TCGTemp *ts;
+    int temp;
+
+    temp = s->reg_to_temp[reg];
+    ts = &s->temps[temp];
+    assert(ts->val_type == TEMP_VAL_REG);
+    if (!ts->mem_coherent && !ts->fixed_reg) {
+        if (!ts->mem_allocated) {
+            temp_allocate_frame(s, temp);
+        }
+        tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
+    }
+    ts->mem_coherent = 1;
+}
+
 /* free register 'reg' by spilling the corresponding temporary if necessary */
 static void tcg_reg_free(TCGContext *s, int reg)
 {
-    TCGTemp *ts;
     int temp;
 
     temp = s->reg_to_temp[reg];
     if (temp != -1) {
-        ts = &s->temps[temp];
-        assert(ts->val_type == TEMP_VAL_REG);
-        if (!ts->mem_coherent) {
-            if (!ts->mem_allocated) 
-                temp_allocate_frame(s, temp);
-            tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-        }
-        ts->val_type = TEMP_VAL_MEM;
+        tcg_reg_sync(s, reg);
+        s->temps[temp].val_type = TEMP_VAL_MEM;
         s->reg_to_temp[reg] = -1;
     }
 }
@@ -1525,31 +1604,45 @@ static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
     tcg_abort();
 }
 
-/* save a temporary to memory. 'allocated_regs' is used in case a
+/* mark a temporary as dead. */
+static inline void temp_dead(TCGContext *s, int temp)
+{
+    TCGTemp *ts;
+
+    ts = &s->temps[temp];
+    if (!ts->fixed_reg) {
+        if (ts->val_type == TEMP_VAL_REG) {
+            s->reg_to_temp[ts->reg] = -1;
+        }
+        if (temp < s->nb_globals || ts->temp_local) {
+            ts->val_type = TEMP_VAL_MEM;
+        } else {
+            ts->val_type = TEMP_VAL_DEAD;
+        }
+    }
+}
+
+/* sync a temporary to memory. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
-static void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
+static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs)
 {
     TCGTemp *ts;
-    int reg;
 
     ts = &s->temps[temp];
     if (!ts->fixed_reg) {
         switch(ts->val_type) {
+        case TEMP_VAL_CONST:
+            ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
+                                    allocated_regs);
+            ts->val_type = TEMP_VAL_REG;
+            s->reg_to_temp[ts->reg] = temp;
+            ts->mem_coherent = 0;
+            tcg_out_movi(s, ts->type, ts->reg, ts->val);
+            /* fallthrough*/
         case TEMP_VAL_REG:
-            tcg_reg_free(s, ts->reg);
+            tcg_reg_sync(s, ts->reg);
             break;
         case TEMP_VAL_DEAD:
-            ts->val_type = TEMP_VAL_MEM;
-            break;
-        case TEMP_VAL_CONST:
-            reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                allocated_regs);
-            if (!ts->mem_allocated) 
-                temp_allocate_frame(s, temp);
-            tcg_out_movi(s, ts->type, reg, ts->val);
-            tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            ts->val_type = TEMP_VAL_MEM;
-            break;
         case TEMP_VAL_MEM:
             break;
         default:
@@ -1558,6 +1651,20 @@ static void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
     }
 }
 
+/* save a temporary to memory. 'allocated_regs' is used in case a
+   temporary registers needs to be allocated to store a constant. */
+static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
+{
+#ifdef USE_LIVENESS_ANALYSIS
+    /* The liveness analysis already ensures that globals are back
+       in memory. Keep an assert for safety. */
+    assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg);
+#else
+    temp_sync(s, temp, allocated_regs);
+    temp_dead(s, temp);
+#endif
+}
+
 /* save globals to their canonical location and assume they can be
    modified be the following code. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
@@ -1570,6 +1677,23 @@ static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
     }
 }
 
+/* sync globals to their canonical location and assume they can be
+   read by the following code. 'allocated_regs' is used in case a
+   temporary registers needs to be allocated to store a constant. */
+static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
+{
+    int i;
+
+    for (i = 0; i < s->nb_globals; i++) {
+#ifdef USE_LIVENESS_ANALYSIS
+        assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg ||
+               s->temps[i].mem_coherent);
+#else
+        temp_sync(s, i, allocated_regs);
+#endif
+    }
+}
+
 /* at the end of a basic block, we assume all temporaries are dead and
    all globals are stored at their canonical location. */
 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
@@ -1582,10 +1706,13 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
         if (ts->temp_local) {
             temp_save(s, i, allocated_regs);
         } else {
-            if (ts->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ts->reg] = -1;
-            }
-            ts->val_type = TEMP_VAL_DEAD;
+#ifdef USE_LIVENESS_ANALYSIS
+            /* The liveness analysis already ensures that temps are dead.
+               Keep an assert for safety. */
+            assert(ts->val_type == TEMP_VAL_DEAD);
+#else
+            temp_dead(s, i);
+#endif
         }
     }
 
@@ -1593,8 +1720,10 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
 }
 
 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
+#define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
 
-static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args)
+static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
+                               uint16_t dead_args, uint8_t sync_args)
 {
     TCGTemp *ots;
     tcg_target_ulong val;
@@ -1613,71 +1742,99 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args)
         ots->val_type = TEMP_VAL_CONST;
         ots->val = val;
     }
+    if (NEED_SYNC_ARG(0)) {
+        temp_sync(s, args[0], s->reserved_regs);
+    }
+    if (IS_DEAD_ARG(0)) {
+        temp_dead(s, args[0]);
+    }
 }
 
 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
-                              const TCGArg *args,
-                              unsigned int dead_args)
+                              const TCGArg *args, uint16_t dead_args,
+                              uint8_t sync_args)
 {
+    TCGRegSet allocated_regs;
     TCGTemp *ts, *ots;
-    int reg;
-    const TCGArgConstraint *arg_ct;
+    const TCGArgConstraint *arg_ct, *oarg_ct;
 
+    tcg_regset_set(allocated_regs, s->reserved_regs);
     ots = &s->temps[args[0]];
     ts = &s->temps[args[1]];
-    arg_ct = &def->args_ct[0];
+    oarg_ct = &def->args_ct[0];
+    arg_ct = &def->args_ct[1];
+
+    /* If the source value is not in a register, and we're going to be
+       forced to have it in a register in order to perform the copy,
+       then copy the SOURCE value into its own register first.  That way
+       we don't have to reload SOURCE the next time it is used. */
+    if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
+        || ts->val_type == TEMP_VAL_MEM) {
+        ts->reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+        if (ts->val_type == TEMP_VAL_MEM) {
+            tcg_out_ld(s, ts->type, ts->reg, ts->mem_reg, ts->mem_offset);
+            ts->mem_coherent = 1;
+        } else if (ts->val_type == TEMP_VAL_CONST) {
+            tcg_out_movi(s, ts->type, ts->reg, ts->val);
+        }
+        s->reg_to_temp[ts->reg] = args[1];
+        ts->val_type = TEMP_VAL_REG;
+    }
 
-    /* XXX: always mark arg dead if IS_DEAD_ARG(1) */
-    if (ts->val_type == TEMP_VAL_REG) {
+    if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
+        /* mov to a non-saved dead register makes no sense (even with
+           liveness analysis disabled). */
+        assert(NEED_SYNC_ARG(0));
+        /* The code above should have moved the temp to a register. */
+        assert(ts->val_type == TEMP_VAL_REG);
+        if (!ots->mem_allocated) {
+            temp_allocate_frame(s, args[0]);
+        }
+        tcg_out_st(s, ots->type, ts->reg, ots->mem_reg, ots->mem_offset);
+        if (IS_DEAD_ARG(1)) {
+            temp_dead(s, args[1]);
+        }
+        temp_dead(s, args[0]);
+    } else if (ts->val_type == TEMP_VAL_CONST) {
+        /* propagate constant */
+        if (ots->val_type == TEMP_VAL_REG) {
+            s->reg_to_temp[ots->reg] = -1;
+        }
+        ots->val_type = TEMP_VAL_CONST;
+        ots->val = ts->val;
+    } else {
+        /* The code in the first if block should have moved the
+           temp to a register. */
+        assert(ts->val_type == TEMP_VAL_REG);
         if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
             /* the mov can be suppressed */
-            if (ots->val_type == TEMP_VAL_REG)
-                s->reg_to_temp[ots->reg] = -1;
-            reg = ts->reg;
-            s->reg_to_temp[reg] = -1;
-            ts->val_type = TEMP_VAL_DEAD;
-        } else {
             if (ots->val_type == TEMP_VAL_REG) {
-                reg = ots->reg;
-            } else {
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs);
-            }
-            if (ts->reg != reg) {
-                tcg_out_mov(s, ots->type, reg, ts->reg);
+                s->reg_to_temp[ots->reg] = -1;
             }
-        }
-    } else if (ts->val_type == TEMP_VAL_MEM) {
-        if (ots->val_type == TEMP_VAL_REG) {
-            reg = ots->reg;
+            ots->reg = ts->reg;
+            temp_dead(s, args[1]);
         } else {
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs);
+            if (ots->val_type != TEMP_VAL_REG) {
+                /* When allocating a new register, make sure to not spill the
+                   input one. */
+                tcg_regset_set_reg(allocated_regs, ts->reg);
+                ots->reg = tcg_reg_alloc(s, oarg_ct->u.regs, allocated_regs);
+            }
+            tcg_out_mov(s, ots->type, ots->reg, ts->reg);
         }
-        tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-    } else if (ts->val_type == TEMP_VAL_CONST) {
-        if (ots->fixed_reg) {
-            reg = ots->reg;
-            tcg_out_movi(s, ots->type, reg, ts->val);
-        } else {
-            /* propagate constant */
-            if (ots->val_type == TEMP_VAL_REG)
-                s->reg_to_temp[ots->reg] = -1;
-            ots->val_type = TEMP_VAL_CONST;
-            ots->val = ts->val;
-            return;
+        ots->val_type = TEMP_VAL_REG;
+        ots->mem_coherent = 0;
+        s->reg_to_temp[ots->reg] = args[0];
+        if (NEED_SYNC_ARG(0)) {
+            tcg_reg_sync(s, ots->reg);
         }
-    } else {
-        tcg_abort();
     }
-    s->reg_to_temp[reg] = args[0];
-    ots->reg = reg;
-    ots->val_type = TEMP_VAL_REG;
-    ots->mem_coherent = 0;
 }
 
 static void tcg_reg_alloc_op(TCGContext *s, 
                              const TCGOpDef *def, TCGOpcode opc,
-                             const TCGArg *args,
-                             unsigned int dead_args)
+                             const TCGArg *args, uint16_t dead_args,
+                             uint8_t sync_args)
 {
     TCGRegSet allocated_regs;
     int i, k, nb_iargs, nb_oargs, reg;
@@ -1757,22 +1914,16 @@ static void tcg_reg_alloc_op(TCGContext *s,
     iarg_end: ;
     }
     
+    /* mark dead temporaries and free the associated registers */
+    for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
+        if (IS_DEAD_ARG(i)) {
+            temp_dead(s, args[i]);
+        }
+    }
+
     if (def->flags & TCG_OPF_BB_END) {
         tcg_reg_alloc_bb_end(s, allocated_regs);
     } else {
-        /* mark dead temporaries and free the associated registers */
-        for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            arg = args[i];
-            if (IS_DEAD_ARG(i)) {
-                ts = &s->temps[arg];
-                if (!ts->fixed_reg) {
-                    if (ts->val_type == TEMP_VAL_REG)
-                        s->reg_to_temp[ts->reg] = -1;
-                    ts->val_type = TEMP_VAL_DEAD;
-                }
-            }
-        }
-        
         if (def->flags & TCG_OPF_CALL_CLOBBER) {
             /* XXX: permit generic clobber register list ? */ 
             for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
@@ -1780,12 +1931,11 @@ static void tcg_reg_alloc_op(TCGContext *s,
                     tcg_reg_free(s, reg);
                 }
             }
-            /* XXX: for load/store we could do that only for the slow path
-               (i.e. when a memory callback is called) */
-            
-            /* store globals and free associated registers (we assume the insn
-               can modify any global. */
-            save_globals(s, allocated_regs);
+        }
+        if (def->flags & TCG_OPF_SIDE_EFFECTS) {
+            /* sync globals if the op has side effects and might trigger
+               an exception. */
+            sync_globals(s, allocated_regs);
         }
         
         /* satisfy the output constraints */
@@ -1809,18 +1959,15 @@ static void tcg_reg_alloc_op(TCGContext *s,
             tcg_regset_set_reg(allocated_regs, reg);
             /* if a fixed register is used, then a move will be done afterwards */
             if (!ts->fixed_reg) {
-                if (ts->val_type == TEMP_VAL_REG)
+                if (ts->val_type == TEMP_VAL_REG) {
                     s->reg_to_temp[ts->reg] = -1;
-                if (IS_DEAD_ARG(i)) {
-                    ts->val_type = TEMP_VAL_DEAD;
-                } else {
-                    ts->val_type = TEMP_VAL_REG;
-                    ts->reg = reg;
-                    /* temp value is modified, so the value kept in memory is
-                       potentially not the same */
-                    ts->mem_coherent = 0;
-                    s->reg_to_temp[reg] = arg;
-               }
+                }
+                ts->val_type = TEMP_VAL_REG;
+                ts->reg = reg;
+                /* temp value is modified, so the value kept in memory is
+                   potentially not the same */
+                ts->mem_coherent = 0;
+                s->reg_to_temp[reg] = arg;
             }
         oarg_end:
             new_args[i] = reg;
@@ -1837,6 +1984,12 @@ static void tcg_reg_alloc_op(TCGContext *s,
         if (ts->fixed_reg && ts->reg != reg) {
             tcg_out_mov(s, ts->type, ts->reg, reg);
         }
+        if (NEED_SYNC_ARG(i)) {
+            tcg_reg_sync(s, reg);
+        }
+        if (IS_DEAD_ARG(i)) {
+            temp_dead(s, args[i]);
+        }
     }
 }
 
@@ -1848,7 +2001,7 @@ static void tcg_reg_alloc_op(TCGContext *s,
 
 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
                               TCGOpcode opc, const TCGArg *args,
-                              unsigned int dead_args)
+                              uint16_t dead_args, uint8_t sync_args)
 {
     int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params;
     TCGArg arg, func_arg;
@@ -1866,7 +2019,7 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
 
     flags = args[nb_oargs + nb_iargs];
 
-    nb_regs = tcg_target_get_call_iarg_regs_count(flags);
+    nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
     if (nb_regs > nb_params)
         nb_regs = nb_params;
 
@@ -1972,14 +2125,8 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
     
     /* mark dead temporaries and free the associated registers */
     for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
-        arg = args[i];
         if (IS_DEAD_ARG(i)) {
-            ts = &s->temps[arg];
-            if (!ts->fixed_reg) {
-                if (ts->val_type == TEMP_VAL_REG)
-                    s->reg_to_temp[ts->reg] = -1;
-                ts->val_type = TEMP_VAL_DEAD;
-            }
+            temp_dead(s, args[i]);
         }
     }
     
@@ -1989,10 +2136,14 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
             tcg_reg_free(s, reg);
         }
     }
-    
-    /* store globals and free associated registers (we assume the call
-       can modify any global. */
-    if (!(flags & TCG_CALL_CONST)) {
+
+    /* Save globals if they might be written by the helper, sync them if
+       they might be read. */
+    if (flags & TCG_CALL_NO_READ_GLOBALS) {
+        /* Nothing to do */
+    } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
+        sync_globals(s, allocated_regs);
+    } else {
         save_globals(s, allocated_regs);
     }
 
@@ -2009,15 +2160,18 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
                 tcg_out_mov(s, ts->type, ts->reg, reg);
             }
         } else {
-            if (ts->val_type == TEMP_VAL_REG)
+            if (ts->val_type == TEMP_VAL_REG) {
                 s->reg_to_temp[ts->reg] = -1;
+            }
+            ts->val_type = TEMP_VAL_REG;
+            ts->reg = reg;
+            ts->mem_coherent = 0;
+            s->reg_to_temp[reg] = arg;
+            if (NEED_SYNC_ARG(i)) {
+                tcg_reg_sync(s, reg);
+            }
             if (IS_DEAD_ARG(i)) {
-                ts->val_type = TEMP_VAL_DEAD;
-            } else {
-                ts->val_type = TEMP_VAL_REG;
-                ts->reg = reg;
-                ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
+                temp_dead(s, args[i]);
             }
         }
     }
@@ -2048,7 +2202,6 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     TCGOpcode opc;
     int op_index;
     const TCGOpDef *def;
-    unsigned int dead_args;
     const TCGArg *args;
 
 #ifdef DEBUG_DISAS
@@ -2059,22 +2212,29 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     }
 #endif
 
+#ifdef CONFIG_PROFILER
+    s->opt_time -= profile_getclock();
+#endif
+
 #ifdef USE_TCG_OPTIMIZATIONS
-    gen_opparam_ptr =
-        tcg_optimize(s, gen_opc_ptr, gen_opparam_buf, tcg_op_defs);
+    s->gen_opparam_ptr =
+        tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs);
 #endif
 
 #ifdef CONFIG_PROFILER
+    s->opt_time += profile_getclock();
     s->la_time -= profile_getclock();
 #endif
+
     tcg_liveness_analysis(s);
+
 #ifdef CONFIG_PROFILER
     s->la_time += profile_getclock();
 #endif
 
 #ifdef DEBUG_DISAS
     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
-        qemu_log("OP after liveness analysis:\n");
+        qemu_log("OP after optimization and liveness analysis:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
     }
@@ -2085,11 +2245,11 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     s->code_buf = gen_code_buf;
     s->code_ptr = gen_code_buf;
 
-    args = gen_opparam_buf;
+    args = s->gen_opparam_buf;
     op_index = 0;
 
     for(;;) {
-        opc = gen_opc_buf[op_index];
+        opc = s->gen_opc_buf[op_index];
 #ifdef CONFIG_PROFILER
         tcg_table_op_count[opc]++;
 #endif
@@ -2101,17 +2261,14 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
         switch(opc) {
         case INDEX_op_mov_i32:
-#if TCG_TARGET_REG_BITS == 64
         case INDEX_op_mov_i64:
-#endif
-            dead_args = s->op_dead_args[op_index];
-            tcg_reg_alloc_mov(s, def, args, dead_args);
+            tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index],
+                              s->op_sync_args[op_index]);
             break;
         case INDEX_op_movi_i32:
-#if TCG_TARGET_REG_BITS == 64
         case INDEX_op_movi_i64:
-#endif
-            tcg_reg_alloc_movi(s, args);
+            tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index],
+                               s->op_sync_args[op_index]);
             break;
         case INDEX_op_debug_insn_start:
             /* debug instruction */
@@ -2125,24 +2282,16 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
             args += args[0];
             goto next;
         case INDEX_op_discard:
-            {
-                TCGTemp *ts;
-                ts = &s->temps[args[0]];
-                /* mark the temporary as dead */
-                if (!ts->fixed_reg) {
-                    if (ts->val_type == TEMP_VAL_REG)
-                        s->reg_to_temp[ts->reg] = -1;
-                    ts->val_type = TEMP_VAL_DEAD;
-                }
-            }
+            temp_dead(s, args[0]);
             break;
         case INDEX_op_set_label:
             tcg_reg_alloc_bb_end(s, s->reserved_regs);
             tcg_out_label(s, args[0], s->code_ptr);
             break;
         case INDEX_op_call:
-            dead_args = s->op_dead_args[op_index];
-            args += tcg_reg_alloc_call(s, def, opc, args, dead_args);
+            args += tcg_reg_alloc_call(s, def, opc, args,
+                                       s->op_dead_args[op_index],
+                                       s->op_sync_args[op_index]);
             goto next;
         case INDEX_op_end:
             goto the_end;
@@ -2154,8 +2303,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
             /* Note: in order to speed up the code, it would be much
                faster to have specialized register allocator functions for
                some common argument patterns */
-            dead_args = s->op_dead_args[op_index];
-            tcg_reg_alloc_op(s, def, opc, args, dead_args);
+            tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index],
+                             s->op_sync_args[op_index]);
             break;
         }
         args += def->nb_args;
@@ -2169,6 +2318,10 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
     }
  the_end:
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* Generate TB finalization at the end of block */
+    tcg_out_tb_finalize(s);
+#endif
     return -1;
 }
 
@@ -2177,7 +2330,7 @@ int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf)
 #ifdef CONFIG_PROFILER
     {
         int n;
-        n = (gen_opc_ptr - gen_opc_buf);
+        n = (s->gen_opc_ptr - s->gen_opc_buf);
         s->op_count += n;
         if (n > s->op_count_max)
             s->op_count_max = n;
@@ -2241,6 +2394,9 @@ void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
                 (double)s->interm_time / tot * 100.0);
     cpu_fprintf(f, "  gen_code time     %0.1f%%\n", 
                 (double)s->code_time / tot * 100.0);
+    cpu_fprintf(f, "optim./code time    %0.1f%%\n",
+                (double)s->opt_time / (s->code_time ? s->code_time : 1)
+                * 100.0);
     cpu_fprintf(f, "liveness/code time  %0.1f%%\n", 
                 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
     cpu_fprintf(f, "cpu_restore count   %" PRId64 "\n",
diff --git a/tcg/tcg.h b/tcg/tcg.h
index d710694e0a9..a4279725a61 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -79,6 +79,7 @@ typedef uint64_t TCGRegSet;
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif
 
 #ifndef TCG_TARGET_deposit_i32_valid
@@ -187,6 +188,24 @@ typedef tcg_target_ulong TCGArg;
    are aliases for target_ulong and host pointer sized values respectively.
  */
 
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* Macros/structures for qemu_ld/st IR code optimization:
+   TCG_MAX_HELPER_LABELS is defined as same as OPC_BUF_SIZE in exec-all.h. */
+#define TCG_MAX_QEMU_LDST       640
+
+typedef struct TCGLabelQemuLdst {
+    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
+    int opc:4;
+    int addrlo_reg;         /* reg index for low word of guest virtual addr */
+    int addrhi_reg;         /* reg index for high word of guest virtual addr */
+    int datalo_reg;         /* reg index for low word to be loaded or stored */
+    int datahi_reg;         /* reg index for high word to be loaded or stored */
+    int mem_index;          /* soft MMU memory index */
+    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
+    uint8_t *label_ptr[2];  /* label pointers to be updated */
+} TCGLabelQemuLdst;
+#endif
+
 #ifdef CONFIG_DEBUG_TCG
 #define DEBUG_TCGV 1
 #endif
@@ -251,32 +270,51 @@ typedef int TCGv_i64;
 #define TCGV_UNUSED_I32(x) x = MAKE_TCGV_I32(-1)
 #define TCGV_UNUSED_I64(x) x = MAKE_TCGV_I64(-1)
 
+#define TCGV_IS_UNUSED_I32(x) (GET_TCGV_I32(x) == -1)
+#define TCGV_IS_UNUSED_I64(x) (GET_TCGV_I64(x) == -1)
+
 /* call flags */
-/* A pure function only reads its arguments and TCG global variables
-   and cannot raise exceptions. Hence a call to a pure function can be
-   safely suppressed if the return value is not used. */
-#define TCG_CALL_PURE           0x0010 
-/* A const function only reads its arguments and does not use TCG
-   global variables. Hence a call to such a function does not
-   save TCG global variables back to their canonical location. */
-#define TCG_CALL_CONST          0x0020
+/* Helper does not read globals (either directly or through an exception). It
+   implies TCG_CALL_NO_WRITE_GLOBALS. */
+#define TCG_CALL_NO_READ_GLOBALS    0x0010
+/* Helper does not write globals */
+#define TCG_CALL_NO_WRITE_GLOBALS   0x0020
+/* Helper can be safely suppressed if the return value is not used. */
+#define TCG_CALL_NO_SIDE_EFFECTS    0x0040
+
+/* convenience version of most used call flags */
+#define TCG_CALL_NO_RWG         TCG_CALL_NO_READ_GLOBALS
+#define TCG_CALL_NO_WG          TCG_CALL_NO_WRITE_GLOBALS
+#define TCG_CALL_NO_SE          TCG_CALL_NO_SIDE_EFFECTS
+#define TCG_CALL_NO_RWG_SE      (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
+#define TCG_CALL_NO_WG_SE       (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
 
 /* used to align parameters */
 #define TCG_CALL_DUMMY_TCGV     MAKE_TCGV_I32(-1)
 #define TCG_CALL_DUMMY_ARG      ((TCGArg)(-1))
 
+/* Conditions.  Note that these are laid out for easy manipulation by
+   the functions below:
+     bit 0 is used for inverting;
+     bit 1 is signed,
+     bit 2 is unsigned,
+     bit 3 is used with bit 0 for swapping signed/unsigned.  */
 typedef enum {
-    TCG_COND_EQ,
-    TCG_COND_NE,
-    TCG_COND_LT,
-    TCG_COND_GE,
-    TCG_COND_LE,
-    TCG_COND_GT,
+    /* non-signed */
+    TCG_COND_NEVER  = 0 | 0 | 0 | 0,
+    TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
+    TCG_COND_EQ     = 8 | 0 | 0 | 0,
+    TCG_COND_NE     = 8 | 0 | 0 | 1,
+    /* signed */
+    TCG_COND_LT     = 0 | 0 | 2 | 0,
+    TCG_COND_GE     = 0 | 0 | 2 | 1,
+    TCG_COND_LE     = 8 | 0 | 2 | 0,
+    TCG_COND_GT     = 8 | 0 | 2 | 1,
     /* unsigned */
-    TCG_COND_LTU,
-    TCG_COND_GEU,
-    TCG_COND_LEU,
-    TCG_COND_GTU,
+    TCG_COND_LTU    = 0 | 4 | 0 | 0,
+    TCG_COND_GEU    = 0 | 4 | 0 | 1,
+    TCG_COND_LEU    = 8 | 4 | 0 | 0,
+    TCG_COND_GTU    = 8 | 4 | 0 | 1,
 } TCGCond;
 
 /* Invert the sense of the comparison.  */
@@ -288,13 +326,34 @@ static inline TCGCond tcg_invert_cond(TCGCond c)
 /* Swap the operands in a comparison.  */
 static inline TCGCond tcg_swap_cond(TCGCond c)
 {
-    int mask = (c < TCG_COND_LT ? 0 : c < TCG_COND_LTU ? 7 : 15);
-    return (TCGCond)(c ^ mask);
+    return c & 6 ? (TCGCond)(c ^ 9) : c;
 }
 
+/* Create an "unsigned" version of a "signed" comparison.  */
 static inline TCGCond tcg_unsigned_cond(TCGCond c)
 {
-    return (c >= TCG_COND_LT && c <= TCG_COND_GT ? c + 4 : c);
+    return c & 2 ? (TCGCond)(c ^ 6) : c;
+}
+
+/* Must a comparison be considered unsigned?  */
+static inline bool is_unsigned_cond(TCGCond c)
+{
+    return (c & 4) != 0;
+}
+
+/* Create a "high" version of a double-word comparison.
+   This removes equality from a LTE or GTE comparison.  */
+static inline TCGCond tcg_high_cond(TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_GE:
+    case TCG_COND_LE:
+    case TCG_COND_GEU:
+    case TCG_COND_LEU:
+        return (TCGCond)(c ^ 8);
+    default:
+        return c;
+    }
 }
 
 #define TEMP_VAL_DEAD  0
@@ -335,7 +394,6 @@ struct TCGContext {
     TCGPool *pool_first, *pool_current, *pool_first_large;
     TCGLabel *labels;
     int nb_labels;
-    TCGTemp *temps; /* globals first, temps after */
     int nb_globals;
     int nb_temps;
     /* index of free temps, -1 if none */
@@ -343,13 +401,16 @@ struct TCGContext {
 
     /* goto_tb support */
     uint8_t *code_buf;
-    unsigned long *tb_next;
+    uintptr_t *tb_next;
     uint16_t *tb_next_offset;
     uint16_t *tb_jmp_offset; /* != NULL if USE_DIRECT_JUMP */
 
     /* liveness analysis */
     uint16_t *op_dead_args; /* for each operation, each bit tells if the
                                corresponding argument is dead */
+    uint8_t *op_sync_args;  /* for each operation, each bit tells if the
+                               corresponding output argument needs to be
+                               sync to memory. */
     
     /* tells in which temporary a given register is. It does not take
        into account fixed registers */
@@ -361,7 +422,7 @@ struct TCGContext {
     int frame_reg;
 
     uint8_t *code_ptr;
-    TCGTemp static_temps[TCG_MAX_TEMPS];
+    TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
 
     TCGHelperInfo *helpers;
     int nb_helpers;
@@ -382,20 +443,34 @@ struct TCGContext {
     int64_t interm_time;
     int64_t code_time;
     int64_t la_time;
+    int64_t opt_time;
     int64_t restore_count;
     int64_t restore_time;
 #endif
 
 #ifdef CONFIG_DEBUG_TCG
     int temps_in_use;
+    int goto_tb_issue_mask;
+#endif
+
+    uint16_t gen_opc_buf[OPC_BUF_SIZE];
+    TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
+
+    uint16_t *gen_opc_ptr;
+    TCGArg *gen_opparam_ptr;
+    target_ulong gen_opc_pc[OPC_BUF_SIZE];
+    uint16_t gen_opc_icount[OPC_BUF_SIZE];
+    uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* labels info for qemu_ld/st IRs
+       The labels help to generate TLB miss case codes at the end of TB */
+    TCGLabelQemuLdst *qemu_ldst_labels;
+    int nb_qemu_ldst_labels;
 #endif
 };
 
 extern TCGContext tcg_ctx;
-extern uint16_t *gen_opc_ptr;
-extern TCGArg *gen_opparam_ptr;
-extern uint16_t gen_opc_buf[];
-extern TCGArg gen_opparam_buf[];
 
 /* pool based memory allocation */
 
@@ -458,11 +533,6 @@ static inline TCGv_i64 tcg_temp_local_new_i64(void)
 void tcg_temp_free_i64(TCGv_i64 arg);
 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg);
 
-static inline bool tcg_arg_is_local(TCGContext *s, TCGArg arg)
-{
-    return s->temps[arg].temp_local;
-}
-
 #if defined(CONFIG_DEBUG_TCG)
 /* If you call tcg_clear_temp_count() at the start of a section of
  * code which is not supposed to leak any TCG temporaries, then
@@ -499,8 +569,8 @@ enum {
     TCG_OPF_BB_END       = 0x01,
     /* Instruction clobbers call registers and potentially update globals.  */
     TCG_OPF_CALL_CLOBBER = 0x02,
-    /* Instruction has side effects: it cannot be removed
-       if its outputs are not used.  */
+    /* Instruction has side effects: it cannot be removed if its outputs
+       are not used, and might trigger exceptions.  */
     TCG_OPF_SIDE_EFFECTS = 0x04,
     /* Instruction operands are 64-bits (otherwise 32-bits).  */
     TCG_OPF_64BIT        = 0x08,
@@ -533,6 +603,15 @@ do {\
     abort();\
 } while (0)
 
+#ifdef CONFIG_DEBUG_TCG
+# define tcg_debug_assert(X) do { assert(X); } while (0)
+#elif QEMU_GNUC_PREREQ(4, 5)
+# define tcg_debug_assert(X) \
+    do { if (!(X)) { __builtin_unreachable(); } } while (0)
+#else
+# define tcg_debug_assert(X) do { (void)(X); } while (0)
+#endif
+
 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
 
 #if TCG_TARGET_REG_BITS == 32
@@ -579,7 +658,7 @@ TCGv_i64 tcg_const_i64(int64_t val);
 TCGv_i32 tcg_const_local_i32(int32_t val);
 TCGv_i64 tcg_const_local_i64(int64_t val);
 
-extern uint8_t code_gen_prologue[];
+extern uint8_t *code_gen_prologue;
 
 /* TCG targets may use a different definition of tcg_qemu_tb_exec. */
 #if !defined(tcg_qemu_tb_exec)
@@ -588,3 +667,8 @@ extern uint8_t code_gen_prologue[];
 #endif
 
 void tcg_register_jit(void *buf, size_t buf_size);
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* Generate TB finalization at the end of block */
+void tcg_out_tb_finalize(TCGContext *s);
+#endif
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index ef8580fc8d8..1707169ea8c 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -69,7 +69,6 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_exit_tb, { NULL } },
     { INDEX_op_goto_tb, { NULL } },
     { INDEX_op_call, { RI } },
-    { INDEX_op_jmp, { RI } },
     { INDEX_op_br, { NULL } },
 
     { INDEX_op_mov_i32, { R, R } },
@@ -123,6 +122,9 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_rotl_i32, { R, RI, RI } },
     { INDEX_op_rotr_i32, { R, RI, RI } },
 #endif
+#if TCG_TARGET_HAS_deposit_i32
+    { INDEX_op_deposit_i32, { R, "0", R } },
+#endif
 
     { INDEX_op_brcond_i32, { R, RI } },
 
@@ -201,6 +203,9 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_rotl_i64, { R, RI, RI } },
     { INDEX_op_rotr_i64, { R, RI, RI } },
 #endif
+#if TCG_TARGET_HAS_deposit_i64
+    { INDEX_op_deposit_i64, { R, "0", R } },
+#endif
     { INDEX_op_brcond_i64, { R, RI } },
 
 #if TCG_TARGET_HAS_ext8s_i64
@@ -300,7 +305,7 @@ static const int tcg_target_reg_alloc_order[] = {
 #endif
 };
 
-#if MAX_OPC_PARAM_IARGS != 4
+#if MAX_OPC_PARAM_IARGS != 5
 # error Fix needed, number of supported input arguments changed!
 #endif
 
@@ -309,16 +314,18 @@ static const int tcg_target_call_iarg_regs[] = {
     TCG_REG_R1,
     TCG_REG_R2,
     TCG_REG_R3,
-#if TCG_TARGET_REG_BITS == 32
-    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
 #if 0 /* used for TCG_REG_CALL_STACK */
     TCG_REG_R4,
 #endif
     TCG_REG_R5,
+#if TCG_TARGET_REG_BITS == 32
+    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
     TCG_REG_R6,
     TCG_REG_R7,
 #if TCG_TARGET_NB_REGS >= 16
     TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
 #else
 # error Too few input registers available
 #endif
@@ -581,9 +588,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_ri(s, const_args[0], args[0]);
         break;
-    case INDEX_op_jmp:
-        TODO();
-        break;
     case INDEX_op_setcond_i32:
         tcg_out_r(s, args[0]);
         tcg_out_r(s, args[1]);
@@ -655,6 +659,15 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_ri32(s, const_args[1], args[1]);
         tcg_out_ri32(s, const_args[2], args[2]);
         break;
+    case INDEX_op_deposit_i32:  /* Optional (TCG_TARGET_HAS_deposit_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
 
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_mov_i64:
@@ -682,6 +695,15 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_ri64(s, const_args[1], args[1]);
         tcg_out_ri64(s, const_args[2], args[2]);
         break;
+    case INDEX_op_deposit_i64:  /* Optional (TCG_TARGET_HAS_deposit_i64). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
     case INDEX_op_div_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
     case INDEX_op_divu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
     case INDEX_op_rem_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
@@ -798,9 +820,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_qemu_st8:
     case INDEX_op_qemu_st16:
     case INDEX_op_qemu_st32:
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_out_r(s, TCG_AREG0);
-#endif
         tcg_out_r(s, *args++);
         tcg_out_r(s, *args++);
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
@@ -811,9 +830,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
 #endif
         break;
     case INDEX_op_qemu_st64:
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_out_r(s, TCG_AREG0);
-#endif
         tcg_out_r(s, *args++);
 #if TCG_TARGET_REG_BITS == 32
         tcg_out_r(s, *args++);
@@ -867,12 +883,6 @@ static int tcg_target_const_match(tcg_target_long val,
     return arg_ct->ct & TCG_CT_CONST;
 }
 
-/* Maximum number of register used for input function arguments. */
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return ARRAY_SIZE(tcg_target_call_iarg_regs);
-}
-
 static void tcg_target_init(TCGContext *s)
 {
 #if defined(CONFIG_DEBUG_TCG_INTERPRETER)
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index 30a0f215960..a832f5cf525 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -67,7 +67,7 @@
 #define TCG_TARGET_HAS_ext8u_i32        1
 #define TCG_TARGET_HAS_ext16u_i32       1
 #define TCG_TARGET_HAS_andc_i32         0
-#define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
@@ -75,12 +75,13 @@
 #define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_rot_i32          1
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
-#define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_deposit_i64      1
 /* Not more than one of the next two defines must be 1. */
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_div2_i64         0
@@ -98,11 +99,9 @@
 #define TCG_TARGET_HAS_not_i64          1
 #define TCG_TARGET_HAS_orc_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-/* Offset to user memory in user mode. */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 /* Number of registers available.
    For 32 bit hosts, we need more than 8 registers (call arguments). */
 /* #define TCG_TARGET_NB_REGS 8 */
diff --git a/tci.c b/tci.c
index c79350d2423..2b2c11f2596 100644
--- a/tci.c
+++ b/tci.c
@@ -25,8 +25,7 @@
 #endif
 
 #include "qemu-common.h"
-#include "dyngen-exec.h"        /* env */
-#include "exec-all.h"           /* MAX_OPC_PARAM_IARGS */
+#include "exec/exec-all.h"           /* MAX_OPC_PARAM_IARGS */
 #include "tcg-op.h"
 
 /* Marker for missing code. */
@@ -37,17 +36,19 @@
         tcg_abort(); \
     } while (0)
 
-#if MAX_OPC_PARAM_IARGS != 4
+#if MAX_OPC_PARAM_IARGS != 5
 # error Fix needed, number of supported input arguments changed!
 #endif
 #if TCG_TARGET_REG_BITS == 32
 typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong,
+                                    tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong);
 #else
 typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
-                                    tcg_target_ulong, tcg_target_ulong);
+                                    tcg_target_ulong, tcg_target_ulong,
+                                    tcg_target_ulong);
 #endif
 
 /* TCI can optionally use a global register variable for env. */
@@ -63,17 +64,6 @@ uintptr_t tci_tb_ptr;
 
 static tcg_target_ulong tci_reg[TCG_TARGET_NB_REGS];
 
-#if !defined(CONFIG_TCG_PASS_AREG0)
-# define helper_ldb_mmu(env, addr, mmu_idx) __ldb_mmu(addr, mmu_idx)
-# define helper_ldw_mmu(env, addr, mmu_idx) __ldw_mmu(addr, mmu_idx)
-# define helper_ldl_mmu(env, addr, mmu_idx) __ldl_mmu(addr, mmu_idx)
-# define helper_ldq_mmu(env, addr, mmu_idx) __ldq_mmu(addr, mmu_idx)
-# define helper_stb_mmu(env, addr, val, mmu_idx) __stb_mmu(addr, val, mmu_idx)
-# define helper_stw_mmu(env, addr, val, mmu_idx) __stw_mmu(addr, val, mmu_idx)
-# define helper_stl_mmu(env, addr, val, mmu_idx) __stl_mmu(addr, val, mmu_idx)
-# define helper_stq_mmu(env, addr, val, mmu_idx) __stq_mmu(addr, val, mmu_idx)
-#endif /* !CONFIG_TCG_PASS_AREG0 */
-
 static tcg_target_ulong tci_read_reg(TCGReg index)
 {
     assert(index < ARRAY_SIZE(tci_reg));
@@ -348,9 +338,9 @@ static uint64_t tci_read_ri64(uint8_t **tb_ptr)
 }
 #endif
 
-static target_ulong tci_read_label(uint8_t **tb_ptr)
+static tcg_target_ulong tci_read_label(uint8_t **tb_ptr)
 {
-    target_ulong label = tci_read_i(tb_ptr);
+    tcg_target_ulong label = tci_read_i(tb_ptr);
     assert(label != 0);
     return label;
 }
@@ -501,18 +491,20 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
                                           tci_read_reg(TCG_REG_R5),
                                           tci_read_reg(TCG_REG_R6),
                                           tci_read_reg(TCG_REG_R7),
-                                          tci_read_reg(TCG_REG_R8));
+                                          tci_read_reg(TCG_REG_R8),
+                                          tci_read_reg(TCG_REG_R9),
+                                          tci_read_reg(TCG_REG_R10));
             tci_write_reg(TCG_REG_R0, tmp64);
             tci_write_reg(TCG_REG_R1, tmp64 >> 32);
 #else
             tmp64 = ((helper_function)t0)(tci_read_reg(TCG_REG_R0),
                                           tci_read_reg(TCG_REG_R1),
                                           tci_read_reg(TCG_REG_R2),
-                                          tci_read_reg(TCG_REG_R3));
+                                          tci_read_reg(TCG_REG_R3),
+                                          tci_read_reg(TCG_REG_R5));
             tci_write_reg(TCG_REG_R0, tmp64);
 #endif
             break;
-        case INDEX_op_jmp:
         case INDEX_op_br:
             label = tci_read_label(&tb_ptr);
             assert(tb_ptr == old_code_ptr + op_size);
@@ -697,6 +689,17 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
             tci_write_reg32(t0, (t1 >> t2) | (t1 << (32 - t2)));
             break;
 #endif
+#if TCG_TARGET_HAS_deposit_i32
+        case INDEX_op_deposit_i32:
+            t0 = *tb_ptr++;
+            t1 = tci_read_r32(&tb_ptr);
+            t2 = tci_read_r32(&tb_ptr);
+            tmp16 = *tb_ptr++;
+            tmp8 = *tb_ptr++;
+            tmp32 = (((1 << tmp8) - 1) << tmp16);
+            tci_write_reg32(t0, (t1 & ~tmp32) | ((t2 << tmp16) & tmp32));
+            break;
+#endif
         case INDEX_op_brcond_i32:
             t0 = tci_read_r32(&tb_ptr);
             t1 = tci_read_ri32(&tb_ptr);
@@ -944,6 +947,17 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
             TODO();
             break;
 #endif
+#if TCG_TARGET_HAS_deposit_i64
+        case INDEX_op_deposit_i64:
+            t0 = *tb_ptr++;
+            t1 = tci_read_r64(&tb_ptr);
+            t2 = tci_read_r64(&tb_ptr);
+            tmp16 = *tb_ptr++;
+            tmp8 = *tb_ptr++;
+            tmp64 = (((1ULL << tmp8) - 1) << tmp16);
+            tci_write_reg64(t0, (t1 & ~tmp64) | ((t2 << tmp16) & tmp64));
+            break;
+#endif
         case INDEX_op_brcond_i64:
             t0 = tci_read_r64(&tb_ptr);
             t1 = tci_read_ri64(&tb_ptr);
diff --git a/tests/.gitignore b/tests/.gitignore
index f9041f3d32d..38c94ef1da3 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -10,4 +10,5 @@ test-qmp-commands.h
 test-qmp-commands
 test-qmp-input-strict
 test-qmp-marshal.c
+test-x86-cpuid
 *-test
diff --git a/tests/Makefile b/tests/Makefile
index 26a67ce6f5b..c681cebd18f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,31 +1,74 @@
 export SRC_PATH
 
 check-unit-y = tests/check-qdict$(EXESUF)
+gcov-files-check-qdict-y = qobject/qdict.c
 check-unit-y += tests/check-qfloat$(EXESUF)
+gcov-files-check-qfloat-y = qobject/qfloat.c
 check-unit-y += tests/check-qint$(EXESUF)
+gcov-files-check-qint-y = qobject/qint.c
 check-unit-y += tests/check-qstring$(EXESUF)
+gcov-files-check-qstring-y = qobject/qstring.c
 check-unit-y += tests/check-qlist$(EXESUF)
+gcov-files-check-qlist-y = qobject/qlist.c
 check-unit-y += tests/check-qjson$(EXESUF)
+gcov-files-check-qjson-y = qobject/qjson.c
 check-unit-y += tests/test-qmp-output-visitor$(EXESUF)
+gcov-files-test-qmp-output-visitor-y = qapi/qmp-output-visitor.c
 check-unit-y += tests/test-qmp-input-visitor$(EXESUF)
+gcov-files-test-qmp-input-visitor-y = qapi/qmp-input-visitor.c
 check-unit-y += tests/test-qmp-input-strict$(EXESUF)
 check-unit-y += tests/test-qmp-commands$(EXESUF)
+gcov-files-test-qmp-commands-y = qapi/qmp-dispatch.c
 check-unit-y += tests/test-string-input-visitor$(EXESUF)
+gcov-files-test-string-input-visitor-y = qapi/string-input-visitor.c
 check-unit-y += tests/test-string-output-visitor$(EXESUF)
+gcov-files-test-string-output-visitor-y = qapi/string-output-visitor.c
 check-unit-y += tests/test-coroutine$(EXESUF)
+ifeq ($(CONFIG_WIN32),y)
+gcov-files-test-coroutine-y = coroutine-win32.c
+else
+ifeq ($(CONFIG_UCONTEXT_COROUTINE),y)
+gcov-files-test-coroutine-y = coroutine-ucontext.c
+else
+ifeq ($(CONFIG_SIGALTSTACK_COROUTINE),y)
+gcov-files-test-coroutine-y = coroutine-sigaltstack.c
+else
+gcov-files-test-coroutine-y = coroutine-gthread.c
+endif
+endif
+endif
 check-unit-y += tests/test-visitor-serialization$(EXESUF)
 check-unit-y += tests/test-iov$(EXESUF)
+gcov-files-test-iov-y = util/iov.c
+check-unit-y += tests/test-aio$(EXESUF)
+gcov-files-test-aio-$(CONFIG_WIN32) = aio-win32.c
+gcov-files-test-aio-$(CONFIG_POSIX) = aio-posix.c
+check-unit-y += tests/test-thread-pool$(EXESUF)
+gcov-files-test-thread-pool-y = thread-pool.c
+gcov-files-test-hbitmap-y = util/hbitmap.c
+check-unit-y += tests/test-hbitmap$(EXESUF)
+check-unit-y += tests/test-x86-cpuid$(EXESUF)
+# all code tested by test-x86-cpuid is inside topology.h
+gcov-files-test-x86-cpuid-y =
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
 # All QTests for now are POSIX-only, but the dependencies are
 # really in libqtest, not in the testcases themselves.
 check-qtest-i386-y = tests/fdc-test$(EXESUF)
+gcov-files-i386-y = hw/fdc.c
 check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
+gcov-files-i386-y += hw/hd-geometry.c
 check-qtest-i386-y += tests/rtc-test$(EXESUF)
 check-qtest-x86_64-y = $(check-qtest-i386-y)
+gcov-files-i386-y += i386-softmmu/hw/mc146818rtc.c
+gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
 check-qtest-sparc-y = tests/m48t59-test$(EXESUF)
 check-qtest-sparc64-y = tests/m48t59-test$(EXESUF)
+gcov-files-sparc-y += hw/m48t59.c
+gcov-files-sparc64-y += hw/m48t59.c
+check-qtest-arm-y = tests/tmp105-test$(EXESUF)
+gcov-files-arm-y += hw/tmp105.c
 
 GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h tests/test-qmp-commands.h
 
@@ -34,22 +77,27 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-coroutine.o tests/test-string-output-visitor.o \
 	tests/test-string-input-visitor.o tests/test-qmp-output-visitor.o \
 	tests/test-qmp-input-visitor.o tests/test-qmp-input-strict.o \
-	tests/test-qmp-commands.o tests/test-visitor-serialization.o
+	tests/test-qmp-commands.o tests/test-visitor-serialization.o \
+	tests/test-x86-cpuid.o
 
-test-qapi-obj-y =  $(qobject-obj-y) $(qapi-obj-y) $(tools-obj-y)
-test-qapi-obj-y += tests/test-qapi-visit.o tests/test-qapi-types.o
-test-qapi-obj-y += module.o
+test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o
 
 $(test-obj-y): QEMU_INCLUDES += -Itests
 
-tests/check-qint$(EXESUF): tests/check-qint.o qint.o $(tools-obj-y)
-tests/check-qstring$(EXESUF): tests/check-qstring.o qstring.o $(tools-obj-y)
-tests/check-qdict$(EXESUF): tests/check-qdict.o qdict.o qfloat.o qint.o qstring.o qbool.o qlist.o $(tools-obj-y)
-tests/check-qlist$(EXESUF): tests/check-qlist.o qlist.o qint.o $(tools-obj-y)
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o qfloat.o $(tools-obj-y)
-tests/check-qjson$(EXESUF): tests/check-qjson.o $(qobject-obj-y) $(tools-obj-y)
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(coroutine-obj-y) $(tools-obj-y)
-tests/test-iov$(EXESUF): tests/test-iov.o iov.o
+tests/test-x86-cpuid.o: QEMU_INCLUDES += -I$(SRC_PATH)/target-i386
+
+tests/check-qint$(EXESUF): tests/check-qint.o libqemuutil.a
+tests/check-qstring$(EXESUF): tests/check-qstring.o libqemuutil.a
+tests/check-qdict$(EXESUF): tests/check-qdict.o libqemuutil.a
+tests/check-qlist$(EXESUF): tests/check-qlist.o libqemuutil.a
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o libqemuutil.a
+tests/check-qjson$(EXESUF): tests/check-qjson.o libqemuutil.a libqemustub.a
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
+tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
+tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
@@ -62,18 +110,19 @@ $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o tests -p "test-" < $<, "  GEN   $@")
 
 
-tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y)
-tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y)
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y)
-tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y)
+tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) qapi-types.o qapi-visit.o libqemuutil.a libqemustub.a
+tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 
-tests/rtc-test$(EXESUF): tests/rtc-test.o $(trace-obj-y)
-tests/m48t59-test$(EXESUF): tests/m48t59-test.o $(trace-obj-y)
-tests/fdc-test$(EXESUF): tests/fdc-test.o tests/libqtest.o $(trace-obj-y)
-tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o tests/libqtest.o $(trace-obj-y)
+tests/rtc-test$(EXESUF): tests/rtc-test.o
+tests/m48t59-test$(EXESUF): tests/m48t59-test.o
+tests/fdc-test$(EXESUF): tests/fdc-test.o
+tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
+tests/tmp105-test$(EXESUF): tests/tmp105-test.o
 
 # QTest rules
 
@@ -81,7 +130,8 @@ TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
 QTEST_TARGETS=$(foreach TARGET,$(TARGETS), $(if $(check-qtest-$(TARGET)-y), $(TARGET),))
 check-qtest-$(CONFIG_POSIX)=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
 
-qtest-obj-y = tests/libqtest.o $(oslib-obj-y) $(tools-obj-y)
+qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
+qtest-obj-y += tests/libi2c.o tests/libi2c-omap.o
 $(check-qtest-y): $(qtest-obj-y)
 
 .PHONY: check-help
@@ -104,17 +154,28 @@ check-help:
 
 SPEED = quick
 GTESTER_OPTIONS = -k $(if $(V),--verbose,-q)
+GCOV_OPTIONS = -n $(if $(V),-f,)
 
 # gtester tests, possibly with verbose output
 
 .PHONY: $(patsubst %, check-qtest-%, $(QTEST_TARGETS))
 $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: $(check-qtest-y)
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
 	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
 		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
 
 .PHONY: $(patsubst %, check-%, $(check-unit-y))
 $(patsubst %, check-%, $(check-unit-y)): check-%: %
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
 	$(call quiet-command,gtester $(GTESTER_OPTIONS) -m=$(SPEED) $*,"GTESTER $*")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
 
 # gtester tests with XML output
 
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index fc0d2765384..dc5f05a85fa 100644
--- a/tests/check-qdict.c
+++ b/tests/check-qdict.c
@@ -11,9 +11,9 @@
  */
 #include <glib.h>
 
-#include "qint.h"
-#include "qdict.h"
-#include "qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qfloat.c b/tests/check-qfloat.c
index cdc66ea10bd..6404ac8df60 100644
--- a/tests/check-qfloat.c
+++ b/tests/check-qfloat.c
@@ -12,7 +12,7 @@
  */
 #include <glib.h>
 
-#include "qfloat.h"
+#include "qapi/qmp/qfloat.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qint.c b/tests/check-qint.c
index 5a27119ae24..86868844abb 100644
--- a/tests/check-qint.c
+++ b/tests/check-qint.c
@@ -11,7 +11,7 @@
  */
 #include <glib.h>
 
-#include "qint.h"
+#include "qapi/qmp/qint.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qjson.c b/tests/check-qjson.c
index 3b896f5f9c9..32ffb436df6 100644
--- a/tests/check-qjson.c
+++ b/tests/check-qjson.c
@@ -10,13 +10,13 @@
  */
 #include <glib.h>
 
-#include "qstring.h"
-#include "qint.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "qjson.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qjson.h"
 
 #include "qemu-common.h"
 
diff --git a/tests/check-qlist.c b/tests/check-qlist.c
index 501ba262da0..b9c05d43fd6 100644
--- a/tests/check-qlist.c
+++ b/tests/check-qlist.c
@@ -11,8 +11,8 @@
  */
 #include <glib.h>
 
-#include "qint.h"
-#include "qlist.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
 
 /*
  * Public Interface test-cases
diff --git a/tests/check-qstring.c b/tests/check-qstring.c
index addad6c6737..95dc9e3e7b8 100644
--- a/tests/check-qstring.c
+++ b/tests/check-qstring.c
@@ -11,7 +11,7 @@
  */
 #include <glib.h>
 
-#include "qstring.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/fdc-test.c b/tests/fdc-test.c
index fa7441110df..4b0301da469 100644
--- a/tests/fdc-test.c
+++ b/tests/fdc-test.c
@@ -48,13 +48,17 @@ enum {
 
 enum {
     CMD_SENSE_INT           = 0x08,
+    CMD_READ_ID             = 0x0a,
     CMD_SEEK                = 0x0f,
+    CMD_VERIFY              = 0x16,
     CMD_READ                = 0xe6,
     CMD_RELATIVE_SEEK_OUT   = 0x8f,
     CMD_RELATIVE_SEEK_IN    = 0xcf,
 };
 
 enum {
+    BUSY    = 0x10,
+    NONDMA  = 0x20,
     RQM     = 0x80,
     DIO     = 0x40,
 
@@ -110,7 +114,7 @@ static void ack_irq(uint8_t *pcn)
     g_assert(!get_irq(FLOPPY_IRQ));
 }
 
-static uint8_t send_read_command(void)
+static uint8_t send_read_command(uint8_t cmd)
 {
     uint8_t drive = 0;
     uint8_t head = 0;
@@ -126,7 +130,7 @@ static uint8_t send_read_command(void)
 
     uint8_t ret = 0;
 
-    floppy_send(CMD_READ);
+    floppy_send(cmd);
     floppy_send(head << 2 | drive);
     g_assert(!get_irq(FLOPPY_IRQ));
     floppy_send(cyl);
@@ -152,7 +156,70 @@ static uint8_t send_read_command(void)
     }
 
     st0 = floppy_recv();
-    if (st0 != 0x60) {
+    if (st0 != 0x40) {
+        ret = 1;
+    }
+
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    return ret;
+}
+
+static uint8_t send_read_no_dma_command(int nb_sect, uint8_t expected_st0)
+{
+    uint8_t drive = 0;
+    uint8_t head = 0;
+    uint8_t cyl = 0;
+    uint8_t sect_addr = 1;
+    uint8_t sect_size = 2;
+    uint8_t eot = nb_sect;
+    uint8_t gap = 0x1b;
+    uint8_t gpl = 0xff;
+
+    uint8_t msr = 0;
+    uint8_t st0;
+
+    uint8_t ret = 0;
+
+    floppy_send(CMD_READ);
+    floppy_send(head << 2 | drive);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(cyl);
+    floppy_send(head);
+    floppy_send(sect_addr);
+    floppy_send(sect_size);
+    floppy_send(eot);
+    floppy_send(gap);
+    floppy_send(gpl);
+
+    uint16_t i = 0;
+    uint8_t n = 2;
+    for (; i < n; i++) {
+        msr = inb(FLOPPY_BASE + reg_msr);
+        if (msr == (BUSY | NONDMA | DIO | RQM)) {
+            break;
+        }
+        sleep(1);
+    }
+
+    if (i >= n) {
+        return 1;
+    }
+
+    /* Non-DMA mode */
+    for (i = 0; i < 512 * 2 * nb_sect; i++) {
+        msr = inb(FLOPPY_BASE + reg_msr);
+        assert_bit_set(msr, BUSY | RQM | DIO);
+        inb(FLOPPY_BASE + reg_fifo);
+    }
+
+    st0 = floppy_recv();
+    if (st0 != expected_st0) {
         ret = 1;
     }
 
@@ -213,11 +280,11 @@ static void test_read_without_media(void)
 {
     uint8_t ret;
 
-    ret = send_read_command();
+    ret = send_read_command(CMD_READ);
     g_assert(ret == 0);
 }
 
-static void test_media_change(void)
+static void test_media_insert(void)
 {
     uint8_t dir;
 
@@ -245,6 +312,13 @@ static void test_media_change(void)
     assert_bit_clear(dir, DSKCHG);
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_clear(dir, DSKCHG);
+}
+
+static void test_media_change(void)
+{
+    uint8_t dir;
+
+    test_media_insert();
 
     /* Eject the floppy and check that DSKCHG is set. Reading it out doesn't
      * reset the bit. */
@@ -320,6 +394,108 @@ static void test_relative_seek(void)
     g_assert(pcn == 0);
 }
 
+static void test_read_id(void)
+{
+    uint8_t drive = 0;
+    uint8_t head = 0;
+    uint8_t cyl;
+    uint8_t st0;
+
+    /* Seek to track 0 and check with READ ID */
+    send_seek(0);
+
+    floppy_send(CMD_READ_ID);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(head << 2 | drive);
+
+    while (!get_irq(FLOPPY_IRQ)) {
+        /* qemu involves a timer with READ ID... */
+        clock_step(1000000000LL / 50);
+    }
+
+    st0 = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    cyl = floppy_recv();
+    head = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    g_assert_cmpint(cyl, ==, 0);
+    g_assert_cmpint(head, ==, 0);
+    g_assert_cmpint(st0, ==, head << 2);
+
+    /* Seek to track 8 on head 1 and check with READ ID */
+    head = 1;
+    cyl = 8;
+
+    floppy_send(CMD_SEEK);
+    floppy_send(head << 2 | drive);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(cyl);
+    g_assert(get_irq(FLOPPY_IRQ));
+    ack_irq(NULL);
+
+    floppy_send(CMD_READ_ID);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(head << 2 | drive);
+
+    while (!get_irq(FLOPPY_IRQ)) {
+        /* qemu involves a timer with READ ID... */
+        clock_step(1000000000LL / 50);
+    }
+
+    st0 = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    cyl = floppy_recv();
+    head = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    g_assert_cmpint(cyl, ==, 8);
+    g_assert_cmpint(head, ==, 1);
+    g_assert_cmpint(st0, ==, head << 2);
+}
+
+static void test_read_no_dma_1(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(1, 0x04);
+    g_assert(ret == 0);
+}
+
+static void test_read_no_dma_18(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(18, 0x04);
+    g_assert(ret == 0);
+}
+
+static void test_read_no_dma_19(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(19, 0x20);
+    g_assert(ret == 0);
+}
+
+static void test_verify(void)
+{
+    uint8_t ret;
+
+    ret = send_read_command(CMD_VERIFY);
+    g_assert(ret == 0);
+}
+
 /* success if no crash or abort */
 static void fuzz_registers(void)
 {
@@ -369,6 +545,12 @@ int main(int argc, char **argv)
     qtest_add_func("/fdc/media_change", test_media_change);
     qtest_add_func("/fdc/sense_interrupt", test_sense_interrupt);
     qtest_add_func("/fdc/relative_seek", test_relative_seek);
+    qtest_add_func("/fdc/read_id", test_read_id);
+    qtest_add_func("/fdc/verify", test_verify);
+    qtest_add_func("/fdc/media_insert", test_media_insert);
+    qtest_add_func("/fdc/read_no_dma_1", test_read_no_dma_1);
+    qtest_add_func("/fdc/read_no_dma_18", test_read_no_dma_18);
+    qtest_add_func("/fdc/read_no_dma_19", test_read_no_dma_19);
     qtest_add_func("/fdc/fuzz-registers", fuzz_registers);
 
     ret = g_test_run();
diff --git a/tests/libi2c-omap.c b/tests/libi2c-omap.c
new file mode 100644
index 00000000000..9be57e92c44
--- /dev/null
+++ b/tests/libi2c-omap.c
@@ -0,0 +1,166 @@
+/*
+ * QTest I2C driver
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libi2c.h"
+
+#include <glib.h>
+#include <string.h>
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+
+enum OMAPI2CRegisters {
+    OMAP_I2C_REV  = 0x00,
+    OMAP_I2C_STAT = 0x08,
+    OMAP_I2C_CNT  = 0x18,
+    OMAP_I2C_DATA = 0x1c,
+    OMAP_I2C_CON  = 0x24,
+    OMAP_I2C_SA   = 0x2c,
+};
+
+enum OMAPI2CSTATBits {
+    OMAP_I2C_STAT_NACK = 1 << 1,
+    OMAP_I2C_STAT_ARDY = 1 << 2,
+    OMAP_I2C_STAT_RRDY = 1 << 3,
+    OMAP_I2C_STAT_XRDY = 1 << 4,
+    OMAP_I2C_STAT_ROVR = 1 << 11,
+    OMAP_I2C_STAT_SBD  = 1 << 15,
+};
+
+enum OMAPI2CCONBits {
+    OMAP_I2C_CON_STT    = 1 << 0,
+    OMAP_I2C_CON_STP    = 1 << 1,
+    OMAP_I2C_CON_TRX    = 1 << 9,
+    OMAP_I2C_CON_MST    = 1 << 10,
+    OMAP_I2C_CON_BE     = 1 << 14,
+    OMAP_I2C_CON_I2C_EN = 1 << 15,
+};
+
+typedef struct OMAPI2C {
+    I2CAdapter parent;
+
+    uint64_t addr;
+} OMAPI2C;
+
+
+static void omap_i2c_set_slave_addr(OMAPI2C *s, uint8_t addr)
+{
+    uint16_t data = addr;
+
+    memwrite(s->addr + OMAP_I2C_SA, &data, 2);
+    memread(s->addr + OMAP_I2C_SA, &data, 2);
+    g_assert_cmphex(data, ==, addr);
+}
+
+static void omap_i2c_send(I2CAdapter *i2c, uint8_t addr,
+                          const uint8_t *buf, uint16_t len)
+{
+    OMAPI2C *s = (OMAPI2C *)i2c;
+    uint16_t data;
+
+    omap_i2c_set_slave_addr(s, addr);
+
+    data = len;
+    memwrite(s->addr + OMAP_I2C_CNT, &data, 2);
+
+    data = OMAP_I2C_CON_I2C_EN |
+           OMAP_I2C_CON_TRX |
+           OMAP_I2C_CON_MST |
+           OMAP_I2C_CON_STT |
+           OMAP_I2C_CON_STP;
+    memwrite(s->addr + OMAP_I2C_CON, &data, 2);
+    memread(s->addr + OMAP_I2C_CON, &data, 2);
+    g_assert((data & OMAP_I2C_CON_STP) != 0);
+
+    memread(s->addr + OMAP_I2C_STAT, &data, 2);
+    g_assert((data & OMAP_I2C_STAT_NACK) == 0);
+
+    while (len > 1) {
+        memread(s->addr + OMAP_I2C_STAT, &data, 2);
+        g_assert((data & OMAP_I2C_STAT_XRDY) != 0);
+
+        memwrite(s->addr + OMAP_I2C_DATA, buf, 2);
+        buf = (uint8_t *)buf + 2;
+        len -= 2;
+    }
+    if (len == 1) {
+        memread(s->addr + OMAP_I2C_STAT, &data, 2);
+        g_assert((data & OMAP_I2C_STAT_XRDY) != 0);
+
+        memwrite(s->addr + OMAP_I2C_DATA, buf, 1);
+    }
+
+    memread(s->addr + OMAP_I2C_CON, &data, 2);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+}
+
+static void omap_i2c_recv(I2CAdapter *i2c, uint8_t addr,
+                          uint8_t *buf, uint16_t len)
+{
+    OMAPI2C *s = (OMAPI2C *)i2c;
+    uint16_t data, stat;
+
+    omap_i2c_set_slave_addr(s, addr);
+
+    data = len;
+    memwrite(s->addr + OMAP_I2C_CNT, &data, 2);
+
+    data = OMAP_I2C_CON_I2C_EN |
+           OMAP_I2C_CON_MST |
+           OMAP_I2C_CON_STT |
+           OMAP_I2C_CON_STP;
+    memwrite(s->addr + OMAP_I2C_CON, &data, 2);
+    memread(s->addr + OMAP_I2C_CON, &data, 2);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+
+    memread(s->addr + OMAP_I2C_STAT, &data, 2);
+    g_assert((data & OMAP_I2C_STAT_NACK) == 0);
+
+    memread(s->addr + OMAP_I2C_CNT, &data, 2);
+    g_assert_cmpuint(data, ==, len);
+
+    while (len > 0) {
+        memread(s->addr + OMAP_I2C_STAT, &data, 2);
+        g_assert((data & OMAP_I2C_STAT_RRDY) != 0);
+        g_assert((data & OMAP_I2C_STAT_ROVR) == 0);
+
+        memread(s->addr + OMAP_I2C_DATA, &data, 2);
+
+        memread(s->addr + OMAP_I2C_STAT, &stat, 2);
+        if (unlikely(len == 1)) {
+            *buf = data & 0xf;
+            buf++;
+            len--;
+        } else {
+            memcpy(buf, &data, 2);
+            buf += 2;
+            len -= 2;
+        }
+    }
+
+    memread(s->addr + OMAP_I2C_CON, &data, 2);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+}
+
+I2CAdapter *omap_i2c_create(uint64_t addr)
+{
+    OMAPI2C *s = g_malloc0(sizeof(*s));
+    I2CAdapter *i2c = (I2CAdapter *)s;
+    uint16_t data;
+
+    s->addr = addr;
+
+    i2c->send = omap_i2c_send;
+    i2c->recv = omap_i2c_recv;
+
+    /* verify the mmio address by looking for a known signature */
+    memread(addr + OMAP_I2C_REV, &data, 2);
+    g_assert_cmphex(data, ==, 0x34);
+
+    return i2c;
+}
diff --git a/tests/libi2c.c b/tests/libi2c.c
new file mode 100644
index 00000000000..13ec85c0cbe
--- /dev/null
+++ b/tests/libi2c.c
@@ -0,0 +1,22 @@
+/*
+ * QTest I2C driver
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libi2c.h"
+#include "libqtest.h"
+
+void i2c_send(I2CAdapter *i2c, uint8_t addr,
+              const uint8_t *buf, uint16_t len)
+{
+    i2c->send(i2c, addr, buf, len);
+}
+
+void i2c_recv(I2CAdapter *i2c, uint8_t addr,
+              uint8_t *buf, uint16_t len)
+{
+    i2c->recv(i2c, addr, buf, len);
+}
diff --git a/tests/libi2c.h b/tests/libi2c.h
new file mode 100644
index 00000000000..1ce9af40535
--- /dev/null
+++ b/tests/libi2c.h
@@ -0,0 +1,30 @@
+/*
+ * I2C libqos
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef LIBQOS_I2C_H
+#define LIBQOS_I2C_H
+
+#include <stdint.h>
+
+typedef struct I2CAdapter I2CAdapter;
+struct I2CAdapter {
+    void (*send)(I2CAdapter *adapter, uint8_t addr,
+                 const uint8_t *buf, uint16_t len);
+    void (*recv)(I2CAdapter *adapter, uint8_t addr,
+                 uint8_t *buf, uint16_t len);
+};
+
+void i2c_send(I2CAdapter *i2c, uint8_t addr,
+              const uint8_t *buf, uint16_t len);
+void i2c_recv(I2CAdapter *i2c, uint8_t addr,
+              uint8_t *buf, uint16_t len);
+
+/* libi2c-omap.c */
+I2CAdapter *omap_i2c_create(uint64_t addr);
+
+#endif
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 02d039218db..913fa0535c8 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -26,8 +26,8 @@
 #include <unistd.h>
 #include <string.h>
 
-#include "compiler.h"
-#include "osdep.h"
+#include "qemu/compiler.h"
+#include "qemu/osdep.h"
 
 #define MAX_IRQ 256
 
@@ -85,6 +85,22 @@ static int socket_accept(int sock)
     return ret;
 }
 
+static pid_t qtest_qemu_pid(QTestState *s)
+{
+    FILE *f;
+    char buffer[1024];
+    pid_t pid = -1;
+
+    f = fopen(s->pid_file, "r");
+    if (f) {
+        if (fgets(buffer, sizeof(buffer), f)) {
+            pid = atoi(buffer);
+        }
+    }
+    fclose(f);
+    return pid;
+}
+
 QTestState *qtest_init(const char *extra_args)
 {
     QTestState *s;
@@ -136,25 +152,21 @@ QTestState *qtest_init(const char *extra_args)
     qtest_qmp(s, "");
     qtest_qmp(s, "{ 'execute': 'qmp_capabilities' }");
 
+    if (getenv("QTEST_STOP")) {
+        kill(qtest_qemu_pid(s), SIGSTOP);
+    }
+
     return s;
 }
 
 void qtest_quit(QTestState *s)
 {
-    FILE *f;
-    char buffer[1024];
-
-    f = fopen(s->pid_file, "r");
-    if (f) {
-        if (fgets(buffer, sizeof(buffer), f)) {
-            pid_t pid = atoi(buffer);
-            int status = 0;
-
-            kill(pid, SIGTERM);
-            waitpid(pid, &status, 0);
-        }
+    int status;
 
-        fclose(f);
+    pid_t pid = qtest_qemu_pid(s);
+    if (pid != -1) {
+        kill(pid, SIGTERM);
+        waitpid(pid, &status, 0);
     }
 
     unlink(s->pid_file);
diff --git a/tests/m48t59-test.c b/tests/m48t59-test.c
index 5179681ca5a..d79f55472dd 100644
--- a/tests/m48t59-test.c
+++ b/tests/m48t59-test.c
@@ -233,6 +233,11 @@ static void fuzz_registers(void)
         reg = (uint8_t)g_test_rand_int_range(0, 16);
         val = (uint8_t)g_test_rand_int_range(0, 256);
 
+        if (reg == 7) {
+            /* watchdog setup register, may trigger system reset, skip */
+            continue;
+        }
+
         cmos_write(reg, val);
         cmos_read(reg);
     }
diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 55b16f81dde..dd4ef11996a 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import time
 import os
 import iotests
 from iotests import qemu_img, qemu_io
@@ -98,6 +99,43 @@ class TestSingleDrive(ImageStreamingTestCase):
                          qemu_io('-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
+    def test_stream_pause(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-pause', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        offset = self.dictpath(result, 'return[0]/offset')
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/offset', offset)
+
+        result = self.vm.qmp('block-job-resume', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+        self.assertEqual(qemu_io('-c', 'map', backing_img),
+                         qemu_io('-c', 'map', test_img),
+                         'image file map does not match backing file after streaming')
+
     def test_stream_partial(self):
         self.assert_no_active_streams()
 
@@ -157,6 +195,226 @@ class TestSmallerBackingFile(ImageStreamingTestCase):
         self.assert_no_active_streams()
         self.vm.shutdown()
 
+class TestErrors(ImageStreamingTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should match STREAM_BUFFER_SIZE/512 in block/stream.c
+    STREAM_BUFFER_SIZE = 512 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.STREAM_BUFFER_SIZE / 512, event, event))
+        file.close()
+
+class TestEIO(TestErrors):
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_ignore(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_stop(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_enospc(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='enospc')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+class TestENOSPC(TestErrors):
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 28)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_enospc(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='enospc')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(result, 'return[0]/io-status', 'nospace')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
 
 class TestStreamStop(ImageStreamingTestCase):
     image_len = 8 * 1024 * 1024 * 1024 # GB
@@ -173,8 +431,6 @@ class TestStreamStop(ImageStreamingTestCase):
         os.remove(backing_img)
 
     def test_stream_stop(self):
-        import time
-
         self.assert_no_active_streams()
 
         result = self.vm.qmp('block-stream', device='drive0')
diff --git a/tests/qemu-iotests/030.out b/tests/qemu-iotests/030.out
index 2f7d3902f23..fa16b5cceff 100644
--- a/tests/qemu-iotests/030.out
+++ b/tests/qemu-iotests/030.out
@@ -1,5 +1,5 @@
-.......
+.............
 ----------------------------------------------------------------------
-Ran 7 tests
+Ran 13 tests
 
 OK
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
new file mode 100755
index 00000000000..aad535a74b3
--- /dev/null
+++ b/tests/qemu-iotests/040
@@ -0,0 +1,280 @@
+#!/usr/bin/env python
+#
+# Tests for image block commit.
+#
+# Copyright (C) 2012 IBM, Corp.
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Test for live block commit
+# Derived from Image Streaming Test 030
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+import struct
+import errno
+
+backing_img = os.path.join(iotests.test_dir, 'backing.img')
+mid_img = os.path.join(iotests.test_dir, 'mid.img')
+test_img = os.path.join(iotests.test_dir, 'test.img')
+
+class ImageCommitTestCase(iotests.QMPTestCase):
+    '''Abstract base class for image commit test cases'''
+
+    def assert_no_active_commit(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+    def cancel_and_wait(self, drive='drive0'):
+        '''Cancel a block job and wait for it to finish'''
+        result = self.vm.qmp('block-job-cancel', device=drive)
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', drive)
+                    cancelled = True
+
+        self.assert_no_active_commit()
+
+    def create_image(self, name, size):
+        file = open(name, 'w')
+        i = 0
+        while i < size:
+            sector = struct.pack('>l504xl', i / 512, i / 512)
+            file.write(sector)
+            i = i + 512
+        file.close()
+
+
+class TestSingleDrive(ImageCommitTestCase):
+    image_len = 1 * 1024 * 1024
+    test_len = 1 * 1024 * 256
+
+    def setUp(self):
+        self.create_image(backing_img, TestSingleDrive.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
+        qemu_io('-c', 'write -P 0xab 0 524288', backing_img)
+        qemu_io('-c', 'write -P 0xef 524288 524288', mid_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(mid_img)
+        os.remove(backing_img)
+
+    def test_commit(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % mid_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_commit()
+        self.vm.shutdown()
+
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_top_same_base(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % backing_img, base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % backing_img)
+
+    def test_top_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='badfile', base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image file badfile not found')
+
+    def test_base_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % mid_img, base='badfile')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'badfile\' not found')
+
+    def test_top_is_active(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % test_img, base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image as the active layer is currently unsupported')
+
+    def test_top_and_base_reversed(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % backing_img, base='%s' % mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % mid_img)
+
+    def test_top_omitted(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', "Parameter 'top' is missing")
+
+class TestRelativePaths(ImageCommitTestCase):
+    image_len = 1 * 1024 * 1024
+    test_len = 1 * 1024 * 256
+
+    dir1 = "dir1"
+    dir2 = "dir2/"
+    dir3 = "dir2/dir3/"
+
+    test_img = os.path.join(iotests.test_dir, dir3, 'test.img')
+    mid_img = "../mid.img"
+    backing_img = "../dir1/backing.img"
+
+    backing_img_abs = os.path.join(iotests.test_dir, dir1, 'backing.img')
+    mid_img_abs = os.path.join(iotests.test_dir, dir2, 'mid.img')
+
+    def setUp(self):
+        try:
+            os.mkdir(os.path.join(iotests.test_dir, self.dir1))
+            os.mkdir(os.path.join(iotests.test_dir, self.dir2))
+            os.mkdir(os.path.join(iotests.test_dir, self.dir3))
+        except OSError as exception:
+            if exception.errno != errno.EEXIST:
+                raise
+        self.create_image(self.backing_img_abs, TestRelativePaths.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.backing_img_abs, self.mid_img_abs)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.mid_img_abs, self.test_img)
+        qemu_img('rebase', '-u', '-b', self.backing_img, self.mid_img_abs)
+        qemu_img('rebase', '-u', '-b', self.mid_img, self.test_img)
+        qemu_io('-c', 'write -P 0xab 0 524288', self.backing_img_abs)
+        qemu_io('-c', 'write -P 0xef 524288 524288', self.mid_img_abs)
+        self.vm = iotests.VM().add_drive(self.test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.test_img)
+        os.remove(self.mid_img_abs)
+        os.remove(self.backing_img_abs)
+        try:
+            os.rmdir(os.path.join(iotests.test_dir, self.dir1))
+            os.rmdir(os.path.join(iotests.test_dir, self.dir3))
+            os.rmdir(os.path.join(iotests.test_dir, self.dir2))
+        except OSError as exception:
+            if exception.errno != errno.EEXIST and exception.errno != errno.ENOTEMPTY:
+                raise
+
+    def test_commit(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_commit()
+        self.vm.shutdown()
+
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_top_same_base(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img, base='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % self.mid_img)
+
+    def test_top_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='badfile', base='%s' % self.backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image file badfile not found')
+
+    def test_base_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img, base='badfile')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'badfile\' not found')
+
+    def test_top_is_active(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.test_img, base='%s' % self.backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image as the active layer is currently unsupported')
+
+    def test_top_and_base_reversed(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.backing_img, base='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % self.mid_img)
+
+
+class TestSetSpeed(ImageCommitTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', backing_img, str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(mid_img)
+        os.remove(backing_img)
+
+    def test_set_speed(self):
+        self.assert_no_active_commit()
+
+        result = self.vm.qmp('block-commit', device='drive0', top=mid_img, speed=1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        # Ensure the speed we set was accepted
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 1024 * 1024)
+
+        self.cancel_and_wait()
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
new file mode 100644
index 00000000000..b6f257674e9
--- /dev/null
+++ b/tests/qemu-iotests/040.out
@@ -0,0 +1,5 @@
+................
+----------------------------------------------------------------------
+Ran 16 tests
+
+OK
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
new file mode 100755
index 00000000000..b040820c51c
--- /dev/null
+++ b/tests/qemu-iotests/041
@@ -0,0 +1,696 @@
+#!/usr/bin/env python
+#
+# Tests for image mirroring.
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+import struct
+
+backing_img = os.path.join(iotests.test_dir, 'backing.img')
+target_backing_img = os.path.join(iotests.test_dir, 'target-backing.img')
+test_img = os.path.join(iotests.test_dir, 'test.img')
+target_img = os.path.join(iotests.test_dir, 'target.img')
+
+class ImageMirroringTestCase(iotests.QMPTestCase):
+    '''Abstract base class for image mirroring test cases'''
+
+    def assert_no_active_mirrors(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+    def cancel_and_wait(self, drive='drive0', wait_ready=True):
+        '''Cancel a block job and wait for it to finish'''
+        if wait_ready:
+            ready = False
+            while not ready:
+                for event in self.vm.get_qmp_events(wait=True):
+                    if event['event'] == 'BLOCK_JOB_READY':
+                        self.assert_qmp(event, 'data/type', 'mirror')
+                        self.assert_qmp(event, 'data/device', drive)
+                        ready = True
+
+        result = self.vm.qmp('block-job-cancel', device=drive,
+                             force=not wait_ready)
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED' or \
+                   event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', drive)
+                    if wait_ready:
+                        self.assertEquals(event['event'], 'BLOCK_JOB_COMPLETED')
+                        self.assert_qmp(event, 'data/offset', self.image_len)
+                        self.assert_qmp(event, 'data/len', self.image_len)
+                    cancelled = True
+
+        self.assert_no_active_mirrors()
+
+    def complete_and_wait(self, drive='drive0', wait_ready=True):
+        '''Complete a block job and wait for it to finish'''
+        if wait_ready:
+            ready = False
+            while not ready:
+                for event in self.vm.get_qmp_events(wait=True):
+                    if event['event'] == 'BLOCK_JOB_READY':
+                        self.assert_qmp(event, 'data/type', 'mirror')
+                        self.assert_qmp(event, 'data/device', drive)
+                        ready = True
+
+        result = self.vm.qmp('block-job-complete', device=drive)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', drive)
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+
+    def create_image(self, name, size):
+        file = open(name, 'w')
+        i = 0
+        while i < size:
+            sector = struct.pack('>l504xl', i / 512, i / 512)
+            file.write(sector)
+            i = i + 512
+        file.close()
+
+    def compare_images(self, img1, img2):
+        try:
+            qemu_img('convert', '-f', iotests.imgfmt, '-O', 'raw', img1, img1 + '.raw')
+            qemu_img('convert', '-f', iotests.imgfmt, '-O', 'raw', img2, img2 + '.raw')
+            file1 = open(img1 + '.raw', 'r')
+            file2 = open(img2 + '.raw', 'r')
+            return file1.read() == file2.read()
+        finally:
+            if file1 is not None:
+                file1.close()
+            if file2 is not None:
+                file2.close()
+            try:
+                os.remove(img1 + '.raw')
+            except OSError:
+                pass
+            try:
+                os.remove(img2 + '.raw')
+            except OSError:
+                pass
+
+class TestSingleDrive(ImageMirroringTestCase):
+    image_len = 1 * 1024 * 1024 # MB
+
+    def setUp(self):
+        self.create_image(backing_img, TestSingleDrive.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_complete(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_cancel(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait(wait_ready=False)
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+
+    def test_cancel_after_ready(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_pause(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-pause', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        offset = self.dictpath(result, 'return[0]/offset')
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/offset', offset)
+
+        result = self.vm.qmp('block-job-resume', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_small_buffer(self):
+        self.assert_no_active_mirrors()
+
+        # A small buffer is rounded up automatically
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             buf_size=4096, target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_small_buffer2(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,size=%d'
+                        % (TestSingleDrive.image_len, TestSingleDrive.image_len), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             buf_size=65536, mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
+                        % (TestSingleDrive.image_len, backing_img), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_medium_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_image_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='nonexistent', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+class TestMirrorNoBacking(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    def complete_and_wait(self, drive='drive0', wait_ready=True):
+        self.create_image(target_backing_img, TestMirrorNoBacking.image_len)
+        return ImageMirroringTestCase.complete_and_wait(self, drive, wait_ready)
+
+    def compare_images(self, img1, img2):
+        self.create_image(target_backing_img, TestMirrorNoBacking.image_len)
+        return ImageMirroringTestCase.compare_images(self, img1, img2)
+
+    def setUp(self):
+        self.create_image(backing_img, TestMirrorNoBacking.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(target_backing_img)
+        os.remove(target_img)
+
+    def test_complete(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_cancel(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        # qemu-img create fails if the image is not there
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'size=%d'
+                        %(TestMirrorNoBacking.image_len), target_backing_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
+                        % (TestMirrorNoBacking.image_len, target_backing_img), target_img)
+        os.remove(target_backing_img)
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+class TestReadErrors(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should be a multiple of twice the default granularity
+    # so that we hit this offset first in state 1
+    MIRROR_GRANULARITY = 1024 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.MIRROR_GRANULARITY / 512, event, event))
+        file.close()
+
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestReadErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        # Write something for tests that use sync='top'
+        qemu_io('-c', 'write %d 512' % (self.MIRROR_GRANULARITY + 65536),
+                        test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(False, 'job completed unexpectedly')
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+    def test_ignore_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, on_source_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'read')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        # Test COW into the target image.  The first half of the
+        # cluster at MIRROR_GRANULARITY has to be copied from
+        # backing_img, even though sync='top'.
+        qemu_img('create', '-f', iotests.imgfmt, '-ocluster_size=131072,backing_file=%s' %(backing_img), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='top',
+                             on_source_error='ignore',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'read')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+        # Detach blkdebug to compare images successfully
+        qemu_img('rebase', '-f', iotests.imgfmt, '-u', '-b', backing_img, test_img)
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_stop_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, on_source_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        ready = False
+        while not ready:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    ready = True
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.assert_qmp(result, 'return[0]/io-status', 'ok')
+
+        self.complete_and_wait(wait_ready=False)
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+class TestWriteErrors(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should be a multiple of twice the default granularity
+    # so that we hit this offset first in state 1
+    MIRROR_GRANULARITY = 1024 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.MIRROR_GRANULARITY / 512, event, event))
+        file.close()
+
+    def setUp(self):
+        self.blkdebug_file = target_img + ".blkdebug"
+        self.create_image(backing_img, TestWriteErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "write_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt, '-obacking_file=%s' %(backing_img), test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.target_img = 'blkdebug:%s:%s' % (self.blkdebug_file, target_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-osize=%d' %(TestWriteErrors.image_len), target_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'write')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(False, 'job completed unexpectedly')
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+    def test_ignore_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img,
+                             on_target_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'write')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+    def test_stop_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img,
+                             on_target_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        ready = False
+        while not ready:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'write')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    ready = True
+
+        self.complete_and_wait(wait_ready=False)
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+class TestSetSpeed(ImageMirroringTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', backing_img, str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(target_img)
+
+    def test_set_speed(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        # Default speed is 0
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 0)
+
+        result = self.vm.qmp('block-job-set-speed', device='drive0', speed=8 * 1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        # Ensure the speed we set was accepted
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 8 * 1024 * 1024)
+
+        self.cancel_and_wait()
+
+        # Check setting speed in drive-mirror works
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, speed=4*1024*1024)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 4 * 1024 * 1024)
+
+        self.cancel_and_wait()
+
+    def test_set_speed_invalid(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, speed=-1)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-set-speed', device='drive0', speed=-1)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.cancel_and_wait()
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/041.out b/tests/qemu-iotests/041.out
new file mode 100644
index 00000000000..84bfd63fba1
--- /dev/null
+++ b/tests/qemu-iotests/041.out
@@ -0,0 +1,5 @@
+......................
+----------------------------------------------------------------------
+Ran 22 tests
+
+OK
diff --git a/tests/qemu-iotests/042 b/tests/qemu-iotests/042
new file mode 100755
index 00000000000..c3c3ca8d021
--- /dev/null
+++ b/tests/qemu-iotests/042
@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Test qemu-img operation on zero size images
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2 qcow qed vmdk
+_supported_proto file
+_supported_os Linux
+
+echo
+echo "== Creating zero size image =="
+
+_make_test_img 0
+_check_test_img
+
+mv $TEST_IMG $TEST_IMG.orig
+
+echo
+echo "== Converting the image =="
+
+$QEMU_IMG convert -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+_check_test_img
+
+echo
+echo "== Converting the image, compressed =="
+
+if [ "$IMGFMT" == "qcow2" ]; then
+    $QEMU_IMG convert -c -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+fi
+_check_test_img
+
+echo
+echo "== Rebasing the image =="
+
+$QEMU_IMG rebase -u -b $TEST_IMG.orig $TEST_IMG
+$QEMU_IMG rebase -b $TEST_IMG.orig $TEST_IMG
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
+
diff --git a/tests/qemu-iotests/042.out b/tests/qemu-iotests/042.out
new file mode 100644
index 00000000000..dc80f4b5b3c
--- /dev/null
+++ b/tests/qemu-iotests/042.out
@@ -0,0 +1,15 @@
+QA output created by 042
+
+== Creating zero size image ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 
+No errors were found on the image.
+
+== Converting the image ==
+No errors were found on the image.
+
+== Converting the image, compressed ==
+No errors were found on the image.
+
+== Rebasing the image ==
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/043 b/tests/qemu-iotests/043
new file mode 100755
index 00000000000..3ba08dc94e9
--- /dev/null
+++ b/tests/qemu-iotests/043
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Test that qemu-img info --backing-chain detects infinite loops
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f $TEST_IMG.[123].base
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# Any format supporting backing files
+_supported_fmt qcow qcow2 vmdk qed
+_supported_proto generic
+_supported_os Linux
+
+
+size=128M
+_make_test_img $size
+$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG
+
+echo
+echo "== backing file references self =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.base
+_make_test_img -b $TEST_IMG.base $size
+$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG.base
+
+echo
+echo "== parent references self =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.1.base
+_make_test_img -b $TEST_IMG.1.base $size
+mv $TEST_IMG $TEST_IMG.2.base
+_make_test_img -b $TEST_IMG.2.base $size
+mv $TEST_IMG $TEST_IMG.3.base
+_make_test_img -b $TEST_IMG.3.base $size
+$QEMU_IMG rebase -u -b $TEST_IMG.2.base $TEST_IMG.1.base
+
+echo
+echo "== ancestor references another ancestor =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.1.base
+_make_test_img -b $TEST_IMG.1.base $size
+mv $TEST_IMG $TEST_IMG.2.base
+_make_test_img -b $TEST_IMG.2.base $size
+
+echo
+echo "== finite chain of length 3 (human) =="
+_img_info --backing-chain
+
+echo
+echo "== finite chain of length 3 (json) =="
+_img_info --backing-chain --output=json
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/043.out b/tests/qemu-iotests/043.out
new file mode 100644
index 00000000000..ad23337db5f
--- /dev/null
+++ b/tests/qemu-iotests/043.out
@@ -0,0 +1,66 @@
+QA output created by 043
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+
+== backing file references self ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+
+== parent references self ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.3.base' 
+
+== ancestor references another ancestor ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT.2.base' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
+
+== finite chain of length 3 (human) ==
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+backing file: TEST_DIR/t.IMGFMT.2.base
+
+image: TEST_DIR/t.IMGFMT.2.base
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+backing file: TEST_DIR/t.IMGFMT.1.base
+
+image: TEST_DIR/t.IMGFMT.1.base
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+
+== finite chain of length 3 (json) ==
+[
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "backing-filename": "TEST_DIR/t.IMGFMT.2.base", 
+        "dirty-flag": false
+    }, 
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT.2.base", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "backing-filename": "TEST_DIR/t.IMGFMT.1.base", 
+        "dirty-flag": false
+    }, 
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT.1.base", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "dirty-flag": false
+    }
+]
+*** done
diff --git a/tests/qemu-iotests/044 b/tests/qemu-iotests/044
new file mode 100755
index 00000000000..11ea0f4d35d
--- /dev/null
+++ b/tests/qemu-iotests/044
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+#
+# Tests growing a large refcount table.
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import qcow2
+from qcow2 import QcowHeader
+import iotests
+from iotests import qemu_img, qemu_img_verbose, qemu_io
+import struct
+import subprocess
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+
+class TestRefcountTableGrowth(iotests.QMPTestCase):
+    '''Abstract base class for image mirroring test cases'''
+
+    def preallocate(self, name):
+        fd = open(name, "r+b")
+        try:
+            off_reftable = 512
+            off_refblock = off_reftable + (512 * 512)
+            off_l1       = off_refblock + (512 * 512 * 64)
+            off_l2       = off_l1 + (512 * 512 * 4 * 8)
+            off_data     = off_l2 + (512 * 512 * 4 * 512)
+
+            # Write a new header
+            h = QcowHeader(fd)
+            h.refcount_table_offset = off_reftable
+            h.refcount_table_clusters = 512
+            h.l1_table_offset = off_l1
+            h.l1_size = 512 * 512 * 4
+            h.update(fd)
+
+            # Write a refcount table
+            fd.seek(off_reftable)
+
+            for i in xrange(0, h.refcount_table_clusters):
+                sector = ''.join(struct.pack('>Q',
+                    off_refblock + i * 64 * 512 + j * 512)
+                    for j in xrange(0, 64))
+                fd.write(sector)
+
+            # Write the refcount blocks
+            assert(fd.tell() == off_refblock)
+            sector = ''.join(struct.pack('>H', 1) for j in xrange(0, 64 * 256))
+            for block in xrange(0, h.refcount_table_clusters):
+                fd.write(sector)
+
+            # Write the L1 table
+            assert(fd.tell() == off_l1)
+            assert(off_l2 + 512 * h.l1_size == off_data)
+            table = ''.join(struct.pack('>Q', (1 << 63) | off_l2 + 512 * j)
+                for j in xrange(0, h.l1_size))
+            fd.write(table)
+
+            # Write the L2 tables
+            assert(fd.tell() == off_l2)
+            img_file_size = h.refcount_table_clusters * 64 * 256 * 512
+            remaining = img_file_size - off_data
+
+            off = off_data
+            while remaining > 1024 * 512:
+                pytable = list((1 << 63) | off + 512 * j
+                    for j in xrange(0, 1024))
+                table = struct.pack('>1024Q', *pytable)
+                fd.write(table)
+                remaining = remaining - 1024 * 512
+                off = off + 1024 * 512
+
+            table = ''.join(struct.pack('>Q', (1 << 63) | off + 512 * j)
+                for j in xrange(0, remaining / 512))
+            fd.write(table)
+
+
+            # Data
+            fd.truncate(img_file_size)
+
+
+        finally:
+            fd.close()
+
+
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=512', test_img, '16G')
+        self.preallocate(test_img)
+        pass
+
+
+    def tearDown(self):
+        os.remove(test_img)
+        pass
+
+    def test_grow_refcount_table(self):
+        qemu_io('-c', 'write 3800M 1M', test_img)
+        qemu_img_verbose('check' , test_img)
+        pass
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/044.out b/tests/qemu-iotests/044.out
new file mode 100644
index 00000000000..7a4007137d8
--- /dev/null
+++ b/tests/qemu-iotests/044.out
@@ -0,0 +1,6 @@
+No errors were found on the image.
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/tests/qemu-iotests/045 b/tests/qemu-iotests/045
new file mode 100755
index 00000000000..2b6f1af27ac
--- /dev/null
+++ b/tests/qemu-iotests/045
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+#
+# Tests for fdsets.
+#
+# Copyright (C) 2012 IBM Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+from iotests import qemu_img
+
+image0 = os.path.join(iotests.test_dir, 'image0')
+image1 = os.path.join(iotests.test_dir, 'image1')
+image2 = os.path.join(iotests.test_dir, 'image2')
+image3 = os.path.join(iotests.test_dir, 'image3')
+image4 = os.path.join(iotests.test_dir, 'image4')
+
+class TestFdSets(iotests.QMPTestCase):
+
+    def setUp(self):
+        self.vm = iotests.VM()
+        qemu_img('create', '-f', iotests.imgfmt, image0, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image1, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image2, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image3, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image4, '128K')
+        self.file0 = open(image0, 'r')
+        self.file1 = open(image1, 'w+')
+        self.file2 = open(image2, 'r')
+        self.file3 = open(image3, 'r')
+        self.file4 = open(image4, 'r')
+        self.vm.add_fd(self.file0.fileno(), 1, 'image0:r')
+        self.vm.add_fd(self.file1.fileno(), 1, 'image1:w+')
+        self.vm.add_fd(self.file2.fileno(), 0, 'image2:r')
+        self.vm.add_fd(self.file3.fileno(), 2, 'image3:r')
+        self.vm.add_fd(self.file4.fileno(), 2, 'image4:r')
+        self.vm.add_drive("/dev/fdset/1")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        self.file0.close()
+        self.file1.close()
+        self.file2.close()
+        self.file3.close()
+        self.file4.close()
+        os.remove(image0)
+        os.remove(image1)
+        os.remove(image2)
+        os.remove(image3)
+        os.remove(image4)
+
+    def test_query_fdset(self):
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 2)
+        self.assert_qmp(result, 'return[1]/fdset-id', 1)
+        self.assert_qmp(result, 'return[2]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image3:r')
+        self.assert_qmp(result, 'return[0]/fds[1]/opaque', 'image4:r')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[1]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[2]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fdset(self):
+        result = self.vm.qmp('remove-fd', fdset_id=2)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 1)
+        self.assert_qmp(result, 'return[1]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[0]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fd(self):
+        result = self.vm.qmp('query-fdsets')
+        fd_image3 = result['return'][0]['fds'][0]['fd']
+        result = self.vm.qmp('remove-fd', fdset_id=2, fd=fd_image3)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 2)
+        self.assert_qmp(result, 'return[1]/fdset-id', 1)
+        self.assert_qmp(result, 'return[2]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image4:r')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[1]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[2]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fd_invalid_fdset(self):
+        result = self.vm.qmp('query-fdsets')
+        fd_image3 = result['return'][0]['fds'][0]['fd']
+        result = self.vm.qmp('remove-fd', fdset_id=3, fd=fd_image3)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            'File descriptor named \'fdset-id:3, fd:%d\' not found' % fd_image3)
+        self.vm.shutdown()
+
+    def test_remove_fd_invalid_fd(self):
+        result = self.vm.qmp('query-fdsets')
+        result = self.vm.qmp('remove-fd', fdset_id=2, fd=999)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            'File descriptor named \'fdset-id:2, fd:999\' not found')
+        self.vm.shutdown()
+
+    def test_add_fd_invalid_fd(self):
+        result = self.vm.qmp('add-fd', fdset_id=2)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+                'No file descriptor supplied via SCM_RIGHTS')
+        self.vm.shutdown()
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['raw'])
diff --git a/tests/qemu-iotests/045.out b/tests/qemu-iotests/045.out
new file mode 100644
index 00000000000..3f8a935a082
--- /dev/null
+++ b/tests/qemu-iotests/045.out
@@ -0,0 +1,5 @@
+......
+----------------------------------------------------------------------
+Ran 6 tests
+
+OK
diff --git a/tests/qemu-iotests/046 b/tests/qemu-iotests/046
new file mode 100755
index 00000000000..e0176f42df8
--- /dev/null
+++ b/tests/qemu-iotests/046
@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Test concurrent cluster allocations
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+CLUSTER_SIZE=64k
+size=128M
+
+echo
+echo "== creating backing file for COW tests =="
+
+_make_test_img $size
+
+function backing_io()
+{
+    local offset=$1
+    local sectors=$2
+    local op=$3
+    local pattern=0
+    local cur_sec=0
+
+    for i in $(seq 0 $((sectors - 1))); do
+        cur_sec=$((offset / 65536 + i))
+        pattern=$(( ( (cur_sec % 128) + (cur_sec / 128)) % 128 ))
+
+        echo "$op -P $pattern $((cur_sec * 64))k 64k"
+    done
+}
+
+backing_io 0 16 write | $QEMU_IO $TEST_IMG | _filter_qemu_io
+
+mv $TEST_IMG $TEST_IMG.base
+
+_make_test_img -b $TEST_IMG.base 6G
+
+echo
+echo "== Some concurrent requests touching the same cluster =="
+
+function overlay_io()
+{
+# Allocate middle of cluster 1, then write to somewhere before and after it
+cat  <<EOF
+break write_aio A
+aio_write -P 10 0x18000 0x2000
+wait_break A
+
+aio_write -P 11 0x12000 0x2000
+aio_write -P 12 0x1c000 0x2000
+
+resume A
+aio_flush
+EOF
+
+# Sequential write case: Alloc middle of cluster 2, then write overlapping
+# to next cluster
+cat  <<EOF
+break write_aio A
+aio_write -P 20 0x28000 0x2000
+wait_break A
+aio_write -P 21 0x2a000 0x10000
+resume A
+aio_flush
+EOF
+
+# The same with a gap between both requests
+cat  <<EOF
+break write_aio A
+aio_write -P 40 0x48000 0x2000
+wait_break A
+aio_write -P 41 0x4c000 0x10000
+resume A
+aio_flush
+EOF
+
+# Sequential write, but the next cluster is already allocated
+cat  <<EOF
+write -P 70 0x76000 0x8000
+aio_flush
+break write_aio A
+aio_write -P 60 0x66000 0x2000
+wait_break A
+aio_write -P 61 0x6a000 0xe000
+resume A
+aio_flush
+EOF
+
+# Sequential write, but the next cluster is already allocated
+# and phyiscally in the right position
+cat  <<EOF
+write -P 89 0x80000 0x1000
+write -P 90 0x96000 0x8000
+aio_flush
+discard 0x80000 0x10000
+aio_flush
+break write_aio A
+aio_write -P 80 0x86000 0x2000
+wait_break A
+aio_write -P 81 0x8a000 0xe000
+resume A
+aio_flush
+EOF
+
+# Sequential write, and the next cluster is compressed
+cat  <<EOF
+write    -P 109 0xa0000 0x1000
+write -c -P 110 0xb0000 0x10000
+aio_flush
+discard 0xa0000 0x10000
+aio_flush
+break write_aio A
+aio_write -P 100 0xa6000 0x2000
+wait_break A
+aio_write -P 101 0xaa000 0xe000
+resume A
+aio_flush
+EOF
+}
+
+overlay_io | $QEMU_IO blkdebug::$TEST_IMG | _filter_qemu_io |\
+	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
+
+echo
+echo "== Verify image content =="
+
+function verify_io()
+{
+    echo read -P 0 0 0x10000
+
+    echo read -P 1  0x10000 0x2000
+    echo read -P 11 0x12000 0x2000
+    echo read -P 1  0x14000 0x4000
+    echo read -P 10 0x18000 0x2000
+    echo read -P 1  0x1a000 0x2000
+    echo read -P 12 0x1c000 0x2000
+    echo read -P 1  0x1e000 0x2000
+
+    echo read -P 2  0x20000 0x8000
+    echo read -P 20 0x28000 0x2000
+    echo read -P 21 0x2a000 0x10000
+    echo read -P 3  0x3a000 0x6000
+
+    echo read -P 4  0x40000 0x8000
+    echo read -P 40 0x48000 0x2000
+    echo read -P 4  0x4a000 0x2000
+    echo read -P 41 0x4c000 0x10000
+    echo read -P 5  0x5c000 0x4000
+
+    echo read -P 6  0x60000 0x6000
+    echo read -P 60 0x66000 0x2000
+    echo read -P 6  0x68000 0x2000
+    echo read -P 61 0x6a000 0xe000
+    echo read -P 70 0x78000 0x6000
+    echo read -P 7  0x7e000 0x2000
+
+    echo read -P 8  0x80000 0x6000
+    echo read -P 80 0x86000 0x2000
+    echo read -P 8  0x88000 0x2000
+    echo read -P 81 0x8a000 0xe000
+    echo read -P 90 0x98000 0x6000
+    echo read -P 9  0x9e000 0x2000
+
+    echo read -P 10  0xa0000 0x6000
+    echo read -P 100 0xa6000 0x2000
+    echo read -P 10  0xa8000 0x2000
+    echo read -P 101 0xaa000 0xe000
+    echo read -P 110 0xb8000 0x8000
+}
+
+verify_io | $QEMU_IO $TEST_IMG | _filter_qemu_io
+
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/046.out b/tests/qemu-iotests/046.out
new file mode 100644
index 00000000000..565360fe603
--- /dev/null
+++ b/tests/qemu-iotests/046.out
@@ -0,0 +1,163 @@
+QA output created by 046
+
+== creating backing file for COW tests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+qemu-io> wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 196608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 262144
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 327680
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 393216
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 458752
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 524288
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 589824
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 655360
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 720896
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 786432
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 851968
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 917504
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 983040
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+
+== Some concurrent requests touching the same cluster ==
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 32768/32768 bytes at offset XXX
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 4096/4096 bytes at offset XXX
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 32768/32768 bytes at offset XXX
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> discard 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 4096/4096 bytes at offset XXX
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> discard 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> 
+== Verify image content ==
+qemu-io> read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 65536
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 73728
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 16384/16384 bytes at offset 81920
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 98304
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 106496
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 114688
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 122880
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 131072
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 163840
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 65536/65536 bytes at offset 172032
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 237568
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 262144
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 294912
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 303104
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 65536/65536 bytes at offset 311296
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 16384/16384 bytes at offset 376832
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 393216
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 417792
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 425984
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 434176
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 491520
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 516096
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 524288
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 548864
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 557056
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 565248
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 622592
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 647168
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 655360
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 679936
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 688128
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 696320
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 753664
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index 1f6fdf5c560..b3aad89e2cc 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -136,6 +136,7 @@ check options
     -vmdk               test vmdk
     -rbd                test rbd
     -sheepdog           test sheepdog
+    -nbd                test nbd
     -xdiff		graphical mode diff
     -nocache		use O_DIRECT on backing file
     -misalign		misalign memory allocations
@@ -197,12 +198,14 @@ testlist options
 	    IMGPROTO=rbd
 	    xpand=false
 	    ;;
-
 	-sheepdog)
 	    IMGPROTO=sheepdog
 	    xpand=false
 	    ;;
-
+	-nbd)
+	    IMGPROTO=nbd
+	    xpand=false
+	    ;;
 	-nocache)
 	    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --nocache"
 	    xpand=false
@@ -350,7 +353,11 @@ fi
 
 [ "$QEMU" = "" ] && _fatal "qemu not found"
 [ "$QEMU_IMG" = "" ] && _fatal "qemu-img not found"
-[ "$QEMU_IO" = "" ] && _fatal "qemu-img not found"
+[ "$QEMU_IO" = "" ] && _fatal "qemu-io not found"
+
+if [ "$IMGPROTO" = "nbd" ] ; then
+    [ "$QEMU_NBD" = "" ] && _fatal "qemu-nbd not found"
+fi
 
 if $valgrind; then
     export REAL_QEMU_IO="$QEMU_IO_PROG"
diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config
index df082e750c9..08a3f100b83 100644
--- a/tests/qemu-iotests/common.config
+++ b/tests/qemu-iotests/common.config
@@ -90,21 +90,23 @@ export PS_ALL_FLAGS="-ef"
 if [ -z "$QEMU_PROG" ]; then
     export QEMU_PROG="`set_prog_path qemu`"
 fi
-[ "$QEMU_PROG" = "" ] && _fatal "qemu not found"
 
 if [ -z "$QEMU_IMG_PROG" ]; then
     export QEMU_IMG_PROG="`set_prog_path qemu-img`"
 fi
-[ "$QEMU_IMG_PROG" = "" ] && _fatal "qemu-img not found"
 
 if [ -z "$QEMU_IO_PROG" ]; then
     export QEMU_IO_PROG="`set_prog_path qemu-io`"
 fi
-[ "$QEMU_IO_PROG" = "" ] && _fatal "qemu-io not found"
+
+if [ -z "$QEMU_NBD_PROG" ]; then
+    export QEMU_NBD_PROG="`set_prog_path qemu-nbd`"
+fi
 
 export QEMU=$QEMU_PROG
-export QEMU_IMG=$QEMU_IMG_PROG 
+export QEMU_IMG=$QEMU_IMG_PROG
 export QEMU_IO="$QEMU_IO_PROG $QEMU_IO_OPTIONS"
+export QEMU_NBD=$QEMU_NBD_PROG
 
 [ -f /etc/qemu-iotest.config ]       && . /etc/qemu-iotest.config
 
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index d534e9466d2..aef5f52b4f4 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -49,6 +49,9 @@ umask 022
 
 if [ "$IMGPROTO" = "file" ]; then
     TEST_IMG=$TEST_DIR/t.$IMGFMT
+elif [ "$IMGPROTO" = "nbd" ]; then
+    TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+    TEST_IMG="nbd:127.0.0.1:10810"
 else
     TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
 fi
@@ -86,6 +89,13 @@ _make_test_img()
     local extra_img_options=""
     local image_size=$*
     local optstr=""
+    local img_name=""
+
+    if [ -n "$TEST_IMG_FILE" ]; then
+        img_name=$TEST_IMG_FILE
+    else
+        img_name=$TEST_IMG
+    fi
 
     if [ -n "$IMGOPTS" ]; then
         optstr=$(_optstr_add "$optstr" "$IMGOPTS")
@@ -104,7 +114,7 @@ _make_test_img()
     fi
 
     # XXX(hch): have global image options?
-    $QEMU_IMG create -f $IMGFMT $extra_img_options $TEST_IMG $image_size | \
+    $QEMU_IMG create -f $IMGFMT $extra_img_options $img_name $image_size | \
         sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
             -e "s#$TEST_DIR#TEST_DIR#g" \
             -e "s#$IMGFMT#IMGFMT#g" \
@@ -115,12 +125,23 @@ _make_test_img()
             -e "s# compat6=\\(on\\|off\\)##g" \
             -e "s# static=\\(on\\|off\\)##g" \
             -e "s# lazy_refcounts=\\(on\\|off\\)##g"
+
+    # Start an NBD server on the image file, which is what we'll be talking to
+    if [ $IMGPROTO = "nbd" ]; then
+        eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810  $TEST_IMG_FILE &"
+        QEMU_NBD_PID=$!
+        sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
+    fi
 }
 
 _cleanup_test_img()
 {
     case "$IMGPROTO" in
 
+        nbd)
+            kill $QEMU_NBD_PID
+            rm -f $TEST_IMG_FILE
+            ;;
         file)
             rm -f $TEST_DIR/t.$IMGFMT
             rm -f $TEST_DIR/t.$IMGFMT.orig
@@ -145,6 +166,16 @@ _check_test_img()
     	sed -e 's/qemu-img\: This image format does not support checks/No errors were found on the image./'
 }
 
+_img_info()
+{
+    $QEMU_IMG info "$@" $TEST_IMG 2>&1 | \
+        sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
+            -e "s#$TEST_DIR#TEST_DIR#g" \
+            -e "s#$IMGFMT#IMGFMT#g" \
+            -e "/^disk size:/ D" \
+            -e "/actual-size/ D"
+}
+
 _get_pids_by_name()
 {
     if [ $# -ne 1 ]
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index ebb5ca4b419..a0307de06bc 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -36,7 +36,7 @@
 027 rw auto quick
 028 rw backing auto
 029 rw auto quick
-030 rw auto
+030 rw auto backing
 031 rw auto quick
 032 rw auto
 033 rw auto
@@ -46,3 +46,10 @@
 037 rw auto backing
 038 rw auto backing
 039 rw auto
+040 rw auto
+041 rw auto backing
+042 rw auto quick
+043 rw auto backing
+044 rw auto
+045 rw auto
+046 rw auto aio
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index e05b1d640b4..569ca3d8046 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -19,6 +19,7 @@
 import os
 import re
 import subprocess
+import string
 import unittest
 import sys; sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'QMP'))
 import qmp
@@ -41,6 +42,10 @@ def qemu_img(*args):
     devnull = open('/dev/null', 'r+')
     return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
 
+def qemu_img_verbose(*args):
+    '''Run qemu-img without suppressing its output and return the exit code'''
+    return subprocess.call(qemu_img_args + list(args))
+
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
@@ -74,6 +79,18 @@ class VM(object):
         self._num_drives += 1
         return self
 
+    def add_fd(self, fd, fdset, opaque, opts=''):
+        '''Pass a file descriptor to the VM'''
+        options = ['fd=%d' % fd,
+                   'set=%d' % fdset,
+                   'opaque=%s' % opaque]
+        if opts:
+            options.append(opts)
+
+        self._args.append('-add-fd')
+        self._args.append(','.join(options))
+        return self
+
     def launch(self):
         '''Launch the VM and establish a QMP connection'''
         devnull = open('/dev/null', 'rb')
@@ -96,9 +113,18 @@ class VM(object):
             os.remove(self._qemu_log_path)
             self._popen = None
 
+    underscore_to_dash = string.maketrans('_', '-')
     def qmp(self, cmd, **args):
         '''Invoke a QMP command and return the result dict'''
-        return self._qmp.cmd(cmd, args=args)
+        qmp_args = dict()
+        for k in args.keys():
+            qmp_args[k.translate(self.underscore_to_dash)] = args[k]
+
+        return self._qmp.cmd(cmd, args=qmp_args)
+
+    def get_qmp_event(self, wait=False):
+        '''Poll for one queued QMP events and return it'''
+        return self._qmp.pull_event(wait=wait)
 
     def get_qmp_events(self, wait=False):
         '''Poll for queued QMP events and return a list of dicts'''
@@ -132,6 +158,13 @@ class QMPTestCase(unittest.TestCase):
                     self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
         return d
 
+    def assert_qmp_absent(self, d, path):
+        try:
+            result = self.dictpath(d, path)
+        except AssertionError:
+            return
+        self.fail('path "%s" has value "%s"' % (path, str(result)))
+
     def assert_qmp(self, d, path, value):
         '''Assert that the value for a specific path in a QMP dict matches'''
         result = self.dictpath(d, path)
@@ -165,4 +198,4 @@ def main(supported_fmts=[]):
     try:
         unittest.main(testRunner=MyTestRunner)
     finally:
-        sys.stderr.write(re.sub(r'Ran (\d+) test[s] in [\d.]+s', r'Ran \1 tests', output.getvalue()))
+        sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', output.getvalue()))
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 97f37707bca..fecf5b9a598 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -233,8 +233,9 @@ def usage():
     for name, handler, num_args, desc in cmds:
         print "    %-20s - %s" % (name, desc)
 
-if len(sys.argv) < 3:
-    usage()
-    sys.exit(1)
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        usage()
+        sys.exit(1)
 
-main(sys.argv[1], sys.argv[2], sys.argv[3:])
+    main(sys.argv[1], sys.argv[2], sys.argv[3:])
diff --git a/tests/rtc-test.c b/tests/rtc-test.c
index f23ac3a6cf1..e7123cafbc8 100644
--- a/tests/rtc-test.c
+++ b/tests/rtc-test.c
@@ -179,6 +179,81 @@ static void check_time(int wiggle)
 
 static int wiggle = 2;
 
+static void set_year_20xx(void)
+{
+    /* Set BCD mode */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_DM);
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_CENTURY, 0x20);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    if (sizeof(time_t) == 4) {
+        return;
+    }
+
+    /* Set a date in 2080 to ensure there is no year-2038 overflow.  */
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x80);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x80);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+}
+
+static void set_year_1980(void)
+{
+    /* Set BCD mode */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_DM);
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x80);
+    cmos_write(RTC_CENTURY, 0x19);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x80);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x19);
+}
+
 static void bcd_check_time(void)
 {
     /* Set BCD mode */
@@ -256,6 +331,45 @@ static void fuzz_registers(void)
     }
 }
 
+static void register_b_set_flag(void)
+{
+    /* Enable binary-coded decimal (BCD) mode and SET flag in Register B*/
+    cmos_write(RTC_REG_B, (cmos_read(RTC_REG_B) & ~REG_B_DM) | REG_B_SET);
+
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_CENTURY, 0x20);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    /* Since SET flag is still enabled, these are equality checks. */
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), ==, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    /* Disable SET flag in Register B */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_SET);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+
+    /* Since SET flag is disabled, this is an inequality check.
+     * We (reasonably) assume that no (sexagesimal) overflow occurs. */
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+}
+
 int main(int argc, char **argv)
 {
     QTestState *s = NULL;
@@ -269,6 +383,9 @@ int main(int argc, char **argv)
     qtest_add_func("/rtc/bcd/check-time", bcd_check_time);
     qtest_add_func("/rtc/dec/check-time", dec_check_time);
     qtest_add_func("/rtc/alarm-time", alarm_time);
+    qtest_add_func("/rtc/set-year/20xx", set_year_20xx);
+    qtest_add_func("/rtc/set-year/1980", set_year_1980);
+    qtest_add_func("/rtc/register_b_set_flag", register_b_set_flag);
     qtest_add_func("/rtc/fuzz-registers", fuzz_registers);
     ret = g_test_run();
 
diff --git a/tests/tcg/Makefile b/tests/tcg/Makefile
index 15e36a208c7..24e3154cae1 100644
--- a/tests/tcg/Makefile
+++ b/tests/tcg/Makefile
@@ -1,13 +1,13 @@
--include ../config-host.mak
+-include ../../config-host.mak
 -include $(SRC_PATH)/rules.mak
 
-$(call set-vpath, $(SRC_PATH)/tests)
+$(call set-vpath, $(SRC_PATH)/tests/tcg)
 
-QEMU=../i386-linux-user/qemu-i386
-QEMU_X86_64=../x86_64-linux-user/qemu-x86_64
+QEMU=../../i386-linux-user/qemu-i386
+QEMU_X86_64=../../x86_64-linux-user/qemu-x86_64
 CC_X86_64=$(CC_I386) -m64
 
-QEMU_INCLUDES += -I..
+QEMU_INCLUDES += -I../..
 CFLAGS=-Wall -O2 -g -fno-strict-aliasing
 #CFLAGS+=-msse2
 LDFLAGS=
@@ -22,6 +22,7 @@ I386_TESTS=hello-i386 \
 	   testthread \
 	   sha1-i386 \
 	   test-i386 \
+	   test-i386-fprem \
 	   test-mmap \
 	   # runcom
 
@@ -36,6 +37,7 @@ TESTS += $(I386_TESTS)
 endif
 
 all: $(patsubst %,run-%,$(TESTS))
+test: all
 
 # rules to run tests
 
@@ -54,6 +56,11 @@ run-test-i386: test-i386
 	-$(QEMU) test-i386 > test-i386.out
 	@if diff -u test-i386.ref test-i386.out ; then echo "Auto Test OK"; fi
 
+run-test-i386-fprem: test-i386-fprem
+	./test-i386-fprem > test-i386-fprem.ref
+	-$(QEMU) test-i386-fprem > test-i386-fprem.out
+	@if diff -u test-i386-fprem.ref test-i386-fprem.out ; then echo "Auto Test OK"; fi
+
 run-test-x86_64: test-x86_64
 	./test-x86_64 > test-x86_64.ref
 	-$(QEMU_X86_64) test-x86_64 > test-x86_64.out
@@ -74,7 +81,10 @@ run-test_path: test_path
 # rules to compile tests
 
 test_path: test_path.o
+	$(CC_I386) $(LDFLAGS) -o $@ $^ $(LIBS)
+
 test_path.o: test_path.c
+	$(CC_I386) $(QEMU_INCLUDES) $(GLIB_CFLAGS) $(CFLAGS) -c -o $@ $^
 
 hello-i386: hello-i386.c
 	$(CC_I386) -nostdlib $(CFLAGS) -static $(LDFLAGS) -o $@ $<
@@ -86,12 +96,15 @@ testthread: testthread.c
 # i386/x86_64 emulation test (test various opcodes) */
 test-i386: test-i386.c test-i386-code16.S test-i386-vm86.S \
            test-i386.h test-i386-shift.h test-i386-muldiv.h
-	$(CC_I386) $(CFLAGS) $(LDFLAGS) -o $@ \
+	$(CC_I386) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ \
               $(<D)/test-i386.c $(<D)/test-i386-code16.S $(<D)/test-i386-vm86.S -lm
 
+test-i386-fprem: test-i386-fprem.c
+	$(CC_I386) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ $^
+
 test-x86_64: test-i386.c \
            test-i386.h test-i386-shift.h test-i386-muldiv.h
-	$(CC_X86_64) $(CFLAGS) $(LDFLAGS) -o $@ $(<D)/test-i386.c -lm
+	$(CC_X86_64) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ $(<D)/test-i386.c -lm
 
 # generic Linux and CPU test
 linux-test: linux-test.c
diff --git a/tests/tcg/cris/crisutils.h b/tests/tcg/cris/crisutils.h
index 29b71cd7b93..3456b9d50da 100644
--- a/tests/tcg/cris/crisutils.h
+++ b/tests/tcg/cris/crisutils.h
@@ -1,3 +1,6 @@
+#ifndef CRISUTILS_H
+#define CRISUTILS_H 1
+
 static char *tst_cc_loc = NULL;
 
 #define cris_tst_cc_init() \
@@ -69,3 +72,5 @@ static inline void cris_tst_cc(const int n, const int z,
 	if (c) cris_tst_cc_c1(); else cris_tst_cc_c0();
 	asm volatile ("" : : "g" (_err));
 }
+
+#endif
diff --git a/tests/tcg/hello-i386.c b/tests/tcg/hello-i386.c
index 86afc34fb13..fa00380de29 100644
--- a/tests/tcg/hello-i386.c
+++ b/tests/tcg/hello-i386.c
@@ -1,6 +1,6 @@
 #include <asm/unistd.h>
 
-static inline volatile void exit(int status)
+static inline void exit(int status)
 {
   int __res;
   __asm__ volatile ("movl %%ecx,%%ebx\n"\
@@ -17,6 +17,7 @@ static inline int write(int fd, const char * buf, int len)
 		    "popl %%ebx\n"\
 		    : "=a" (status) \
 		    : "0" (__NR_write),"S" ((long)(fd)),"c" ((long)(buf)),"d" ((long)(len)));
+  return status;
 }
 
 void _start(void)
diff --git a/tests/tcg/linux-test.c b/tests/tcg/linux-test.c
index 2e4a746ac3b..83cb32ddb91 100644
--- a/tests/tcg/linux-test.c
+++ b/tests/tcg/linux-test.c
@@ -16,6 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#define _GNU_SOURCE
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -38,6 +39,7 @@
 #include <dirent.h>
 #include <setjmp.h>
 #include <sys/shm.h>
+#include <sched.h>
 
 #define TESTPATH "/tmp/linux-test.tmp"
 #define TESTPORT 7654
diff --git a/tests/tcg/mips/mips32-dsp/Makefile b/tests/tcg/mips/mips32-dsp/Makefile
new file mode 100644
index 00000000000..c3a0a009446
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/Makefile
@@ -0,0 +1,136 @@
+-include ../../config-host.mak
+
+CROSS=mips64el-unknown-linux-gnu-
+
+SIM=qemu-mipsel
+SIM_FLAGS=-cpu 74Kf
+
+CC      = $(CROSS)gcc
+CFLAGS  = -mabi=32 -march=mips32r2 -mgp32 -mdsp -static
+
+TESTCASES = absq_s_ph.tst
+TESTCASES += absq_s_w.tst
+TESTCASES += addq_ph.tst
+TESTCASES += addq_s_ph.tst
+TESTCASES += addq_s_w.tst
+TESTCASES += addsc.tst
+TESTCASES += addu_qb.tst
+TESTCASES += addu_s_qb.tst
+TESTCASES += addwc.tst
+TESTCASES += bitrev.tst
+TESTCASES += bposge32.tst
+TESTCASES += cmp_eq_ph.tst
+TESTCASES += cmpgu_eq_qb.tst
+TESTCASES += cmpgu_le_qb.tst
+TESTCASES += cmpgu_lt_qb.tst
+TESTCASES += cmp_le_ph.tst
+TESTCASES += cmp_lt_ph.tst
+TESTCASES += cmpu_eq_qb.tst
+TESTCASES += cmpu_le_qb.tst
+TESTCASES += cmpu_lt_qb.tst
+TESTCASES += dpaq_sa_l_w.tst
+TESTCASES += dpaq_s_w_ph.tst
+TESTCASES += dpau_h_qbl.tst
+TESTCASES += dpau_h_qbr.tst
+TESTCASES += dpsq_sa_l_w.tst
+TESTCASES += dpsq_s_w_ph.tst
+TESTCASES += dpsu_h_qbl.tst
+TESTCASES += dpsu_h_qbr.tst
+TESTCASES += extp.tst
+TESTCASES += extpdp.tst
+TESTCASES += extpdpv.tst
+TESTCASES += extpv.tst
+TESTCASES += extr_rs_w.tst
+TESTCASES += extr_r_w.tst
+TESTCASES += extr_s_h.tst
+TESTCASES += extrv_rs_w.tst
+TESTCASES += extrv_r_w.tst
+TESTCASES += extrv_s_h.tst
+TESTCASES += extrv_w.tst
+TESTCASES += extr_w.tst
+TESTCASES += insv.tst
+TESTCASES += lbux.tst
+TESTCASES += lhx.tst
+TESTCASES += lwx.tst
+TESTCASES += madd.tst
+TESTCASES += maddu.tst
+TESTCASES += maq_sa_w_phl.tst
+TESTCASES += maq_sa_w_phr.tst
+TESTCASES += maq_s_w_phl.tst
+TESTCASES += maq_s_w_phr.tst
+TESTCASES += mfhi.tst
+TESTCASES += mflo.tst
+TESTCASES += modsub.tst
+TESTCASES += msub.tst
+TESTCASES += msubu.tst
+TESTCASES += mthi.tst
+TESTCASES += mthlip.tst
+TESTCASES += mtlo.tst
+TESTCASES += muleq_s_w_phl.tst
+TESTCASES += muleq_s_w_phr.tst
+TESTCASES += muleu_s_ph_qbl.tst
+TESTCASES += muleu_s_ph_qbr.tst
+TESTCASES += mulq_rs_ph.tst
+TESTCASES += mult.tst
+TESTCASES += multu.tst
+TESTCASES += packrl_ph.tst
+TESTCASES += pick_ph.tst
+TESTCASES += pick_qb.tst
+TESTCASES += precequ_ph_qbla.tst
+TESTCASES += precequ_ph_qbl.tst
+TESTCASES += precequ_ph_qbra.tst
+TESTCASES += precequ_ph_qbr.tst
+TESTCASES += preceq_w_phl.tst
+TESTCASES += preceq_w_phr.tst
+TESTCASES += preceu_ph_qbla.tst
+TESTCASES += preceu_ph_qbl.tst
+TESTCASES += preceu_ph_qbra.tst
+TESTCASES += preceu_ph_qbr.tst
+TESTCASES += precrq_ph_w.tst
+TESTCASES += precrq_qb_ph.tst
+TESTCASES += precrq_rs_ph_w.tst
+TESTCASES += precrqu_s_qb_ph.tst
+TESTCASES += raddu_w_qb.tst
+TESTCASES += rddsp.tst
+TESTCASES += repl_ph.tst
+TESTCASES += repl_qb.tst
+TESTCASES += replv_ph.tst
+TESTCASES += replv_qb.tst
+TESTCASES += shilo.tst
+TESTCASES += shilov.tst
+TESTCASES += shll_ph.tst
+TESTCASES += shll_qb.tst
+TESTCASES += shll_s_ph.tst
+TESTCASES += shll_s_w.tst
+TESTCASES += shllv_ph.tst
+TESTCASES += shllv_qb.tst
+TESTCASES += shllv_s_ph.tst
+TESTCASES += shllv_s_w.tst
+TESTCASES += shra_ph.tst
+TESTCASES += shra_r_ph.tst
+TESTCASES += shra_r_w.tst
+TESTCASES += shrav_ph.tst
+TESTCASES += shrav_r_ph.tst
+TESTCASES += shrav_r_w.tst
+TESTCASES += shrl_qb.tst
+TESTCASES += shrlv_qb.tst
+TESTCASES += subq_ph.tst
+TESTCASES += subq_s_ph.tst
+TESTCASES += subq_s_w.tst
+TESTCASES += subu_qb.tst
+TESTCASES += subu_s_qb.tst
+TESTCASES += wrdsp.tst
+
+all: $(TESTCASES)
+
+%.tst: %.c
+	$(CC) $(CFLAGS) $< -o $@
+
+check: $(TESTCASES)
+	@for case in $(TESTCASES); do \
+        echo $(SIM) $(SIM_FLAGS) ./$$case;\
+        $(SIM) $(SIM_FLAGS) ./$$case; \
+	done
+
+clean:
+	$(RM) -rf $(TESTCASES)
diff --git a/tests/tcg/mips/mips32-dsp/absq_s_ph.c b/tests/tcg/mips/mips32-dsp/absq_s_ph.c
new file mode 100644
index 00000000000..aa8411202e2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/absq_s_ph.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x10017EFD;
+    result = 0x10017EFD;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x8000A536;
+    result = 0x7FFF5ACA;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/absq_s_w.c b/tests/tcg/mips/mips32-dsp/absq_s_w.c
new file mode 100644
index 00000000000..3f52a48039c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/absq_s_w.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x80000000;
+    result = 0x7FFFFFFF;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x80030000;
+    result = 0x7FFD0000;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x31036080;
+    result = 0x31036080;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_ph.c b/tests/tcg/mips/mips32-dsp/addq_ph.c
new file mode 100644
index 00000000000..96a549637b4
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_ph.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x374333AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x803033AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm("rddsp %0\n\t"
+          : "=r"(dsp)
+         );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_s_ph.c b/tests/tcg/mips/mips32-dsp/addq_s_ph.c
new file mode 100644
index 00000000000..5f865f6cffd
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_s_ph.c
@@ -0,0 +1,69 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x37438000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x7fff8000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0x8030847D;
+    rt     = 0x8a00AF2D;
+    result = 0x80008000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_s_w.c b/tests/tcg/mips/mips32-dsp/addq_s_w.c
new file mode 100644
index 00000000000..1e13acf68fb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_s_w.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rt     = 0x10017EFD;
+    rs     = 0x11111111;
+    result = 0x2112900e;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x80017EFD;
+    rs     = 0x81111111;
+    result = 0x80000000;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x7fffffff;
+    rs     = 0x01111111;
+    result = 0x7fffffff;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addsc.c b/tests/tcg/mips/mips32-dsp/addsc.c
new file mode 100644
index 00000000000..ace749f667c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addsc.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x0000000F;
+    rt     = 0x00000001;
+    result = 0x00000010;
+    __asm
+        ("addsc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x00001110;
+    __asm
+        ("addsc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 13) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addu_qb.c b/tests/tcg/mips/mips32-dsp/addu_qb.c
new file mode 100644
index 00000000000..23ba2e90d1c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addu_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x00000000;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFF011112;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addu_s_qb.c b/tests/tcg/mips/mips32-dsp/addu_s_qb.c
new file mode 100644
index 00000000000..fe7fd3e6aa2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addu_s_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    result = 0x20FF01FF;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x1) == 1);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFF1112;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x1) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addwc.c b/tests/tcg/mips/mips32-dsp/addwc.c
new file mode 100644
index 00000000000..8a8d81fab4c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addwc.c
@@ -0,0 +1,49 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dspi, dspo;
+    int result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    dspi   = 0x00002000;
+    result = 0x21000201;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    dspi   = 0x00;
+    result = 0x00011112;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+
+    rs     = 0x8FFF1111;
+    rt     = 0x80020001;
+    dspi   = 0x00;
+    result = 0x10011112;
+    __asm
+        ("wrdsp %4\n"
+         "addwc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspo)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+    assert(((dspo >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/bitrev.c b/tests/tcg/mips/mips32-dsp/bitrev.c
new file mode 100644
index 00000000000..04d8a3844eb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/bitrev.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x00001E6A;
+
+    __asm
+        ("bitrev %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/bposge32.c b/tests/tcg/mips/mips32-dsp/bposge32.c
new file mode 100644
index 00000000000..d25417ea77b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/bposge32.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp, sum;
+    int result;
+
+    dsp =  0x20;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    assert(sum == result);
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    assert(sum == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c b/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c
new file mode 100644
index 00000000000..957bd88ce01
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x00;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_le_ph.c b/tests/tcg/mips/mips32-dsp/cmp_le_ph.c
new file mode 100644
index 00000000000..356f156c5d1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_le_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c b/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c
new file mode 100644
index 00000000000..3fb4827ad76
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c
new file mode 100644
index 00000000000..2615c84c75c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c
new file mode 100644
index 00000000000..65d0813c3b1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0F;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x09;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c
new file mode 100644
index 00000000000..7dddad98532
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0D;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x00;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c
new file mode 100644
index 00000000000..680f2a1999d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c
new file mode 100644
index 00000000000..43cfa509ca3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c
new file mode 100644
index 00000000000..074ca5b402d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c b/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c
new file mode 100644
index 00000000000..a6425b6edc8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 0, acl = 0;
+    int resulth, resultl, resultdsp;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x00;
+    resultl   = 0x800003FB;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %1, $ac1\n\t"
+         "dpaq_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = dsp >> 17 & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c b/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c
new file mode 100644
index 00000000000..ce864844d96
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c
@@ -0,0 +1,77 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 0, acl = 0;
+    int resulth, resultl, resultdsp;
+
+    rs        = 0x80000000;
+    rt        = 0x80000000;
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x12;
+    acl = 0x48;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x741532A0;
+    acl = 0xfceabb08;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7fffffff;
+    resultl   = 0xffffffff;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c b/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c
new file mode 100644
index 00000000000..6017b5e73ab
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 3;
+    int resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x4003;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbl $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c b/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c
new file mode 100644
index 00000000000..e4abb2e2af0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 3;
+    int resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x0201;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbr $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c b/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c
new file mode 100644
index 00000000000..22ab4d57ba3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c
@@ -0,0 +1,45 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xEE9794A3;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x1424Ef1f;
+    acl = 0x1035219A;
+    rs      = 0x800083AD;
+    rt      = 0x80003721;
+    resulth = 0x1424ef1e;
+    resultl = 0x577ed901;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c b/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c
new file mode 100644
index 00000000000..b7b73fdb660
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0xfdf4cbe0;
+    resultl = 0xd138776b;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x54321123;
+    acl = 5;
+    rs      = 0x80000000;
+    rt      = 0x80000000;
+
+    resulth = 0xd4321123;
+    resultl = 0x06;
+    resultdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c b/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c
new file mode 100644
index 00000000000..94e2bf62549
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFEE5;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c b/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c
new file mode 100644
index 00000000000..a1e66356310
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFE233;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extp.c b/tests/tcg/mips/mips32-dsp/extp.c
new file mode 100644
index 00000000000..21a67af2165
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extp.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpdp.c b/tests/tcg/mips/mips32-dsp/extpdp.c
new file mode 100644
index 00000000000..15ba0828fb6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpdp.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp, pos, efi;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    assert(pos == 3);
+    assert(efi == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    efi = (dsp >> 14) & 0x01;
+    assert(efi == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpdpv.c b/tests/tcg/mips/mips32-dsp/extpdpv.c
new file mode 100644
index 00000000000..f5774eed3ca
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpdpv.c
@@ -0,0 +1,47 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp, pos, efi;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    assert(pos == 3);
+    assert(efi == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    efi = (dsp >> 14) & 0x01;
+    assert(efi == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpv.c b/tests/tcg/mips/mips32-dsp/extpv.c
new file mode 100644
index 00000000000..401b94afadf
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpv.c
@@ -0,0 +1,45 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ac, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    ac  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_r_w.c b/tests/tcg/mips/mips32-dsp/extr_r_w.c
new file mode 100644
index 00000000000..02e022427a4
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_r_w.c
@@ -0,0 +1,71 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_rs_w.c b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
new file mode 100644
index 00000000000..c3a22ee7048
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
@@ -0,0 +1,71 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_s_h.c b/tests/tcg/mips/mips32-dsp/extr_s_h.c
new file mode 100644
index 00000000000..9bc2a63cc2d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_s_h.c
@@ -0,0 +1,86 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x00007FFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xFFFF8000;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x08\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x123;
+    acl = 0x87654321;
+    result = 0x1238;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 28\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_w.c b/tests/tcg/mips/mips32-dsp/extr_w.c
new file mode 100644
index 00000000000..bd6b0b95c23
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_w.c
@@ -0,0 +1,71 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_r_w.c b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
new file mode 100644
index 00000000000..2403b3afe40
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
@@ -0,0 +1,79 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 31;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_rs_w.c b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
new file mode 100644
index 00000000000..ccceeb9f4ce
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
@@ -0,0 +1,77 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    rs = 0x03;
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x1F;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_s_h.c b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
new file mode 100644
index 00000000000..feac3e2e334
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
@@ -0,0 +1,88 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x00007FFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    rs = 0x08;
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xFFFF8000;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    rs = 0x1C;
+    ach = 0x123;
+    acl = 0x87654321;
+    result = 0x1238;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_w.c b/tests/tcg/mips/mips32-dsp/extrv_w.c
new file mode 100644
index 00000000000..9e8b238a04d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_w.c
@@ -0,0 +1,80 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 31;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/insv.c b/tests/tcg/mips/mips32-dsp/insv.c
new file mode 100644
index 00000000000..243b00733d4
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/insv.c
@@ -0,0 +1,23 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, dsp;
+    int result;
+
+    /* msb = 10, lsb = 5 */
+    dsp    = 0x305;
+    rt     = 0x12345678;
+    rs     = 0x87654321;
+    result = 0x12345438;
+    __asm
+        ("wrdsp %2, 0x03\n\t"
+         "insv  %0, %1\n\t"
+         : "+r"(rt)
+         : "r"(rs), "r"(dsp)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lbux.c b/tests/tcg/mips/mips32-dsp/lbux.c
new file mode 100644
index 00000000000..2337abea2ac
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lbux.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = value & 0xFF;
+    __asm
+        ("lbux %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lhx.c b/tests/tcg/mips/mips32-dsp/lhx.c
new file mode 100644
index 00000000000..10be3b385fe
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lhx.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = 0xFFFFF389;
+    __asm
+        ("lhx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lwx.c b/tests/tcg/mips/mips32-dsp/lwx.c
new file mode 100644
index 00000000000..e6543c9e7ea
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lwx.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = 0xBCDEF389;
+    __asm
+        ("lwx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/madd.c b/tests/tcg/mips/mips32-dsp/madd.c
new file mode 100644
index 00000000000..af4bfcfe9df
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/madd.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int acho, aclo;
+    int resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maddu.c b/tests/tcg/mips/mips32-dsp/maddu.c
new file mode 100644
index 00000000000..af4bfcfe9df
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maddu.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int acho, aclo;
+    int resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/main.c b/tests/tcg/mips/mips32-dsp/main.c
new file mode 100644
index 00000000000..b296b20c925
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/main.c
@@ -0,0 +1,6 @@
+#include<stdio.h>
+
+int main()
+{
+    printf("hello world\n");
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c b/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c
new file mode 100644
index 00000000000..292d68566d8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF060000;
+    rt  = 0xCB000000;
+    resulth = 0x04;
+    resultl = 0x947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x6;
+    resultl = 0x8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == resdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c b/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c
new file mode 100644
index 00000000000..7b2ef2ab714
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x04;
+    resultl = 0x947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x6;
+    resultl = 0x8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == resdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c b/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c
new file mode 100644
index 00000000000..a7569917236
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF060000;
+    rt = 0xCB000000;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c b/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c
new file mode 100644
index 00000000000..d6498f8dfd6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mfhi.c b/tests/tcg/mips/mips32-dsp/mfhi.c
new file mode 100644
index 00000000000..43a80669d1e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mfhi.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acho;
+    int result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    assert(result == acho);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mflo.c b/tests/tcg/mips/mips32-dsp/mflo.c
new file mode 100644
index 00000000000..caeafdb05ce
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mflo.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int acli, aclo;
+    int result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    assert(result == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/modsub.c b/tests/tcg/mips/mips32-dsp/modsub.c
new file mode 100644
index 00000000000..c294eebb519
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/modsub.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x000000FF;
+    result = 0xFFFFFF00;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x00000000;
+    rt     = 0x00CD1FFF;
+    result = 0x0000CD1F;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/msub.c b/tests/tcg/mips/mips32-dsp/msub.c
new file mode 100644
index 00000000000..5779e6f47ac
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/msub.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acli, rs, rt;
+    int acho, aclo;
+    int resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFF81F29;
+    resultl = 0xB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msub $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(acho == resulth);
+    assert(aclo == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/msubu.c b/tests/tcg/mips/mips32-dsp/msubu.c
new file mode 100644
index 00000000000..e0f9b5a77ae
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/msubu.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acli, rs, rt;
+    int acho, aclo;
+    int resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFF81F29;
+    resultl = 0xB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msubu $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(acho == resulth);
+    assert(aclo == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mthi.c b/tests/tcg/mips/mips32-dsp/mthi.c
new file mode 100644
index 00000000000..43a80669d1e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mthi.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acho;
+    int result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    assert(result == acho);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mthlip.c b/tests/tcg/mips/mips32-dsp/mthlip.c
new file mode 100644
index 00000000000..9549aae36a3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mthlip.c
@@ -0,0 +1,58 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, ach, acl, dsp;
+    int result, resulth, resultl;
+
+    dsp = 0x07;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x27;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    assert(dsp == result);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    dsp = 0x3f;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x3f;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    assert(dsp == result);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mtlo.c b/tests/tcg/mips/mips32-dsp/mtlo.c
new file mode 100644
index 00000000000..caeafdb05ce
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mtlo.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int acli, aclo;
+    int result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    assert(result == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c b/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c
new file mode 100644
index 00000000000..b3a5370fe59
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c
@@ -0,0 +1,41 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80001234;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x12349988;
+    rt = 0x43219988;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c b/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c
new file mode 100644
index 00000000000..8066d7d02a1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x8000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x1234;
+    rt = 0x4321;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c
new file mode 100644
index 00000000000..66a382806a2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c
new file mode 100644
index 00000000000..4cc6c8f7cfe
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x80004321;
+    result = 0xFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c b/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c
new file mode 100644
index 00000000000..c7206039ea7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098C;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mult.c b/tests/tcg/mips/mips32-dsp/mult.c
new file mode 100644
index 00000000000..15e6fde92c0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mult.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("mult $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/multu.c b/tests/tcg/mips/mips32-dsp/multu.c
new file mode 100644
index 00000000000..85d36c1b621
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/multu.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("multu $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/packrl_ph.c b/tests/tcg/mips/mips32-dsp/packrl_ph.c
new file mode 100644
index 00000000000..1f8e6999250
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/packrl_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x56788765;
+
+    __asm
+        ("packrl.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/pick_ph.c b/tests/tcg/mips/mips32-dsp/pick_ph.c
new file mode 100644
index 00000000000..929a002e754
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/pick_ph.c
@@ -0,0 +1,49 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0A000000;
+    result = 0x12344321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x03000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0x87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/pick_qb.c b/tests/tcg/mips/mips32-dsp/pick_qb.c
new file mode 100644
index 00000000000..a7904752466
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/pick_qb.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0f000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0x87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceq_w_phl.c b/tests/tcg/mips/mips32-dsp/preceq_w_phl.c
new file mode 100644
index 00000000000..bf70bf7d3ae
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceq_w_phl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x87650000;
+
+    __asm
+        ("preceq.w.phl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceq_w_phr.c b/tests/tcg/mips/mips32-dsp/preceq_w_phr.c
new file mode 100644
index 00000000000..3f885ef584e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceq_w_phr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43210000;
+
+    __asm
+        ("preceq.w.phr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c
new file mode 100644
index 00000000000..63b7a956834
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43803280;
+
+    __asm
+        ("precequ.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c
new file mode 100644
index 00000000000..31627f0bd60
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43802180;
+
+    __asm
+        ("precequ.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c
new file mode 100644
index 00000000000..b6f72d3cbfd
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x21801080;
+
+    __asm
+        ("precequ.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c
new file mode 100644
index 00000000000..4764fd031d8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x32801080;
+
+    __asm
+        ("precequ.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c
new file mode 100644
index 00000000000..fa95c26cc48
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00870065;
+
+    __asm
+        ("preceu.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c
new file mode 100644
index 00000000000..021f21a7444
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00870043;
+
+    __asm
+        ("preceu.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c
new file mode 100644
index 00000000000..03df18c72ca
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00430021;
+
+    __asm
+        ("preceu.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c
new file mode 100644
index 00000000000..634327618cc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00650021;
+
+    __asm
+        ("preceu.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_ph_w.c b/tests/tcg/mips/mips32-dsp/precrq_ph_w.c
new file mode 100644
index 00000000000..25d45f1a9aa
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_ph_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c b/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c
new file mode 100644
index 00000000000..fe23acce8c4
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12568743;
+
+    __asm
+        ("precrq.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c b/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c
new file mode 100644
index 00000000000..3535b37a58c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq_rs.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs = 0x7fffC678;
+    rt = 0x865432A0;
+    result = 0x7fff8654;
+
+    __asm
+        ("precrq_rs.ph.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(((dsp >> 22) & 0x01) == 1);
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c b/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c
new file mode 100644
index 00000000000..7481d5af3a5
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87657FFF;
+    result = 0x24AC00FF;
+
+    __asm
+        ("precrqu_s.qb.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+    assert(((dsp >> 22) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/raddu_w_qb.c b/tests/tcg/mips/mips32-dsp/raddu_w_qb.c
new file mode 100644
index 00000000000..77a983c0d20
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/raddu_w_qb.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs;
+    int result;
+
+    rs = 0x12345678;
+    result = 0x114;
+
+    __asm
+        ("raddu.w.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/rddsp.c b/tests/tcg/mips/mips32-dsp/rddsp.c
new file mode 100644
index 00000000000..2f302850332
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/rddsp.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp_i, dsp_o;
+    int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+
+    ccond_i   = 0x0000000C;  /* 4 */
+    outflag_i = 0x0000001B;  /* 3 */
+    efi_i     = 0x00000001;  /* 5 */
+    c_i       = 0x00000001;  /* 2 */
+    scount_i  = 0x0000000F;  /* 1 */
+    pos_i     = 0x0000000C;  /* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    assert(ccond_o   == ccond_i);
+    assert(outflag_o == outflag_i);
+    assert(efi_o     == efi_i);
+    assert(c_o       == c_i);
+    assert(scount_o  == scount_i);
+    assert(pos_o     == pos_i);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/repl_ph.c b/tests/tcg/mips/mips32-dsp/repl_ph.c
new file mode 100644
index 00000000000..21074953bdd
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/repl_ph.c
@@ -0,0 +1,23 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, result;
+
+    result = 0x01BF01BF;
+    __asm
+        ("repl.ph %0, 0x1BF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    result = 0x01FF01FF;
+    __asm
+        ("repl.ph %0, 0x01FF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/repl_qb.c b/tests/tcg/mips/mips32-dsp/repl_qb.c
new file mode 100644
index 00000000000..6631393ea13
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/repl_qb.c
@@ -0,0 +1,16 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, result;
+
+    result = 0xBFBFBFBF;
+    __asm
+        ("repl.qb %0, 0xBF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/replv_ph.c b/tests/tcg/mips/mips32-dsp/replv_ph.c
new file mode 100644
index 00000000000..07fb15f1f79
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/replv_ph.c
@@ -0,0 +1,19 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x56785678;
+    __asm
+        ("replv.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/replv_qb.c b/tests/tcg/mips/mips32-dsp/replv_qb.c
new file mode 100644
index 00000000000..dd1271fedff
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/replv_qb.c
@@ -0,0 +1,19 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x78787878;
+    __asm
+        ("replv.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shilo.c b/tests/tcg/mips/mips32-dsp/shilo.c
new file mode 100644
index 00000000000..ce8ebc69c2a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shilo.c
@@ -0,0 +1,45 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int ach, acl;
+    int resulth, resultl;
+
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0x99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, 0x0F\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+
+    ach = 0x1;
+    acl = 0x80000000;
+
+    resulth = 0x3;
+    resultl = 0x0;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, -1\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shilov.c b/tests/tcg/mips/mips32-dsp/shilov.c
new file mode 100644
index 00000000000..e1d6cea4b6d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shilov.c
@@ -0,0 +1,49 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, ach, acl;
+    int resulth, resultl;
+
+    rs  = 0x0F;
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0x99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+
+    rs  = 0xffffffff;
+    ach = 0x1;
+    acl = 0x80000000;
+
+    resulth = 0x3;
+    resultl = 0x0;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_ph.c b/tests/tcg/mips/mips32-dsp/shll_ph.c
new file mode 100644
index 00000000000..b8f1ff52850
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0xA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shll.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_qb.c b/tests/tcg/mips/mips32-dsp/shll_qb.c
new file mode 100644
index 00000000000..8c1b91c635d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_qb.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt     = 0x87654321;
+    result  = 0x87654321;
+    resultdsp = 0x00;
+
+    __asm
+        ("shll.qb %0, %2, 0x00\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll.qb %0, %2, 0x03\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_s_ph.c b/tests/tcg/mips/mips32-dsp/shll_s_ph.c
new file mode 100644
index 00000000000..910fea3b317
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_s_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_s_w.c b/tests/tcg/mips/mips32-dsp/shll_s_w.c
new file mode 100644
index 00000000000..628c7521027
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_s_w.c
@@ -0,0 +1,52 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x82345678;
+    result    = 0x82345678;
+    resultdsp = 0x00;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rt        = 0x82345678;
+    result    = 0x80000000;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_ph.c b/tests/tcg/mips/mips32-dsp/shllv_ph.c
new file mode 100644
index 00000000000..f98a6322dc1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0xA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_qb.c b/tests/tcg/mips/mips32-dsp/shllv_qb.c
new file mode 100644
index 00000000000..6d8ff4a259b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_qb.c
@@ -0,0 +1,38 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_s_ph.c b/tests/tcg/mips/mips32-dsp/shllv_s_ph.c
new file mode 100644
index 00000000000..fc9bd32765b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_s_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x0;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_s_w.c b/tests/tcg/mips/mips32-dsp/shllv_s_w.c
new file mode 100644
index 00000000000..350c25617ad
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_s_w.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_ph.c b/tests/tcg/mips/mips32-dsp/shra_ph.c
new file mode 100644
index 00000000000..5b2d840a6bc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0EC0864;
+
+    __asm
+        ("shra.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra.ph %0, %1, 0x00\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_r_ph.c b/tests/tcg/mips/mips32-dsp/shra_r_ph.c
new file mode 100644
index 00000000000..adc4ae68bdc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_r_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0ED0864;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x00\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_r_w.c b/tests/tcg/mips/mips32-dsp/shra_r_w.c
new file mode 100644
index 00000000000..ec0cf2c72cc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_r_w.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0ECA864;
+
+    __asm
+        ("shra_r.w %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra_r.w %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_ph.c b/tests/tcg/mips/mips32-dsp/shrav_ph.c
new file mode 100644
index 00000000000..6e42aaf8e1a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_ph.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0EC0864;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_r_ph.c b/tests/tcg/mips/mips32-dsp/shrav_r_ph.c
new file mode 100644
index 00000000000..f03b978d054
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_r_ph.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0ED0864;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_r_w.c b/tests/tcg/mips/mips32-dsp/shrav_r_w.c
new file mode 100644
index 00000000000..2ab03bb5daa
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_r_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0ECA864;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x40000000;
+    result = 0x40000000;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    assert(rd == result);
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrl_qb.c b/tests/tcg/mips/mips32-dsp/shrl_qb.c
new file mode 100644
index 00000000000..a7e4e6a5e4f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrl_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrl.qb %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x12345678;
+    result = 0x12345678;
+
+    __asm
+        ("shrl.qb %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrlv_qb.c b/tests/tcg/mips/mips32-dsp/shrlv_qb.c
new file mode 100644
index 00000000000..db77f6d0e11
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrlv_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x12345678;
+    result = 0x12345678;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_ph.c b/tests/tcg/mips/mips32-dsp/subq_ph.c
new file mode 100644
index 00000000000..fdd7b38b647
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x77777777;
+    rt = 0x67654321;
+    result    = 0x10123456;
+    resultdsp = 0x0;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x8ACF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_s_ph.c b/tests/tcg/mips/mips32-dsp/subq_s_ph.c
new file mode 100644
index 00000000000..8e36dadef9c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_s_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x12348000;
+    rt = 0x87657000;
+    result    = 0x7FFF8000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_s_w.c b/tests/tcg/mips/mips32-dsp/subq_s_w.c
new file mode 100644
index 00000000000..09022e9c85e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_s_w.c
@@ -0,0 +1,58 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x66666;
+    rt = 0x55555;
+    result    = 0x11111;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+
+#if 0
+    rs = 0x35555555;
+    rt = 0xf5555555;
+    result    = 0x80000000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+#endif
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subu_qb.c b/tests/tcg/mips/mips32-dsp/subu_qb.c
new file mode 100644
index 00000000000..42090961552
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subu_qb.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x8BCF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subu_s_qb.c b/tests/tcg/mips/mips32-dsp/subu_s_qb.c
new file mode 100644
index 00000000000..3d650533d17
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subu_s_qb.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x00001357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/wrdsp.c b/tests/tcg/mips/mips32-dsp/wrdsp.c
new file mode 100644
index 00000000000..dc54943a99e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/wrdsp.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp_i, dsp_o;
+    int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+
+    ccond_i   = 0x000000BC;  /* 4 */
+    outflag_i = 0x0000001B;  /* 3 */
+    efi_i     = 0x00000001;  /* 5 */
+    c_i       = 0x00000001;  /* 2 */
+    scount_i  = 0x0000000F;  /* 1 */
+    pos_i     = 0x0000000C;  /* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    assert(ccond_o   == (ccond_i & 0x0F));
+    assert(outflag_o == outflag_i);
+    assert(efi_o     == efi_i);
+    assert(c_o       == c_i);
+    assert(scount_o  == scount_i);
+    assert(pos_o     == pos_i);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/Makefile b/tests/tcg/mips/mips32-dspr2/Makefile
new file mode 100644
index 00000000000..ed19581c7eb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/Makefile
@@ -0,0 +1,71 @@
+-include ../../config-host.mak
+
+CROSS=mips64el-unknown-linux-gnu-
+
+SIM=qemu-mipsel
+SIM_FLAGS=-cpu 74Kf
+
+CC      = $(CROSS)gcc
+CFLAGS  = -mabi=32 -march=mips32r2 -mgp32 -mdspr2 -static
+
+TESTCASES = absq_s_qb.tst
+TESTCASES += addqh_ph.tst
+TESTCASES += addqh_r_ph.tst
+TESTCASES += addqh_r_w.tst
+TESTCASES += addqh_w.tst
+TESTCASES += adduh_qb.tst
+TESTCASES += adduh_r_qb.tst
+TESTCASES += addu_ph.tst
+TESTCASES += addu_s_ph.tst
+TESTCASES += append.tst
+TESTCASES += balign.tst
+TESTCASES += cmpgdu_eq_qb.tst
+TESTCASES += cmpgdu_le_qb.tst
+TESTCASES += cmpgdu_lt_qb.tst
+TESTCASES += dpaqx_sa_w_ph.tst
+TESTCASES += dpa_w_ph.tst
+TESTCASES += dpax_w_ph.tst
+TESTCASES += dpaqx_s_w_ph.tst
+TESTCASES += dpsqx_sa_w_ph.tst
+TESTCASES += dpsqx_s_w_ph.tst
+TESTCASES += dps_w_ph.tst
+TESTCASES += dpsx_w_ph.tst
+TESTCASES += mul_ph.tst
+TESTCASES += mulq_rs_w.tst
+TESTCASES += mulq_s_ph.tst
+TESTCASES += mulq_s_w.tst
+TESTCASES += mulsaq_s_w_ph.tst
+TESTCASES += mulsa_w_ph.tst
+TESTCASES += mul_s_ph.tst
+TESTCASES += precr_qb_ph.tst
+TESTCASES += precr_sra_ph_w.tst
+TESTCASES += precr_sra_r_ph_w.tst
+TESTCASES += prepend.tst
+TESTCASES += shra_qb.tst
+TESTCASES += shra_r_qb.tst
+TESTCASES += shrav_qb.tst
+TESTCASES += shrav_r_qb.tst
+TESTCASES += shrl_ph.tst
+TESTCASES += shrlv_ph.tst
+TESTCASES += subqh_ph.tst
+TESTCASES += subqh_r_ph.tst
+TESTCASES += subqh_r_w.tst
+TESTCASES += subqh_w.tst
+TESTCASES += subuh_qb.tst
+TESTCASES += subuh_r_qb.tst
+TESTCASES += subu_ph.tst
+TESTCASES += subu_s_ph.tst
+
+all: $(TESTCASES)
+
+%.tst: %.c
+	$(CC) $(CFLAGS) $< -o $@
+
+check: $(TESTCASES)
+	@for case in $(TESTCASES); do \
+        echo $(SIM) $(SIM_FLAGS) ./$$case;\
+		$(SIM) $(SIM_FLAGS) ./$$case; \
+	done
+
+clean:
+	$(RM) -rf $(TESTCASES)
diff --git a/tests/tcg/mips/mips32-dspr2/absq_s_qb.c b/tests/tcg/mips/mips32-dspr2/absq_s_qb.c
new file mode 100644
index 00000000000..af4683f3047
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/absq_s_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int input, result, dsp;
+    int hope;
+
+    input = 0x701BA35E;
+    hope  = 0x701B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %1\n\t"
+         : "=r"(result)
+         : "r"(input)
+        );
+    assert(result == hope);
+
+
+    input = 0x801BA35E;
+    hope  = 0x7F1B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(result), "=r"(dsp)
+         : "r"(input)
+        );
+    dsp = dsp >> 20;
+    dsp &= 0x01;
+    assert(dsp == 1);
+    assert(result == hope);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_ph.c b/tests/tcg/mips/mips32-dspr2/addqh_ph.c
new file mode 100644
index 00000000000..921f0eaf33c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0x81000100;
+    rt     = 0xc2000100;
+    result = 0xa1800100;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c b/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c
new file mode 100644
index 00000000000..213ba372500
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0x81010100;
+    rt     = 0xc2000100;
+    result = 0xa1810100;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_r_w.c b/tests/tcg/mips/mips32-dspr2/addqh_r_w.c
new file mode 100644
index 00000000000..75a75c50f32
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_r_w.c
@@ -0,0 +1,34 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000009;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0x00000000;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_w.c b/tests/tcg/mips/mips32-dspr2/addqh_w.c
new file mode 100644
index 00000000000..de6926ebbb7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_w.c
@@ -0,0 +1,34 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000008;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0xFFFFFFFF;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addu_ph.c b/tests/tcg/mips/mips32-dspr2/addu_ph.c
new file mode 100644
index 00000000000..1d7a25a2a77
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addu_ph.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x01000100;
+    __asm
+        ("addu.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0x00011112;
+    __asm
+        ("addu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addu_s_ph.c b/tests/tcg/mips/mips32-dspr2/addu_s_ph.c
new file mode 100644
index 00000000000..979651bfc9a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addu_s_ph.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FE00FE;
+    rt     = 0x00020001;
+    result = 0x010000FF;
+    __asm
+        ("addu_s.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFF1112;
+    __asm
+        ("addu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/adduh_qb.c b/tests/tcg/mips/mips32-dspr2/adduh_qb.c
new file mode 100644
index 00000000000..a1f5d631b52
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/adduh_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x80094B62;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x7F800888;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c b/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c
new file mode 100644
index 00000000000..81e98c190d2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x01112211;
+    result = 0x80093C5E;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x80800888;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/append.c b/tests/tcg/mips/mips32-dspr2/append.c
new file mode 100644
index 00000000000..9a91e1650db
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/append.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x02268436;
+    __asm
+        ("append %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x0010111F;
+    __asm
+        ("append %0, %1, 0x04\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/balign.c b/tests/tcg/mips/mips32-dspr2/balign.c
new file mode 100644
index 00000000000..537cf0451cc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/balign.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x13421BFF;
+    __asm
+        ("balign %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x11FFFF0F;
+    __asm
+        ("balign %0, %1, 0x03\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c
new file mode 100644
index 00000000000..2d6340d6fb1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x11777066;
+    rt = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c
new file mode 100644
index 00000000000..a0ecdca2ac2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11707066;
+    result = 0x0B;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c
new file mode 100644
index 00000000000..dba99e392c2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
new file mode 100644
index 00000000000..fae49f10eb9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x06;
+    resultl = 0x206;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c
new file mode 100644
index 00000000000..ce878302468
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c
@@ -0,0 +1,79 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x05;
+    resultl = 0x80000202;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x05FF;
+    /***********************************************************
+     * Because of we set outflag at last time, although this
+     * time we set nothing, but it is stay the last time value.
+     **********************************************************/
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x800000FF;
+    rt     = 0x00028000;
+    resulth = 0x05;
+    resultl = 0x80000400;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c
new file mode 100644
index 00000000000..798c4da5cab
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(dsp >> (16 + 1) == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 9;
+    acl = 0xb;
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(dsp >> (16 + 1) == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
new file mode 100644
index 00000000000..514797cfd1b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xFFFFFF06;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dps_w_ph.c b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
new file mode 100644
index 00000000000..f51f9b9d13d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x04;
+    resultl = 0xFFFFFD08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xFFFFFE08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c
new file mode 100644
index 00000000000..14cdd7c0f50
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xAEA3E09B;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x99f13005;
+    acl = 0x51730062;
+    rs      = 0x80008000;
+    rt      = 0x80008000;
+
+    resulth = 0x99f13004;
+    resultl = 0x51730064;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c
new file mode 100644
index 00000000000..7da278eacc0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x8c0b354A;
+    acl = 0xbbc02249;
+    rs = 0x800023AD;
+    rt = 0x01648000;
+    resulth = 0xffffffff;
+    resultl = 0x80000000;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
new file mode 100644
index 00000000000..bb49a4031d3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x05;
+    resultl = 0xE72F050;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsx.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mul_ph.c b/tests/tcg/mips/mips32-dspr2/mul_ph.c
new file mode 100644
index 00000000000..c7e9d60d12e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mul_ph.c
@@ -0,0 +1,47 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0xF504F4B4;
+    resultdsp = 1;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00210010;
+    rt = 0x00110005;
+    result = 0x2310050;
+    resultdsp = 0;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mul_s_ph.c b/tests/tcg/mips/mips32-dspr2/mul_s_ph.c
new file mode 100644
index 00000000000..33da110de86
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mul_s_ph.c
@@ -0,0 +1,62 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0x7fff7FFF;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x7fffff00;
+    rt = 0xff007fff;
+    result = 0x80008000;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00320001;
+    rt = 0x00210002;
+    result = 0x06720002;
+    resultdsp = 0;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c b/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c
new file mode 100644
index 00000000000..669405faf1b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x80005555;
+
+    __asm
+        ("mulq_rs.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd  == result);
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c b/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c
new file mode 100644
index 00000000000..d0f7674a38e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098B;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_s_w.c b/tests/tcg/mips/mips32-dspr2/mulq_s_w.c
new file mode 100644
index 00000000000..df148b7ffbf
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_s_w.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x80005555;
+
+    __asm
+        ("mulq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd  == result);
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c b/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c
new file mode 100644
index 00000000000..a6940939ca5
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c
@@ -0,0 +1,29 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x3BF5E918;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsa.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c
new file mode 100644
index 00000000000..06c91a43e7e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c
@@ -0,0 +1,29 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x772ff463;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsaq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c b/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c
new file mode 100644
index 00000000000..3a2b3fde057
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x34786521;
+
+    __asm
+        ("precr.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c b/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c
new file mode 100644
index 00000000000..5c9baab03d0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFF0000;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c b/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c
new file mode 100644
index 00000000000..6474a108c05
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFF0000;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/prepend.c b/tests/tcg/mips/mips32-dspr2/prepend.c
new file mode 100644
index 00000000000..f6bcd47b2da
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/prepend.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x87654321;
+    __asm
+        ("prepend %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xACF10ECA;
+    __asm
+        ("prepend %0, %1, 0x0F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shra_qb.c b/tests/tcg/mips/mips32-dspr2/shra_qb.c
new file mode 100644
index 00000000000..48193de87ad
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shra_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt = 0x87654321;
+    result = 0xF00C0804;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shra_r_qb.c b/tests/tcg/mips/mips32-dspr2/shra_r_qb.c
new file mode 100644
index 00000000000..29afa0e4b07
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shra_r_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrav_qb.c b/tests/tcg/mips/mips32-dspr2/shrav_qb.c
new file mode 100644
index 00000000000..b21e1b7ca64
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrav_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xF00C0804;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c b/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c
new file mode 100644
index 00000000000..9ea8aa0cbb9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrl_ph.c b/tests/tcg/mips/mips32-dspr2/shrl_ph.c
new file mode 100644
index 00000000000..724b9a7a469
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrl_ph.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrl.ph %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrlv_ph.c b/tests/tcg/mips/mips32-dspr2/shrlv_ph.c
new file mode 100644
index 00000000000..ac79aa69acf
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrlv_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrlv.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_ph.c b/tests/tcg/mips/mips32-dspr2/subqh_ph.c
new file mode 100644
index 00000000000..dbc096734ce
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456709AB;
+
+    __asm
+        ("subqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c b/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c
new file mode 100644
index 00000000000..24ef0f1aeb9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456809AC;
+
+    __asm
+        ("subqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_r_w.c b/tests/tcg/mips/mips32-dspr2/subqh_r_w.c
new file mode 100644
index 00000000000..d460f8630fa
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_r_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AC;
+
+    __asm
+        ("subqh_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_w.c b/tests/tcg/mips/mips32-dspr2/subqh_w.c
new file mode 100644
index 00000000000..42be3deb80a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AB;
+
+    __asm
+        ("subqh.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subu_ph.c b/tests/tcg/mips/mips32-dspr2/subu_ph.c
new file mode 100644
index 00000000000..0d39a017c79
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subu_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x11111111;
+    result    = 0x76543210;
+    resultdsp = 0x00;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x7531ECA9;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subu_s_ph.c b/tests/tcg/mips/mips32-dspr2/subu_s_ph.c
new file mode 100644
index 00000000000..8e4da4f3e54
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subu_s_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x75310000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subuh_qb.c b/tests/tcg/mips/mips32-dspr2/subuh_qb.c
new file mode 100644
index 00000000000..92cfc764b8c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subuh_qb.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC5E7092B;
+
+    __asm
+        ("subuh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c b/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c
new file mode 100644
index 00000000000..dac81d47db6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC6E80A2C;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs = 0xBEFC292A;
+    rt = 0x9205C1B4;
+    result = 0x167cb4bb;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/Makefile b/tests/tcg/mips/mips64-dsp/Makefile
new file mode 100644
index 00000000000..b2ac6b3ffd8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/Makefile
@@ -0,0 +1,306 @@
+
+CROSS_COMPILE	?= mips64el-unknown-linux-gnu-
+
+SIM = qemu-system-mips64el
+SIMFLAGS = -nographic -cpu mips64dspr2 -kernel
+
+AS      = $(CROSS_COMPILE)as
+LD      = $(CROSS_COMPILE)ld
+CC      = $(CROSS_COMPILE)gcc
+AR      = $(CROSS_COMPILE)ar
+NM      = $(CROSS_COMPILE)nm
+STRIP       = $(CROSS_COMPILE)strip
+RANLIB      = $(CROSS_COMPILE)ranlib
+OBJCOPY     = $(CROSS_COMPILE)objcopy
+OBJDUMP     = $(CROSS_COMPILE)objdump
+
+VECTORS_OBJ ?= ./head.o ./printf.o
+
+HEAD_FLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -pipe \
+              -msoft-float -march=mips64 -Wa,-mips64 -Wa,--trap \
+              -msym32 -DKBUILD_64BIT_SYM32 -I./
+
+CFLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -fno-builtin  \
+          -pipe -march=mips64r2 -mgp64 -mdsp -static -Wa,--trap -msym32 \
+          -DKBUILD_64BIT_SYM32 -I./
+
+LDFLAGS = -T./mips_boot.lds -L./
+FLAGS = -nostdlib -mabi=64 -march=mips64r2 -mgp64 -mdsp
+
+
+#TESTCASES = absq_s_ob.tst
+TESTCASES = absq_s_ph.tst
+TESTCASES += absq_s_pw.tst
+TESTCASES += absq_s_qh.tst
+TESTCASES += absq_s_w.tst
+TESTCASES += addq_ph.tst
+TESTCASES += addq_pw.tst
+TESTCASES += addq_qh.tst
+TESTCASES += addq_s_ph.tst
+TESTCASES += addq_s_pw.tst
+TESTCASES += addq_s_qh.tst
+TESTCASES += addq_s_w.tst
+TESTCASES += addsc.tst
+TESTCASES += addu_ob.tst
+TESTCASES += addu_qb.tst
+TESTCASES += addu_s_ob.tst
+TESTCASES += addu_s_qb.tst
+TESTCASES += addwc.tst
+TESTCASES += bitrev.tst
+TESTCASES += bposge32.tst
+TESTCASES += bposge64.tst
+TESTCASES += cmp_eq_ph.tst
+TESTCASES += cmp_eq_pw.tst
+TESTCASES += cmp_eq_qh.tst
+TESTCASES += cmpgu_eq_ob.tst
+TESTCASES += cmpgu_eq_qb.tst
+TESTCASES += cmpgu_le_ob.tst
+TESTCASES += cmpgu_le_qb.tst
+TESTCASES += cmpgu_lt_ob.tst
+TESTCASES += cmpgu_lt_qb.tst
+TESTCASES += cmp_le_ph.tst
+TESTCASES += cmp_le_pw.tst
+TESTCASES += cmp_le_qh.tst
+TESTCASES += cmp_lt_ph.tst
+TESTCASES += cmp_lt_pw.tst
+TESTCASES += cmp_lt_qh.tst
+TESTCASES += cmpu_eq_ob.tst
+TESTCASES += cmpu_eq_qb.tst
+TESTCASES += cmpu_le_ob.tst
+TESTCASES += cmpu_le_qb.tst
+TESTCASES += cmpu_lt_ob.tst
+TESTCASES += cmpu_lt_qb.tst
+#TESTCASES += dappend.tst
+TESTCASES += dextp.tst
+TESTCASES += dextpdp.tst
+TESTCASES += dextpdpv.tst
+TESTCASES += dextpv.tst
+TESTCASES += dextr_l.tst
+TESTCASES += dextr_r_l.tst
+TESTCASES += dextr_rs_l.tst
+TESTCASES += dextr_rs_w.tst
+TESTCASES += dextr_r_w.tst
+TESTCASES += dextr_s_h.tst
+TESTCASES += dextrv_l.tst
+TESTCASES += dextrv_r_l.tst
+TESTCASES += dextrv_rs_l.tst
+TESTCASES += dextrv_rs_w.tst
+TESTCASES += dextrv_r_w.tst
+TESTCASES += dextrv_s_h.tst
+TESTCASES += dextrv_w.tst
+TESTCASES += dextr_w.tst
+TESTCASES += dinsv.tst
+TESTCASES += dmadd.tst
+TESTCASES += dmaddu.tst
+TESTCASES += dmsub.tst
+TESTCASES += dmsubu.tst
+TESTCASES += dmthlip.tst
+TESTCASES += dpaq_sa_l_pw.tst
+TESTCASES += dpaq_sa_l_w.tst
+TESTCASES += dpaq_s_w_ph.tst
+TESTCASES += dpaq_s_w_qh.tst
+TESTCASES += dpau_h_obl.tst
+TESTCASES += dpau_h_obr.tst
+TESTCASES += dpau_h_qbl.tst
+TESTCASES += dpau_h_qbr.tst
+TESTCASES += dpsq_sa_l_pw.tst
+TESTCASES += dpsq_sa_l_w.tst
+TESTCASES += dpsq_s_w_ph.tst
+TESTCASES += dpsq_s_w_qh.tst
+TESTCASES += dpsu_h_obl.tst
+TESTCASES += dpsu_h_obr.tst
+TESTCASES += dpsu_h_qbl.tst
+TESTCASES += dpsu_h_qbr.tst
+TESTCASES += dshilo.tst
+TESTCASES += dshilov.tst
+TESTCASES += extp.tst
+TESTCASES += extpdp.tst
+TESTCASES += extpdpv.tst
+TESTCASES += extpv.tst
+TESTCASES += extr_rs_w.tst
+TESTCASES += extr_r_w.tst
+TESTCASES += extr_s_h.tst
+TESTCASES += extrv_rs_w.tst
+TESTCASES += extrv_r_w.tst
+TESTCASES += extrv_s_h.tst
+TESTCASES += extrv_w.tst
+TESTCASES += extr_w.tst
+TESTCASES += insv.tst
+TESTCASES += lbux.tst
+TESTCASES += lhx.tst
+TESTCASES += lwx.tst
+TESTCASES += ldx.tst
+TESTCASES += madd.tst
+TESTCASES += maddu.tst
+TESTCASES += maq_sa_w_phl.tst
+TESTCASES += maq_sa_w_phr.tst
+TESTCASES += maq_sa_w_qhll.tst
+TESTCASES += maq_sa_w_qhlr.tst
+TESTCASES += maq_sa_w_qhrl.tst
+TESTCASES += maq_sa_w_qhrr.tst
+TESTCASES += maq_s_l_pwl.tst
+TESTCASES += maq_s_l_pwr.tst
+TESTCASES += maq_s_w_phl.tst
+TESTCASES += maq_s_w_phr.tst
+TESTCASES += maq_s_w_qhll.tst
+TESTCASES += maq_s_w_qhlr.tst
+TESTCASES += maq_s_w_qhrl.tst
+TESTCASES += maq_s_w_qhrr.tst
+TESTCASES += mfhi.tst
+TESTCASES += mflo.tst
+TESTCASES += modsub.tst
+TESTCASES += msub.tst
+TESTCASES += msubu.tst
+TESTCASES += mthi.tst
+TESTCASES += mthlip.tst
+TESTCASES += mtlo.tst
+TESTCASES += muleq_s_pw_qhl.tst
+TESTCASES += muleq_s_pw_qhr.tst
+TESTCASES += muleq_s_w_phl.tst
+TESTCASES += muleq_s_w_phr.tst
+TESTCASES += muleu_s_ph_qbl.tst
+TESTCASES += muleu_s_ph_qbr.tst
+TESTCASES += muleu_s_qh_obl.tst
+TESTCASES += muleu_s_qh_obr.tst
+TESTCASES += mulq_rs_ph.tst
+TESTCASES += mulq_rs_qh.tst
+TESTCASES += mulsaq_s_l_pw.tst
+TESTCASES += mulsaq_s_w_qh.tst
+TESTCASES += mult.tst
+TESTCASES += multu.tst
+TESTCASES += packrl_ph.tst
+TESTCASES += packrl_pw.tst
+TESTCASES += pick_ob.tst
+TESTCASES += pick_ph.tst
+TESTCASES += pick_pw.tst
+TESTCASES += pick_qb.tst
+TESTCASES += pick_qh.tst
+#TESTCASES += preceq_l_pwl.tst
+#TESTCASES += preceq_l_pwr.tst
+TESTCASES += preceq_pw_qhla.tst
+TESTCASES += preceq_pw_qhl.tst
+TESTCASES += preceq_pw_qhra.tst
+TESTCASES += preceq_pw_qhr.tst
+TESTCASES += precequ_ph_qbla.tst
+TESTCASES += precequ_ph_qbl.tst
+TESTCASES += precequ_ph_qbra.tst
+TESTCASES += precequ_ph_qbr.tst
+#TESTCASES += precequ_qh_obla.tst
+#TESTCASES += precequ_qh_obl.tst
+#TESTCASES += precequ_qh_obra.tst
+#TESTCASES += precequ_qh_obr.tst
+TESTCASES += preceq_w_phl.tst
+TESTCASES += preceq_w_phr.tst
+TESTCASES += preceu_ph_qbla.tst
+TESTCASES += preceu_ph_qbl.tst
+TESTCASES += preceu_ph_qbra.tst
+TESTCASES += preceu_ph_qbr.tst
+TESTCASES += preceu_qh_obla.tst
+TESTCASES += preceu_qh_obl.tst
+TESTCASES += preceu_qh_obra.tst
+TESTCASES += preceu_qh_obr.tst
+#TESTCASES += precr_ob_qh.tst
+TESTCASES += precrq_ob_qh.tst
+TESTCASES += precrq_ph_w.tst
+TESTCASES += precrq_pw_l.tst
+TESTCASES += precrq_qb_ph.tst
+TESTCASES += precrq_qh_pw.tst
+TESTCASES += precrq_rs_ph_w.tst
+TESTCASES += precrq_rs_qh_pw.tst
+TESTCASES += precrqu_s_ob_qh.tst
+TESTCASES += precrqu_s_qb_ph.tst
+#TESTCASES += precr_sra_qh_pw.tst
+#TESTCASES += precr_sra_r_qh_pw.tst
+#TESTCASES += prependd.tst
+#TESTCASES += prependw.tst
+#TESTCASES += raddu_l_ob.tst
+TESTCASES += raddu_w_qb.tst
+TESTCASES += rddsp.tst
+TESTCASES += repl_ob.tst
+TESTCASES += repl_ph.tst
+TESTCASES += repl_pw.tst
+TESTCASES += repl_qb.tst
+TESTCASES += repl_qh.tst
+TESTCASES += replv_ob.tst
+TESTCASES += replv_ph.tst
+TESTCASES += replv_pw.tst
+TESTCASES += replv_qb.tst
+TESTCASES += shilo.tst
+TESTCASES += shilov.tst
+TESTCASES += shll_ob.tst
+TESTCASES += shll_ph.tst
+TESTCASES += shll_pw.tst
+TESTCASES += shll_qb.tst
+TESTCASES += shll_qh.tst
+TESTCASES += shll_s_ph.tst
+TESTCASES += shll_s_pw.tst
+TESTCASES += shll_s_qh.tst
+TESTCASES += shll_s_w.tst
+TESTCASES += shllv_ob.tst
+TESTCASES += shllv_ph.tst
+TESTCASES += shllv_pw.tst
+TESTCASES += shllv_qb.tst
+TESTCASES += shllv_qh.tst
+TESTCASES += shllv_s_ph.tst
+TESTCASES += shllv_s_pw.tst
+TESTCASES += shllv_s_qh.tst
+TESTCASES += shllv_s_w.tst
+#TESTCASES += shra_ob.tst
+TESTCASES += shra_ph.tst
+TESTCASES += shra_pw.tst
+TESTCASES += shra_qh.tst
+#TESTCASES += shra_r_ob.tst
+TESTCASES += shra_r_ph.tst
+TESTCASES += shra_r_pw.tst
+TESTCASES += shra_r_qh.tst
+TESTCASES += shra_r_w.tst
+TESTCASES += shrav_ph.tst
+TESTCASES += shrav_pw.tst
+TESTCASES += shrav_qh.tst
+TESTCASES += shrav_r_ph.tst
+TESTCASES += shrav_r_pw.tst
+TESTCASES += shrav_r_qh.tst
+TESTCASES += shrav_r_w.tst
+TESTCASES += shrl_ob.tst
+TESTCASES += shrl_qb.tst
+#TESTCASES += shrl_qh.tst
+TESTCASES += shrlv_ob.tst
+TESTCASES += shrlv_qb.tst
+#TESTCASES += shrlv_qh.tst
+TESTCASES += subq_ph.tst
+TESTCASES += subq_pw.tst
+TESTCASES += subq_qh.tst
+TESTCASES += subq_s_ph.tst
+TESTCASES += subq_s_pw.tst
+TESTCASES += subq_s_qh.tst
+TESTCASES += subq_s_w.tst
+TESTCASES += subu_ob.tst
+TESTCASES += subu_qb.tst
+TESTCASES += subu_s_ob.tst
+TESTCASES += subu_s_qb.tst
+TESTCASES += wrdsp.tst
+
+all: build
+
+head.o : head.S
+	$(Q)$(CC) $(HEAD_FLAGS) -D"STACK_TOP=0xffffffff80200000" -c $< -o $@
+
+%.o  : %.S
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.o  : %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.tst: %.o $(VECTORS_OBJ)
+	$(CC) $(VECTORS_OBJ) $(FLAGS) $(LDFLAGS) $< -o $@
+
+build: $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+
+check:  $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+	@for case in $(TESTCASES); do \
+		echo $(SIM) $(SIMFLAGS) ./$$case; \
+		$(SIM) $(SIMFLAGS) ./$$case & (sleep 1; killall $(SIM)); \
+	done
+
+clean:
+	$(Q)rm -f *.o *.tst *.a
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_ob.c b/tests/tcg/mips/mips64-dsp/absq_s_ob.c
new file mode 100644
index 00000000000..62140315786
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_ob.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        (".set mips64\n\t"
+         "absq_s.ob %0 %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.ob test 1 error\n");
+
+        return -1;
+    }
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 0) {
+        printf("absq_s.ob test 1 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    rt = 0x80FFFFFFFFFFFFFF;
+    result = 0x7F01010101010101;
+
+    __asm
+        ("absq_s.ob %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.ob test 2 error\n");
+
+        return -1;
+    }
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 1) {
+        printf("absq_s.ob test 2 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_ph.c b/tests/tcg/mips/mips64-dsp/absq_s_ph.c
new file mode 100644
index 00000000000..238416d438c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_ph.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x10017EFD;
+    result = 0x10017EFD;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x8000A536;
+    result = 0x7FFF5ACA;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_pw.c b/tests/tcg/mips/mips64-dsp/absq_s_pw.c
new file mode 100644
index 00000000000..48fc763b4fd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_pw.c
@@ -0,0 +1,66 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rd = 0;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.pw test 1 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 0) {
+        printf("absq_s.pw test 1 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rt = 0x80000000FFFFFFFF;
+    result = 0x7FFFFFFF00000001;
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.pw test 2 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 1) {
+        printf("absq_s.pw test 2 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_qh.c b/tests/tcg/mips/mips64-dsp/absq_s_qh.c
new file mode 100644
index 00000000000..9001a9e1642
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_qh.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rd = 0;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        ("absq_s.qh %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.qh test 1 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rt = 0x8000FFFFFFFFFFFF;
+    result = 0x7FFF000100000001;
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.rw test 2 error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_w.c b/tests/tcg/mips/mips64-dsp/absq_s_w.c
new file mode 100644
index 00000000000..414c8bd3f60
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_w.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x80000000;
+    result = 0x7FFFFFFF;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x80030000;
+    result = 0x7FFD0000;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x31036080;
+    result = 0x31036080;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_ph.c b/tests/tcg/mips/mips64-dsp/addq_ph.c
new file mode 100644
index 00000000000..22a36d98051
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_ph.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x374333AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("2 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0xffffffff803033AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm("rddsp %0\n\t"
+          : "=r"(dsp)
+         );
+
+    if (rd != result || (((dsp >> 20) & 0x01) != 1)) {
+        printf("3 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_pw.c b/tests/tcg/mips/mips64-dsp/addq_pw.c
new file mode 100644
index 00000000000..99a7668c0c6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456787FFFFFFF;
+    rt = 0x1111111100000101;
+    result = 0x2345678980000100;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x1234567880FFFFFF;
+    rt = 0x1111111180000001;
+    result = 0x2345678901000000;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_qh.c b/tests/tcg/mips/mips64-dsp/addq_qh.c
new file mode 100644
index 00000000000..4b874afb8a4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_qh.c
@@ -0,0 +1,28 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456787FFF8010;
+    rt = 0x1111111100018000;
+    result = 0x2345678980000010;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_ph.c b/tests/tcg/mips/mips64-dsp/addq_s_ph.c
new file mode 100644
index 00000000000..ad84cdcfe0c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_ph.c
@@ -0,0 +1,84 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x37438000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x7fff8000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("3 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x8030847D;
+    rt     = 0x8a00AF2D;
+    result = 0xffffffff80008000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("4 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_pw.c b/tests/tcg/mips/mips64-dsp/addq_s_pw.c
new file mode 100644
index 00000000000..2e380bbfc52
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456787FFFFFFF;
+    rt = 0x1111111100000001;
+    result = 0x234567897FFFFFFF;
+    dspresult = 0x1;
+
+    __asm
+        ("addq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x80FFFFFFE00000FF;
+    rt = 0x80000001200000DD;
+    result = 0x80000000000001DC;
+    dspresult = 0x01;
+
+    __asm
+        ("addq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_qh.c b/tests/tcg/mips/mips64-dsp/addq_s_qh.c
new file mode 100644
index 00000000000..b638a2b93a3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_qh.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456787FFF8000;
+    rt = 0x1111111100028000;
+    result = 0x234567897FFF8000;
+    dspresult = 0x1;
+
+    __asm
+        ("addq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_w.c b/tests/tcg/mips/mips64-dsp/addq_s_w.c
new file mode 100644
index 00000000000..3e08f5d482d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_w.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rt     = 0x10017EFD;
+    rs     = 0x11111111;
+    result = 0x2112900e;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    rt     = 0x80017EFD;
+    rs     = 0x81111111;
+    result = 0xffffffff80000000;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    rt     = 0x7fffffff;
+    rs     = 0x01111111;
+    result = 0x7fffffff;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addsc.c b/tests/tcg/mips/mips64-dsp/addsc.c
new file mode 100644
index 00000000000..4b684b9b99b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addsc.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x0000000F;
+    rt     = 0x00000001;
+    result = 0x00000010;
+    __asm
+        ("addsc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addsc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x00001110;
+    __asm
+        ("addsc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 13) & 0x01) != 1)) {
+        printf("2 addsc wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_ob.c b/tests/tcg/mips/mips64-dsp/addu_ob.c
new file mode 100644
index 00000000000..17f9c668c04
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_ob.c
@@ -0,0 +1,28 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x3456123498DEF390;
+    result = 0x468A68AC329AD180;
+    dspresult = 0x01;
+
+    __asm
+        ("addu.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_qb.c b/tests/tcg/mips/mips64-dsp/addu_qb.c
new file mode 100644
index 00000000000..3b9b5fc5bbc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_qb.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x00000000;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("1 addu.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFF011112;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addu.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_s_ob.c b/tests/tcg/mips/mips64-dsp/addu_s_ob.c
new file mode 100644
index 00000000000..e89a4638b71
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_s_ob.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x3456123498DEF390;
+    result = 0x468A68ACFFFFFFFF;
+    dspresult = 0x01;
+
+    __asm
+        ("addu_s.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu_s.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_s_qb.c b/tests/tcg/mips/mips64-dsp/addu_s_qb.c
new file mode 100644
index 00000000000..cb84293ade6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_s_qb.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    result = 0x20FF01FF;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x1) != 1)) {
+        printf("1 addu_s.qb error 1\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFFFFFFFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFFFF1112;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x1) != 1)) {
+        printf("2 addu_s.qb error 2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addwc.c b/tests/tcg/mips/mips64-dsp/addwc.c
new file mode 100644
index 00000000000..5929cd2f5ca
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addwc.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dspi, dspo;
+    long long result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    dspi   = 0x00002000;
+    result = 0x21000201;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if (rd != result) {
+        printf("1 addwc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    dspi   = 0x00;
+    result = 0x00011112;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if (rd != result) {
+        printf("2 addwc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x8FFF1111;
+    rt     = 0x80020001;
+    dspi   = 0x00;
+    result = 0x10011112;
+    __asm
+        ("wrdsp %4\n"
+         "addwc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspo)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if ((rd != result) || (((dspo >> 20) & 0x01) != 1)) {
+        printf("3 addwc wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bitrev.c b/tests/tcg/mips/mips64-dsp/bitrev.c
new file mode 100644
index 00000000000..ac24ef3f5c0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bitrev.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x00001E6A;
+
+    __asm
+        ("bitrev %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("bitrev wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bposge32.c b/tests/tcg/mips/mips64-dsp/bposge32.c
new file mode 100644
index 00000000000..97bce44602f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bposge32.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp, sum;
+    long long result;
+
+    dsp =  0x20;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge32 wrong\n");
+
+        return -1;
+    }
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge32 wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bposge64.c b/tests/tcg/mips/mips64-dsp/bposge64.c
new file mode 100644
index 00000000000..36161ad852d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bposge64.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp, sum;
+    long long result;
+
+    dsp =  0x40;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge64 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge64 wrong\n");
+
+        return -1;
+    }
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge64 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge64 wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c b/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c
new file mode 100644
index 00000000000..63069d0dabc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x00;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c b/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c
new file mode 100644
index 00000000000..bae4c06ccb5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+  long long rs, rt, dspreg, dspresult;
+
+  rs = 0x123456789ABCDEFF;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x03;
+
+  __asm
+      ("cmp.eq.pw %1, %2\n\t"
+       "rddsp %0\n\t"
+       : "=r"(dspreg)
+       : "r"(rs), "r"(rt)
+      );
+
+  dspreg = ((dspreg >> 24) & 0x03);
+
+  if (dspreg != dspresult) {
+    printf("1 cmp.eq.pw error\n");
+
+    return -1;
+  }
+
+  rs = 0x123456799ABCDEFe;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x00;
+
+  __asm
+      ("cmp.eq.pw %1, %2\n\t"
+       "rddsp %0\n\t"
+       : "=r"(dspreg)
+       : "r"(rs), "r"(rt)
+      );
+
+  dspreg = ((dspreg >> 24) & 0x03);
+
+  if (dspreg != dspresult) {
+    printf("2 cmp.eq.pw error\n");
+
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c b/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c
new file mode 100644
index 00000000000..49ea271003e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+  long long rs, rt, dspreg, dspresult;
+
+  rs = 0x123456789ABCDEF0;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x0E;
+
+  __asm
+      ("cmp.eq.qh %1, %2\n\t"
+        "rddsp %0\n\t"
+        : "=r"(dspreg)
+        : "r"(rs), "r"(rt)
+       );
+
+  dspreg = ((dspreg >> 24) & 0x0F);
+
+  if (dspreg != dspresult) {
+    printf("cmp.eq.qh error\n");
+
+    return -1;
+  }
+
+  rs = 0x12355a789A4CD3F0;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x00;
+
+  __asm
+      ("cmp.eq.qh %1, %2\n\t"
+        "rddsp %0\n\t"
+        : "=r"(dspreg)
+        : "r"(rs), "r"(rt)
+       );
+
+  dspreg = ((dspreg >> 24) & 0x0F);
+
+  if (dspreg != dspresult) {
+    printf("cmp.eq.qh error\n");
+
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_ph.c b/tests/tcg/mips/mips64-dsp/cmp_le_ph.c
new file mode 100644
index 00000000000..12d24f1783e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_ph.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.le.ph wrong\n");
+
+        return -1;
+    }
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.le.ph wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_pw.c b/tests/tcg/mips/mips64-dsp/cmp_le_pw.c
new file mode 100644
index 00000000000..6acc43cd5b5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x03;
+
+    __asm
+        ("cmp.le.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("1 cmp.le.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x123456799ABCEEFF;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x00;
+
+    __asm
+        ("cmp.le.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("2 cmp.le.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_qh.c b/tests/tcg/mips/mips64-dsp/cmp_le_qh.c
new file mode 100644
index 00000000000..c9ce21667f3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0F;
+
+    __asm
+        ("cmp.le.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.le.qh error\n");
+
+        return -1;
+    }
+
+    rs = 0x823456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0f;
+
+    __asm
+        ("cmp.le.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.le.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c b/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c
new file mode 100644
index 00000000000..1d91228c3ab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.lt.ph wrong\n");
+
+        return -1;
+    }
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.lt.ph2 wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c b/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c
new file mode 100644
index 00000000000..87e74caf381
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmp.lt.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x123456779ABCDEFf;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x02;
+
+    __asm
+        ("cmp.lt.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c b/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c
new file mode 100644
index 00000000000..0a13a5eaaed
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123558789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmp.lt.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.qh error\n");
+
+        return -1;
+    }
+
+    rs = 0x123356779ABbDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0f;
+
+    __asm
+        ("cmp.lt.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c
new file mode 100644
index 00000000000..697d73dd1a5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFE;
+
+    __asm
+        ("cmpgu.eq.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x133456789ABCDEF0;
+    rt = 0x123556789ABCDEFF;
+    result = 0x3E;
+
+    __asm
+        ("cmpgu.eq.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c
new file mode 100644
index 00000000000..b41c4430fdc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c
new file mode 100644
index 00000000000..8b65f18c00c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFF;
+
+    __asm
+        ("cmpgu.le.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.le.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823556789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x3F;
+
+    __asm
+        ("cmpgu.le.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.le.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c
new file mode 100644
index 00000000000..dd2b091f612
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0F;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x09;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c
new file mode 100644
index 00000000000..3e5c9dd6dab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x01;
+
+    __asm
+        ("cmpgu.lt.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823455789ABCDEF0;
+    rt = 0x123356789ABCDEFF;
+    result = 0x21;
+
+    __asm
+        ("cmpgu.lt.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.ob error\n");
+
+        return -1;
+    }
+
+   return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c
new file mode 100644
index 00000000000..a467cb78db1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0D;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x00;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c
new file mode 100644
index 00000000000..4d1983e5ea9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0xFE;
+
+    __asm
+        ("cmpu.eq.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if (dspreg != dspresult) {
+        printf("cmpu.eq.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x133516713A0CD1F0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x00;
+
+    __asm
+        ("cmpu.eq.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if (dspreg != dspresult) {
+        printf("cmpu.eq.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c
new file mode 100644
index 00000000000..28f3bec2523
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.eq.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.eq.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c
new file mode 100644
index 00000000000..8acbd1c4ba5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0xFF;
+
+    __asm
+        ("cmpu.le.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.le.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823656789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x3F;
+
+    __asm
+        ("cmpu.le.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.le.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c
new file mode 100644
index 00000000000..8a17a085133
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.le.qb wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c
new file mode 100644
index 00000000000..34e312d8183
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmpu.lt.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.lt.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823156789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x41;
+
+    __asm
+        ("cmpu.lt.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.lt.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c
new file mode 100644
index 00000000000..adb75eed526
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dappend.c b/tests/tcg/mips/mips64-dsp/dappend.c
new file mode 100644
index 00000000000..ba8e12182e7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dappend.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("dappend %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dappend error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x2345678876543215;
+    __asm
+        ("dappend %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dappend error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextp.c b/tests/tcg/mips/mips64-dsp/dextp.c
new file mode 100644
index 00000000000..a469cc0366a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextp.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+    int rs;
+
+    rs = 0xabcd1234;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextp error\n");
+        return -1;
+    }
+
+    rs = 0xabcd1200;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if (dsp != resdsp) {
+        printf("dextp error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpdp.c b/tests/tcg/mips/mips64-dsp/dextpdp.c
new file mode 100644
index 00000000000..a2361e2d42d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpdp.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp, resdsppos;
+    int rs;
+    int tmp1, tmp2;
+
+    rs = 0xabcd1234;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+    resdsppos = 0x2c;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    tmp1 = (dsp >> 14) & 0x1;
+    tmp2 = dsp & 0x3f;
+
+    if ((tmp1 != resdsp) || (rt != res) || (tmp2 != resdsppos)) {
+        printf("dextpdp error\n");
+        return -1;
+    }
+
+    rs = 0xabcd1200;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    tmp1 = (dsp >> 14) & 0x1;
+
+    if (tmp1 != resdsp) {
+        printf("dextpdp error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpdpv.c b/tests/tcg/mips/mips64-dsp/dextpdpv.c
new file mode 100644
index 00000000000..09c0b5b412c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpdpv.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp, resdsppos;
+    int rsdsp;
+    int tmp1, tmp2;
+
+    rsdsp = 0xabcd1234;
+    rs = 0x7;
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+    resdsppos = 0x2c;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+
+    tmp1 = (dsp >> 14) & 0x1;
+    tmp2 = dsp & 0x3f;
+
+    if ((tmp1 != resdsp) || (rt != res) || (tmp2 != resdsppos)) {
+        printf("dextpdpv error\n");
+        return -1;
+    }
+
+    rsdsp = 0xabcd1200;
+    rs = 0x7;
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+
+    tmp1 = (dsp >> 14) & 0x1;
+
+    if (tmp1 != resdsp) {
+        printf("dextpdpv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpv.c b/tests/tcg/mips/mips64-dsp/dextpv.c
new file mode 100644
index 00000000000..2626f3d98c8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpv.c
@@ -0,0 +1,58 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+    int rsdsp;
+
+    rsdsp = 0xabcd1234;
+    rs = 0x7;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextpv error\n");
+        return -1;
+    }
+
+    rsdsp = 0xabcd1200;
+    rs = 0x7;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if (dsp != resdsp) {
+        printf("dextpv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_l.c b/tests/tcg/mips/mips64-dsp/dextr_l.c
new file mode 100644
index 00000000000..538846df181
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_l.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x2100000000123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.l %0, $ac1, 0x8\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.l %0, $ac1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_r_l.c b/tests/tcg/mips/mips64-dsp/dextr_r_l.c
new file mode 100644
index 00000000000..a10a9ab40ee
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_r_l.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x2100000000123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.l %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.l %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_r_w.c b/tests/tcg/mips/mips64-dsp/dextr_r_w.c
new file mode 100644
index 00000000000..2774e9bfcc9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_r_w.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.w %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_rs_l.c b/tests/tcg/mips/mips64-dsp/dextr_rs_l.c
new file mode 100644
index 00000000000..1a202fefa25
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_rs_l.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x8000000000000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.l %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.l error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.l %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_rs_w.c b/tests/tcg/mips/mips64-dsp/dextr_rs_w.c
new file mode 100644
index 00000000000..ebe5f99db09
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_rs_w.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0xffffffff80000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.w error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+
+    res = 0x123456;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_s_h.c b/tests/tcg/mips/mips64-dsp/dextr_s_h.c
new file mode 100644
index 00000000000..1adb5549a98
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_s_h.c
@@ -0,0 +1,73 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0xffffffffffff8000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("1 dextr_s.h error\n");
+        return -1;
+    }
+
+    achi = 0x77654321;
+    acli = 0x12345678;
+
+    res = 0x7fff;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("2 dextr_s.h error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x78;
+
+    res = 0x7;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("3 dextr_s.h error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_w.c b/tests/tcg/mips/mips64-dsp/dextr_w.c
new file mode 100644
index 00000000000..79bed5da35f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_w.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.w %0, $ac1, 0x8\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.w %0, $ac1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_l.c b/tests/tcg/mips/mips64-dsp/dextrv_l.c
new file mode 100644
index 00000000000..2e6187f729a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_l.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x2100000000123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.l %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.l %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_r_l.c b/tests/tcg/mips/mips64-dsp/dextrv_r_l.c
new file mode 100644
index 00000000000..b47a0177d4f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_r_l.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp, rs;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x2100000000123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_r_w.c b/tests/tcg/mips/mips64-dsp/dextrv_r_w.c
new file mode 100644
index 00000000000..cd201deb211
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_r_w.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c b/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c
new file mode 100644
index 00000000000..6ce41854620
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x8000000000000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.l error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c b/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c
new file mode 100644
index 00000000000..a65183c0304
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0xffffffff80000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.w error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_s_h.c b/tests/tcg/mips/mips64-dsp/dextrv_s_h.c
new file mode 100644
index 00000000000..87d3aeedce5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_s_h.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0xffffffffffff8000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_s.h %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_s.h error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_w.c b/tests/tcg/mips/mips64-dsp/dextrv_w.c
new file mode 100644
index 00000000000..973765c1c0a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_w.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.w %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.w %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dinsv.c b/tests/tcg/mips/mips64-dsp/dinsv.c
new file mode 100644
index 00000000000..f6192188c01
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dinsv.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long res;
+
+    rs = 0x1234567887654321;
+    rt = 0x1234567812345678;
+    dsp = 0x2222;
+    res = 0x1234567812345678;
+    __asm
+        ("wrdsp  %1, 0x3\n\t"
+         "wrdsp %1\n\t"
+         "dinsv %0, %2\n\t"
+         : "+r"(rt)
+         : "r"(dsp), "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dinsv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmadd.c b/tests/tcg/mips/mips64-dsp/dmadd.c
new file mode 100644
index 00000000000..fb22614725b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmadd.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x5;
+    __asm
+       ("mthi %2, $ac1 \t\n"
+        "mtlo %3, $ac1 \t\n"
+        "dmadd $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1 \t\n"
+        "mflo %1, $ac1 \t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+      );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmadd error\n");
+
+        return -1;
+    }
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    resh = 0x0000000000000000;
+    resl = 0xffffffffca860b63;
+
+    __asm
+       ("mthi %2, $ac1 \t\n"
+        "mtlo %3, $ac1 \t\n"
+        "dmadd $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1 \t\n"
+        "mflo %1, $ac1 \t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+      );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmadd error\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmaddu.c b/tests/tcg/mips/mips64-dsp/dmaddu.c
new file mode 100644
index 00000000000..39ab0c10db6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmaddu.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x2;
+
+    rs = 0x0000000200000002;
+    rt = 0x0000000200000002;
+    resh = 0x1;
+    resl = 0xa;
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmaddu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmaddu error\n");
+
+        return -1;
+    }
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    resh = 0x0000000000000002;
+    resl = 0xffffffffca860b63;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmaddu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmaddu error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmsub.c b/tests/tcg/mips/mips64-dsp/dmsub.c
new file mode 100644
index 00000000000..16be6170e4f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmsub.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x8;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsub $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmsub error\n");
+
+        return -1;
+    }
+
+    achi = 0xfffffffF;
+    acli = 0xfffffffF;
+
+    rs = 0x8888999977776666;
+    rt = 0x9999888877776666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x789aae13;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsub $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmsub error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmsubu.c b/tests/tcg/mips/mips64-dsp/dmsubu.c
new file mode 100644
index 00000000000..cc4838ad5ff
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmsubu.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x8;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsubu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmsubu error\n");
+
+        return -1;
+    }
+
+    achi = 0xfffffffF;
+    acli = 0xfffffffF;
+
+    rs = 0x8888999977776666;
+    rt = 0x9999888877776666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x789aae13;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsubu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmsubu error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmthlip.c b/tests/tcg/mips/mips64-dsp/dmthlip.c
new file mode 100644
index 00000000000..027555fb535
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmthlip.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, dsp;
+    long long achi, acli;
+
+    long long rsdsp;
+    long long acho, aclo;
+
+    long long res;
+    long long reshi, reslo;
+
+
+    rs = 0xaaaabbbbccccdddd;
+    achi = 0x87654321;
+    acli = 0x12345678;
+    dsp = 0x22;
+
+    res = 0x62;
+    reshi = 0x12345678;
+    reslo = 0xffffffffccccdddd;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "wrdsp %5\n\t"
+         "dmthlip %6, $ac1\n\t"
+         "rddsp %0\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         : "=r"(rsdsp), "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(dsp), "r"(rs)
+        );
+    if ((rsdsp != res) || (acho != reshi) || (aclo != reslo)) {
+        printf("dmthlip error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c b/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c
new file mode 100644
index 00000000000..1bca935008b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 0, acl = 0;
+    long long resulth, resultl, resultdsp;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x00;
+    resultl   = 0xFFFFFFFF800003FB;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %1, $ac1\n\t"
+         "dpaq_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = dsp >> 17 & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_w.w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c
new file mode 100644
index 00000000000..844a3474292
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c
@@ -0,0 +1,57 @@
+#include"io.h"
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+    rs = 0x0001000100010001;
+    rt = 0x0002000200020002;
+    resh = 0x1;
+    resl = 0x11;
+
+    __asm
+        ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dpaq_s.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpaq_s.w.qh error\n");
+
+        return -1;
+    }
+
+    achi = 0xffffffff;
+    acli = 0xaaaaaaaa;
+
+    rs = 0x1111222233334444;
+    rt = 0xffffeeeeddddcccc;
+
+    resh = 0x00;
+    resl = 0xffffffffd27ad82e;
+
+    __asm
+        ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dpaq_s.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpaq_s.w.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c
new file mode 100644
index 00000000000..1bb2ec2f260
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c
@@ -0,0 +1,88 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long dsp;
+    long long resh, resl;
+    long long resdsp;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0xffffffffffffffff;
+    resl = 0x0;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi        %3, $ac1\n\t"
+         "mtlo        %4, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %5, %6\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl) || ((dsp >> (16 + 1)) != resdsp)) {
+        printf("1 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x3333444455556666;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888887;
+    resl = 0xffffffff9e2661da;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x8000000080000000;
+    rt = 0x8000000080000000;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x00;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi        %3, $ac1\n\t"
+         "mtlo        %4, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %5, %6\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl) || ((dsp >> (16 + 1)) != resdsp)) {
+        printf("2 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c
new file mode 100644
index 00000000000..f840cdd761c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c
@@ -0,0 +1,82 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 0, acl = 0;
+    long long resulth, resultl, resultdsp;
+
+    rs        = 0x80000000;
+    rt        = 0x80000000;
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xffffffffFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    ach = 0x12;
+    acl = 0x48;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xffffffffFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    ach = 0x741532A0;
+    acl = 0xfceabb08;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7fffffff;
+    resultl   = 0xffffffffffffffff;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_obl.c b/tests/tcg/mips/mips64-dsp/dpau_h_obl.c
new file mode 100644
index 00000000000..54905e8f933
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_obl.c
@@ -0,0 +1,59 @@
+
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0x1;
+    resl = 0x3;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obl $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obl error\n");
+
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x3333444455556666;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888888;
+    resl = 0x66670d7a;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obl $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_obr.c b/tests/tcg/mips/mips64-dsp/dpau_h_obr.c
new file mode 100644
index 00000000000..d7aa60b4b10
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_obr.c
@@ -0,0 +1,59 @@
+
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0x1;
+    resl = 0x3;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obr $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obr error\n");
+
+        return -1;
+    }
+
+    rs = 0xccccddddaaaabbbb;
+    rt = 0x5555666633334444;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888888;
+    resl = 0x66670d7a;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obr $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c b/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c
new file mode 100644
index 00000000000..fcfd7643104
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 3;
+    long long resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x4003;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbl $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpau.h.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c b/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c
new file mode 100644
index 00000000000..3282461a7d1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 3;
+    long long resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x0201;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbr $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpau.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c b/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c
new file mode 100644
index 00000000000..7660f037daf
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFEE9794A3;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("1 dpsq_s.w.ph wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x1424Ef1f;
+    acl = 0x1035219A;
+    rs      = 0x800083AD;
+    rt      = 0x80003721;
+    resulth = 0x1424ef1e;
+    resultl = 0x577ed901;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("2 dpsq_s.w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c
new file mode 100644
index 00000000000..2cc50c577ed
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0xffffeeeeddddcccc;
+    rt = 0x9999888877776666;
+    achi = 0x67576;
+    acli = 0x98878;
+
+    resh = 0x67576;
+    resl = 0x5b1682c4;
+    __asm
+        ("mthi  %2, $ac1\n\t"
+         "mtlo  %3, $ac1\n\t"
+         "dpsq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpsq_s.w.qh wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x8000800080008000;
+    rt = 0x8000800080008000;
+    achi = 0x67576;
+    acli = 0x98878;
+
+    resh = 0x67575;
+    resl = 0x0009887c;
+
+    __asm
+        ("mthi  %2, $ac1\n\t"
+         "mtlo  %3, $ac1\n\t"
+         "dpsq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpsq_s.w.qh wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c
new file mode 100644
index 00000000000..7fc2503fc58
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c
@@ -0,0 +1,76 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long resh, resl, resdsp;
+
+    rs = 0x89789BC0123AD;
+    rt = 0x5467591643721;
+
+    achi = 0x98765437;
+    acli = 0x65489709;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x00;
+
+    resdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.pw $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(achi), "+r"(acli), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resdsp) || (achi != resh) || (acli != resl)) {
+        printf("1 dpsq_sa.l.pw wrong\n");
+
+        return -1;
+    }
+
+    /* clear dspcontrol reg for next test use. */
+    dsp = 0;
+    __asm
+        ("wrdsp %0"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x8B78980000000;
+    rt = 0x5867580000000;
+
+    achi = 0x98765437;
+    acli = 0x65489709;
+
+    resh = 0xffffffff98765436;
+    resl = 0x11d367d0;
+
+    resdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.pw $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(achi), "+r"(acli), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resdsp) || (achi != resh) || (acli != resl)) {
+        printf("2 dpsq_sa.l.pw wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c
new file mode 100644
index 00000000000..f55afc9095e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+
+    resulth = 0xfffffffffdf4cbe0;
+    resultl = 0xFFFFFFFFd138776b;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("1 dpsq_sa.l.w wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x54321123;
+    acl = 5;
+    rs = 0x80000000;
+    rt = 0x80000000;
+
+    resulth = 0xffffffffd4321123;
+    resultl = 0x06;
+    resultdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("2 dpsq_sa.l.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c b/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c
new file mode 100644
index 00000000000..c0a8f4d7aaf
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0x88886666BC0123AD;
+    rt      = 0x9999888801643721;
+
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFEF115;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.obl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.obl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c b/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c
new file mode 100644
index 00000000000..aa0d47a0651
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0x7878878888886666;
+    rt      = 0x9865454399998888;
+
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFeF115;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.obr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c b/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c
new file mode 100644
index 00000000000..da6dbb61549
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFFEE5;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c b/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c
new file mode 100644
index 00000000000..bf00b70aa7c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFE233;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dshilo.c b/tests/tcg/mips/mips64-dsp/dshilo.c
new file mode 100644
index 00000000000..f50584b9c42
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dshilo.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli;
+    long long acho, aclo;
+    long long reshi, reslo;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0xfffffffff8765432;
+    reslo = 0x1234567;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilo $ac1, 0x4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("1 dshilo error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0x1234567;
+    reslo = 0x00;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilo $ac1, -60\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("2 dshilo error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dshilov.c b/tests/tcg/mips/mips64-dsp/dshilov.c
new file mode 100644
index 00000000000..792bd237301
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dshilov.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs;
+    long long acho, aclo;
+    long long reshi, reslo;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x4;
+
+    reshi = 0xfffffffff8765432;
+    reslo = 0x1234567;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilov $ac1, %4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("dshilov error\n");
+        return -1;
+    }
+
+    rs = 0x44;
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0x1234567;
+    reslo = 0x00;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilov $ac1, %4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("dshilov error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extp.c b/tests/tcg/mips/mips64-dsp/extp.c
new file mode 100644
index 00000000000..c72f54bace8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extp.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extp wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if (dsp != 1) {
+        printf("extp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpdp.c b/tests/tcg/mips/mips64-dsp/extpdp.c
new file mode 100644
index 00000000000..f430193841c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpdp.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp, pos, efi;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    if ((pos != 3) || (efi != 0) || (result != rt)) {
+        printf("extpdp wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    efi = (dsp >> 14) & 0x01;
+    if (efi != 1) {
+        printf("extpdp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpdpv.c b/tests/tcg/mips/mips64-dsp/extpdpv.c
new file mode 100644
index 00000000000..ba57426d228
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpdpv.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp, pos, efi;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    if ((pos != 3) || (efi != 0) || (result != rt)) {
+        printf("extpdpv wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    efi = (dsp >> 14) & 0x01;
+    if (efi != 1) {
+        printf("extpdpv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpv.c b/tests/tcg/mips/mips64-dsp/extpv.c
new file mode 100644
index 00000000000..158472bf938
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpv.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ac, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    ac  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extpv wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if (dsp != 1) {
+        printf("extpv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_r_w.c b/tests/tcg/mips/mips64-dsp/extr_r_w.c
new file mode 100644
index 00000000000..94572ad1546
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_r_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xFFFFFFFFA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extr_r.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extr_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_rs_w.c b/tests/tcg/mips/mips64-dsp/extr_rs_w.c
new file mode 100644
index 00000000000..73551f96b3f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_rs_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extr_rs.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extr_rs.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_s_h.c b/tests/tcg/mips/mips64-dsp/extr_s_h.c
new file mode 100644
index 00000000000..de10cb57a5e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_s_h.c
@@ -0,0 +1,71 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x00007FFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xffffffffFFFF8000;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x08\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_w.c b/tests/tcg/mips/mips64-dsp/extr_w.c
new file mode 100644
index 00000000000..bd69576687c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xFFFFFFFFA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extr.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_r_w.c b/tests/tcg/mips/mips64-dsp/extrv_r_w.c
new file mode 100644
index 00000000000..8379729787d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_r_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xFFFFFFFFA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_r.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_rs_w.c b/tests/tcg/mips/mips64-dsp/extrv_rs_w.c
new file mode 100644
index 00000000000..8707cd1174b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_rs_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x7FFFFFFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extrv_rs.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extrv_rs.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_s_h.c b/tests/tcg/mips/mips64-dsp/extrv_s_h.c
new file mode 100644
index 00000000000..b6dcaebcbc2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_s_h.c
@@ -0,0 +1,79 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x00007FFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x08;
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xffffffffFFFF8000;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_w.c b/tests/tcg/mips/mips64-dsp/extrv_w.c
new file mode 100644
index 00000000000..8adffb3954a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xFFFFFFFFA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/head.S b/tests/tcg/mips/mips64-dsp/head.S
new file mode 100644
index 00000000000..9a099ae42fb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/head.S
@@ -0,0 +1,16 @@
+/*
+ *  Startup Code for MIPS64 CPU-core
+ *
+ */
+.text
+.globl _start
+.align 4
+_start:
+    ori    $2, $2, 0xffff
+    sll    $2, $2, 16
+    ori    $2, $2, 0xffff
+    mtc0   $2, $12, 0
+    jal    main
+
+end:
+    b end
diff --git a/tests/tcg/mips/mips64-dsp/insv.c b/tests/tcg/mips/mips64-dsp/insv.c
new file mode 100644
index 00000000000..fc5696f4c4a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/insv.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long result;
+
+    /* msb = 10, lsb = 5 */
+    dsp    = 0x305;
+    rt     = 0x12345678;
+    rs     = 0xffffffff87654321;
+    result = 0x12345338;
+    __asm
+        ("wrdsp %2, 0x03\n\t"
+         "insv  %0, %1\n\t"
+         : "+r"(rt)
+         : "r"(rs), "r"(dsp)
+        );
+    if (rt != result) {
+        printf("insv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/io.h b/tests/tcg/mips/mips64-dsp/io.h
new file mode 100644
index 00000000000..b7db61d7c19
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/io.h
@@ -0,0 +1,22 @@
+#ifndef _ASM_IO_H
+#define _ASM_IO_H
+extern int printf(const char *fmt, ...);
+extern unsigned long get_ticks(void);
+
+#define _read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "mfc0\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#define __read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "move\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#endif
diff --git a/tests/tcg/mips/mips64-dsp/lbux.c b/tests/tcg/mips/mips64-dsp/lbux.c
new file mode 100644
index 00000000000..dbdc87bffed
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lbux.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = value & 0xFF;
+    __asm
+        ("lbux %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lbux wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/ldx.c b/tests/tcg/mips/mips64-dsp/ldx.c
new file mode 100644
index 00000000000..787d9f00b49
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/ldx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xBCDEF389;
+    __asm
+        ("ldx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lwx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/lhx.c b/tests/tcg/mips/mips64-dsp/lhx.c
new file mode 100644
index 00000000000..2020e56866f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lhx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xFFFFFFFFFFFFF389;
+    __asm
+        ("lhx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lhx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/lwx.c b/tests/tcg/mips/mips64-dsp/lwx.c
new file mode 100644
index 00000000000..6a81414d653
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lwx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xFFFFFFFFBCDEF389;
+    __asm
+        ("lwx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lwx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/madd.c b/tests/tcg/mips/mips64-dsp/madd.c
new file mode 100644
index 00000000000..de6e44fbc5c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/madd.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("madd wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maddu.c b/tests/tcg/mips/mips64-dsp/maddu.c
new file mode 100644
index 00000000000..e9f426a3742
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maddu.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maddu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c b/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c
new file mode 100644
index 00000000000..c196b435378
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x98765432FF060000;
+    rt  = 0xfdeca987CB000000;
+    resulth = 0x05;
+    resultl = 0x18278587;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.l.pwl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s_l.w.pwl wrong 1\n");
+
+        return -1;
+    }
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x80000000FF060000;
+    rt  = 0x80000000CB000000;
+    resulth = 0x05;
+    resultl = 0xb4ca;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.l.pwl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s_l.w.pwl wrong 2\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c b/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c
new file mode 100644
index 00000000000..e2af69fe2cb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x87898765432;
+    rt  = 0x7878fdeca987;
+    resulth = 0x05;
+    resultl = 0x18278587;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.l.pwr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.pwr wrong\n");
+
+        return -1;
+    }
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x89899980000000;
+    rt  = 0x88780000000;
+    resulth = 0x05;
+    resultl = 0xb4ca;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.l.pwr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.pwr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c b/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c
new file mode 100644
index 00000000000..7dba8746e50
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF060000;
+    rt  = 0xCB000000;
+    resulth = 0x04;
+    resultl = 0xffffffff947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.phl error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x6;
+    resultl = 0xffffffff8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != resdsp)) {
+        printf("2 maq_s.w.phl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c
new file mode 100644
index 00000000000..138ee2a691f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x04;
+    resultl = 0xffffffff947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.phr error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x6;
+    resultl = 0xffffffff8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != resdsp)) {
+        printf("2 maq_s.w.phr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c
new file mode 100644
index 00000000000..234a0af2939
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888899990000;
+    rt  = 0x9876888899990000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhll $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.qhll wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899990000;
+    rt  = 0x8000888899990000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhll $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.qhll wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c
new file mode 100644
index 00000000000..8768cbaa3d9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987698760000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhlr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhlr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080000000;
+    rt  = 0x8000800080000000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhlr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c
new file mode 100644
index 00000000000..5006e2be34b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888812340000;
+    rt  = 0x9876888898760000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhrl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8888999980000000;
+    rt  = 0x8888999980000000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c
new file mode 100644
index 00000000000..1d213a51b57
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888812341234;
+    rt  = 0x9876888898769876;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhrr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899998000;
+    rt  = 0x8000888899998000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c
new file mode 100644
index 00000000000..5530ffbe69d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF060000;
+    rt = 0xCB000000;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff947438cb;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.phl error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != 0x01)) {
+        printf("2 maq_sa.w.phl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c
new file mode 100644
index 00000000000..b611cfa91ab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF06;
+    rt = 0xCB00;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff947438cb;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.phr error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != 0x01)) {
+        printf("2 maq_sa.w.phr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c
new file mode 100644
index 00000000000..136ff2d77be
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888899990000;
+    rt  = 0x9876888899990000;
+
+    resulth = 0x00;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.qhll $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhll wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899990000;
+    rt  = 0x8000888899990000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhll $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhll wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c
new file mode 100644
index 00000000000..dd0ae1cca2e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987699990000;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_sa.w.qhlr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800099990000;
+    rt  = 0x8000800099990000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_sa.w.qhlr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c
new file mode 100644
index 00000000000..a3de6f8e2e8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987698760000;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080000000;
+    rt  = 0x8000800080000000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhrl wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c
new file mode 100644
index 00000000000..f02173736a4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412341234;
+    rt  = 0x9876987698769876;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080008000;
+    rt  = 0x8000800080008000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhrr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mfhi.c b/tests/tcg/mips/mips64-dsp/mfhi.c
new file mode 100644
index 00000000000..ee915f796e6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mfhi.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acho;
+    long long result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    if (result != acho) {
+        printf("mfhi wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mflo.c b/tests/tcg/mips/mips64-dsp/mflo.c
new file mode 100644
index 00000000000..cdc646b5fc7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mflo.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long acli, aclo;
+    long long result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mtlo %1, $ac1\n\t"
+         "mflo %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    if (result != aclo) {
+        printf("mflo wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mips_boot.lds b/tests/tcg/mips/mips64-dsp/mips_boot.lds
new file mode 100644
index 00000000000..bd7c0c0f3fd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mips_boot.lds
@@ -0,0 +1,31 @@
+OUTPUT_ARCH(mips)
+SECTIONS
+{
+    . = 0xffffffff80100000;
+    . = ALIGN((1 << 13));
+    .text :
+    {
+        *(.text)
+        *(.rodata)
+        *(.rodata.*)
+    }
+
+    __init_begin = .;
+    . = ALIGN((1 << 12));
+    .init.text : AT(ADDR(.init.text) - 0)
+    {
+        *(.init.text)
+    }
+    .init.data : AT(ADDR(.init.data) - 0)
+    {
+        *(.init.data)
+    }
+    . = ALIGN((1 << 12));
+    __init_end = .;
+
+    . = ALIGN((1 << 13));
+    .data :
+    {
+        *(.data)
+    }
+}
diff --git a/tests/tcg/mips/mips64-dsp/modsub.c b/tests/tcg/mips/mips64-dsp/modsub.c
new file mode 100644
index 00000000000..2c91cb4c59c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/modsub.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x000000FF;
+    result = 0xFFFFFF00;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("modsub wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x00000000;
+    rt     = 0x00CD1FFF;
+    result = 0x0000CD1F;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("modsub wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/msub.c b/tests/tcg/mips/mips64-dsp/msub.c
new file mode 100644
index 00000000000..75066b5916d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/msub.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs, rt;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFFFFFFFFFF81F29;
+    resultl = 0xFFFFFFFFB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msub $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resulth) || (aclo != resultl)) {
+        printf("msub wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/msubu.c b/tests/tcg/mips/mips64-dsp/msubu.c
new file mode 100644
index 00000000000..55f8ae046f6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/msubu.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs, rt;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFFFFFFFFFF81F29;
+    resultl = 0xFFFFFFFFB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msubu $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resulth) || (aclo != resultl)) {
+        printf("msubu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mthi.c b/tests/tcg/mips/mips64-dsp/mthi.c
new file mode 100644
index 00000000000..857005139b4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mthi.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acho;
+    long long result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    if (result != acho) {
+        printf("mthi wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mthlip.c b/tests/tcg/mips/mips64-dsp/mthlip.c
new file mode 100644
index 00000000000..957cd426f00
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mthlip.c
@@ -0,0 +1,61 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, ach, acl, dsp;
+    long long result, resulth, resultl;
+
+    dsp = 0x07;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x27;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    if ((dsp != result) || (ach != resulth) || (acl != resultl)) {
+        printf("mthlip wrong\n");
+
+        return -1;
+    }
+
+    dsp = 0x3f;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x3f;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    if ((dsp != result) || (ach != resulth) || (acl != resultl)) {
+        printf("mthlip wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mtlo.c b/tests/tcg/mips/mips64-dsp/mtlo.c
new file mode 100644
index 00000000000..304fffbe7c4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mtlo.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long acli, aclo;
+    long long result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    if (result != aclo) {
+        printf("mtlo wrong\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c
new file mode 100644
index 00000000000..6c68d45afe6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0;
+    rs = 0x45BCFFFF12345678;
+    rt = 0x98529AD287654321;
+    result = 0x52fbec7035a2ca5c;
+
+    __asm
+        ("muleq_s.pw.qhl %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("1 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rs = 0x45BC800012345678;
+    rt = 0x9852800087654321;
+    result = 0x52fbec707FFFFFFF;
+
+    __asm
+        ("muleq_s.pw.qhl %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("2 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 21;
+    rd = rd & 0x1;
+
+    if (rd != 1) {
+        printf("3 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c
new file mode 100644
index 00000000000..fa8b41fd395
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rd = 0;
+    rs = 0x1234567845BCFFFF;
+    rt = 0x8765432198529AD2;
+    result = 0x52fbec7035a2ca5c;
+
+    __asm
+        ("muleq_s.pw.qhr %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("1 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rs = 0x1234567845BC8000;
+    rt = 0x8765432198528000;
+    result = 0x52fbec707FFFFFFF;
+
+    __asm
+        ("muleq_s.pw.qhr %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("2 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 21;
+    rd = rd & 0x1;
+
+    if (rd != 1) {
+        printf("3 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c b/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c
new file mode 100644
index 00000000000..997a9f64d94
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80009988;
+    rt = 0x80009988;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phl wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12343322;
+    rt = 0x43213322;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c b/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c
new file mode 100644
index 00000000000..0e594794dda
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x8000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phr wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x1234;
+    rt = 0x4321;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c
new file mode 100644
index 00000000000..2f444c9f85a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFFFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleu_s.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c
new file mode 100644
index 00000000000..8bd0e9942c5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x80004321;
+    result = 0xFFFFFFFFFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleu_s.ph.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c
new file mode 100644
index 00000000000..db0d386e885
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c
@@ -0,0 +1,30 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long resdsp, result;
+
+    rd = 0;
+    rs = 0x1234567802020202;
+    rt = 0x0034432112344321;
+    result = 0x03A8FFFFFFFFFFFF;
+    resdsp = 0x01;
+
+    __asm
+        ("muleu_s.qh.obl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd != result) || (resdsp != dsp)) {
+        printf("muleu_s.qh.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c
new file mode 100644
index 00000000000..52ed9c095a8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c
@@ -0,0 +1,31 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long resdsp, result;
+
+    rd = 0;
+    rs = 0x0202020212345678;
+
+    rt = 0x0034432112344321;
+    result = 0x03A8FFFFFFFFFFFF;
+    resdsp = 0x01;
+
+    __asm
+        ("muleu_s.qh.obr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd != result) || (resdsp != dsp)) {
+        printf("muleu_s.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c b/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c
new file mode 100644
index 00000000000..fd6233d4dfc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098C;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("mulq_rs.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c b/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c
new file mode 100644
index 00000000000..7863c059120
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dsp, dspresult;
+    rt = 0x80003698CE8F9201;
+    rs = 0x800034634BCDE321;
+    result = 0x7fff16587a530313;
+
+    dspresult = 0x01;
+
+    __asm
+        ("mulq_rs.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != result) {
+        printf("mulq_rs.qh error\n");
+
+        return -1;
+    }
+
+    dsp = (dsp >> 21) & 0x01;
+    if (dsp != dspresult) {
+        printf("mulq_rs.qh DSPControl Reg ouflag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c b/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c
new file mode 100644
index 00000000000..02548f85cf8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resl, resh;
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x1234567887654321;
+    rt = 0x8765432112345678;
+
+    resh = 0x4;
+    resl = 0x4;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "mulsaq_s.l.pw $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 mulsaq_s.l.pw wrong\n");
+
+        return -1;
+    }
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x8000000087654321;
+    rt = 0x8000000012345678;
+
+    resh = 0x4;
+    resl = 0x1e8ee513;
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "mulsaq_s.l.pw $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (acho != resh) || (aclo != resl)) {
+        printf("2 mulsaq_s.l.pw wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c
new file mode 100644
index 00000000000..92d7a0b4f2b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resl, resh;
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x5678123443218765;
+    rt = 0x4321876556781234;
+
+    resh = 0x4;
+    resl = 0x342fcbd4;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "mulsaq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 mulsaq_s.w.qh wrong\n");
+        return -1;
+    }
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x8000800087654321;
+    rt = 0x8000800012345678;
+
+    resh = 0x3;
+    resl = 0xffffffffe5e81a1c;
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "mulsaq_s.w.qh $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (acho != resh) || (aclo != resl)) {
+        printf("2 mulsaq_s.w.qh wrong\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mult.c b/tests/tcg/mips/mips64-dsp/mult.c
new file mode 100644
index 00000000000..4a294d1a0c4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mult.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("mult $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("mult wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/multu.c b/tests/tcg/mips/mips64-dsp/multu.c
new file mode 100644
index 00000000000..21a8a7c77d7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/multu.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("multu $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("multu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/packrl_ph.c b/tests/tcg/mips/mips64-dsp/packrl_ph.c
new file mode 100644
index 00000000000..3722b0ae6c2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/packrl_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x56788765;
+
+    __asm
+        ("packrl.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("packrl.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/packrl_pw.c b/tests/tcg/mips/mips64-dsp/packrl_pw.c
new file mode 100644
index 00000000000..78074188342
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/packrl_pw.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567887654321;
+    rt = 0xabcdef9812345678;
+
+    res = 0x87654321abcdef98;
+
+    __asm
+        ("packrl.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != res) {
+        printf("packrl.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_ob.c b/tests/tcg/mips/mips64-dsp/pick_ob.c
new file mode 100644
index 00000000000..160049ffd40
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_ob.c
@@ -0,0 +1,66 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("1 pick.ob error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("2 pick.ob error\n");
+        return -1;
+    }
+
+    dsp = 0x34000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765567887344321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("3 pick.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_ph.c b/tests/tcg/mips/mips64-dsp/pick_ph.c
new file mode 100644
index 00000000000..8800c14d1ef
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_ph.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0A000000;
+    result = 0x12344321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("1 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x03000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("2 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0xffffffff87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("3 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_pw.c b/tests/tcg/mips/mips64-dsp/pick_pw.c
new file mode 100644
index 00000000000..24d80f551a1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_pw.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.pw %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.pw error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.pw %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_qb.c b/tests/tcg/mips/mips64-dsp/pick_qb.c
new file mode 100644
index 00000000000..0d5de9db9ed
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_qb.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0f000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("pick.qb wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0xffffffff87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("pick.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_qh.c b/tests/tcg/mips/mips64-dsp/pick_qh.c
new file mode 100644
index 00000000000..aa2e2938af9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_qh.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.qh %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.qh error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.qh %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c b/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c
new file mode 100644
index 00000000000..64551007cf3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+    rt = 0xFFFFFFFF11111111;
+    result = 0xFFFFFFFF00000000;
+
+    __asm
+        ("preceq.l.pwl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.l.pwl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c b/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c
new file mode 100644
index 00000000000..1e05339d23f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+    rt = 0xFFFFFFFF11111111;
+    result = 0x1111111100000000;
+
+    __asm
+        ("preceq.l.pwl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.l.pwr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c
new file mode 100644
index 00000000000..f44b9404925
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x0123456789ABCDEF;
+    result = 0x0123000045670000;
+
+    __asm
+        ("preceq.pw.qhl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.pw.qhl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c
new file mode 100644
index 00000000000..f0f78f43c56
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x123400009ABC0000;
+
+    __asm
+        ("preceq.pw.qhla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.pw.qhla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c
new file mode 100644
index 00000000000..709d4f90044
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x0123456789ABCDEF;
+    result = 0x89AB0000CDEF0000;
+
+    __asm
+        ("preceq.pw.qhr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.pw.qhr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c
new file mode 100644
index 00000000000..4d071ec8633
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x56780000DEF00000;
+
+    __asm
+        ("preceq.pw.qhra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.pw.qhra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_w_phl.c b/tests/tcg/mips/mips64-dsp/preceq_w_phl.c
new file mode 100644
index 00000000000..4ed3fc030ce
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_w_phl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0xFFFFFFFF87650000;
+
+    __asm
+        ("preceq.w.phl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.w.phl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_w_phr.c b/tests/tcg/mips/mips64-dsp/preceq_w_phr.c
new file mode 100644
index 00000000000..e2ea0933b29
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_w_phr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43210000;
+
+    __asm
+        ("preceq.w.phr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.w.phr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c
new file mode 100644
index 00000000000..17b73311dc4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43803280;
+
+    __asm
+        ("precequ.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c
new file mode 100644
index 00000000000..15e94946b10
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43802180;
+
+    __asm
+        ("precequ.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbla wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c
new file mode 100644
index 00000000000..495368ce0b0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x21801080;
+
+    __asm
+        ("precequ.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c
new file mode 100644
index 00000000000..7c6636975c2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x32801080;
+
+    __asm
+        ("precequ.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbra wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c
new file mode 100644
index 00000000000..176d2365a88
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x09001A002B003C00;
+
+    __asm
+        ("precequ.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c
new file mode 100644
index 00000000000..93a36a4855f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x09002B004D006F00;
+
+    __asm
+        ("precequ.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c
new file mode 100644
index 00000000000..121473083bb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x4D005E006F007000;
+
+    __asm
+        ("precequ.qh.obr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c
new file mode 100644
index 00000000000..3aa0e096cac
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x1A003C005D007000;
+
+    __asm
+        ("precequ.qh.obra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c
new file mode 100644
index 00000000000..81f7917c190
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00870065;
+
+    __asm
+        ("preceu.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c
new file mode 100644
index 00000000000..38cf6a62b9b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00870043;
+
+    __asm
+        ("preceu.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbla wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c
new file mode 100644
index 00000000000..70c32b6716e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00430021;
+
+    __asm
+        ("preceu.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbr wrong");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c
new file mode 100644
index 00000000000..c6638aaafd4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00650021;
+
+    __asm
+        ("preceu.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbra wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c
new file mode 100644
index 00000000000..63f9373b08e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x0012003400560078;
+
+    __asm
+        ("preceu.qh.obl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c
new file mode 100644
index 00000000000..5fb65e40492
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x00120056009A00DE;
+
+    __asm
+        ("preceu.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c
new file mode 100644
index 00000000000..9af3b6372e3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x009A00BC00DE00F0;
+
+    __asm
+        ("preceu.qh.obr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c
new file mode 100644
index 00000000000..fd04083371b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x0034007800BC00F0;
+
+    __asm
+        ("preceu.qh.obra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_ob_qh.c b/tests/tcg/mips/mips64-dsp/precr_ob_qh.c
new file mode 100644
index 00000000000..ce2da79af8d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_ob_qh.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x3478347865216521;
+
+    __asm
+        ("precr.ob.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precr.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c b/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c
new file mode 100644
index 00000000000..8bb16de9af0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long res;
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x4321432156785678;
+
+    __asm
+        ("precr_sra.qh.pw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra.qh.pw error\n");
+        return -1;
+    }
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x5432543245674567;
+
+    __asm
+        ("precr_sra.qh.pw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra.qh.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c b/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c
new file mode 100644
index 00000000000..734ac322e7f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long res;
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x4321432156785678;
+
+    __asm
+        ("precr_sra_r.qh.pw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra_r.qh.pw error\n");
+        return -1;
+    }
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x5432543245684568;
+
+    __asm
+        ("precr_sra_r.qh.pw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra_r.qh.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c b/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c
new file mode 100644
index 00000000000..4f61b1709e3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1256125687438743;
+
+    __asm
+        ("precrq.ob.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_ph_w.c b/tests/tcg/mips/mips64-dsp/precrq_ph_w.c
new file mode 100644
index 00000000000..f0946abdcb9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_ph_w.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precrq.ph.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_pw_l.c b/tests/tcg/mips/mips64-dsp/precrq_pw_l.c
new file mode 100644
index 00000000000..da957c07439
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_pw_l.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567887654321;
+
+    __asm
+        ("precrq.pw.l %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.pw.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c b/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c
new file mode 100644
index 00000000000..f417c9f342b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12568743;
+
+    __asm
+        ("precrq.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precrq.qb.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c b/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c
new file mode 100644
index 00000000000..4a4ffef8ea1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234123487658765;
+
+    __asm
+        ("precrq.qh.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.qh.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c b/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c
new file mode 100644
index 00000000000..61da3331fbe
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq_rs.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("1 precrq_rs.ph.w wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x7fffC678;
+    rt = 0x865432A0;
+    result = 0x7fff8654;
+
+    __asm
+        ("precrq_rs.ph.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((result != rd) || (((dsp >> 22) & 0x01) != 1)) {
+        printf("2 precrq_rs.ph.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c b/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c
new file mode 100644
index 00000000000..ac78728abf3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234123487658765;
+
+    __asm
+        ("precrq_rs.qh.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq_rs.qh.pw error\n");
+        return -1;
+    }
+
+    rs = 0x7fffC67812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x7fff123487658765;
+
+    __asm
+        ("precrq_rs.qh.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq_rs.qh.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c b/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c
new file mode 100644
index 00000000000..e27c36b7fd5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res, resdsp;
+
+    rs = 0x7fff567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0xffac24ac00860086;
+    resdsp = 0x1;
+
+    __asm
+        ("precrqu_s.ob.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x1;
+    if ((rd != res) || (dsp != resdsp)) {
+        printf("precrq_s.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c b/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c
new file mode 100644
index 00000000000..cb1fee4508f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87657fff;
+    result = 0x24AC00FF;
+
+    __asm
+        ("precrqu_s.qb.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((result != rd) || (((dsp >> 22) & 0x01) != 0x01)) {
+        printf("precrqu_s.qb.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/prependd.c b/tests/tcg/mips/mips64-dsp/prependd.c
new file mode 100644
index 00000000000..b4208c2dabc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/prependd.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("prependd %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependd error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0xd876512345678876;
+    __asm
+        ("prependd %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependd error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/prependw.c b/tests/tcg/mips/mips64-dsp/prependw.c
new file mode 100644
index 00000000000..d91bd2023c9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/prependw.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("prependw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x5123456788765432;
+    __asm
+        ("prependw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/printf.c b/tests/tcg/mips/mips64-dsp/printf.c
new file mode 100644
index 00000000000..cf8676d3909
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/printf.c
@@ -0,0 +1,266 @@
+
+typedef unsigned long va_list;
+
+#define ACC    4
+#define __read(source)                    \
+({ va_list __res;                    \
+    __asm__ __volatile__(                \
+        "move\t%0, " #source "\n\t"        \
+        : "=r" (__res));            \
+    __res;                        \
+})
+
+enum format_type {
+    FORMAT_TYPE_NONE,
+    FORMAT_TYPE_HEX,
+    FORMAT_TYPE_ULONG,
+    FORMAT_TYPE_FLOAT
+};
+
+struct printf_spec {
+    char    type;
+};
+
+static int format_decode(char *fmt, struct printf_spec *spec)
+{
+    char *start = fmt;
+
+    for (; *fmt ; ++fmt) {
+        if (*fmt == '%') {
+            break;
+        }
+    }
+
+    switch (*++fmt) {
+    case 'x':
+        spec->type = FORMAT_TYPE_HEX;
+        break;
+
+    case 'd':
+        spec->type = FORMAT_TYPE_ULONG;
+        break;
+
+    case 'f':
+        spec->type = FORMAT_TYPE_FLOAT;
+        break;
+
+    default:
+        spec->type = FORMAT_TYPE_NONE;
+    }
+
+    return ++fmt - start;
+}
+
+void *memcpy(void *dest, void *src, int n)
+{
+    int i;
+    char *s = src;
+    char *d = dest;
+
+    for (i = 0; i < n; i++) {
+        d[i] = s[i];
+    }
+    return dest;
+}
+
+char *number(char *buf, va_list num)
+{
+    int i;
+    char *str = buf;
+    static char digits[16] = "0123456789abcdef";
+    str = str + sizeof(num) * 2;
+
+    for (i = 0; i < sizeof(num) * 2; i++) {
+        *--str = digits[num & 15];
+        num >>= 4;
+    }
+
+    return buf + sizeof(num) * 2;
+}
+
+char *__number(char *buf, va_list num)
+{
+    int i;
+    va_list mm = num;
+    char *str = buf;
+
+    if (!num) {
+        *str++ = '0';
+        return str;
+    }
+
+    for (i = 0; mm; mm = mm/10, i++) {
+        /* Do nothing. */
+    }
+
+    str = str + i;
+
+    while (num) {
+        *--str = num % 10 + 48;
+        num = num / 10;
+    }
+
+    return str + i;
+}
+
+va_list modf(va_list args, va_list *integer, va_list *num)
+{
+    int i;
+    double dot_v = 0;
+    va_list E, DOT, DOT_V;
+
+    if (!args) {
+        return 0;
+    }
+
+    for (i = 0, args = args << 1 >> 1; i < 52; i++) {
+        if ((args >> i) & 0x1) {
+            break;
+        }
+    }
+
+    *integer = 0;
+
+    if ((args >> 56 != 0x3f) || (args >> 52 == 0x3ff)) {
+        E = (args >> 52) - 1023;
+        DOT = 52 - E - i;
+        DOT_V = args << (12 + E) >> (12 + E) >> i;
+        *integer = ((args << 12 >> 12) >> (i + DOT)) | (1 << E);
+    } else {
+        E = ~((args >> 52) - 1023) + 1;
+        DOT_V = args << 12 >> 12;
+
+        dot_v += 1.0 / (1 << E);
+
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (52 - i)) & 0x1) {
+                dot_v += 1.0 / (1 << E + i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+    }
+
+    *num = E;
+
+    return dot_v;
+    }
+
+    if (args & 0xf) {
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+        }
+
+        *num = E;
+
+        return dot_v;
+    } else if (DOT) {
+        for (i = 1; i <= DOT; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1; i <= ACC; i++) {
+            dot_v = dot_v * 10;
+        }
+
+    return dot_v;
+    }
+
+    return 0;
+}
+
+int vsnprintf(char *buf, int size, char *fmt, va_list args)
+{
+    char *str, *mm;
+    struct printf_spec spec = {0};
+
+    str = mm = buf;
+
+    while (*fmt) {
+        char *old_fmt = fmt;
+        int read = format_decode(fmt, &spec);
+
+        fmt += read;
+
+        switch (spec.type) {
+        case FORMAT_TYPE_NONE: {
+            memcpy(str, old_fmt, read);
+            str += read;
+            break;
+        }
+        case FORMAT_TYPE_HEX: {
+            memcpy(str, old_fmt, read);
+            str = number(str + read, args);
+            for (; *mm ; ++mm) {
+                if (*mm == '%') {
+                    *mm = '0';
+                break;
+                }
+            }
+        break;
+        }
+        case FORMAT_TYPE_ULONG: {
+            memcpy(str, old_fmt, read - 2);
+            str = __number(str + read - 2, args);
+            break;
+        }
+        case FORMAT_TYPE_FLOAT: {
+            va_list integer, dot_v, num;
+            dot_v = modf(args, &integer, &num);
+            memcpy(str, old_fmt, read - 2);
+            str += read - 2;
+            if ((args >> 63 & 0x1)) {
+                *str++ = '-';
+            }
+            str = __number(str, integer);
+            if (dot_v) {
+                *str++ = '.';
+                while (num--) {
+                    *str++ = '0';
+                }
+                str = __number(str, dot_v);
+            }
+            break;
+        }
+        }
+    }
+    *str = '\0';
+
+    return str - buf;
+}
+
+static void serial_out(char *str)
+{
+    while (*str) {
+        *(char *)0xffffffffb80003f8 = *str++;
+    }
+}
+
+int vprintf(char *fmt, va_list args)
+{
+    int printed_len = 0;
+    static char printf_buf[512];
+    printed_len = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args);
+    serial_out(printf_buf);
+    return printed_len;
+}
+
+int printf(char *fmt, ...)
+{
+    return vprintf(fmt, __read($5));
+}
diff --git a/tests/tcg/mips/mips64-dsp/raddu_l_ob.c b/tests/tcg/mips/mips64-dsp/raddu_l_ob.c
new file mode 100644
index 00000000000..76ddf25fb99
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/raddu_l_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, result;
+    rs = 0x12345678ABCDEF0;
+    result = 0x000000000001E258;
+
+    __asm
+        ("raddu.l.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+
+    if (rd != result) {
+        printf("raddu.l.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/raddu_w_qb.c b/tests/tcg/mips/mips64-dsp/raddu_w_qb.c
new file mode 100644
index 00000000000..c9d6535bba8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/raddu_w_qb.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs;
+    long long result;
+
+    rs = 0x12345678;
+    result = 0x114;
+
+    __asm
+        ("raddu.w.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+    if (rd != result) {
+        printf("raddu.w.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/rddsp.c b/tests/tcg/mips/mips64-dsp/rddsp.c
new file mode 100644
index 00000000000..71655724350
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/rddsp.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp_i, dsp_o;
+    long long ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    long long ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    long long ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i   = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i     = 0x00000001;/* 5 */
+    c_i       = 0x00000001;/* 2 */
+    scount_i  = 0x0000000F;/* 1 */
+    pos_i     = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    ccond_r   = ccond_i;
+    outflag_r = outflag_i;
+    efi_r     = efi_i;
+    c_r       = c_i;
+    scount_r  = scount_i;
+    pos_r     = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    if ((ccond_o != ccond_r) || (outflag_o != outflag_r) || (efi_o != efi_r) \
+            || (c_o != c_r) || (scount_o != scount_r) || (pos_o != pos_r)) {
+        printf("rddsp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_ob.c b/tests/tcg/mips/mips64-dsp/repl_ob.c
new file mode 100644
index 00000000000..20cb7801366
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_ob.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("repl.ob %0, 0xFF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_ph.c b/tests/tcg/mips/mips64-dsp/repl_ph.c
new file mode 100644
index 00000000000..11d29bdbc24
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_ph.c
@@ -0,0 +1,30 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+
+    result = 0x01BF01BF;
+    __asm
+        ("repl.ph %0, 0x1BF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.ph wrong\n");
+
+        return -1;
+    }
+
+    result = 0x01FF01FF;
+    __asm
+        ("repl.ph %0, 0x01FF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_pw.c b/tests/tcg/mips/mips64-dsp/repl_pw.c
new file mode 100644
index 00000000000..d35376a2a32
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_pw.c
@@ -0,0 +1,34 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0x000001FF000001FF;
+
+    __asm
+        ("repl.pw %0, 0x1FF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.pw error1\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    result = 0xFFFFFE00FFFFFE00;
+    __asm
+        ("repl.pw %0, 0xFFFFFFFFFFFFFE00\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.pw error2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_qb.c b/tests/tcg/mips/mips64-dsp/repl_qb.c
new file mode 100644
index 00000000000..592feaecb0a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_qb.c
@@ -0,0 +1,19 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+
+    result = 0xFFFFFFFFBFBFBFBF;
+    __asm
+        ("repl.qb %0, 0xBF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_qh.c b/tests/tcg/mips/mips64-dsp/repl_qh.c
new file mode 100644
index 00000000000..82afc371676
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_qh.c
@@ -0,0 +1,34 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0x01FF01FF01FF01FF;
+
+    __asm
+        ("repl.qh %0, 0x1FF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.qh error 1\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    result = 0xFE00FE00FE00FE00;
+    __asm
+        ("repl.qh %0, 0xFFFFFFFFFFFFFE00\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.qh error 2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_ob.c b/tests/tcg/mips/mips64-dsp/replv_ob.c
new file mode 100644
index 00000000000..31ff3186d35
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0xFF;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("replv.ob %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("replv.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_ph.c b/tests/tcg/mips/mips64-dsp/replv_ph.c
new file mode 100644
index 00000000000..0af7a36b402
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_ph.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x12345678;
+    result = 0x56785678;
+    __asm
+        ("replv.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("replv.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_pw.c b/tests/tcg/mips/mips64-dsp/replv_pw.c
new file mode 100644
index 00000000000..e1789af4c83
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_pw.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rd = 0;
+    rt = 0xFFFFFFFF;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("replv.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("replv.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_qb.c b/tests/tcg/mips/mips64-dsp/replv_qb.c
new file mode 100644
index 00000000000..d99298c31ec
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_qb.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x78787878;
+    __asm
+        ("replv.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("replv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shilo.c b/tests/tcg/mips/mips64-dsp/shilo.c
new file mode 100644
index 00000000000..5f454f69e0a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shilo.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long ach, acl;
+    long long resulth, resultl;
+
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0xFFFFFFFF99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, 0x0F\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("shilo wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shilov.c b/tests/tcg/mips/mips64-dsp/shilov.c
new file mode 100644
index 00000000000..e82615a8c90
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shilov.c
@@ -0,0 +1,31 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, ach, acl;
+    long long resulth, resultl;
+
+    rs  = 0x0F;
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0xFFFFFFFF99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("shilov wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_ob.c b/tests/tcg/mips/mips64-dsp/shll_ob.c
new file mode 100644
index 00000000000..7dcb58ff46a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_ob.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll.ob %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.ob error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0xd840b0a098283848;
+    resdsp = 0x1;
+    __asm
+        ("shll.ob %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_ph.c b/tests/tcg/mips/mips64-dsp/shll_ph.c
new file mode 100644
index 00000000000..42b462d20de
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_ph.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll.ph %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.ph wrong\n");
+
+        return -1;
+    }
+
+    rt        = 0x12345678;
+    result    = 0xFFFFFFFFA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shll.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.ph wrong1\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_pw.c b/tests/tcg/mips/mips64-dsp/shll_pw.c
new file mode 100644
index 00000000000..d7878b2792c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll.pw %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    result    = 0x6543210034567800;
+    resultdsp = 1;
+
+    __asm
+        ("shll.pw %0, %2, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_qb.c b/tests/tcg/mips/mips64-dsp/shll_qb.c
new file mode 100644
index 00000000000..c21ab6698ac
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_qb.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll.qb %0, %2, 0x03\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if (rd != result) {
+        printf("shll.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_qh.c b/tests/tcg/mips/mips64-dsp/shll_qh.c
new file mode 100644
index 00000000000..1380825a327
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_qh.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll.qh %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.qh error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0xdd40b2a09a283c48;
+    resdsp = 0x1;
+    __asm
+        ("shll.qh %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_ph.c b/tests/tcg/mips/mips64-dsp/shll_s_ph.c
new file mode 100644
index 00000000000..1cf5d6da6e9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_ph.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x0;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_pw.c b/tests/tcg/mips/mips64-dsp/shll_s_pw.c
new file mode 100644
index 00000000000..e38f6860c75
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll_s.pw %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    result    = 0x800000007fffffff;
+    resultdsp = 1;
+
+    __asm
+        ("shll_s.pw %0, %2, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_qh.c b/tests/tcg/mips/mips64-dsp/shll_s_qh.c
new file mode 100644
index 00000000000..f2f57fa27a5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll_s.qh %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll_s.qh error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0x80007fff7fff7fff;
+    resdsp = 0x1;
+    __asm
+        ("shll_s.qh %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll_s.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_w.c b/tests/tcg/mips/mips64-dsp/shll_s_w.c
new file mode 100644
index 00000000000..57800615d27
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_w.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_ob.c b/tests/tcg/mips/mips64-dsp/shllv_ob.c
new file mode 100644
index 00000000000..96a2e6f55fc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_ob.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ob wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x7050301020406080;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ob wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_ph.c b/tests/tcg/mips/mips64-dsp/shllv_ph.c
new file mode 100644
index 00000000000..532291f3fb7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0xFFFFFFFFA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_pw.c b/tests/tcg/mips/mips64-dsp/shllv_pw.c
new file mode 100644
index 00000000000..8d4ec295bda
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.pw wrong\n");
+        return -1;
+    }
+
+
+    rt        = 0x8765432112345678;
+    rs = 0x8;
+    result    = 0x6543210034567800;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_qb.c b/tests/tcg/mips/mips64-dsp/shllv_qb.c
new file mode 100644
index 00000000000..e49356b8ec9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if (rd != result) {
+        printf("shllv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_qh.c b/tests/tcg/mips/mips64-dsp/shllv_qh.c
new file mode 100644
index 00000000000..0de4077e7d0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_qh.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.qh wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x7650321023406780;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.qh wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_ph.c b/tests/tcg/mips/mips64-dsp/shllv_s_ph.c
new file mode 100644
index 00000000000..7e69f941f44
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_pw.c b/tests/tcg/mips/mips64-dsp/shllv_s_pw.c
new file mode 100644
index 00000000000..f8dc8d29641
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x8;
+    result    = 0x800000007fffffff;
+    resultdsp = 1;
+
+    __asm
+        ("shllv_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_qh.c b/tests/tcg/mips/mips64-dsp/shllv_s_qh.c
new file mode 100644
index 00000000000..db3832d0911
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_qh.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.qh wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x80007fff7fff7fff;
+    resultdsp = 1;
+
+    __asm
+        ("shllv_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.qh wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_w.c b/tests/tcg/mips/mips64-dsp/shllv_s_w.c
new file mode 100644
index 00000000000..5f6af8b8c0a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_w.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_ob.c b/tests/tcg/mips/mips64-dsp/shra_ob.c
new file mode 100644
index 00000000000..d7fcfa816b4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xbc98756abc654389;
+    res = 0xfbf9f7f6fb0604f8;
+
+    __asm
+        ("shra.ob %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_ph.c b/tests/tcg/mips/mips64-dsp/shra_ph.c
new file mode 100644
index 00000000000..a2dc014742d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_ph.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0EC0864;
+
+    __asm
+        ("shra.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_pw.c b/tests/tcg/mips/mips64-dsp/shra_pw.c
new file mode 100644
index 00000000000..33b1b8fe72e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_pw.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x1234567887654321;
+    res = 0x01234567f8765432;
+
+    __asm
+        ("shra.pw %0, %1, 0x4"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shra.pw %0, %1, 0x0"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_qh.c b/tests/tcg/mips/mips64-dsp/shra_qh.c
new file mode 100644
index 00000000000..85dbfef3ea2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_qh.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8512345654323454;
+    res = 0xf851034505430345;
+
+    __asm
+        ("shra.qh %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shra.qh %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_ob.c b/tests/tcg/mips/mips64-dsp/shra_r_ob.c
new file mode 100644
index 00000000000..184709443e9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xbc98756abc654389;
+    res = 0xfcfaf8f7fc0705f9;
+
+    __asm
+        ("shra_r.ob %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_ph.c b/tests/tcg/mips/mips64-dsp/shra_r_ph.c
new file mode 100644
index 00000000000..e0943ad474e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_ph.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ED0864;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_pw.c b/tests/tcg/mips/mips64-dsp/shra_r_pw.c
new file mode 100644
index 00000000000..6a86e684b8a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_pw.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x1234567887654321;
+    res = 0x01234568f8765432;
+
+    __asm
+        ("shra_r.pw %0, %1, 0x4"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shra_r.pw %0, %1, 0x0"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_qh.c b/tests/tcg/mips/mips64-dsp/shra_r_qh.c
new file mode 100644
index 00000000000..d5c2110efea
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_qh.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8512345654323454;
+    res = 0xf0a2068b0a86068b;
+
+    __asm
+        ("shra_r.qh %0, %1, 0x3\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shra_r.qh %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_w.c b/tests/tcg/mips/mips64-dsp/shra_r_w.c
new file mode 100644
index 00000000000..36d2c9c8871
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_w.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ECA864;
+
+    __asm
+        ("shra_r.w %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_ph.c b/tests/tcg/mips/mips64-dsp/shrav_ph.c
new file mode 100644
index 00000000000..1b4e9837460
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0EC0864;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_pw.c b/tests/tcg/mips/mips64-dsp/shrav_pw.c
new file mode 100644
index 00000000000..e19d5157977
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_pw.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0x01234567f8765432;
+
+    __asm
+        ("shrav.pw %0, %1, %2"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0x0;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shrav.pw %0, %1, %2"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.pw error1\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_qh.c b/tests/tcg/mips/mips64-dsp/shrav_qh.c
new file mode 100644
index 00000000000..dc92e09d44b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_qh.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8512345654323454;
+    rs = 0x4;
+    res = 0xf851034505430345;
+
+    __asm
+        ("shrav.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    rs = 0x0;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shrav.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_ph.c b/tests/tcg/mips/mips64-dsp/shrav_r_ph.c
new file mode 100644
index 00000000000..350d5294fc6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ED0864;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_pw.c b/tests/tcg/mips/mips64-dsp/shrav_r_pw.c
new file mode 100644
index 00000000000..25b0545931a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_pw.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0x01234568f8765432;
+
+    __asm
+        ("shrav_r.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0x0;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shrav_r.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != res) {
+        printf("shrav_r.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_qh.c b/tests/tcg/mips/mips64-dsp/shrav_r_qh.c
new file mode 100644
index 00000000000..fd187a1e0b6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_qh.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8512345654323454;
+    rs = 0x3;
+    res = 0xf0a2068b0a86068b;
+
+    __asm
+        ("shrav_r.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.qh error\n");
+        return -1;
+    }
+
+    rt = 0x400000000000000;
+    rs = 0x0;
+    res = 0x400000000000000;
+
+    __asm
+        ("shrav_r.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_w.c b/tests/tcg/mips/mips64-dsp/shrav_r_w.c
new file mode 100644
index 00000000000..3766c7255c7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_w.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ECA864;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_ob.c b/tests/tcg/mips/mips64-dsp/shrl_ob.c
new file mode 100644
index 00000000000..a1145713a08
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_ob.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xab76543212345678;
+    res = 0x150e0a0602060a0f;
+
+    __asm
+        ("shrl.ob %0, %1, 0x3\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.ob error\n");
+        return -1;
+    }
+
+    rt = 0xab76543212345678;
+    res = 0xab76543212345678;
+
+    __asm
+        ("shrl.ob %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.ob error\n");
+        return -1;
+    }
+
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_qb.c b/tests/tcg/mips/mips64-dsp/shrl_qb.c
new file mode 100644
index 00000000000..c0e36dba10a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_qb.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrl.qb %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shrl.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_qh.c b/tests/tcg/mips/mips64-dsp/shrl_qh.c
new file mode 100644
index 00000000000..c1562463e07
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_qh.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8765679abc543786;
+    res = 0x087606790bc50378;
+
+    __asm
+        ("shrl.qh %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_ob.c b/tests/tcg/mips/mips64-dsp/shrlv_ob.c
new file mode 100644
index 00000000000..cb39c46716a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_ob.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0xab76543212345678;
+    rs = 0x3;
+    res = 0x150e0a0602060a0f;
+
+    __asm
+        ("shrlv.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.ob error\n");
+        return -1;
+    }
+
+    rt = 0xab76543212345678;
+    rs = 0x0;
+    res = 0xab76543212345678;
+
+    __asm
+        ("shrlv.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_qb.c b/tests/tcg/mips/mips64-dsp/shrlv_qb.c
new file mode 100644
index 00000000000..5616aa9c5bf
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_qb.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrlv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_qh.c b/tests/tcg/mips/mips64-dsp/shrlv_qh.c
new file mode 100644
index 00000000000..05de2fd6c8b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_qh.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8765679abc543786;
+    rs = 0x4;
+    res = 0x087606790bc50378;
+
+    __asm
+        ("shrlv.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_ph.c b/tests/tcg/mips/mips64-dsp/subq_ph.c
new file mode 100644
index 00000000000..6a1b18610ca
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0xFFFFFFFF8ACF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_pw.c b/tests/tcg/mips/mips64-dsp/subq_pw.c
new file mode 100644
index 00000000000..32f96ba4bde
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_pw.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x123456789ABCDEF0;
+    rs = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.pw error1\n\t");
+
+        return -1;
+    }
+
+    rt = 0x123456789ABCDEF1;
+    rs = 0x123456789ABCDEF2;
+    result =  0x0000000000000001;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.pw error2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_qh.c b/tests/tcg/mips/mips64-dsp/subq_qh.c
new file mode 100644
index 00000000000..76d5f0a10a2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_qh.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x123456789ABCDEF0;
+    rs = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.qh error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_ph.c b/tests/tcg/mips/mips64-dsp/subq_s_ph.c
new file mode 100644
index 00000000000..0b162f07eab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_pw.c b/tests/tcg/mips/mips64-dsp/subq_s_pw.c
new file mode 100644
index 00000000000..e8e0b0567e4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_pw.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x9FFFFFFD9FFFFFFD;
+    rs = 0x4000000080000000;
+    result = 0x7fffffffe0000003;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.pw error1\n");
+
+        return -1;
+    }
+
+    rt = 0x123456789ABCDEF1;
+    rs = 0x123456789ABCDEF2;
+    result =  0x0000000000000001;
+    /* This time we do not set dspctrl, but it setted in pre-action. */
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.pw error2\n");
+
+        return -1;
+    }
+
+    rt = 0x8000000080000000;
+    rs = 0x7000000070000000;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((dspreg != dspresult)) {
+        printf("subq_s.pw error3\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_qh.c b/tests/tcg/mips/mips64-dsp/subq_s_qh.c
new file mode 100644
index 00000000000..4053b6b8844
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_qh.c
@@ -0,0 +1,61 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error1\n");
+
+        return -1;
+    }
+
+    rs = 0x4000000080000000;
+    rt = 0x9FFD00009FFC0000;
+    result =  0x7FFF0000E0040000;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error2\n");
+
+        return -1;
+    }
+
+    rs = 0x8000000000000000;
+    rt = 0x7000000000000000;
+    result =  0x8000000000000000;
+    dspresult = 0x1;
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error3\n");
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_w.c b/tests/tcg/mips/mips64-dsp/subq_s_w.c
new file mode 100644
index 00000000000..91d32da1724
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_w.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_ob.c b/tests/tcg/mips/mips64-dsp/subu_ob.c
new file mode 100644
index 00000000000..f6709671138
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_ob.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x6F6F6F6F6F6F6F6F;
+    rt = 0x5E5E5E5E5E5E5E5E;
+    result = 0x1111111111111111;
+    dspresult = 0x0;
+
+    __asm
+        ("subu.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subu_qb.c b/tests/tcg/mips/mips64-dsp/subu_qb.c
new file mode 100644
index 00000000000..9eb80df3790
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0xFFFFFFFF8BCF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subu.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_s_ob.c b/tests/tcg/mips/mips64-dsp/subu_s_ob.c
new file mode 100644
index 00000000000..5df64e5ff01
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_s_ob.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x12345678ABCDEF0;
+    rt = 0x12345678ABCDEF1;
+    result = 0x00000000000;
+    dspresult = 0x01;
+
+    __asm
+        ("subu_s.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_s_qb.c b/tests/tcg/mips/mips64-dsp/subu_s_qb.c
new file mode 100644
index 00000000000..9de76f4a1fa
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_s_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x00001357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subu_s_qb wrong");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/wrdsp.c b/tests/tcg/mips/mips64-dsp/wrdsp.c
new file mode 100644
index 00000000000..3033fd88d1a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/wrdsp.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp_i, dsp_o;
+    long long ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    long long ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    long long ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i = 0x00000001;/* 5 */
+    c_i = 0x00000001;/* 2 */
+    scount_i = 0x0000000F;/* 1 */
+    pos_i = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i << 24) | (outflag_i << 16) | (efi_i << 14) | (c_i << 13)
+            | (scount_i << 7) | pos_i;
+
+    ccond_r = ccond_i;
+    outflag_r = outflag_i;
+    efi_r = efi_i;
+    c_r = c_i;
+    scount_r = scount_i;
+    pos_r = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o = (dsp_o >> 14) & 0x01;
+    c_o = (dsp_o >> 14) & 0x01;
+    scount_o = (dsp_o >> 7) & 0x3F;
+    pos_o = dsp_o & 0x1F;
+
+    if ((ccond_o != ccond_r) || (outflag_o != outflag_r) || (efi_o != efi_r) \
+            || (c_o != c_r) || (scount_o != scount_r) || (pos_o != pos_r)) {
+        printf("wrddsp wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/.directory b/tests/tcg/mips/mips64-dspr2/.directory
new file mode 100644
index 00000000000..c75a91451ce
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/.directory
@@ -0,0 +1,2 @@
+[Dolphin]
+Timestamp=2012,8,3,16,41,52
diff --git a/tests/tcg/mips/mips64-dspr2/Makefile b/tests/tcg/mips/mips64-dspr2/Makefile
new file mode 100644
index 00000000000..ba44bb9c0e5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/Makefile
@@ -0,0 +1,116 @@
+CROSS_COMPILE	?= mips64el-unknown-linux-gnu-
+
+SIM = qemu-system-mips64el
+SIMFLAGS = -nographic -cpu mips64dspr2 -kernel
+
+AS      = $(CROSS_COMPILE)as
+LD      = $(CROSS_COMPILE)ld
+CC      = $(CROSS_COMPILE)gcc
+AR      = $(CROSS_COMPILE)ar
+NM      = $(CROSS_COMPILE)nm
+STRIP       = $(CROSS_COMPILE)strip
+RANLIB      = $(CROSS_COMPILE)ranlib
+OBJCOPY     = $(CROSS_COMPILE)objcopy
+OBJDUMP     = $(CROSS_COMPILE)objdump
+
+VECTORS_OBJ ?= ./head.o ./printf.o
+
+HEAD_FLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -pipe \
+              -msoft-float -march=mips64 -Wa,-mips64 -Wa,--trap \
+              -msym32 -DKBUILD_64BIT_SYM32 -I./
+
+CFLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -fno-builtin  \
+          -pipe -march=mips64r2 -mgp64 -mdspr2 -static -Wa,--trap -msym32 \
+          -DKBUILD_64BIT_SYM32 -I./
+
+LDFLAGS = -T./mips_boot.lds -L./
+FLAGS = -nostdlib -mabi=64 -march=mips64r2 -mgp64 -mdspr2
+
+TESTCASES = absq_s_qb.tst
+TESTCASES += addqh_ph.tst
+TESTCASES += addqh_r_ph.tst
+TESTCASES += addqh_r_w.tst
+TESTCASES += addqh_w.tst
+#TESTCASES += adduh_ob.tst
+TESTCASES += adduh_qb.tst
+#TESTCASES += adduh_r_ob.tst
+TESTCASES += adduh_r_qb.tst
+TESTCASES += addu_ph.tst
+#TESTCASES += addu_qh.tst
+TESTCASES += addu_s_ph.tst
+#TESTCASES += addu_s_qh.tst
+TESTCASES += append.tst
+TESTCASES += balign.tst
+#TESTCASES += cmpgdu_eq_ob.tst
+TESTCASES += cmpgdu_eq_qb.tst
+#TESTCASES += cmpgdu_le_ob.tst
+TESTCASES += cmpgdu_le_qb.tst
+#TESTCASES += cmpgdu_lt_ob.tst
+TESTCASES += cmpgdu_lt_qb.tst
+#TESTCASES += dbalign.tst
+TESTCASES += dpaqx_sa_w_ph.tst
+TESTCASES += dpaqx_s_w_ph.tst
+TESTCASES += dpa_w_ph.tst
+#TESTCASES += dpa_w_qh.tst
+TESTCASES += dpax_w_ph.tst
+TESTCASES += dpsqx_sa_w_ph.tst
+TESTCASES += dpsqx_s_w_ph.tst
+TESTCASES += dps_w_ph.tst
+#TESTCASES += dps_w_qh.tst
+TESTCASES += dpsx_w_ph.tst
+TESTCASES += mul_ph.tst
+TESTCASES += mulq_rs_w.tst
+TESTCASES += mulq_s_ph.tst
+TESTCASES += mulq_s_w.tst
+TESTCASES += mulsaq_s_w_ph.tst
+TESTCASES += mulsa_w_ph.tst
+TESTCASES += mul_s_ph.tst
+TESTCASES += precr_qb_ph.tst
+TESTCASES += precr_sra_ph_w.tst
+TESTCASES += precr_sra_r_ph_w.tst
+TESTCASES += prepend.tst
+TESTCASES += shra_qb.tst
+TESTCASES += shra_r_qb.tst
+#TESTCASES += shrav_ob.tst
+TESTCASES += shrav_qb.tst
+#TESTCASES += shrav_r_ob.tst
+TESTCASES += shrav_r_qb.tst
+TESTCASES += shrl_ph.tst
+TESTCASES += shrlv_ph.tst
+TESTCASES += subqh_ph.tst
+TESTCASES += subqh_r_ph.tst
+TESTCASES += subqh_r_w.tst
+TESTCASES += subqh_w.tst
+#TESTCASES += subuh_ob.tst
+TESTCASES += subuh_qb.tst
+#TESTCASES += subuh_r_ob.tst
+TESTCASES += subuh_r_qb.tst
+TESTCASES += subu_ph.tst
+#TESTCASES += subu_qh.tst
+TESTCASES += subu_s_ph.tst
+#TESTCASES += subu_s_qh.tst
+
+all: build
+
+head.o : head.S
+	$(Q)$(CC) $(HEAD_FLAGS) -D"STACK_TOP=0xffffffff80200000" -c $< -o $@
+
+%.o  : %.S
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.o  : %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.tst: %.o $(VECTORS_OBJ)
+	$(CC) $(VECTORS_OBJ) $(FLAGS) $(LDFLAGS) $< -o $@
+
+build: $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+
+check:  $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+	@for case in $(TESTCASES); do \
+		echo $(SIM) $(SIMFLAGS) ./$$case; \
+		$(SIM) $(SIMFLAGS) ./$$case & (sleep 1; killall $(SIM)); \
+	done
+
+clean:
+	$(Q)rm -f *.o *.tst *.a
diff --git a/tests/tcg/mips/mips64-dspr2/absq_s_qb.c b/tests/tcg/mips/mips64-dspr2/absq_s_qb.c
new file mode 100644
index 00000000000..f7aec3e568f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/absq_s_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+int main()
+{
+    long long input, result, dsp;
+    long long hope;
+
+    input = 0x701BA35E;
+    hope  = 0x701B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %1\n\t"
+         : "=r"(result)
+         : "r"(input)
+        );
+    if (result != hope) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    input = 0x801BA35E;
+    hope  = 0x7F1B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(result), "=r"(dsp)
+         : "r"(input)
+        );
+    dsp = dsp >> 20;
+    dsp &= 0x01;
+    if (result != hope) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    if (dsp != 1) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_ph.c b/tests/tcg/mips/mips64-dspr2/addqh_ph.c
new file mode 100644
index 00000000000..6b43cb8d066
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_ph.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("addqh.ph error!\n");
+        return -1;
+    }
+
+    rs     = 0x81000100;
+    rt     = 0xc2000100;
+    result = 0xffffffffa1800100;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("addqh.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c b/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c
new file mode 100644
index 00000000000..890ec98d9be
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addqh_r.ph error\n");
+        return -1;
+    }
+
+    rs     = 0x81010100;
+    rt     = 0xc2000100;
+    result = 0xffffffffa1810100;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addqh_r.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_r_w.c b/tests/tcg/mips/mips64-dspr2/addqh_r_w.c
new file mode 100644
index 00000000000..d324decbd3e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_r_w.c
@@ -0,0 +1,38 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000009;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh_r.w error!\n");
+        return -1;
+    }
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0x00000000;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh_r.w error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_w.c b/tests/tcg/mips/mips64-dspr2/addqh_w.c
new file mode 100644
index 00000000000..78559e67842
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_w.c
@@ -0,0 +1,39 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000008;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh.w wrong\n");
+        return -1;
+    }
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh.w wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_ph.c b/tests/tcg/mips/mips64-dspr2/addu_ph.c
new file mode 100644
index 00000000000..d64c8cde8a8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_ph.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x01000100;
+    __asm
+        ("addu.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addu.ph error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0x00011112;
+    __asm
+        ("addu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addu.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_qh.c b/tests/tcg/mips/mips64-dspr2/addu_qh.c
new file mode 100644
index 00000000000..edcbf342ce6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg;
+    long long result, dspresult;
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180000000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x0;
+
+    __asm("addu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.qh error\n");
+        return -1;
+    }
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180020000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x01;
+
+    __asm("addu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.qh overflow error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_s_ph.c b/tests/tcg/mips/mips64-dspr2/addu_s_ph.c
new file mode 100644
index 00000000000..9250edb45c9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_s_ph.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FE00FE;
+    rt     = 0x00020001;
+    result = 0x010000FF;
+    __asm
+        ("addu_s.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addu_s.ph error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFFFF1112;
+    __asm
+        ("addu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("addu_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_s_qh.c b/tests/tcg/mips/mips64-dspr2/addu_s_qh.c
new file mode 100644
index 00000000000..b0c16262513
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_s_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg;
+    long long result, dspresult;
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180000000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x0;
+
+    __asm("addu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("1 addu_s.qh error\n");
+        return -1;
+    }
+
+    rs = 0x12345678FFFF0000;
+    rt = 0x11111111000F0000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x01;
+
+    __asm("addu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("2 addu_s.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_ob.c b/tests/tcg/mips/mips64-dspr2/adduh_ob.c
new file mode 100644
index 00000000000..9b309f6f16e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_ob.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+    rs = 0xFF987CDEBCEF2356;
+    rt = 0xFF987CDEBCEF2354;
+    result = 0xFF987CDEBCEF2355;
+
+    __asm("adduh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("adduh.ob error\n\t");
+        return -1;
+    }
+
+    rs = 0xac50691729945316;
+    rt = 0xb9234ca3f5573162;
+    result = 0xb2395a5d8f75423c;
+
+    __asm("adduh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("adduh.ob error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_qb.c b/tests/tcg/mips/mips64-dspr2/adduh_qb.c
new file mode 100644
index 00000000000..796b409a86d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0xffffffff80094B62;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh.qb error\n");
+        return -1;
+    }
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x7F800888;
+
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c b/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c
new file mode 100644
index 00000000000..832de833ef5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+    rs = 0xFF987CDEBCEF2356;
+    rt = 0xFF987CDEBCEF2355;
+    result = 0xFF987CDEBCEF2356;
+
+    __asm("adduh_r.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("1 adduh_r.ob error\n\t");
+        return -1;
+    }
+
+    rs = 0xac50691729945316;
+    rt = 0xb9234ca3f5573162;
+    result = 0xb33a5b5d8f76423c;
+
+    __asm("adduh_r.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("2 adduh_r.ob error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c b/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c
new file mode 100644
index 00000000000..ae65fa5e186
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x01112211;
+    result = 0xffffffff80093C5E;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh_r.qb error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0xffffffff80800888;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh_r.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/append.c b/tests/tcg/mips/mips64-dspr2/append.c
new file mode 100644
index 00000000000..68a7cecc4b6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/append.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x02268436;
+    __asm
+        ("append %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("append error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x0010111F;
+    __asm
+        ("append %0, %1, 0x04\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("append error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/balign.c b/tests/tcg/mips/mips64-dspr2/balign.c
new file mode 100644
index 00000000000..7fbe8157829
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/balign.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x13421BFF;
+    __asm
+        ("balign %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("balign error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x11FFFF0F;
+    __asm
+        ("balign %0, %1, 0x03\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("balign error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c
new file mode 100644
index 00000000000..61217f38cf4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFE;
+    dspresult = 0xFE;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("1 cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+    rs = 0x133256789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x3E;
+    dspresult = 0x3E;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("2 cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+   return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c
new file mode 100644
index 00000000000..c63f6480eb1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c
@@ -0,0 +1,41 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if ((rd != result) || (dsp != result)) {
+        printf("cmpgdu.eq.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+
+    if ((rd != result) || (dsp != result)) {
+        printf("cmpgdu.eq.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c
new file mode 100644
index 00000000000..b3da0981896
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789abcdef0;
+    rt = 0x123456789abcdeff;
+    dspresult = 0xff;
+    result = 0xff;
+
+    __asm("cmpgdu.le.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xff);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.le.ob error\n");
+        return -1;
+    }
+
+    rs = 0x113556789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xBE;
+    dspresult = 0xFE;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c
new file mode 100644
index 00000000000..f0a60ea4e0b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c
@@ -0,0 +1,48 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11707066;
+    result = 0x0B;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c
new file mode 100644
index 00000000000..d80b4e6ab9b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+    result = 0x01;
+
+    __asm("cmpgdu.lt.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.lt.ob error\n");
+        return -1;
+    }
+
+    rs = 0x143356789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x41;
+    result = 0x41;
+
+    __asm("cmpgdu.lt.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.lt.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c
new file mode 100644
index 00000000000..a71e4e307fc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c
@@ -0,0 +1,48 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dbalign.c b/tests/tcg/mips/mips64-dspr2/dbalign.c
new file mode 100644
index 00000000000..c7431b18579
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dbalign.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd1234;
+
+    res = 0x34567887654321ab;
+
+    asm ("dbalign %0, %1, 0x1\n"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dbalign error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd1234;
+
+    res = 0x7887654321abcd12;
+
+    asm ("dbalign %0, %1, 0x3\n"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dbalign error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c
new file mode 100644
index 00000000000..39dc99aa551
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c
@@ -0,0 +1,47 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs = 0x00FF00FF;
+    rt = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("1 dpa.w.ph error\n");
+        return -1;
+    }
+
+    ach = 6, acl = 7;
+    rs = 0xFFFF00FF;
+    rt = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xfffffffffffe0206;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("2 dpa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c b/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c
new file mode 100644
index 00000000000..1411e44be3d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c
@@ -0,0 +1,56 @@
+#include"io.h"
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0x0001000100010001;
+    rt = 0x0002000200020002;
+
+    resh = 0x1;
+    resl = 0x9;
+
+    asm("mthi %2, $ac1\t\n"
+        "mtlo %3, $ac1\t\n"
+        "dpa.w.qh $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1\t\n"
+        "mflo %1, $ac1\t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+       );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpa.w.qh error\n");
+        return -1;
+    }
+
+
+    achi = 0xffffffff;
+    acli = 0xaaaaaaaa;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x7777888899996666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x320cdf02;
+
+    asm("mthi %2, $ac1\t\n"
+        "mtlo %3, $ac1\t\n"
+        "dpa.w.qh $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1\t\n"
+        "mflo %1, $ac1\t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+       );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpa.w.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c
new file mode 100644
index 00000000000..51252fb980e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c
@@ -0,0 +1,97 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x05;
+    resultl = 0xFFFFFFFF80000202;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (ach != resulth) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x05FF;
+    /***********************************************************
+     * Because of we set outflag at last time, although this
+     * time we set nothing, but it is stay the last time value.
+     **********************************************************/
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (ach != resulth) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x800000FF;
+    rt     = 0x00028000;
+    resulth = 0x05;
+    resultl = 0xffffffff80000400;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c
new file mode 100644
index 00000000000..18d6b3a98ef
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main()
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((dsp >> (16 + 1) != resultdsp) || (ach != resulth) ||
+        (acl != resultl)) {
+        printf("dpaqx_sa.w.ph errror\n");
+    }
+
+    ach = 9;
+    acl = 0xb;
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((dsp >> (16 + 1) != resultdsp) || (ach != resulth) ||
+        (acl != resultl)) {
+        printf("dpaqx_sa.w.ph errror\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c
new file mode 100644
index 00000000000..9d595fc14ab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c
@@ -0,0 +1,32 @@
+#include"io.h"
+
+int main(void)
+{
+    long rs, rt;
+    long ach = 5, acl = 5;
+    long resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth) {
+        printf("dpax.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpax.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dps_w_ph.c b/tests/tcg/mips/mips64-dspr2/dps_w_ph.c
new file mode 100644
index 00000000000..99f292ecb21
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dps_w_ph.c
@@ -0,0 +1,28 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFFFD08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("dps.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dps_w_qh.c b/tests/tcg/mips/mips64-dspr2/dps_w_qh.c
new file mode 100644
index 00000000000..61277eb30c5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dps_w_qh.c
@@ -0,0 +1,55 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x8;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    asm ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dps.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dps.w.qh error\n");
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    achi = 0x88888888;
+    achi = 0x55555555;
+
+    resh = 0xfffffffff7777777;
+    resl = 0x0a38b181;
+
+    asm ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dps.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dps.w.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c
new file mode 100644
index 00000000000..ba46a926984
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c
@@ -0,0 +1,55 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFAEA3E09B;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach = 0x99f13005;
+    acl = 0x51730062;
+    rs = 0x80008000;
+    rt = 0x80008000;
+
+    resulth = 0xffffffff99f13004;
+    resultl = 0x51730064;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c
new file mode 100644
index 00000000000..24c888134d7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include"io.h"
+int main()
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_sa.w.ph error\n");
+        return -1;
+    }
+
+    ach = 0x8c0b354A;
+    acl = 0xbbc02249;
+    rs      = 0x800023AD;
+    rt      = 0x01648000;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff80000000;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_sa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c
new file mode 100644
index 00000000000..b6291b5eb64
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c
@@ -0,0 +1,28 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFD751F050;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsx.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("dpsx.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/head.S b/tests/tcg/mips/mips64-dspr2/head.S
new file mode 100644
index 00000000000..9a099ae42fb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/head.S
@@ -0,0 +1,16 @@
+/*
+ *  Startup Code for MIPS64 CPU-core
+ *
+ */
+.text
+.globl _start
+.align 4
+_start:
+    ori    $2, $2, 0xffff
+    sll    $2, $2, 16
+    ori    $2, $2, 0xffff
+    mtc0   $2, $12, 0
+    jal    main
+
+end:
+    b end
diff --git a/tests/tcg/mips/mips64-dspr2/io.h b/tests/tcg/mips/mips64-dspr2/io.h
new file mode 100644
index 00000000000..b7db61d7c19
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/io.h
@@ -0,0 +1,22 @@
+#ifndef _ASM_IO_H
+#define _ASM_IO_H
+extern int printf(const char *fmt, ...);
+extern unsigned long get_ticks(void);
+
+#define _read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "mfc0\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#define __read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "move\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#endif
diff --git a/tests/tcg/mips/mips64-dspr2/mips_boot.lds b/tests/tcg/mips/mips64-dspr2/mips_boot.lds
new file mode 100644
index 00000000000..bd7c0c0f3fd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mips_boot.lds
@@ -0,0 +1,31 @@
+OUTPUT_ARCH(mips)
+SECTIONS
+{
+    . = 0xffffffff80100000;
+    . = ALIGN((1 << 13));
+    .text :
+    {
+        *(.text)
+        *(.rodata)
+        *(.rodata.*)
+    }
+
+    __init_begin = .;
+    . = ALIGN((1 << 12));
+    .init.text : AT(ADDR(.init.text) - 0)
+    {
+        *(.init.text)
+    }
+    .init.data : AT(ADDR(.init.data) - 0)
+    {
+        *(.init.data)
+    }
+    . = ALIGN((1 << 12));
+    __init_end = .;
+
+    . = ALIGN((1 << 13));
+    .data :
+    {
+        *(.data)
+    }
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mul_ph.c b/tests/tcg/mips/mips64-dspr2/mul_ph.c
new file mode 100644
index 00000000000..5a3d05cb290
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mul_ph.c
@@ -0,0 +1,50 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0xFFFFFFFFF504F4B4;
+    resultdsp = 1;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mul.ph wrong\n");
+        return -1;
+    }
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00210010;
+    rt = 0x00110005;
+    result = 0x2310050;
+    resultdsp = 0;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mul.ph wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mul_s_ph.c b/tests/tcg/mips/mips64-dspr2/mul_s_ph.c
new file mode 100644
index 00000000000..7c8b2c718f5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mul_s_ph.c
@@ -0,0 +1,67 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0x7fff7FFF;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("1 mul_s.ph error\n");
+        return -1;
+    }
+
+    rs = 0x7fffff00;
+    rt = 0xff007fff;
+    result = 0xffffffff80008000;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("2 mul_s.ph error\n");
+        return -1;
+    }
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00320001;
+    rt = 0x00210002;
+    result = 0x06720002;
+    resultdsp = 0;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("3 mul_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c b/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c
new file mode 100644
index 00000000000..ffdc66d54ae
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c
@@ -0,0 +1,40 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFF80005555;
+
+    __asm
+        ("mulq_rs.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("mulq_rs.w error!\n");
+        return -1;
+    }
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("mulq_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c b/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c
new file mode 100644
index 00000000000..b8c20c68cc6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c
@@ -0,0 +1,26 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098B;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mulq_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_s_w.c b/tests/tcg/mips/mips64-dspr2/mulq_s_w.c
new file mode 100644
index 00000000000..db74b713f23
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_s_w.c
@@ -0,0 +1,40 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFF80005555;
+
+    __asm
+        ("mulq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("mulq_s.w error\n");
+        return -1;
+    }
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("mulq_s.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c b/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c
new file mode 100644
index 00000000000..5b22a60a8d8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c
@@ -0,0 +1,30 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x3BF5E918;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsa.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("mulsa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c
new file mode 100644
index 00000000000..835a73d4797
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c
@@ -0,0 +1,30 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x772ff463;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsaq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("mulsaq_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c b/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c
new file mode 100644
index 00000000000..80d5e8dce98
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x34786521;
+
+    __asm
+        ("precr.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precr.qb.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c b/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c
new file mode 100644
index 00000000000..b1d7bcdf8e4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra.ph.w error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFFFFF0000;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra.ph.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c b/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c
new file mode 100644
index 00000000000..62d220dcaea
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra_r.ph.w error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFFFFF0000;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra_r.ph.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/prepend.c b/tests/tcg/mips/mips64-dspr2/prepend.c
new file mode 100644
index 00000000000..4ab083e9691
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/prepend.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFF87654321;
+    __asm
+        ("prepend %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("prepend error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFACF10ECA;
+    __asm
+        ("prepend %0, %1, 0x0F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("prepend error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/printf.c b/tests/tcg/mips/mips64-dspr2/printf.c
new file mode 100644
index 00000000000..cf8676d3909
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/printf.c
@@ -0,0 +1,266 @@
+
+typedef unsigned long va_list;
+
+#define ACC    4
+#define __read(source)                    \
+({ va_list __res;                    \
+    __asm__ __volatile__(                \
+        "move\t%0, " #source "\n\t"        \
+        : "=r" (__res));            \
+    __res;                        \
+})
+
+enum format_type {
+    FORMAT_TYPE_NONE,
+    FORMAT_TYPE_HEX,
+    FORMAT_TYPE_ULONG,
+    FORMAT_TYPE_FLOAT
+};
+
+struct printf_spec {
+    char    type;
+};
+
+static int format_decode(char *fmt, struct printf_spec *spec)
+{
+    char *start = fmt;
+
+    for (; *fmt ; ++fmt) {
+        if (*fmt == '%') {
+            break;
+        }
+    }
+
+    switch (*++fmt) {
+    case 'x':
+        spec->type = FORMAT_TYPE_HEX;
+        break;
+
+    case 'd':
+        spec->type = FORMAT_TYPE_ULONG;
+        break;
+
+    case 'f':
+        spec->type = FORMAT_TYPE_FLOAT;
+        break;
+
+    default:
+        spec->type = FORMAT_TYPE_NONE;
+    }
+
+    return ++fmt - start;
+}
+
+void *memcpy(void *dest, void *src, int n)
+{
+    int i;
+    char *s = src;
+    char *d = dest;
+
+    for (i = 0; i < n; i++) {
+        d[i] = s[i];
+    }
+    return dest;
+}
+
+char *number(char *buf, va_list num)
+{
+    int i;
+    char *str = buf;
+    static char digits[16] = "0123456789abcdef";
+    str = str + sizeof(num) * 2;
+
+    for (i = 0; i < sizeof(num) * 2; i++) {
+        *--str = digits[num & 15];
+        num >>= 4;
+    }
+
+    return buf + sizeof(num) * 2;
+}
+
+char *__number(char *buf, va_list num)
+{
+    int i;
+    va_list mm = num;
+    char *str = buf;
+
+    if (!num) {
+        *str++ = '0';
+        return str;
+    }
+
+    for (i = 0; mm; mm = mm/10, i++) {
+        /* Do nothing. */
+    }
+
+    str = str + i;
+
+    while (num) {
+        *--str = num % 10 + 48;
+        num = num / 10;
+    }
+
+    return str + i;
+}
+
+va_list modf(va_list args, va_list *integer, va_list *num)
+{
+    int i;
+    double dot_v = 0;
+    va_list E, DOT, DOT_V;
+
+    if (!args) {
+        return 0;
+    }
+
+    for (i = 0, args = args << 1 >> 1; i < 52; i++) {
+        if ((args >> i) & 0x1) {
+            break;
+        }
+    }
+
+    *integer = 0;
+
+    if ((args >> 56 != 0x3f) || (args >> 52 == 0x3ff)) {
+        E = (args >> 52) - 1023;
+        DOT = 52 - E - i;
+        DOT_V = args << (12 + E) >> (12 + E) >> i;
+        *integer = ((args << 12 >> 12) >> (i + DOT)) | (1 << E);
+    } else {
+        E = ~((args >> 52) - 1023) + 1;
+        DOT_V = args << 12 >> 12;
+
+        dot_v += 1.0 / (1 << E);
+
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (52 - i)) & 0x1) {
+                dot_v += 1.0 / (1 << E + i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+    }
+
+    *num = E;
+
+    return dot_v;
+    }
+
+    if (args & 0xf) {
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+        }
+
+        *num = E;
+
+        return dot_v;
+    } else if (DOT) {
+        for (i = 1; i <= DOT; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1; i <= ACC; i++) {
+            dot_v = dot_v * 10;
+        }
+
+    return dot_v;
+    }
+
+    return 0;
+}
+
+int vsnprintf(char *buf, int size, char *fmt, va_list args)
+{
+    char *str, *mm;
+    struct printf_spec spec = {0};
+
+    str = mm = buf;
+
+    while (*fmt) {
+        char *old_fmt = fmt;
+        int read = format_decode(fmt, &spec);
+
+        fmt += read;
+
+        switch (spec.type) {
+        case FORMAT_TYPE_NONE: {
+            memcpy(str, old_fmt, read);
+            str += read;
+            break;
+        }
+        case FORMAT_TYPE_HEX: {
+            memcpy(str, old_fmt, read);
+            str = number(str + read, args);
+            for (; *mm ; ++mm) {
+                if (*mm == '%') {
+                    *mm = '0';
+                break;
+                }
+            }
+        break;
+        }
+        case FORMAT_TYPE_ULONG: {
+            memcpy(str, old_fmt, read - 2);
+            str = __number(str + read - 2, args);
+            break;
+        }
+        case FORMAT_TYPE_FLOAT: {
+            va_list integer, dot_v, num;
+            dot_v = modf(args, &integer, &num);
+            memcpy(str, old_fmt, read - 2);
+            str += read - 2;
+            if ((args >> 63 & 0x1)) {
+                *str++ = '-';
+            }
+            str = __number(str, integer);
+            if (dot_v) {
+                *str++ = '.';
+                while (num--) {
+                    *str++ = '0';
+                }
+                str = __number(str, dot_v);
+            }
+            break;
+        }
+        }
+    }
+    *str = '\0';
+
+    return str - buf;
+}
+
+static void serial_out(char *str)
+{
+    while (*str) {
+        *(char *)0xffffffffb80003f8 = *str++;
+    }
+}
+
+int vprintf(char *fmt, va_list args)
+{
+    int printed_len = 0;
+    static char printf_buf[512];
+    printed_len = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args);
+    serial_out(printf_buf);
+    return printed_len;
+}
+
+int printf(char *fmt, ...)
+{
+    return vprintf(fmt, __read($5));
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shra_qb.c b/tests/tcg/mips/mips64-dspr2/shra_qb.c
new file mode 100644
index 00000000000..cac31023555
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shra_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.qb error\n");
+        return -1;
+    }
+
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF00C0804;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shra_r_qb.c b/tests/tcg/mips/mips64-dspr2/shra_r_qb.c
new file mode 100644
index 00000000000..9c64f75bd46
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shra_r_qb.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.qb wrong\n");
+        return -1;
+    }
+
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_ob.c b/tests/tcg/mips/mips64-dspr2/shrav_ob.c
new file mode 100644
index 00000000000..fbdfbab35a2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0xf1f3f5f7f8060402;
+
+    asm ("shrav.ob %0, %1, %2"
+        : "=r"(rd)
+        : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shra.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_qb.c b/tests/tcg/mips/mips64-dspr2/shrav_qb.c
new file mode 100644
index 00000000000..a716203d806
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.qb error\n");
+        return -1;
+    }
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF00C0804;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c b/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c
new file mode 100644
index 00000000000..b80100a7c2e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0xe3e7ebf0f1ede9e5;
+
+    asm ("shrav_r.ob %0, %1, %2"
+        : "=r"(rd)
+        : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shra_r.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c b/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c
new file mode 100644
index 00000000000..009080b2a72
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.qb error\n");
+        return -1;
+    }
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF10D0804;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrl_ph.c b/tests/tcg/mips/mips64-dspr2/shrl_ph.c
new file mode 100644
index 00000000000..e32d9766253
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrl_ph.c
@@ -0,0 +1,22 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrl.ph %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shrl.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrlv_ph.c b/tests/tcg/mips/mips64-dspr2/shrlv_ph.c
new file mode 100644
index 00000000000..58c5488b588
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrlv_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrlv.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrlv.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_ph.c b/tests/tcg/mips/mips64-dspr2/subqh_ph.c
new file mode 100644
index 00000000000..90374019aee
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456709AB;
+
+    __asm
+        ("subqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c b/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c
new file mode 100644
index 00000000000..b8f9d2fee60
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456809AC;
+
+    __asm
+        ("subqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh_r.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_r_w.c b/tests/tcg/mips/mips64-dspr2/subqh_r_w.c
new file mode 100644
index 00000000000..b025e40a354
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_r_w.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AC;
+
+    __asm
+        ("subqh_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_w.c b/tests/tcg/mips/mips64-dspr2/subqh_w.c
new file mode 100644
index 00000000000..65f17603d83
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_w.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AB;
+
+    __asm
+        ("subqh.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_ph.c b/tests/tcg/mips/mips64-dspr2/subu_ph.c
new file mode 100644
index 00000000000..60a6b1b7da7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_ph.c
@@ -0,0 +1,26 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x7531ECA9;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if (dsp != resultdsp || rd  != result) {
+        printf("subu.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_qh.c b/tests/tcg/mips/mips64-dspr2/subu_qh.c
new file mode 100644
index 00000000000..911cb349d45
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_qh.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEF1;
+    result = 0x000000000000000F;
+    dspresult = 0x01;
+
+    __asm("subu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_s_ph.c b/tests/tcg/mips/mips64-dspr2/subu_s_ph.c
new file mode 100644
index 00000000000..ae32cc06f5d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_s_ph.c
@@ -0,0 +1,25 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x75310000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if (dsp != resultdsp || rd  != result) {
+        printf("subu_s.ph error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_s_qh.c b/tests/tcg/mips/mips64-dspr2/subu_s_qh.c
new file mode 100644
index 00000000000..de7a29e7753
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_s_qh.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x1111111111111111;
+    rt = 0x2222222222222222;
+    result = 0x1111111111111111;
+    dspresult = 0x00;
+
+    __asm("subu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.qh error\n\t");
+        return -1;
+    }
+
+
+    rs = 0x8888888888888888;
+    rt = 0xa888a888a888a888;
+    result = 0x0000000000000000;
+    dspresult = 0x01;
+
+    __asm("subu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.qh error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_ob.c b/tests/tcg/mips/mips64-dspr2/subuh_ob.c
new file mode 100644
index 00000000000..3fc452bf8ed
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_ob.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0x0;
+    rs = 0x246856789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x091A000000000000;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+
+    rs = 0x246856789ABCDEF0;
+    rt = 0x1131517191B1D1F1;
+    result = 0x1b4f2d2d51637577;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_qb.c b/tests/tcg/mips/mips64-dspr2/subuh_qb.c
new file mode 100644
index 00000000000..aac7a834ee8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_qb.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC5E7092B;
+
+    __asm
+        ("subuh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subuh.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c b/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c
new file mode 100644
index 00000000000..fc20ffd09eb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0x0;
+    rs = 0x246956789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x091B000000000000;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c b/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c
new file mode 100644
index 00000000000..66d46804403
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC6E80A2C;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 subuh_r.qb wrong\n");
+        return -1;
+    }
+
+    rs = 0xBEFC292A;
+    rt = 0x9205C1B4;
+    result = 0x167cb4bb;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("2 subuh_r.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/test-i386-fprem.c b/tests/tcg/test-i386-fprem.c
new file mode 100644
index 00000000000..e91fb1ae93c
--- /dev/null
+++ b/tests/tcg/test-i386-fprem.c
@@ -0,0 +1,353 @@
+/*
+ *  x86 FPREM test - executes the FPREM and FPREM1 instructions with corner case
+ *  operands and prints the operands, result and FPU status word.
+ *
+ *  Run this on real hardware, then under QEMU, and diff the outputs, to compare
+ *  QEMU's implementation to your hardware. The 'run-test-i386-fprem' make
+ *  target does this.
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *  Copyright (c) 2012 Catalin Patulea
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "qemu/compiler.h"
+#include "qemu/osdep.h"
+#include <stdio.h>
+#include <inttypes.h>
+
+/*
+ * Inspired by <ieee754.h>'s union ieee854_long_double, but with single
+ * long long mantissa fields and assuming little-endianness for simplicity.
+ */
+union float80u {
+    long double d;
+
+    /* This is the IEEE 854 double-extended-precision format.  */
+    struct {
+        unsigned long long mantissa:63;
+        unsigned int one:1;
+        unsigned int exponent:15;
+        unsigned int negative:1;
+        unsigned int empty:16;
+    } QEMU_PACKED ieee;
+
+    /* This is for NaNs in the IEEE 854 double-extended-precision format.  */
+    struct {
+        unsigned long long mantissa:62;
+        unsigned int quiet_nan:1;
+        unsigned int one:1;
+        unsigned int exponent:15;
+        unsigned int negative:1;
+        unsigned int empty:16;
+    } QEMU_PACKED ieee_nan;
+};
+
+#define IEEE854_LONG_DOUBLE_BIAS 0x3fff
+
+static const union float80u q_nan = {
+    .ieee_nan.negative = 0,  /* X */
+    .ieee_nan.exponent = 0x7fff,
+    .ieee_nan.one = 1,
+    .ieee_nan.quiet_nan = 1,
+    .ieee_nan.mantissa = 0,
+};
+
+static const union float80u s_nan = {
+    .ieee_nan.negative = 0,  /* X */
+    .ieee_nan.exponent = 0x7fff,
+    .ieee_nan.one = 1,
+    .ieee_nan.quiet_nan = 0,
+    .ieee_nan.mantissa = 1,  /* nonzero */
+};
+
+static const union float80u pos_inf = {
+    .ieee.negative = 0,
+    .ieee.exponent = 0x7fff,
+    .ieee.one = 1,
+    .ieee.mantissa = 0,
+};
+
+static const union float80u pseudo_pos_inf = {  /* "unsupported" */
+    .ieee.negative = 0,
+    .ieee.exponent = 0x7fff,
+    .ieee.one = 0,
+    .ieee.mantissa = 0,
+};
+
+static const union float80u pos_denorm = {
+    .ieee.negative = 0,
+    .ieee.exponent = 0,
+    .ieee.one = 0,
+    .ieee.mantissa = 1,
+};
+
+static const union float80u smallest_positive_norm = {
+    .ieee.negative = 0,
+    .ieee.exponent = 1,
+    .ieee.one = 1,
+    .ieee.mantissa = 0,
+};
+
+static void fninit()
+{
+    asm volatile ("fninit\n");
+}
+
+static long double fprem(long double a, long double b, uint16_t *sw)
+{
+    long double result;
+    asm volatile ("fprem\n"
+                  "fnstsw %1\n"
+                  : "=t" (result), "=m" (*sw)
+                  : "0" (a), "u" (b)
+                  : "st(1)");
+    return result;
+}
+
+static long double fprem1(long double a, long double b, uint16_t *sw)
+{
+    long double result;
+    asm volatile ("fprem1\n"
+                  "fnstsw %1\n"
+                  : "=t" (result), "=m" (*sw)
+                  : "0" (a), "u" (b)
+                  : "st(1)");
+    return result;
+}
+
+#define FPUS_IE (1 << 0)
+#define FPUS_DE (1 << 1)
+#define FPUS_ZE (1 << 2)
+#define FPUS_OE (1 << 3)
+#define FPUS_UE (1 << 4)
+#define FPUS_PE (1 << 5)
+#define FPUS_SF (1 << 6)
+#define FPUS_SE (1 << 7)
+#define FPUS_C0 (1 << 8)
+#define FPUS_C1 (1 << 9)
+#define FPUS_C2 (1 << 10)
+#define FPUS_TOP 0x3800
+#define FPUS_C3 (1 << 14)
+#define FPUS_B  (1 << 15)
+
+#define FPUS_EMASK 0x007f
+
+#define FPUC_EM 0x3f
+
+static void psw(uint16_t sw)
+{
+    printf("SW:  C3 TopC2C1C0\n");
+    printf("SW: %c %d %3d %d %d %d %c %c %c %c %c %c %c %c\n",
+           sw & FPUS_B ? 'B' : 'b',
+           !!(sw & FPUS_C3),
+           (sw & FPUS_TOP) >> 11,
+           !!(sw & FPUS_C2),
+           !!(sw & FPUS_C1),
+           !!(sw & FPUS_C0),
+           (sw & FPUS_SE) ? 'S' : 's',
+           (sw & FPUS_SF) ? 'F' : 'f',
+           (sw & FPUS_PE) ? 'P' : 'p',
+           (sw & FPUS_UE) ? 'U' : 'u',
+           (sw & FPUS_OE) ? 'O' : 'o',
+           (sw & FPUS_ZE) ? 'Z' : 'z',
+           (sw & FPUS_DE) ? 'D' : 'd',
+           (sw & FPUS_IE) ? 'I' : 'i');
+}
+
+static void do_fprem(long double a, long double b)
+{
+    const union float80u au = {.d = a};
+    const union float80u bu = {.d = b};
+    union float80u ru;
+    uint16_t sw;
+
+    printf("A: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           au.ieee.negative, au.ieee.exponent, au.ieee.one,
+           au.ieee_nan.quiet_nan, (unsigned long long)au.ieee.mantissa,
+           a);
+    printf("B: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           bu.ieee.negative, bu.ieee.exponent, bu.ieee.one,
+           bu.ieee_nan.quiet_nan, (unsigned long long)bu.ieee.mantissa,
+           b);
+    fflush(stdout);
+
+    fninit();
+    ru.d = fprem(a, b, &sw);
+    psw(sw);
+
+    printf("R : S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+
+    fninit();
+    ru.d = fprem1(a, b, &sw);
+    psw(sw);
+
+    printf("R1: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+
+    printf("\n");
+}
+
+static void do_fprem_stack_underflow(void)
+{
+    const long double a = 1.0;
+    union float80u ru;
+    uint16_t sw;
+
+    fninit();
+    asm volatile ("fprem\n"
+                  "fnstsw %1\n"
+                  : "=t" (ru.d), "=m" (sw)
+                  : "0" (a)
+                  : "st(1)");
+    psw(sw);
+
+    printf("R: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+    printf("\n");
+}
+
+static void test_fprem_cases(void)
+{
+    printf("= stack underflow =\n");
+    do_fprem_stack_underflow();
+
+    printf("= invalid operation =\n");
+    do_fprem(s_nan.d, 1.0);
+    do_fprem(1.0, 0.0);
+    do_fprem(pos_inf.d, 1.0);
+    do_fprem(pseudo_pos_inf.d, 1.0);
+
+    printf("= denormal =\n");
+    do_fprem(pos_denorm.d, 1.0);
+    do_fprem(1.0, pos_denorm.d);
+
+    /* printf("= underflow =\n"); */
+    /* TODO: Is there a case where FPREM raises underflow? */
+}
+
+static void test_fprem_pairs(void)
+{
+    unsigned long long count;
+
+    unsigned int negative_index_a = 0;
+    unsigned int negative_index_b = 0;
+    static const unsigned int negative_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int exponent_index_a = 0;
+    unsigned int exponent_index_b = 0;
+    static const unsigned int exponent_values[] = {
+        0,
+        1,
+        2,
+        IEEE854_LONG_DOUBLE_BIAS - 1,
+        IEEE854_LONG_DOUBLE_BIAS,
+        IEEE854_LONG_DOUBLE_BIAS + 1,
+        0x7ffd,
+        0x7ffe,
+        0x7fff,
+    };
+
+    unsigned int one_index_a = 0;
+    unsigned int one_index_b = 0;
+    static const unsigned int one_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int quiet_nan_index_a = 0;
+    unsigned int quiet_nan_index_b = 0;
+    static const unsigned int quiet_nan_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int mantissa_index_a = 0;
+    unsigned int mantissa_index_b = 0;
+    static const unsigned long long mantissa_values[] = {
+        0,
+        1,
+        2,
+        0x3ffffffffffffffdULL,
+        0x3ffffffffffffffeULL,
+        0x3fffffffffffffffULL,
+    };
+
+    for (count = 0; ; ++count) {
+#define INIT_FIELD(var, field) \
+            .ieee_nan.field = field##_values[field##_index_##var]
+        const union float80u a = {
+            INIT_FIELD(a, negative),
+            INIT_FIELD(a, exponent),
+            INIT_FIELD(a, one),
+            INIT_FIELD(a, quiet_nan),
+            INIT_FIELD(a, mantissa),
+        };
+        const union float80u b = {
+            INIT_FIELD(b, negative),
+            INIT_FIELD(b, exponent),
+            INIT_FIELD(b, one),
+            INIT_FIELD(b, quiet_nan),
+            INIT_FIELD(b, mantissa),
+        };
+#undef INIT_FIELD
+
+        do_fprem(a.d, b.d);
+
+        int carry = 1;
+#define CARRY_INTO(var, field) do { \
+            if (carry) { \
+                if (++field##_index_##var == ARRAY_SIZE(field##_values)) { \
+                    field##_index_##var = 0; \
+                } else { \
+                    carry = 0; \
+                } \
+            } \
+        } while (0)
+        CARRY_INTO(b, mantissa);
+        CARRY_INTO(b, quiet_nan);
+        CARRY_INTO(b, one);
+        CARRY_INTO(b, exponent);
+        CARRY_INTO(b, negative);
+        CARRY_INTO(a, mantissa);
+        CARRY_INTO(a, quiet_nan);
+        CARRY_INTO(a, one);
+        CARRY_INTO(a, exponent);
+        CARRY_INTO(a, negative);
+#undef CARRY_INTO
+
+        if (carry) {
+            break;
+        }
+    }
+
+    fprintf(stderr, "test-i386-fprem: tested %llu cases\n", count);
+}
+
+int main(int argc, char **argv)
+{
+    test_fprem_cases();
+    test_fprem_pairs();
+    return 0;
+}
diff --git a/tests/tcg/test-i386.c b/tests/tcg/test-i386.c
index 8e64bbaf38a..6dc730d882f 100644
--- a/tests/tcg/test-i386.c
+++ b/tests/tcg/test-i386.c
@@ -17,6 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #define _GNU_SOURCE
+#include "qemu/compiler.h"
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -784,7 +785,7 @@ void fpu_clear_exceptions(void)
         long double fpregs[8];
     } float_env32;
 
-    asm volatile ("fnstenv %0\n" : : "m" (float_env32));
+    asm volatile ("fnstenv %0\n" : "=m" (float_env32));
     float_env32.fpus &= ~0x7f;
     asm volatile ("fldenv %0\n" : : "m" (float_env32));
 }
@@ -1827,7 +1828,7 @@ void test_exceptions(void)
     printf("lock nop exception:\n");
     if (setjmp(jmp_env) == 0) {
         /* now execute an invalid instruction */
-        asm volatile("lock nop");
+        asm volatile(".byte 0xf0, 0x90"); /* lock nop */
     }
 
     printf("INT exception:\n");
diff --git a/tests/tcg/test-mmap.c b/tests/tcg/test-mmap.c
index c418b676605..3982fa2c725 100644
--- a/tests/tcg/test-mmap.c
+++ b/tests/tcg/test-mmap.c
@@ -429,6 +429,12 @@ void check_file_fixed_mmaps(void)
 	fprintf (stderr, " passed\n");
 }
 
+void checked_write(int fd, const void *buf, size_t count)
+{
+    ssize_t rc = write(fd, buf, count);
+    fail_unless(rc == count);
+}
+
 int main(int argc, char **argv)
 {
 	char tempname[] = "/tmp/.cmmapXXXXXX";
@@ -450,13 +456,15 @@ int main(int argc, char **argv)
 	unlink(tempname);
 
 	/* Fill the file with int's counting from zero and up.  */
-	for (i = 0; i < (pagesize * 4) / sizeof i; i++)
-		write (test_fd, &i, sizeof i);
+    for (i = 0; i < (pagesize * 4) / sizeof i; i++) {
+        checked_write(test_fd, &i, sizeof i);
+    }
+
 	/* Append a few extra writes to make the file end at non 
 	   page boundary.  */
-	write (test_fd, &i, sizeof i); i++;
-	write (test_fd, &i, sizeof i); i++;
-	write (test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
 
 	test_fsize = lseek(test_fd, 0, SEEK_CUR);
 
diff --git a/tests/tcg/test_path.c b/tests/tcg/test_path.c
index 7265a9445d0..a064eea8fb9 100644
--- a/tests/tcg/test_path.c
+++ b/tests/tcg/test_path.c
@@ -1,11 +1,12 @@
 /* Test path override code */
-#include "../config-host.h"
-#include "../qemu-malloc.c"
-#include "../cutils.c"
-#include "../path.c"
-#include "../trace.c"
+#define _GNU_SOURCE
+#include "config-host.h"
+#include "iov.c"
+#include "cutils.c"
+#include "path.c"
+#include "trace.c"
 #ifdef CONFIG_TRACE_SIMPLE
-#include "../simpletrace.c"
+#include "../trace/simple.c"
 #endif
 
 #include <stdarg.h>
diff --git a/tests/tcg/testthread.c b/tests/tcg/testthread.c
index 27e4825bc6d..2679af119ac 100644
--- a/tests/tcg/testthread.c
+++ b/tests/tcg/testthread.c
@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -8,6 +9,12 @@
 #include <sys/wait.h>
 #include <sched.h>
 
+void checked_write(int fd, const void *buf, size_t count)
+{
+    ssize_t rc = write(fd, buf, count);
+    assert(rc == count);
+}
+
 void *thread1_func(void *arg)
 {
     int i;
@@ -15,7 +22,7 @@ void *thread1_func(void *arg)
 
     for(i=0;i<10;i++) {
         snprintf(buf, sizeof(buf), "thread1: %d %s\n", i, (char *)arg);
-        write(1, buf, strlen(buf));
+        checked_write(1, buf, strlen(buf));
         usleep(100 * 1000);
     }
     return NULL;
@@ -27,7 +34,7 @@ void *thread2_func(void *arg)
     char buf[512];
     for(i=0;i<20;i++) {
         snprintf(buf, sizeof(buf), "thread2: %d %s\n", i, (char *)arg);
-        write(1, buf, strlen(buf));
+        checked_write(1, buf, strlen(buf));
         usleep(150 * 1000);
     }
     return NULL;
diff --git a/tests/tcg/xtensa/Makefile b/tests/tcg/xtensa/Makefile
index 0ff0ccfb8ce..002fd871d95 100644
--- a/tests/tcg/xtensa/Makefile
+++ b/tests/tcg/xtensa/Makefile
@@ -42,9 +42,11 @@ endif
 TESTCASES += test_quo.tst
 TESTCASES += test_rem.tst
 TESTCASES += test_rst0.tst
+TESTCASES += test_s32c1i.tst
 TESTCASES += test_sar.tst
 TESTCASES += test_sext.tst
 TESTCASES += test_shift.tst
+TESTCASES += test_sr.tst
 TESTCASES += test_timer.tst
 TESTCASES += test_windowed.tst
 
diff --git a/tests/tcg/xtensa/macros.inc b/tests/tcg/xtensa/macros.inc
index 23bf3e96c8b..c9be1ce5163 100644
--- a/tests/tcg/xtensa/macros.inc
+++ b/tests/tcg/xtensa/macros.inc
@@ -1,7 +1,7 @@
 .macro test_suite name
 .data
 status: .word result
-result: .space 20
+result: .space 256
 .text
 .global main
 .align 4
diff --git a/tests/tcg/xtensa/test_s32c1i.S b/tests/tcg/xtensa/test_s32c1i.S
new file mode 100644
index 00000000000..4536015a845
--- /dev/null
+++ b/tests/tcg/xtensa/test_s32c1i.S
@@ -0,0 +1,39 @@
+.include "macros.inc"
+
+test_suite s32c1i
+
+test s32c1i_nowrite
+    movi    a2, 1f
+    movi    a3, 1
+    wsr     a3, scompare1
+    movi    a1, 2
+    s32c1i  a1, a2, 0
+    assert  ne, a1, a3
+    l32i    a1, a2, 0
+    assert  eqi, a1, 3
+
+.data
+.align 4
+1:
+    .word   3
+.text
+test_end
+
+test s32c1i_write
+    movi    a2, 1f
+    movi    a3, 3
+    wsr     a3, scompare1
+    movi    a1, 2
+    s32c1i  a1, a2, 0
+    assert  eq, a1, a3
+    l32i    a1, a2, 0
+    assert  eqi, a1, 2
+
+.data
+.align 4
+1:
+    .word   3
+.text
+test_end
+
+test_suite_end
diff --git a/tests/tcg/xtensa/test_sr.S b/tests/tcg/xtensa/test_sr.S
new file mode 100644
index 00000000000..470c03dae2a
--- /dev/null
+++ b/tests/tcg/xtensa/test_sr.S
@@ -0,0 +1,90 @@
+.include "macros.inc"
+
+test_suite sr
+
+.macro  sr_op sym, op_sym, op_byte, sr
+    .if \sym
+    \op_sym a4, \sr
+    .else
+    .byte 0x40, \sr, \op_byte
+    .endif
+.endm
+
+.macro 	test_sr_op sym, mask, op, op_byte, sr
+    movi    a4, 0
+    .if (\mask)
+    set_vector kernel, 0
+    sr_op   \sym, \op, \op_byte, \sr
+    .else
+    set_vector kernel, 2f
+1:
+    sr_op   \sym, \op, \op_byte, \sr
+    test_fail
+2:
+    reset_ps
+    rsr     a2, exccause
+    assert  eqi, a2, 0
+    rsr     a2, epc1
+    movi    a3, 1b
+    assert  eq, a2, a3
+    .endif
+.endm
+
+.macro  test_sr_mask sr, sym, mask
+test \sr
+    test_sr_op \sym, \mask & 1, rsr, 0x03, \sr
+    test_sr_op \sym, \mask & 2, wsr, 0x13, \sr
+    test_sr_op \sym, \mask & 4, xsr, 0x61, \sr
+test_end
+.endm
+
+.macro  test_sr sr, conf
+    test_sr_mask    \sr, \conf, 7
+.endm
+
+test_sr acchi, 1
+test_sr acclo, 1
+test_sr_mask /*atomctl*/99, 0, 0
+test_sr_mask /*br*/4, 0, 0
+test_sr_mask /*cacheattr*/98, 0, 0
+test_sr ccompare0, 1
+test_sr ccount, 1
+test_sr cpenable, 1
+test_sr dbreaka0, 1
+test_sr dbreakc0, 1
+test_sr_mask debugcause, 1, 1
+test_sr depc, 1
+test_sr dtlbcfg, 1
+test_sr epc1, 1
+test_sr epc2, 1
+test_sr eps2, 1
+test_sr exccause, 1
+test_sr excsave1, 1
+test_sr excsave2, 1
+test_sr excvaddr, 1
+test_sr ibreaka0, 1
+test_sr ibreakenable, 1
+test_sr icount, 1
+test_sr icountlevel, 1
+test_sr_mask /*intclear*/227, 0, 2
+test_sr_mask /*interrupt*/226, 0, 3
+test_sr intenable, 1
+test_sr itlbcfg, 1
+test_sr lbeg, 1
+test_sr lcount, 1
+test_sr lend, 1
+test_sr litbase, 1
+test_sr m0, 1
+test_sr misc0, 1
+test_sr_mask /*prefctl*/40, 0, 0
+test_sr_mask /*prid*/235, 0, 1
+test_sr ps, 1
+test_sr ptevaddr, 1
+test_sr rasid, 1
+test_sr sar, 1
+test_sr scompare1, 1
+test_sr vecbase, 1
+test_sr windowbase, 1
+test_sr windowstart, 1
+
+test_suite_end
diff --git a/tests/test-aio.c b/tests/test-aio.c
new file mode 100644
index 00000000000..c1738706cd2
--- /dev/null
+++ b/tests/test-aio.c
@@ -0,0 +1,666 @@
+/*
+ * AioContext tests
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Paolo Bonzini    <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <glib.h>
+#include "block/aio.h"
+
+AioContext *ctx;
+
+/* Wait until there are no more BHs or AIO requests */
+static void wait_for_aio(void)
+{
+    while (aio_poll(ctx, true)) {
+        /* Do nothing */
+    }
+}
+
+/* Simple callbacks for testing.  */
+
+typedef struct {
+    QEMUBH *bh;
+    int n;
+    int max;
+} BHTestData;
+
+static void bh_test_cb(void *opaque)
+{
+    BHTestData *data = opaque;
+    if (++data->n < data->max) {
+        qemu_bh_schedule(data->bh);
+    }
+}
+
+static void bh_delete_cb(void *opaque)
+{
+    BHTestData *data = opaque;
+    if (++data->n < data->max) {
+        qemu_bh_schedule(data->bh);
+    } else {
+        qemu_bh_delete(data->bh);
+        data->bh = NULL;
+    }
+}
+
+typedef struct {
+    EventNotifier e;
+    int n;
+    int active;
+    bool auto_set;
+} EventNotifierTestData;
+
+static int event_active_cb(EventNotifier *e)
+{
+    EventNotifierTestData *data = container_of(e, EventNotifierTestData, e);
+    return data->active > 0;
+}
+
+static void event_ready_cb(EventNotifier *e)
+{
+    EventNotifierTestData *data = container_of(e, EventNotifierTestData, e);
+    g_assert(event_notifier_test_and_clear(e));
+    data->n++;
+    if (data->active > 0) {
+        data->active--;
+    }
+    if (data->auto_set && data->active) {
+        event_notifier_set(e);
+    }
+}
+
+/* Tests using aio_*.  */
+
+static void test_notify(void)
+{
+    g_assert(!aio_poll(ctx, false));
+    aio_notify(ctx);
+    g_assert(!aio_poll(ctx, true));
+    g_assert(!aio_poll(ctx, false));
+}
+
+static void test_bh_schedule(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(aio_poll(ctx, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_schedule10(void)
+{
+    BHTestData data = { .n = 0, .max = 10 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(aio_poll(ctx, true));
+    g_assert_cmpint(data.n, ==, 2);
+
+    wait_for_aio();
+    g_assert_cmpint(data.n, ==, 10);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 10);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_cancel(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_cancel(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_delete(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_delete(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+}
+
+static void test_bh_delete_from_cb(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+
+    qemu_bh_schedule(data1.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+
+    wait_for_aio();
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert(data1.bh == NULL);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert(!aio_poll(ctx, true));
+}
+
+static void test_bh_delete_from_cb_many(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+    BHTestData data2 = { .n = 0, .max = 3 };
+    BHTestData data3 = { .n = 0, .max = 2 };
+    BHTestData data4 = { .n = 0, .max = 4 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+    data2.bh = aio_bh_new(ctx, bh_delete_cb, &data2);
+    data3.bh = aio_bh_new(ctx, bh_delete_cb, &data3);
+    data4.bh = aio_bh_new(ctx, bh_delete_cb, &data4);
+
+    qemu_bh_schedule(data1.bh);
+    qemu_bh_schedule(data2.bh);
+    qemu_bh_schedule(data3.bh);
+    qemu_bh_schedule(data4.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+    g_assert_cmpint(data2.n, ==, 0);
+    g_assert_cmpint(data3.n, ==, 0);
+    g_assert_cmpint(data4.n, ==, 0);
+
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data1.n, ==, 1);
+    g_assert_cmpint(data2.n, ==, 1);
+    g_assert_cmpint(data3.n, ==, 1);
+    g_assert_cmpint(data4.n, ==, 1);
+    g_assert(data1.bh == NULL);
+
+    wait_for_aio();
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert_cmpint(data2.n, ==, data2.max);
+    g_assert_cmpint(data3.n, ==, data3.max);
+    g_assert_cmpint(data4.n, ==, data4.max);
+    g_assert(data1.bh == NULL);
+    g_assert(data2.bh == NULL);
+    g_assert(data3.bh == NULL);
+    g_assert(data4.bh == NULL);
+}
+
+static void test_bh_flush(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    wait_for_aio();
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_set_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 0 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_wait_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 1 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_flush_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 10);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 9);
+    g_assert(aio_poll(ctx, false));
+
+    wait_for_aio();
+    g_assert_cmpint(data.n, ==, 10);
+    g_assert_cmpint(data.active, ==, 0);
+    g_assert(!aio_poll(ctx, false));
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_wait_event_notifier_noflush(void)
+{
+    EventNotifierTestData data = { .n = 0 };
+    EventNotifierTestData dummy = { .n = 0, .active = 1 };
+
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, NULL);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    /* Until there is an active descriptor, aio_poll may or may not call
+     * event_ready_cb.  Still, it must not block.  */
+    event_notifier_set(&data.e);
+    g_assert(!aio_poll(ctx, true));
+    data.n = 0;
+
+    /* An active event notifier forces aio_poll to look at EventNotifiers.  */
+    event_notifier_init(&dummy.e, false);
+    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb, event_active_cb);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_set(&dummy.e);
+    wait_for_aio();
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert_cmpint(dummy.n, ==, 1);
+    g_assert_cmpint(dummy.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &dummy.e, NULL, NULL);
+    event_notifier_cleanup(&dummy.e);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_cleanup(&data.e);
+}
+
+/* Now the same tests, using the context as a GSource.  They are
+ * very similar to the ones above, with g_main_context_iteration
+ * replacing aio_poll.  However:
+ * - sometimes both the AioContext and the glib main loop wake
+ *   themselves up.  Hence, some "g_assert(!aio_poll(ctx, false));"
+ *   are replaced by "while (g_main_context_iteration(NULL, false));".
+ * - there is no exact replacement for a blocking wait.
+ *   "while (g_main_context_iteration(NULL, true)" seems to work,
+ *   but it is not documented _why_ it works.  For these tests a
+ *   non-blocking loop like "while (g_main_context_iteration(NULL, false)"
+ *   works well, and that's what I am using.
+ */
+
+static void test_source_notify(void)
+{
+    while (g_main_context_iteration(NULL, false));
+    aio_notify(ctx);
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_flush(void)
+{
+    g_assert(!g_main_context_iteration(NULL, false));
+    aio_notify(ctx);
+    while (g_main_context_iteration(NULL, false));
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_bh_schedule(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_schedule10(void)
+{
+    BHTestData data = { .n = 0, .max = 10 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 2);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_cancel(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_cancel(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_delete(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_delete(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+}
+
+static void test_source_bh_delete_from_cb(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+
+    qemu_bh_schedule(data1.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+
+    g_main_context_iteration(NULL, true);
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert(data1.bh == NULL);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_bh_delete_from_cb_many(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+    BHTestData data2 = { .n = 0, .max = 3 };
+    BHTestData data3 = { .n = 0, .max = 2 };
+    BHTestData data4 = { .n = 0, .max = 4 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+    data2.bh = aio_bh_new(ctx, bh_delete_cb, &data2);
+    data3.bh = aio_bh_new(ctx, bh_delete_cb, &data3);
+    data4.bh = aio_bh_new(ctx, bh_delete_cb, &data4);
+
+    qemu_bh_schedule(data1.bh);
+    qemu_bh_schedule(data2.bh);
+    qemu_bh_schedule(data3.bh);
+    qemu_bh_schedule(data4.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+    g_assert_cmpint(data2.n, ==, 0);
+    g_assert_cmpint(data3.n, ==, 0);
+    g_assert_cmpint(data4.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data1.n, ==, 1);
+    g_assert_cmpint(data2.n, ==, 1);
+    g_assert_cmpint(data3.n, ==, 1);
+    g_assert_cmpint(data4.n, ==, 1);
+    g_assert(data1.bh == NULL);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert_cmpint(data2.n, ==, data2.max);
+    g_assert_cmpint(data3.n, ==, data3.max);
+    g_assert_cmpint(data4.n, ==, data4.max);
+    g_assert(data1.bh == NULL);
+    g_assert(data2.bh == NULL);
+    g_assert(data3.bh == NULL);
+    g_assert(data4.bh == NULL);
+}
+
+static void test_source_bh_flush(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_set_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 0 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_wait_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 1 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_flush_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 10);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 9);
+    g_assert(g_main_context_iteration(NULL, false));
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+    g_assert_cmpint(data.active, ==, 0);
+    g_assert(!g_main_context_iteration(NULL, false));
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_wait_event_notifier_noflush(void)
+{
+    EventNotifierTestData data = { .n = 0 };
+    EventNotifierTestData dummy = { .n = 0, .active = 1 };
+
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, NULL);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    /* Until there is an active descriptor, glib may or may not call
+     * event_ready_cb.  Still, it must not block.  */
+    event_notifier_set(&data.e);
+    g_main_context_iteration(NULL, true);
+    data.n = 0;
+
+    /* An active event notifier forces aio_poll to look at EventNotifiers.  */
+    event_notifier_init(&dummy.e, false);
+    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb, event_active_cb);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_set(&dummy.e);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert_cmpint(dummy.n, ==, 1);
+    g_assert_cmpint(dummy.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &dummy.e, NULL, NULL);
+    event_notifier_cleanup(&dummy.e);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_cleanup(&data.e);
+}
+
+/* End of tests.  */
+
+int main(int argc, char **argv)
+{
+    GSource *src;
+
+    ctx = aio_context_new();
+    src = aio_get_g_source(ctx);
+    g_source_attach(src, NULL);
+    g_source_unref(src);
+
+    while (g_main_context_iteration(NULL, false));
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/aio/notify",                  test_notify);
+    g_test_add_func("/aio/bh/schedule",             test_bh_schedule);
+    g_test_add_func("/aio/bh/schedule10",           test_bh_schedule10);
+    g_test_add_func("/aio/bh/cancel",               test_bh_cancel);
+    g_test_add_func("/aio/bh/delete",               test_bh_delete);
+    g_test_add_func("/aio/bh/callback-delete/one",  test_bh_delete_from_cb);
+    g_test_add_func("/aio/bh/callback-delete/many", test_bh_delete_from_cb_many);
+    g_test_add_func("/aio/bh/flush",                test_bh_flush);
+    g_test_add_func("/aio/event/add-remove",        test_set_event_notifier);
+    g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
+    g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
+    g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
+
+    g_test_add_func("/aio-gsource/notify",                  test_source_notify);
+    g_test_add_func("/aio-gsource/flush",                   test_source_flush);
+    g_test_add_func("/aio-gsource/bh/schedule",             test_source_bh_schedule);
+    g_test_add_func("/aio-gsource/bh/schedule10",           test_source_bh_schedule10);
+    g_test_add_func("/aio-gsource/bh/cancel",               test_source_bh_cancel);
+    g_test_add_func("/aio-gsource/bh/delete",               test_source_bh_delete);
+    g_test_add_func("/aio-gsource/bh/callback-delete/one",  test_source_bh_delete_from_cb);
+    g_test_add_func("/aio-gsource/bh/callback-delete/many", test_source_bh_delete_from_cb_many);
+    g_test_add_func("/aio-gsource/bh/flush",                test_source_bh_flush);
+    g_test_add_func("/aio-gsource/event/add-remove",        test_source_set_event_notifier);
+    g_test_add_func("/aio-gsource/event/wait",              test_source_wait_event_notifier);
+    g_test_add_func("/aio-gsource/event/wait/no-flush-cb",  test_source_wait_event_notifier_noflush);
+    g_test_add_func("/aio-gsource/event/flush",             test_source_flush_event_notifier);
+    return g_test_run();
+}
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index e5d14eb696b..4c6cc81fb98 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -12,7 +12,7 @@
  */
 
 #include <glib.h>
-#include "qemu-coroutine.h"
+#include "block/coroutine.h"
 
 /*
  * Check that qemu_in_coroutine() works
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
new file mode 100644
index 00000000000..8c902f2055f
--- /dev/null
+++ b/tests/test-hbitmap.c
@@ -0,0 +1,401 @@
+/*
+ * Hierarchical bitmap unit-tests.
+ *
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <stdarg.h>
+#include "qemu/hbitmap.h"
+
+#define LOG_BITS_PER_LONG          (BITS_PER_LONG == 32 ? 5 : 6)
+
+#define L1                         BITS_PER_LONG
+#define L2                         (BITS_PER_LONG * L1)
+#define L3                         (BITS_PER_LONG * L2)
+
+typedef struct TestHBitmapData {
+    HBitmap       *hb;
+    unsigned long *bits;
+    size_t         size;
+    int            granularity;
+} TestHBitmapData;
+
+
+/* Check that the HBitmap and the shadow bitmap contain the same data,
+ * ignoring the same "first" bits.
+ */
+static void hbitmap_test_check(TestHBitmapData *data,
+                               uint64_t first)
+{
+    uint64_t count = 0;
+    size_t pos;
+    int bit;
+    HBitmapIter hbi;
+    int64_t i, next;
+
+    hbitmap_iter_init(&hbi, data->hb, first);
+
+    i = first;
+    for (;;) {
+        next = hbitmap_iter_next(&hbi);
+        if (next < 0) {
+            next = data->size;
+        }
+
+        while (i < next) {
+            pos = i >> LOG_BITS_PER_LONG;
+            bit = i & (BITS_PER_LONG - 1);
+            i++;
+            g_assert_cmpint(data->bits[pos] & (1UL << bit), ==, 0);
+        }
+
+        if (next == data->size) {
+            break;
+        }
+
+        pos = i >> LOG_BITS_PER_LONG;
+        bit = i & (BITS_PER_LONG - 1);
+        i++;
+        count++;
+        g_assert_cmpint(data->bits[pos] & (1UL << bit), !=, 0);
+    }
+
+    if (first == 0) {
+        g_assert_cmpint(count << data->granularity, ==, hbitmap_count(data->hb));
+    }
+}
+
+/* This is provided instead of a test setup function so that the sizes
+   are kept in the test functions (and not in main()) */
+static void hbitmap_test_init(TestHBitmapData *data,
+                              uint64_t size, int granularity)
+{
+    size_t n;
+    data->hb = hbitmap_alloc(size, granularity);
+
+    n = (size + BITS_PER_LONG - 1) / BITS_PER_LONG;
+    if (n == 0) {
+        n = 1;
+    }
+    data->bits = g_new0(unsigned long, n);
+    data->size = size;
+    data->granularity = granularity;
+    if (size) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+static void hbitmap_test_teardown(TestHBitmapData *data,
+                                  const void *unused)
+{
+    if (data->hb) {
+        hbitmap_free(data->hb);
+        data->hb = NULL;
+    }
+    if (data->bits) {
+        g_free(data->bits);
+        data->bits = NULL;
+    }
+}
+
+/* Set a range in the HBitmap and in the shadow "simple" bitmap.
+ * The two bitmaps are then tested against each other.
+ */
+static void hbitmap_test_set(TestHBitmapData *data,
+                             uint64_t first, uint64_t count)
+{
+    hbitmap_set(data->hb, first, count);
+    while (count-- != 0) {
+        size_t pos = first >> LOG_BITS_PER_LONG;
+        int bit = first & (BITS_PER_LONG - 1);
+        first++;
+
+        data->bits[pos] |= 1UL << bit;
+    }
+
+    if (data->granularity == 0) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+/* Reset a range in the HBitmap and in the shadow "simple" bitmap.
+ */
+static void hbitmap_test_reset(TestHBitmapData *data,
+                               uint64_t first, uint64_t count)
+{
+    hbitmap_reset(data->hb, first, count);
+    while (count-- != 0) {
+        size_t pos = first >> LOG_BITS_PER_LONG;
+        int bit = first & (BITS_PER_LONG - 1);
+        first++;
+
+        data->bits[pos] &= ~(1UL << bit);
+    }
+
+    if (data->granularity == 0) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+static void hbitmap_test_check_get(TestHBitmapData *data)
+{
+    uint64_t count = 0;
+    uint64_t i;
+
+    for (i = 0; i < data->size; i++) {
+        size_t pos = i >> LOG_BITS_PER_LONG;
+        int bit = i & (BITS_PER_LONG - 1);
+        unsigned long val = data->bits[pos] & (1UL << bit);
+        count += hbitmap_get(data->hb, i);
+        g_assert_cmpint(hbitmap_get(data->hb, i), ==, val != 0);
+    }
+    g_assert_cmpint(count, ==, hbitmap_count(data->hb));
+}
+
+static void test_hbitmap_zero(TestHBitmapData *data,
+                               const void *unused)
+{
+    hbitmap_test_init(data, 0, 0);
+}
+
+static void test_hbitmap_unaligned(TestHBitmapData *data,
+                                   const void *unused)
+{
+    hbitmap_test_init(data, L3 + 23, 0);
+    hbitmap_test_set(data, 0, 1);
+    hbitmap_test_set(data, L3 + 22, 1);
+}
+
+static void test_hbitmap_iter_empty(TestHBitmapData *data,
+                                    const void *unused)
+{
+    hbitmap_test_init(data, L1, 0);
+}
+
+static void test_hbitmap_iter_partial(TestHBitmapData *data,
+                                      const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+    hbitmap_test_check(data, 1);
+    hbitmap_test_check(data, L1 - 1);
+    hbitmap_test_check(data, L1);
+    hbitmap_test_check(data, L1 * 2 - 1);
+    hbitmap_test_check(data, L2 - 1);
+    hbitmap_test_check(data, L2);
+    hbitmap_test_check(data, L2 + 1);
+    hbitmap_test_check(data, L2 + L1);
+    hbitmap_test_check(data, L2 + L1 * 2 - 1);
+    hbitmap_test_check(data, L2 * 2 - 1);
+    hbitmap_test_check(data, L2 * 2);
+    hbitmap_test_check(data, L2 * 2 + 1);
+    hbitmap_test_check(data, L2 * 2 + L1);
+    hbitmap_test_check(data, L2 * 2 + L1 * 2 - 1);
+    hbitmap_test_check(data, L3 / 2);
+}
+
+static void test_hbitmap_set_all(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+}
+
+static void test_hbitmap_get_all(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+    hbitmap_test_check_get(data);
+}
+
+static void test_hbitmap_get_some(TestHBitmapData *data,
+                                  const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, 10, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L1 - 1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L2 - 1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L2, 1);
+    hbitmap_test_check_get(data);
+}
+
+static void test_hbitmap_set_one(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, 10, 1);
+    hbitmap_test_set(data, L1 - 1, 1);
+    hbitmap_test_set(data, L1, 1);
+    hbitmap_test_set(data, L2 - 1, 1);
+    hbitmap_test_set(data, L2, 1);
+}
+
+static void test_hbitmap_set_two_elem(TestHBitmapData *data,
+                                      const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, L1 - 1, 2);
+    hbitmap_test_set(data, L1 * 2 - 1, 4);
+    hbitmap_test_set(data, L1 * 4, L1 + 1);
+    hbitmap_test_set(data, L1 * 8 - 1, L1 + 1);
+    hbitmap_test_set(data, L2 - 1, 2);
+    hbitmap_test_set(data, L2 + L1 - 1, 8);
+    hbitmap_test_set(data, L2 + L1 * 4, L1 + 1);
+    hbitmap_test_set(data, L2 + L1 * 8 - 1, L1 + 1);
+}
+
+static void test_hbitmap_set(TestHBitmapData *data,
+                             const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 3 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 5, L1 * 2 + 1);
+    hbitmap_test_set(data, L1 * 8 - 1, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 - 1, L1 + 2);
+    hbitmap_test_set(data, L2 + L1 * 2 - 1, L1 + 2);
+    hbitmap_test_set(data, L2 + L1 * 4, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 + L1 * 7 - 1, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 * 2 - 1, L3 * 2 - L2 * 2);
+}
+
+static void test_hbitmap_set_twice(TestHBitmapData *data,
+                                   const void *unused)
+{
+    hbitmap_test_init(data, L1 * 3, 0);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_set(data, L1, 1);
+}
+
+static void test_hbitmap_set_overlap(TestHBitmapData *data,
+                                     const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 2 - 1, L1 * 2 + 2);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_set(data, L1 * 8 - 1, L2);
+    hbitmap_test_set(data, L2, L1);
+    hbitmap_test_set(data, L2 - L1 - 1, L1 * 8 + 2);
+    hbitmap_test_set(data, L2, L3 - L2 + 1);
+    hbitmap_test_set(data, L3 - L1, L1 * 3);
+    hbitmap_test_set(data, L3 - 1, 3);
+    hbitmap_test_set(data, L3 - 1, L2);
+}
+
+static void test_hbitmap_reset_empty(TestHBitmapData *data,
+                                     const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_reset(data, 0, L3);
+}
+
+static void test_hbitmap_reset(TestHBitmapData *data,
+                               const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_reset(data, L1 * 2 - 1, L1 * 2 + 2);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_reset(data, L1 * 8 - 1, L2);
+    hbitmap_test_set(data, L2, L1);
+    hbitmap_test_reset(data, L2 - L1 - 1, L1 * 8 + 2);
+    hbitmap_test_set(data, L2, L3 - L2 + 1);
+    hbitmap_test_reset(data, L3 - L1, L1 * 3);
+    hbitmap_test_set(data, L3 - 1, 3);
+    hbitmap_test_reset(data, L3 - 1, L2);
+    hbitmap_test_set(data, 0, L3 * 2);
+    hbitmap_test_reset(data, 0, L1);
+    hbitmap_test_reset(data, 0, L2);
+    hbitmap_test_reset(data, L3, L3);
+    hbitmap_test_set(data, L3 / 2, L3);
+}
+
+static void test_hbitmap_granularity(TestHBitmapData *data,
+                                     const void *unused)
+{
+    /* Note that hbitmap_test_check has to be invoked manually in this test.  */
+    hbitmap_test_init(data, L1, 1);
+    hbitmap_test_set(data, 0, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
+    hbitmap_test_check(data, 0);
+    hbitmap_test_set(data, 2, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
+    hbitmap_test_check(data, 0);
+    hbitmap_test_set(data, 0, 3);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
+    hbitmap_test_reset(data, 0, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
+}
+
+static void test_hbitmap_iter_granularity(TestHBitmapData *data,
+                                          const void *unused)
+{
+    HBitmapIter hbi;
+
+    /* Note that hbitmap_test_check has to be invoked manually in this test.  */
+    hbitmap_test_init(data, 131072 << 7, 7);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_test_set(data, ((L2 + L1 + 1) << 7) + 8, 8);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, (L2 + L1 + 1) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_test_set(data, (131072 << 7) - 8, 8);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, (L2 + L1 + 1) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, 131071 << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, 131071 << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+}
+
+static void hbitmap_test_add(const char *testpath,
+                                   void (*test_func)(TestHBitmapData *data, const void *user_data))
+{
+    g_test_add(testpath, TestHBitmapData, NULL, NULL, test_func,
+               hbitmap_test_teardown);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    hbitmap_test_add("/hbitmap/size/0", test_hbitmap_zero);
+    hbitmap_test_add("/hbitmap/size/unaligned", test_hbitmap_unaligned);
+    hbitmap_test_add("/hbitmap/iter/empty", test_hbitmap_iter_empty);
+    hbitmap_test_add("/hbitmap/iter/partial", test_hbitmap_iter_partial);
+    hbitmap_test_add("/hbitmap/iter/granularity", test_hbitmap_iter_granularity);
+    hbitmap_test_add("/hbitmap/get/all", test_hbitmap_get_all);
+    hbitmap_test_add("/hbitmap/get/some", test_hbitmap_get_some);
+    hbitmap_test_add("/hbitmap/set/all", test_hbitmap_set_all);
+    hbitmap_test_add("/hbitmap/set/one", test_hbitmap_set_one);
+    hbitmap_test_add("/hbitmap/set/two-elem", test_hbitmap_set_two_elem);
+    hbitmap_test_add("/hbitmap/set/general", test_hbitmap_set);
+    hbitmap_test_add("/hbitmap/set/twice", test_hbitmap_set_twice);
+    hbitmap_test_add("/hbitmap/set/overlap", test_hbitmap_set_overlap);
+    hbitmap_test_add("/hbitmap/reset/empty", test_hbitmap_reset_empty);
+    hbitmap_test_add("/hbitmap/reset/general", test_hbitmap_reset);
+    hbitmap_test_add("/hbitmap/granularity", test_hbitmap_granularity);
+    g_test_run();
+
+    return 0;
+}
diff --git a/tests/test-iov.c b/tests/test-iov.c
index cbe7a8955ca..46e4dddc555 100644
--- a/tests/test-iov.c
+++ b/tests/test-iov.c
@@ -1,7 +1,7 @@
 #include <glib.h>
 #include "qemu-common.h"
-#include "iov.h"
-#include "qemu_socket.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
 
 /* create a randomly-sized iovec with random vectors */
 static void iov_random(struct iovec **iovp, unsigned *iov_cntp)
@@ -250,11 +250,161 @@ static void test_io(void)
 #endif
 }
 
+static void test_discard_front(void)
+{
+    struct iovec *iov;
+    struct iovec *iov_tmp;
+    unsigned int iov_cnt;
+    unsigned int iov_cnt_tmp;
+    void *old_base;
+    size_t size;
+    size_t ret;
+
+    /* Discard zero bytes */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, 0);
+    g_assert(ret == 0);
+    g_assert(iov_tmp == iov);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    iov_free(iov, iov_cnt);
+
+    /* Discard more bytes than vector size */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size + 1);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire vector */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within first element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov->iov_base;
+    size = g_test_rand_int_range(1, iov->iov_len);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_tmp == iov);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    g_assert(iov_tmp->iov_base == old_base + size);
+    iov_tmp->iov_base = old_base; /* undo before g_free() */
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire first element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, iov->iov_len);
+    g_assert(ret == iov->iov_len);
+    g_assert(iov_tmp == iov + 1);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within second element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[1].iov_base;
+    size = iov->iov_len + g_test_rand_int_range(1, iov[1].iov_len);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_tmp == iov + 1);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    g_assert(iov_tmp->iov_base == old_base + (size - iov->iov_len));
+    iov_tmp->iov_base = old_base; /* undo before g_free() */
+    iov_free(iov, iov_cnt);
+}
+
+static void test_discard_back(void)
+{
+    struct iovec *iov;
+    unsigned int iov_cnt;
+    unsigned int iov_cnt_tmp;
+    void *old_base;
+    size_t size;
+    size_t ret;
+
+    /* Discard zero bytes */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_back(iov, &iov_cnt_tmp, 0);
+    g_assert(ret == 0);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    iov_free(iov, iov_cnt);
+
+    /* Discard more bytes than vector size */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size + 1);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire vector */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 1].iov_base;
+    size = g_test_rand_int_range(1, iov[iov_cnt - 1].iov_len);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    g_assert(iov[iov_cnt - 1].iov_base == old_base);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 1].iov_base;
+    size = iov[iov_cnt - 1].iov_len;
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within second-to-last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 2].iov_base;
+    size = iov[iov_cnt - 1].iov_len +
+           g_test_rand_int_range(1, iov[iov_cnt - 2].iov_len);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    g_assert(iov[iov_cnt - 2].iov_base == old_base);
+    iov_free(iov, iov_cnt);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
     g_test_rand_int();
     g_test_add_func("/basic/iov/from-to-buf", test_to_from_buf);
     g_test_add_func("/basic/iov/io", test_io);
+    g_test_add_func("/basic/iov/discard-front", test_discard_front);
+    g_test_add_func("/basic/iov/discard-back", test_discard_back);
     return g_test_run();
 }
diff --git a/tests/test-qmp-commands.c b/tests/test-qmp-commands.c
index dc3c507f2b1..5a3e82a854f 100644
--- a/tests/test-qmp-commands.c
+++ b/tests/test-qmp-commands.c
@@ -1,8 +1,9 @@
 #include <glib.h>
-#include "qemu-objects.h"
+#include "qemu-common.h"
+#include "qapi/qmp/types.h"
 #include "test-qmp-commands.h"
-#include "qapi/qmp-core.h"
-#include "module.h"
+#include "qapi/qmp/dispatch.h"
+#include "qemu/module.h"
 #include "qapi/qmp-input-visitor.h"
 #include "tests/test-qapi-types.h"
 #include "tests/test-qapi-visit.h"
diff --git a/tests/test-qmp-input-strict.c b/tests/test-qmp-input-strict.c
index f6df8cbe1ed..6f68963a3d1 100644
--- a/tests/test-qmp-input-strict.c
+++ b/tests/test-qmp-input-strict.c
@@ -14,10 +14,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
diff --git a/tests/test-qmp-input-visitor.c b/tests/test-qmp-input-visitor.c
index 8f5a5095821..955a4c0b0af 100644
--- a/tests/test-qmp-input-visitor.c
+++ b/tests/test-qmp-input-visitor.c
@@ -13,10 +13,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
diff --git a/tests/test-qmp-output-visitor.c b/tests/test-qmp-output-visitor.c
index 24a63595047..71367e6efa6 100644
--- a/tests/test-qmp-output-visitor.c
+++ b/tests/test-qmp-output-visitor.c
@@ -12,10 +12,11 @@
 
 #include <glib.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestOutputVisitorData {
     QmpOutputVisitor *qov;
diff --git a/tests/test-string-input-visitor.c b/tests/test-string-input-visitor.c
index 5370e32041a..f6b0093554f 100644
--- a/tests/test-string-input-visitor.c
+++ b/tests/test-string-input-visitor.c
@@ -13,10 +13,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/string-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     StringInputVisitor *siv;
@@ -164,6 +165,53 @@ static void test_visitor_in_enum(TestInputVisitorData *data,
     data->siv = NULL;
 }
 
+/* Try to crash the visitors */
+static void test_visitor_in_fuzz(TestInputVisitorData *data,
+                                 const void *unused)
+{
+    int64_t ires;
+    bool bres;
+    double nres;
+    char *sres;
+    EnumOne eres;
+    Error *errp = NULL;
+    Visitor *v;
+    unsigned int i;
+    char buf[10000];
+
+    for (i = 0; i < 100; i++) {
+        unsigned int j;
+
+        j = g_test_rand_int_range(0, sizeof(buf) - 1);
+
+        buf[j] = '\0';
+
+        if (j != 0) {
+            for (j--; j != 0; j--) {
+                buf[j - 1] = (char)g_test_rand_int_range(0, 256);
+            }
+        }
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_int(v, &ires, NULL, &errp);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_bool(v, &bres, NULL, &errp);
+        visitor_input_teardown(data, NULL);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_number(v, &nres, NULL, &errp);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_str(v, &sres, NULL, &errp);
+        g_free(sres);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_EnumOne(v, &eres, NULL, &errp);
+        visitor_input_teardown(data, NULL);
+    }
+}
+
 static void input_visitor_test_add(const char *testpath,
                                    TestInputVisitorData *data,
                                    void (*test_func)(TestInputVisitorData *data, const void *user_data))
@@ -188,6 +236,8 @@ int main(int argc, char **argv)
                             &in_visitor_data, test_visitor_in_string);
     input_visitor_test_add("/string-visitor/input/enum",
                             &in_visitor_data, test_visitor_in_enum);
+    input_visitor_test_add("/string-visitor/input/fuzz",
+                            &in_visitor_data, test_visitor_in_fuzz);
 
     g_test_run();
 
diff --git a/tests/test-string-output-visitor.c b/tests/test-string-output-visitor.c
index 608f14a5deb..79d815f8885 100644
--- a/tests/test-string-output-visitor.c
+++ b/tests/test-string-output-visitor.c
@@ -12,10 +12,11 @@
 
 #include <glib.h>
 
+#include "qemu-common.h"
 #include "qapi/string-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestOutputVisitorData {
     StringOutputVisitor *sov;
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
new file mode 100644
index 00000000000..9998e031f28
--- /dev/null
+++ b/tests/test-thread-pool.c
@@ -0,0 +1,224 @@
+#include <glib.h>
+#include "qemu-common.h"
+#include "block/aio.h"
+#include "block/thread-pool.h"
+#include "block/block.h"
+
+static int active;
+
+typedef struct {
+    BlockDriverAIOCB *aiocb;
+    int n;
+    int ret;
+} WorkerTestData;
+
+static int worker_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+    return __sync_fetch_and_add(&data->n, 1);
+}
+
+static int long_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+    __sync_fetch_and_add(&data->n, 1);
+    g_usleep(2000000);
+    __sync_fetch_and_add(&data->n, 1);
+    return 0;
+}
+
+static void done_cb(void *opaque, int ret)
+{
+    WorkerTestData *data = opaque;
+    g_assert_cmpint(data->ret, ==, -EINPROGRESS);
+    data->ret = ret;
+    data->aiocb = NULL;
+
+    /* Callbacks are serialized, so no need to use atomic ops.  */
+    active--;
+}
+
+/* A non-blocking poll of the main AIO context (we cannot use aio_poll
+ * because we do not know the AioContext).
+ */
+static void qemu_aio_wait_nonblocking(void)
+{
+    qemu_notify_event();
+    qemu_aio_wait();
+}
+
+/* Wait until all aio and bh activity has finished */
+static void qemu_aio_wait_all(void)
+{
+    while (qemu_aio_wait()) {
+        /* Do nothing */
+    }
+}
+
+static void test_submit(void)
+{
+    WorkerTestData data = { .n = 0 };
+    thread_pool_submit(worker_cb, &data);
+    qemu_aio_wait_all();
+    g_assert_cmpint(data.n, ==, 1);
+}
+
+static void test_submit_aio(void)
+{
+    WorkerTestData data = { .n = 0, .ret = -EINPROGRESS };
+    data.aiocb = thread_pool_submit_aio(worker_cb, &data, done_cb, &data);
+
+    /* The callbacks are not called until after the first wait.  */
+    active = 1;
+    g_assert_cmpint(data.ret, ==, -EINPROGRESS);
+    qemu_aio_wait_all();
+    g_assert_cmpint(active, ==, 0);
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.ret, ==, 0);
+}
+
+static void co_test_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+
+    active = 1;
+    data->n = 0;
+    data->ret = -EINPROGRESS;
+    thread_pool_submit_co(worker_cb, data);
+
+    /* The test continues in test_submit_co, after qemu_coroutine_enter... */
+
+    g_assert_cmpint(data->n, ==, 1);
+    data->ret = 0;
+    active--;
+
+    /* The test continues in test_submit_co, after qemu_aio_wait_all... */
+}
+
+static void test_submit_co(void)
+{
+    WorkerTestData data;
+    Coroutine *co = qemu_coroutine_create(co_test_cb);
+
+    qemu_coroutine_enter(co, &data);
+
+    /* Back here once the worker has started.  */
+
+    g_assert_cmpint(active, ==, 1);
+    g_assert_cmpint(data.ret, ==, -EINPROGRESS);
+
+    /* qemu_aio_wait_all will execute the rest of the coroutine.  */
+
+    qemu_aio_wait_all();
+
+    /* Back here after the coroutine has finished.  */
+
+    g_assert_cmpint(active, ==, 0);
+    g_assert_cmpint(data.ret, ==, 0);
+}
+
+static void test_submit_many(void)
+{
+    WorkerTestData data[100];
+    int i;
+
+    /* Start more work items than there will be threads.  */
+    for (i = 0; i < 100; i++) {
+        data[i].n = 0;
+        data[i].ret = -EINPROGRESS;
+        thread_pool_submit_aio(worker_cb, &data[i], done_cb, &data[i]);
+    }
+
+    active = 100;
+    while (active > 0) {
+        qemu_aio_wait();
+    }
+    for (i = 0; i < 100; i++) {
+        g_assert_cmpint(data[i].n, ==, 1);
+        g_assert_cmpint(data[i].ret, ==, 0);
+    }
+}
+
+static void test_cancel(void)
+{
+    WorkerTestData data[100];
+    int num_canceled;
+    int i;
+
+    /* Start more work items than there will be threads, to ensure
+     * the pool is full.
+     */
+    test_submit_many();
+
+    /* Start long running jobs, to ensure we can cancel some.  */
+    for (i = 0; i < 100; i++) {
+        data[i].n = 0;
+        data[i].ret = -EINPROGRESS;
+        data[i].aiocb = thread_pool_submit_aio(long_cb, &data[i],
+                                               done_cb, &data[i]);
+    }
+
+    /* Starting the threads may be left to a bottom half.  Let it
+     * run, but do not waste too much time...
+     */
+    active = 100;
+    qemu_aio_wait_nonblocking();
+
+    /* Wait some time for the threads to start, with some sanity
+     * testing on the behavior of the scheduler...
+     */
+    g_assert_cmpint(active, ==, 100);
+    g_usleep(1000000);
+    g_assert_cmpint(active, >, 50);
+
+    /* Cancel the jobs that haven't been started yet.  */
+    num_canceled = 0;
+    for (i = 0; i < 100; i++) {
+        if (__sync_val_compare_and_swap(&data[i].n, 0, 3) == 0) {
+            data[i].ret = -ECANCELED;
+            bdrv_aio_cancel(data[i].aiocb);
+            active--;
+            num_canceled++;
+        }
+    }
+    g_assert_cmpint(active, >, 0);
+    g_assert_cmpint(num_canceled, <, 100);
+
+    /* Canceling the others will be a blocking operation.  */
+    for (i = 0; i < 100; i++) {
+        if (data[i].n != 3) {
+            bdrv_aio_cancel(data[i].aiocb);
+        }
+    }
+
+    /* Finish execution and execute any remaining callbacks.  */
+    qemu_aio_wait_all();
+    g_assert_cmpint(active, ==, 0);
+    for (i = 0; i < 100; i++) {
+        if (data[i].n == 3) {
+            g_assert_cmpint(data[i].ret, ==, -ECANCELED);
+            g_assert(data[i].aiocb != NULL);
+        } else {
+            g_assert_cmpint(data[i].n, ==, 2);
+            g_assert_cmpint(data[i].ret, ==, 0);
+            g_assert(data[i].aiocb == NULL);
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    /* These should be removed once each AioContext has its thread pool.
+     * The test should create its own AioContext.
+     */
+    qemu_init_main_loop();
+    bdrv_init();
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/thread-pool/submit", test_submit);
+    g_test_add_func("/thread-pool/submit-aio", test_submit_aio);
+    g_test_add_func("/thread-pool/submit-co", test_submit_co);
+    g_test_add_func("/thread-pool/submit-many", test_submit_many);
+    g_test_add_func("/thread-pool/cancel", test_cancel);
+    return g_test_run();
+}
diff --git a/tests/test-visitor-serialization.c b/tests/test-visitor-serialization.c
index b8ad16fc9e2..3c6b8df6074 100644
--- a/tests/test-visitor-serialization.c
+++ b/tests/test-visitor-serialization.c
@@ -14,9 +14,11 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <float.h>
+
+#include "qemu-common.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/string-input-visitor.h"
diff --git a/tests/test-x86-cpuid.c b/tests/test-x86-cpuid.c
new file mode 100644
index 00000000000..8d9f96a113d
--- /dev/null
+++ b/tests/test-x86-cpuid.c
@@ -0,0 +1,110 @@
+/*
+ *  Test code for x86 CPUID and Topology functions
+ *
+ *  Copyright (c) 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <glib.h>
+
+#include "topology.h"
+
+static void test_topo_bits(void)
+{
+    /* simple tests for 1 thread per core, 1 core per socket */
+    g_assert_cmpuint(apicid_smt_width(1, 1), ==, 0);
+    g_assert_cmpuint(apicid_core_width(1, 1), ==, 0);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 0), ==, 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 1), ==, 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 2), ==, 2);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 3), ==, 3);
+
+
+    /* Test field width calculation for multiple values
+     */
+    g_assert_cmpuint(apicid_smt_width(1, 2), ==, 1);
+    g_assert_cmpuint(apicid_smt_width(1, 3), ==, 2);
+    g_assert_cmpuint(apicid_smt_width(1, 4), ==, 2);
+
+    g_assert_cmpuint(apicid_smt_width(1, 14), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 15), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 16), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 17), ==, 5);
+
+
+    g_assert_cmpuint(apicid_core_width(30, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(31, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(32, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(33, 2), ==, 6);
+
+
+    /* build a weird topology and see if IDs are calculated correctly
+     */
+
+    /* This will use 2 bits for thread ID and 3 bits for core ID
+     */
+    g_assert_cmpuint(apicid_smt_width(6, 3), ==, 2);
+    g_assert_cmpuint(apicid_core_width(6, 3), ==, 3);
+    g_assert_cmpuint(apicid_pkg_offset(6, 3), ==, 5);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 0), ==, 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1), ==, 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2), ==, 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 0), ==,
+                     (1 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 1), ==,
+                     (1 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 2), ==,
+                     (1 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 0), ==,
+                     (2 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 1), ==,
+                     (2 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 2), ==,
+                     (2 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 0), ==,
+                     (5 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 1), ==,
+                     (5 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 2), ==,
+                     (5 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 6 * 3 + 0 * 3 + 0), ==,
+                     (1 << 5));
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 6 * 3 + 1 * 3 + 1), ==,
+                     (1 << 5) | (1 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 3 * 6 * 3 + 5 * 3 + 2), ==,
+                     (3 << 5) | (5 << 2) | 2);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/cpuid/topology/basic", test_topo_bits);
+
+    g_test_run();
+
+    return 0;
+}
diff --git a/tests/tmp105-test.c b/tests/tmp105-test.c
new file mode 100644
index 00000000000..a6ad213de80
--- /dev/null
+++ b/tests/tmp105-test.c
@@ -0,0 +1,76 @@
+/*
+ * QTest testcase for the TMP105 temperature sensor
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libqtest.h"
+#include "libi2c.h"
+#include "hw/tmp105_regs.h"
+
+#include <glib.h>
+
+#define OMAP2_I2C_1_BASE 0x48070000
+
+#define N8X0_ADDR 0x48
+
+static I2CAdapter *i2c;
+static uint8_t addr;
+
+static void send_and_receive(void)
+{
+    uint8_t cmd[3];
+    uint8_t resp[2];
+
+    cmd[0] = TMP105_REG_TEMPERATURE;
+    i2c_send(i2c, addr, cmd, 1);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmpuint(((uint16_t)resp[0] << 8) | resp[1], ==, 0);
+
+    cmd[0] = TMP105_REG_CONFIG;
+    cmd[1] = 0x0; /* matches the reset value */
+    i2c_send(i2c, addr, cmd, 2);
+    i2c_recv(i2c, addr, resp, 1);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+
+    cmd[0] = TMP105_REG_T_LOW;
+    cmd[1] = 0x12;
+    cmd[2] = 0x34;
+    i2c_send(i2c, addr, cmd, 3);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+    g_assert_cmphex(resp[1], ==, cmd[2]);
+
+    cmd[0] = TMP105_REG_T_HIGH;
+    cmd[1] = 0x42;
+    cmd[2] = 0x31;
+    i2c_send(i2c, addr, cmd, 3);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+    g_assert_cmphex(resp[1], ==, cmd[2]);
+}
+
+int main(int argc, char **argv)
+{
+    QTestState *s = NULL;
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    s = qtest_start("-display none -machine n800");
+    i2c = omap_i2c_create(OMAP2_I2C_1_BASE);
+    addr = N8X0_ADDR;
+
+    qtest_add_func("/tmp105/tx-rx", send_and_receive);
+
+    ret = g_test_run();
+
+    if (s) {
+        qtest_quit(s);
+    }
+    g_free(i2c);
+
+    return ret;
+}
diff --git a/thread-pool.c b/thread-pool.c
new file mode 100644
index 00000000000..e3ca64d790a
--- /dev/null
+++ b/thread-pool.c
@@ -0,0 +1,289 @@
+/*
+ * QEMU block layer thread pool
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "qemu/osdep.h"
+#include "block/coroutine.h"
+#include "trace.h"
+#include "block/block_int.h"
+#include "qemu/event_notifier.h"
+#include "block/thread-pool.h"
+
+static void do_spawn_thread(void);
+
+typedef struct ThreadPoolElement ThreadPoolElement;
+
+enum ThreadState {
+    THREAD_QUEUED,
+    THREAD_ACTIVE,
+    THREAD_DONE,
+    THREAD_CANCELED,
+};
+
+struct ThreadPoolElement {
+    BlockDriverAIOCB common;
+    ThreadPoolFunc *func;
+    void *arg;
+
+    /* Moving state out of THREAD_QUEUED is protected by lock.  After
+     * that, only the worker thread can write to it.  Reads and writes
+     * of state and ret are ordered with memory barriers.
+     */
+    enum ThreadState state;
+    int ret;
+
+    /* Access to this list is protected by lock.  */
+    QTAILQ_ENTRY(ThreadPoolElement) reqs;
+
+    /* Access to this list is protected by the global mutex.  */
+    QLIST_ENTRY(ThreadPoolElement) all;
+};
+
+static EventNotifier notifier;
+static QemuMutex lock;
+static QemuCond check_cancel;
+static QemuSemaphore sem;
+static int max_threads = 64;
+static QEMUBH *new_thread_bh;
+
+/* The following variables are protected by the global mutex.  */
+static QLIST_HEAD(, ThreadPoolElement) head;
+
+/* The following variables are protected by lock.  */
+static QTAILQ_HEAD(, ThreadPoolElement) request_list;
+static int cur_threads;
+static int idle_threads;
+static int new_threads;     /* backlog of threads we need to create */
+static int pending_threads; /* threads created but not running yet */
+static int pending_cancellations; /* whether we need a cond_broadcast */
+
+static void *worker_thread(void *unused)
+{
+    qemu_mutex_lock(&lock);
+    pending_threads--;
+    do_spawn_thread();
+
+    while (1) {
+        ThreadPoolElement *req;
+        int ret;
+
+        do {
+            idle_threads++;
+            qemu_mutex_unlock(&lock);
+            ret = qemu_sem_timedwait(&sem, 10000);
+            qemu_mutex_lock(&lock);
+            idle_threads--;
+        } while (ret == -1 && !QTAILQ_EMPTY(&request_list));
+        if (ret == -1) {
+            break;
+        }
+
+        req = QTAILQ_FIRST(&request_list);
+        QTAILQ_REMOVE(&request_list, req, reqs);
+        req->state = THREAD_ACTIVE;
+        qemu_mutex_unlock(&lock);
+
+        ret = req->func(req->arg);
+
+        req->ret = ret;
+        /* Write ret before state.  */
+        smp_wmb();
+        req->state = THREAD_DONE;
+
+        qemu_mutex_lock(&lock);
+        if (pending_cancellations) {
+            qemu_cond_broadcast(&check_cancel);
+        }
+
+        event_notifier_set(&notifier);
+    }
+
+    cur_threads--;
+    qemu_mutex_unlock(&lock);
+    return NULL;
+}
+
+static void do_spawn_thread(void)
+{
+    QemuThread t;
+
+    /* Runs with lock taken.  */
+    if (!new_threads) {
+        return;
+    }
+
+    new_threads--;
+    pending_threads++;
+
+    qemu_thread_create(&t, worker_thread, NULL, QEMU_THREAD_DETACHED);
+}
+
+static void spawn_thread_bh_fn(void *opaque)
+{
+    qemu_mutex_lock(&lock);
+    do_spawn_thread();
+    qemu_mutex_unlock(&lock);
+}
+
+static void spawn_thread(void)
+{
+    cur_threads++;
+    new_threads++;
+    /* If there are threads being created, they will spawn new workers, so
+     * we don't spend time creating many threads in a loop holding a mutex or
+     * starving the current vcpu.
+     *
+     * If there are no idle threads, ask the main thread to create one, so we
+     * inherit the correct affinity instead of the vcpu affinity.
+     */
+    if (!pending_threads) {
+        qemu_bh_schedule(new_thread_bh);
+    }
+}
+
+static void event_notifier_ready(EventNotifier *notifier)
+{
+    ThreadPoolElement *elem, *next;
+
+    event_notifier_test_and_clear(notifier);
+restart:
+    QLIST_FOREACH_SAFE(elem, &head, all, next) {
+        if (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
+            continue;
+        }
+        if (elem->state == THREAD_DONE) {
+            trace_thread_pool_complete(elem, elem->common.opaque, elem->ret);
+        }
+        if (elem->state == THREAD_DONE && elem->common.cb) {
+            QLIST_REMOVE(elem, all);
+            /* Read state before ret.  */
+            smp_rmb();
+            elem->common.cb(elem->common.opaque, elem->ret);
+            qemu_aio_release(elem);
+            goto restart;
+        } else {
+            /* remove the request */
+            QLIST_REMOVE(elem, all);
+            qemu_aio_release(elem);
+        }
+    }
+}
+
+static int thread_pool_active(EventNotifier *notifier)
+{
+    return !QLIST_EMPTY(&head);
+}
+
+static void thread_pool_cancel(BlockDriverAIOCB *acb)
+{
+    ThreadPoolElement *elem = (ThreadPoolElement *)acb;
+
+    trace_thread_pool_cancel(elem, elem->common.opaque);
+
+    qemu_mutex_lock(&lock);
+    if (elem->state == THREAD_QUEUED &&
+        /* No thread has yet started working on elem. we can try to "steal"
+         * the item from the worker if we can get a signal from the
+         * semaphore.  Because this is non-blocking, we can do it with
+         * the lock taken and ensure that elem will remain THREAD_QUEUED.
+         */
+        qemu_sem_timedwait(&sem, 0) == 0) {
+        QTAILQ_REMOVE(&request_list, elem, reqs);
+        elem->state = THREAD_CANCELED;
+        event_notifier_set(&notifier);
+    } else {
+        pending_cancellations++;
+        while (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
+            qemu_cond_wait(&check_cancel, &lock);
+        }
+        pending_cancellations--;
+    }
+    qemu_mutex_unlock(&lock);
+}
+
+static const AIOCBInfo thread_pool_aiocb_info = {
+    .aiocb_size         = sizeof(ThreadPoolElement),
+    .cancel             = thread_pool_cancel,
+};
+
+BlockDriverAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    ThreadPoolElement *req;
+
+    req = qemu_aio_get(&thread_pool_aiocb_info, NULL, cb, opaque);
+    req->func = func;
+    req->arg = arg;
+    req->state = THREAD_QUEUED;
+
+    QLIST_INSERT_HEAD(&head, req, all);
+
+    trace_thread_pool_submit(req, arg);
+
+    qemu_mutex_lock(&lock);
+    if (idle_threads == 0 && cur_threads < max_threads) {
+        spawn_thread();
+    }
+    QTAILQ_INSERT_TAIL(&request_list, req, reqs);
+    qemu_mutex_unlock(&lock);
+    qemu_sem_post(&sem);
+    return &req->common;
+}
+
+typedef struct ThreadPoolCo {
+    Coroutine *co;
+    int ret;
+} ThreadPoolCo;
+
+static void thread_pool_co_cb(void *opaque, int ret)
+{
+    ThreadPoolCo *co = opaque;
+
+    co->ret = ret;
+    qemu_coroutine_enter(co->co, NULL);
+}
+
+int coroutine_fn thread_pool_submit_co(ThreadPoolFunc *func, void *arg)
+{
+    ThreadPoolCo tpc = { .co = qemu_coroutine_self(), .ret = -EINPROGRESS };
+    assert(qemu_in_coroutine());
+    thread_pool_submit_aio(func, arg, thread_pool_co_cb, &tpc);
+    qemu_coroutine_yield();
+    return tpc.ret;
+}
+
+void thread_pool_submit(ThreadPoolFunc *func, void *arg)
+{
+    thread_pool_submit_aio(func, arg, NULL, NULL);
+}
+
+static void thread_pool_init(void)
+{
+    QLIST_INIT(&head);
+    event_notifier_init(&notifier, false);
+    qemu_mutex_init(&lock);
+    qemu_cond_init(&check_cancel);
+    qemu_sem_init(&sem, 0);
+    qemu_aio_set_event_notifier(&notifier, event_notifier_ready,
+                                thread_pool_active);
+
+    QTAILQ_INIT(&request_list);
+    new_thread_bh = qemu_bh_new(spawn_thread_bh_fn, NULL);
+}
+
+block_init(thread_pool_init)
diff --git a/thunk.c b/thunk.c
index 8ebbbb46b68..3cca047509e 100644
--- a/thunk.c
+++ b/thunk.c
@@ -21,7 +21,7 @@
 #include <stdarg.h>
 
 #include "qemu.h"
-#include "thunk.h"
+#include "exec/user/thunk.h"
 
 //#define DEBUG
 
diff --git a/trace-events b/trace-events
index 04b07239270..2b28076e452 100644
--- a/trace-events
+++ b/trace-events
@@ -74,10 +74,29 @@ bdrv_co_do_copy_on_readv(void *bs, int64_t sector_num, int nb_sectors, int64_t c
 # block/stream.c
 stream_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
 stream_start(void *bs, void *base, void *s, void *co, void *opaque) "bs %p base %p s %p co %p opaque %p"
+commit_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
+commit_start(void *bs, void *base, void *top, void *s, void *co, void *opaque) "bs %p base %p top %p s %p co %p opaque %p"
+
+# block/mirror.c
+mirror_start(void *bs, void *s, void *co, void *opaque) "bs %p s %p co %p opaque %p"
+mirror_restart_iter(void *s, int64_t cnt) "s %p dirty count %"PRId64
+mirror_before_flush(void *s) "s %p"
+mirror_before_drain(void *s, int64_t cnt) "s %p dirty count %"PRId64
+mirror_before_sleep(void *s, int64_t cnt, int synced) "s %p dirty count %"PRId64" synced %d"
+mirror_one_iteration(void *s, int64_t sector_num, int nb_sectors) "s %p sector_num %"PRId64" nb_sectors %d"
+mirror_cow(void *s, int64_t sector_num) "s %p sector_num %"PRId64
+mirror_iteration_done(void *s, int64_t sector_num, int nb_sectors, int ret) "s %p sector_num %"PRId64" nb_sectors %d ret %d"
+mirror_yield(void *s, int64_t cnt, int buf_free_count, int in_flight) "s %p dirty count %"PRId64" free buffers %d in_flight %d"
+mirror_yield_in_flight(void *s, int64_t sector_num, int in_flight) "s %p sector_num %"PRId64" in_flight %d"
+mirror_yield_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
+mirror_break_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
 
 # blockdev.c
 qmp_block_job_cancel(void *job) "job %p"
-block_stream_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
+qmp_block_job_pause(void *job) "job %p"
+qmp_block_job_resume(void *job) "job %p"
+qmp_block_job_complete(void *job) "job %p"
+block_job_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
 qmp_block_stream(void *bs, void *job) "bs %p job %p"
 
 # hw/virtio-blk.c
@@ -86,6 +105,20 @@ virtio_blk_rw_complete(void *req, int ret) "req %p ret %d"
 virtio_blk_handle_write(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 
+# hw/dataplane/virtio-blk.c
+virtio_blk_data_plane_start(void *s) "dataplane %p"
+virtio_blk_data_plane_stop(void *s) "dataplane %p"
+virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
+virtio_blk_data_plane_complete_request(void *s, unsigned int head, int ret) "dataplane %p head %u ret %d"
+
+# hw/dataplane/vring.c
+vring_setup(uint64_t physical, void *desc, void *avail, void *used) "vring physical %#"PRIx64" desc %p avail %p used %p"
+
+# thread-pool.c
+thread_pool_submit(void *req, void *opaque) "req %p opaque %p"
+thread_pool_complete(void *req, void *opaque, int ret) "req %p opaque %p ret %d"
+thread_pool_cancel(void *req, void *opaque) "req %p opaque %p"
+
 # posix-aio-compat.c
 paio_submit(void *acb, void *opaque, int64_t sector_num, int nb_sectors, int type) "acb %p opaque %p sector_num %"PRId64" nb_sectors %d type %d"
 paio_complete(void *acb, void *opaque, int ret) "acb %p opaque %p ret %d"
@@ -141,6 +174,13 @@ ecc_mem_readl_ecr1(uint32_t ret) "Read event count 2 %08x"
 ecc_diag_mem_writeb(uint64_t addr, uint32_t val) "Write diagnostic %"PRId64" = %02x"
 ecc_diag_mem_readb(uint64_t addr, uint32_t ret) "Read diagnostic %"PRId64"= %02x"
 
+# hw/fw_cfg.c
+fw_cfg_write(void *s, uint8_t value) "%p %d"
+fw_cfg_select(void *s, uint16_t key, int ret) "%p key %d = %d"
+fw_cfg_read(void *s, uint8_t ret) "%p = %d"
+fw_cfg_add_file_dupe(void *s, char *name) "%p %s"
+fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
+
 # hw/hd-geometry.c
 hd_geometry_lchs_guess(void *bs, int cyls, int heads, int secs) "bs %p LCHS %d %d %d"
 hd_geometry_guess(void *bs, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "bs %p CHS %u %u %u trans %d"
@@ -243,9 +283,12 @@ usb_port_release(int bus, const char *port) "bus %d, port %s"
 
 # hw/usb/hcd-ehci.c
 usb_ehci_reset(void) "=== RESET ==="
-usb_ehci_mmio_readl(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
-usb_ehci_mmio_writel(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
-usb_ehci_mmio_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
+usb_ehci_opreg_read(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
+usb_ehci_opreg_write(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
+usb_ehci_opreg_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
+usb_ehci_portsc_read(uint32_t addr, uint32_t port, uint32_t val) "rd mmio %04x [port %d] = %x"
+usb_ehci_portsc_write(uint32_t addr, uint32_t port, uint32_t val) "wr mmio %04x [port %d] = %x"
+usb_ehci_portsc_change(uint32_t addr, uint32_t port, uint32_t new, uint32_t old) "ch mmio %04x [port %d] = %x (old: %x)"
 usb_ehci_usbsts(const char *sts, int state) "usbsts %s %d"
 usb_ehci_state(const char *schedule, const char *state) "%s schedule %s"
 usb_ehci_qh_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t c_qtd, uint32_t n_qtd, uint32_t a_qtd) "q %p - QH @ %08x: next %08x qtds %08x,%08x,%08x"
@@ -263,6 +306,10 @@ usb_ehci_data(int rw, uint32_t cpage, uint32_t offset, uint32_t addr, uint32_t l
 usb_ehci_queue_action(void *q, const char *action) "q %p: %s"
 usb_ehci_packet_action(void *q, void *p, const char *action) "q %p p %p: %s"
 usb_ehci_irq(uint32_t level, uint32_t frindex, uint32_t sts, uint32_t mask) "level %d, frindex 0x%04x, sts 0x%x, mask 0x%x"
+usb_ehci_guest_bug(const char *reason) "%s"
+usb_ehci_doorbell_ring(void) ""
+usb_ehci_doorbell_ack(void) ""
+usb_ehci_dma_error(void) ""
 
 # hw/usb/hcd-uhci.c
 usb_uhci_reset(void) "=== RESET ==="
@@ -274,10 +321,8 @@ usb_uhci_frame_loop_stop_idle(void) ""
 usb_uhci_frame_loop_continue(void) ""
 usb_uhci_mmio_readw(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%04x"
 usb_uhci_mmio_writew(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%04x"
-usb_uhci_mmio_readl(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%08x"
-usb_uhci_mmio_writel(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%08x"
 usb_uhci_queue_add(uint32_t token) "token 0x%x"
-usb_uhci_queue_del(uint32_t token) "token 0x%x"
+usb_uhci_queue_del(uint32_t token, const char *reason) "token 0x%x: %s"
 usb_uhci_packet_add(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
 usb_uhci_packet_link_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
 usb_uhci_packet_unlink_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
@@ -310,8 +355,13 @@ usb_xhci_runtime_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
 usb_xhci_doorbell_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
 usb_xhci_irq_intx(uint32_t level) "level %d"
 usb_xhci_irq_msi(uint32_t nr) "nr %d"
-usb_xhci_queue_event(uint32_t idx, const char *name, uint64_t param, uint32_t status, uint32_t control) "idx %d, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_irq_msix(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_use(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_unuse(uint32_t nr) "nr %d"
+usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char *trb, const char *evt, uint64_t param, uint32_t status, uint32_t control) "v %d, idx %d, %s, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
 usb_xhci_fetch_trb(uint64_t addr, const char *name, uint64_t param, uint32_t status, uint32_t control) "addr %016" PRIx64 ", %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_port_reset(uint32_t port) "port %d"
+usb_xhci_port_link(uint32_t port, uint32_t pls) "port %d, pls %d"
 usb_xhci_slot_enable(uint32_t slotid) "slotid %d"
 usb_xhci_slot_disable(uint32_t slotid) "slotid %d"
 usb_xhci_slot_address(uint32_t slotid) "slotid %d"
@@ -320,10 +370,11 @@ usb_xhci_slot_evaluate(uint32_t slotid) "slotid %d"
 usb_xhci_slot_reset(uint32_t slotid) "slotid %d"
 usb_xhci_ep_enable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_disable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_set_dequeue(uint32_t slotid, uint32_t epid, uint64_t param) "slotid %d, epid %d, ptr %016" PRIx64
 usb_xhci_ep_kick(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_stop(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_reset(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid, uint32_t length) "%p: slotid %d, epid %d, length %d"
+usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid) "%p: slotid %d, epid %d"
 usb_xhci_xfer_async(void *xfer) "%p"
 usb_xhci_xfer_nak(void *xfer) "%p"
 usb_xhci_xfer_retry(void *xfer) "%p"
@@ -336,6 +387,7 @@ usb_desc_device_qualifier(int addr, int len, int ret) "dev %d query device quali
 usb_desc_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
 usb_desc_other_speed_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
 usb_desc_string(int addr, int index, int len, int ret) "dev %d query string %d, len %d, ret %d"
+usb_desc_bos(int addr, int len, int ret) "dev %d bos, len %d, ret %d"
 usb_set_addr(int addr) "dev %d"
 usb_set_config(int addr, int config, int ret) "dev %d, config %d, ret %d"
 usb_set_interface(int addr, int iface, int alt, int ret) "dev %d, interface %d, altsetting %d, ret %d"
@@ -378,7 +430,7 @@ usb_host_claim_interfaces(int bus, int addr, int config, int nif) "dev %d:%d, co
 usb_host_release_interfaces(int bus, int addr) "dev %d:%d"
 usb_host_req_control(int bus, int addr, void *p, int req, int value, int index) "dev %d:%d, packet %p, req 0x%x, value %d, index %d"
 usb_host_req_data(int bus, int addr, void *p, int in, int ep, int size) "dev %d:%d, packet %p, in %d, ep %d, size %d"
-usb_host_req_complete(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
+usb_host_req_complete(int bus, int addr, void *p, int status, int length) "dev %d:%d, packet %p, status %d, length %d"
 usb_host_req_emulated(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
 usb_host_req_canceled(int bus, int addr, void *p) "dev %d:%d, packet %p"
 usb_host_urb_submit(int bus, int addr, void *aurb, int length, int more) "dev %d:%d, aurb %p, length %d, more %d"
@@ -504,6 +556,7 @@ spice_vmc_write(ssize_t out, int len) "spice wrottn %zd of requested %d"
 spice_vmc_read(int bytes, int len) "spice read %d of requested %d"
 spice_vmc_register_interface(void *scd) "spice vmc registered interface %p"
 spice_vmc_unregister_interface(void *scd) "spice vmc unregistered interface %p"
+spice_vmc_event(int event) "spice vmc event %d"
 
 # hw/lm32_pic.c
 lm32_pic_raise_irq(void) "Raise CPU interrupt"
@@ -694,6 +747,14 @@ mipsnet_read(uint64_t addr, uint32_t val) "read addr=0x%" PRIx64 " val=0x%x"
 mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64 ""
 mipsnet_irq(uint32_t isr, uint32_t intctl) "set irq to %d (%02x)"
 
+# hw/pc87312.c
+pc87312_io_read(uint32_t addr, uint32_t val) "read addr=%x val=%x"
+pc87312_io_write(uint32_t addr, uint32_t val) "write addr=%x val=%x"
+pc87312_info_floppy(uint32_t base) "base 0x%x"
+pc87312_info_ide(uint32_t base) "base 0x%x"
+pc87312_info_parallel(uint32_t base, uint32_t irq) "base 0x%x, irq %u"
+pc87312_info_serial(int n, uint32_t base, uint32_t irq) "id=%d, base 0x%x, irq %u"
+
 # xen-all.c
 xen_ram_alloc(unsigned long ram_addr, unsigned long size) "requested: %#lx, size %#lx"
 xen_client_set_memory(uint64_t start_addr, unsigned long size, bool log_dirty) "%#"PRIx64" size %#lx, log_dirty %i"
@@ -906,6 +967,10 @@ ppm_save(const char *filename, void *display_surface) "%s surface=%p"
 savevm_section_start(void) ""
 savevm_section_end(unsigned int section_id) "section_id %u"
 
+# arch_init.c
+migration_bitmap_sync_start(void) ""
+migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""
+
 # hw/qxl.c
 disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
 disable qxl_io_write_vga(int qid, const char *mode, uint32_t addr, uint32_t val) "%d %s addr=%u val=%u"
@@ -924,8 +989,9 @@ qxl_interface_update_area_complete_rest(int qid, uint32_t num_updated_rects) "%d
 qxl_interface_update_area_complete_overflow(int qid, int max) "%d max=%d"
 qxl_interface_update_area_complete_schedule_bh(int qid, uint32_t num_dirty) "%d #dirty=%d"
 qxl_io_destroy_primary_ignored(int qid, const char *mode) "%d %s"
+qxl_io_log(int qid, const uint8_t *log_buf) "%d %s"
 qxl_io_read_unexpected(int qid) "%d"
-qxl_io_unexpected_vga_mode(int qid, uint32_t io_port, const char *desc) "%d 0x%x (%s)"
+qxl_io_unexpected_vga_mode(int qid, uint64_t addr, uint64_t val, const char *desc) "%d 0x%"PRIx64"=%"PRIu64" (%s)"
 qxl_io_write(int qid, const char *mode, uint64_t addr, uint64_t val, unsigned size, int async) "%d %s addr=%"PRIu64 " val=%"PRIu64" size=%u async=%d"
 qxl_memslot_add_guest(int qid, uint32_t slot_id, uint64_t guest_start, uint64_t guest_end) "%d %u: guest phys 0x%"PRIx64 " - 0x%" PRIx64
 qxl_post_load(int qid, const char *mode) "%d %s"
@@ -956,6 +1022,7 @@ qxl_spice_destroy_surfaces(int qid, int async) "%d async=%d"
 qxl_spice_destroy_surface_wait_complete(int qid, uint32_t id) "%d sid=%d"
 qxl_spice_destroy_surface_wait(int qid, uint32_t id, int async) "%d sid=%d async=%d"
 qxl_spice_flush_surfaces_async(int qid, uint32_t surface_count, uint32_t num_free_res) "%d s#=%d, res#=%d"
+qxl_spice_monitors_config(int qid) "%d"
 qxl_spice_loadvm_commands(int qid, void *ext, uint32_t count) "%d ext=%p count=%d"
 qxl_spice_oom(int qid) "%d"
 qxl_spice_reset_cursor(int qid) "%d"
@@ -964,6 +1031,15 @@ qxl_spice_reset_memslots(int qid) "%d"
 qxl_spice_update_area(int qid, uint32_t surface_id, uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "%d sid=%d [%d,%d,%d,%d]"
 qxl_spice_update_area_rest(int qid, uint32_t num_dirty_rects, uint32_t clear_dirty_region) "%d #d=%d clear=%d"
 qxl_surfaces_dirty(int qid, int surface, int offset, int size) "%d surface=%d offset=%d size=%d"
+qxl_send_events(int qid, uint32_t events) "%d %d"
+qxl_send_events_vm_stopped(int qid, uint32_t events) "%d %d"
+qxl_set_guest_bug(int qid) "%d"
+qxl_interrupt_client_monitors_config(int qid, int num_heads, void *heads) "%d %d %p"
+qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void *client_monitors_config) "%d %X %p"
+qxl_client_monitors_config_unsupported_by_device(int qid, int revision) "%d revision=%d"
+qxl_client_monitors_config_capped(int qid, int requested, int limit) "%d %d %d"
+qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32) "%d %u %u"
+qxl_set_client_capabilities_unsupported_by_revision(int qid, int revision) "%d revision=%d"
 
 # hw/qxl-render.c
 qxl_render_blit_guest_primary_initialized(void) ""
@@ -978,3 +1054,21 @@ spapr_pci_rtas_ibm_change_msi(unsigned func, unsigned req) "func %u, requested %
 spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr) "queries for #%u, IRQ%u"
 spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq) "@%"PRIx64"<=%"PRIx64" IRQ %u"
 spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
+
+# hw/xics.c
+xics_icp_check_ipi(int server, uint8_t mfrr) "CPU %d can take IPI mfrr=%#x"
+xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr) "icp_accept: XIRR %#"PRIx32"->%#"PRIx32
+xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr) "icp_eoi: server %d given XIRR %#"PRIx32" new XIRR %#"PRIx32
+xics_icp_irq(int server, int nr, uint8_t priority) "cpu %d trying to deliver irq %#"PRIx32" priority %#x"
+xics_icp_raise(uint32_t xirr, uint8_t pending_priority) "raising IRQ new XIRR=%#x new pending priority=%#x"
+xics_set_irq_msi(int srcno, int nr) "set_irq_msi: srcno %d [irq %#x]"
+xics_masked_pending(void) "set_irq_msi: masked pending"
+xics_set_irq_lsi(int srcno, int nr) "set_irq_lsi: srcno %d [irq %#x]"
+xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority) "ics_write_xive: irq %#x [src %d] server %#x prio %#x"
+xics_ics_reject(int nr, int srcno) "reject irq %#x [src %d]"
+xics_ics_eoi(int nr) "ics_eoi: irq %#x"
+
+# hbitmap.c
+hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
+hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
+hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
diff --git a/trace/Makefile.objs b/trace/Makefile.objs
new file mode 100644
index 00000000000..27fe26b5c27
--- /dev/null
+++ b/trace/Makefile.objs
@@ -0,0 +1,58 @@
+# -*- mode: makefile -*-
+
+######################################################################
+# Auto-generated header for tracing routines
+
+$(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
+$(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=h \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+######################################################################
+# Auto-generated tracing routines (non-DTrace)
+
+ifneq ($(TRACE_BACKEND),dtrace)
+$(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
+$(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=c \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-tracers.o: $(obj)/generated-tracers.c $(obj)/generated-tracers.h
+endif
+
+
+######################################################################
+# Auto-generated DTrace code
+
+# Normal practice is to name DTrace probe file with a '.d' extension
+# but that gets picked up by QEMU's Makefile as an external dependency
+# rule file. So we use '.dtrace' instead
+ifeq ($(TRACE_BACKEND),dtrace)
+$(obj)/generated-tracers.dtrace: $(obj)/generated-tracers.dtrace-timestamp
+$(obj)/generated-tracers.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=d \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-tracers-dtrace.h: $(obj)/generated-tracers.dtrace
+	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   $@")
+
+$(obj)/generated-tracers.o: $(obj)/generated-tracers.dtrace
+endif
+
+######################################################################
+# Backend code
+
+util-obj-$(CONFIG_TRACE_DEFAULT) += default.o
+util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o
+util-obj-$(CONFIG_TRACE_STDERR) += stderr.o
+util-obj-y += control.o
+util-obj-y += generated-tracers.o
diff --git a/trace/control.c b/trace/control.c
index 22d5863eebf..be05efb99b8 100644
--- a/trace/control.c
+++ b/trace/control.c
@@ -12,6 +12,8 @@
 
 void trace_backend_init_events(const char *fname)
 {
+    int ret;
+
     if (fname == NULL) {
         return;
     }
@@ -30,7 +32,12 @@ void trace_backend_init_events(const char *fname)
             if ('#' == line_buf[0]) { /* skip commented lines */
                 continue;
             }
-            if (!trace_event_set_state(line_buf, true)) {
+            if ('-' == line_buf[0]) {
+                ret = trace_event_set_state(line_buf+1, false);
+            } else {
+                ret = trace_event_set_state(line_buf, true);
+            }
+            if (!ret) {
                 fprintf(stderr,
                         "error: trace event '%s' does not exist\n", line_buf);
                 exit(1);
diff --git a/trace/simple.c b/trace/simple.c
index d83681b2270..ce17d64bd7b 100644
--- a/trace/simple.c
+++ b/trace/simple.c
@@ -16,7 +16,7 @@
 #include <signal.h>
 #include <pthread.h>
 #endif
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "trace.h"
 #include "trace/control.h"
 
diff --git a/translate-all.c b/translate-all.c
index 5bd2d3711a2..d367fc4d11d 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -16,6 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/types.h>
+#include <sys/mman.h>
+#endif
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -24,21 +30,119 @@
 
 #include "config.h"
 
+#include "qemu-common.h"
 #define NO_CPU_IO_DEFS
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#if defined(CONFIG_USER_ONLY)
+#include "qemu.h"
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+#include <sys/param.h>
+#if __FreeBSD_version >= 700104
+#define HAVE_KINFO_GETVMMAP
+#define sigqueue sigqueue_freebsd  /* avoid redefinition */
+#include <sys/time.h>
+#include <sys/proc.h>
+#include <machine/profile.h>
+#define _KERNEL
+#include <sys/user.h>
+#undef _KERNEL
+#undef sigqueue
+#include <libutil.h>
+#endif
+#endif
+#endif
+
+#include "exec/cputlb.h"
+#include "translate-all.h"
+
+//#define DEBUG_TB_INVALIDATE
+//#define DEBUG_FLUSH
+/* make various TB consistency checks */
+//#define DEBUG_TB_CHECK
+
+#if !defined(CONFIG_USER_ONLY)
+/* TB consistency checks only implemented for usermode emulation.  */
+#undef DEBUG_TB_CHECK
+#endif
+
+#define SMC_BITMAP_USE_THRESHOLD 10
+
+/* Code generation and translation blocks */
+static TranslationBlock *tbs;
+static int code_gen_max_blocks;
+TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
+static int nb_tbs;
+/* any access to the tbs or the page table must use this lock */
+spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
+
+uint8_t *code_gen_prologue;
+static uint8_t *code_gen_buffer;
+static size_t code_gen_buffer_size;
+/* threshold to flush the translated code buffer */
+static size_t code_gen_buffer_max_size;
+static uint8_t *code_gen_ptr;
+
+typedef struct PageDesc {
+    /* list of TBs intersecting this ram page */
+    TranslationBlock *first_tb;
+    /* in order to optimize self modifying code, we count the number
+       of lookups we do to a given page to use a bitmap */
+    unsigned int code_write_count;
+    uint8_t *code_bitmap;
+#if defined(CONFIG_USER_ONLY)
+    unsigned long flags;
+#endif
+} PageDesc;
+
+/* In system mode we want L1_MAP to be based on ram offsets,
+   while in user mode we want it to be based on virtual addresses.  */
+#if !defined(CONFIG_USER_ONLY)
+#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
+# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
+#endif
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
+#endif
+
+/* The bits remaining after N lower levels of page tables.  */
+#define V_L1_BITS_REM \
+    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
+
+#if V_L1_BITS_REM < 4
+#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
+#else
+#define V_L1_BITS  V_L1_BITS_REM
+#endif
+
+#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
+
+#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
+
+uintptr_t qemu_real_host_page_size;
+uintptr_t qemu_host_page_size;
+uintptr_t qemu_host_page_mask;
+
+/* This is a multi-level map on the virtual address space.
+   The bottom level has pointers to PageDesc.  */
+static void *l1_map[V_L1_SIZE];
+
+/* statistics */
+static int tb_flush_count;
+static int tb_phys_invalidate_count;
 
 /* code generation context */
 TCGContext tcg_ctx;
 
-uint16_t gen_opc_buf[OPC_BUF_SIZE];
-TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
-
-target_ulong gen_opc_pc[OPC_BUF_SIZE];
-uint16_t gen_opc_icount[OPC_BUF_SIZE];
-uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2);
+static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
 
 void cpu_gen_init(void)
 {
@@ -108,8 +212,8 @@ int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr
 
 /* The cpu state corresponding to 'searched_pc' is restored.
  */
-int cpu_restore_state(TranslationBlock *tb,
-                      CPUArchState *env, uintptr_t searched_pc)
+static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
+                                     uintptr_t searched_pc)
 {
     TCGContext *s = &tcg_ctx;
     int j;
@@ -149,9 +253,10 @@ int cpu_restore_state(TranslationBlock *tb,
     if (j < 0)
         return -1;
     /* now find start of instruction before */
-    while (gen_opc_instr_start[j] == 0)
+    while (s->gen_opc_instr_start[j] == 0) {
         j--;
-    env->icount_decr.u16.low -= gen_opc_icount[j];
+    }
+    env->icount_decr.u16.low -= s->gen_opc_icount[j];
 
     restore_state_to_opc(env, tb, j);
 
@@ -161,3 +266,1622 @@ int cpu_restore_state(TranslationBlock *tb,
 #endif
     return 0;
 }
+
+bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_pc(retaddr);
+    if (tb) {
+        cpu_restore_state_from_tb(tb, env, retaddr);
+        return true;
+    }
+    return false;
+}
+
+#ifdef _WIN32
+static inline void map_exec(void *addr, long size)
+{
+    DWORD old_protect;
+    VirtualProtect(addr, size,
+                   PAGE_EXECUTE_READWRITE, &old_protect);
+}
+#else
+static inline void map_exec(void *addr, long size)
+{
+    unsigned long start, end, page_size;
+
+    page_size = getpagesize();
+    start = (unsigned long)addr;
+    start &= ~(page_size - 1);
+
+    end = (unsigned long)addr + size;
+    end += page_size - 1;
+    end &= ~(page_size - 1);
+
+    mprotect((void *)start, end - start,
+             PROT_READ | PROT_WRITE | PROT_EXEC);
+}
+#endif
+
+static void page_init(void)
+{
+    /* NOTE: we can always suppose that qemu_host_page_size >=
+       TARGET_PAGE_SIZE */
+#ifdef _WIN32
+    {
+        SYSTEM_INFO system_info;
+
+        GetSystemInfo(&system_info);
+        qemu_real_host_page_size = system_info.dwPageSize;
+    }
+#else
+    qemu_real_host_page_size = getpagesize();
+#endif
+    if (qemu_host_page_size == 0) {
+        qemu_host_page_size = qemu_real_host_page_size;
+    }
+    if (qemu_host_page_size < TARGET_PAGE_SIZE) {
+        qemu_host_page_size = TARGET_PAGE_SIZE;
+    }
+    qemu_host_page_mask = ~(qemu_host_page_size - 1);
+
+#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
+    {
+#ifdef HAVE_KINFO_GETVMMAP
+        struct kinfo_vmentry *freep;
+        int i, cnt;
+
+        freep = kinfo_getvmmap(getpid(), &cnt);
+        if (freep) {
+            mmap_lock();
+            for (i = 0; i < cnt; i++) {
+                unsigned long startaddr, endaddr;
+
+                startaddr = freep[i].kve_start;
+                endaddr = freep[i].kve_end;
+                if (h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+                    } else {
+#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
+                        endaddr = ~0ul;
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+#endif
+                    }
+                }
+            }
+            free(freep);
+            mmap_unlock();
+        }
+#else
+        FILE *f;
+
+        last_brk = (unsigned long)sbrk(0);
+
+        f = fopen("/compat/linux/proc/self/maps", "r");
+        if (f) {
+            mmap_lock();
+
+            do {
+                unsigned long startaddr, endaddr;
+                int n;
+
+                n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
+
+                if (n == 2 && h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                    } else {
+                        endaddr = ~0ul;
+                    }
+                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+                }
+            } while (!feof(f));
+
+            fclose(f);
+            mmap_unlock();
+        }
+#endif
+    }
+#endif
+}
+
+static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
+{
+    PageDesc *pd;
+    void **lp;
+    int i;
+
+#if defined(CONFIG_USER_ONLY)
+    /* We can't use g_malloc because it may recurse into a locked mutex. */
+# define ALLOC(P, SIZE)                                 \
+    do {                                                \
+        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
+                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
+    } while (0)
+#else
+# define ALLOC(P, SIZE) \
+    do { P = g_malloc0(SIZE); } while (0)
+#endif
+
+    /* Level 1.  Always allocated.  */
+    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
+
+    /* Level 2..N-1.  */
+    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
+        void **p = *lp;
+
+        if (p == NULL) {
+            if (!alloc) {
+                return NULL;
+            }
+            ALLOC(p, sizeof(void *) * L2_SIZE);
+            *lp = p;
+        }
+
+        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+    }
+
+    pd = *lp;
+    if (pd == NULL) {
+        if (!alloc) {
+            return NULL;
+        }
+        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
+        *lp = pd;
+    }
+
+#undef ALLOC
+
+    return pd + (index & (L2_SIZE - 1));
+}
+
+static inline PageDesc *page_find(tb_page_addr_t index)
+{
+    return page_find_alloc(index, 0);
+}
+
+#if !defined(CONFIG_USER_ONLY)
+#define mmap_lock() do { } while (0)
+#define mmap_unlock() do { } while (0)
+#endif
+
+#if defined(CONFIG_USER_ONLY)
+/* Currently it is not recommended to allocate big chunks of data in
+   user mode. It will change when a dedicated libc will be used.  */
+/* ??? 64-bit hosts ought to have no problem mmaping data outside the
+   region in which the guest needs to run.  Revisit this.  */
+#define USE_STATIC_CODE_GEN_BUFFER
+#endif
+
+/* ??? Should configure for this, not list operating systems here.  */
+#if (defined(__linux__) \
+    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
+    || defined(__DragonFly__) || defined(__OpenBSD__) \
+    || defined(__NetBSD__))
+# define USE_MMAP
+#endif
+
+/* Minimum size of the code gen buffer.  This number is randomly chosen,
+   but not so small that we can't have a fair number of TB's live.  */
+#define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
+
+/* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
+   indicated, this is constrained by the range of direct branches on the
+   host cpu, as used by the TCG implementation of goto_tb.  */
+#if defined(__x86_64__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__sparc__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__arm__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (16u * 1024 * 1024)
+#elif defined(__s390x__)
+  /* We have a +- 4GB range on the branches; leave some slop.  */
+# define MAX_CODE_GEN_BUFFER_SIZE  (3ul * 1024 * 1024 * 1024)
+#else
+# define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
+#endif
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE \
+  (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
+   ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
+
+static inline size_t size_code_gen_buffer(size_t tb_size)
+{
+    /* Size the buffer.  */
+    if (tb_size == 0) {
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+        tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
+#else
+        /* ??? Needs adjustments.  */
+        /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
+           static buffer, we could size this on RESERVED_VA, on the text
+           segment size of the executable, or continue to use the default.  */
+        tb_size = (unsigned long)(ram_size / 4);
+#endif
+    }
+    if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MIN_CODE_GEN_BUFFER_SIZE;
+    }
+    if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MAX_CODE_GEN_BUFFER_SIZE;
+    }
+    code_gen_buffer_size = tb_size;
+    return tb_size;
+}
+
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
+    __attribute__((aligned(CODE_GEN_ALIGN)));
+
+static inline void *alloc_code_gen_buffer(void)
+{
+    map_exec(static_code_gen_buffer, code_gen_buffer_size);
+    return static_code_gen_buffer;
+}
+#elif defined(USE_MMAP)
+static inline void *alloc_code_gen_buffer(void)
+{
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+    uintptr_t start = 0;
+    void *buf;
+
+    /* Constrain the position of the buffer based on the host cpu.
+       Note that these addresses are chosen in concert with the
+       addresses assigned in the relevant linker script file.  */
+# if defined(__PIE__) || defined(__PIC__)
+    /* Don't bother setting a preferred location if we're building
+       a position-independent executable.  We're more likely to get
+       an address near the main executable if we let the kernel
+       choose the address.  */
+# elif defined(__x86_64__) && defined(MAP_32BIT)
+    /* Force the memory down into low memory with the executable.
+       Leave the choice of exact location with the kernel.  */
+    flags |= MAP_32BIT;
+    /* Cannot expect to map more than 800MB in low memory.  */
+    if (code_gen_buffer_size > 800u * 1024 * 1024) {
+        code_gen_buffer_size = 800u * 1024 * 1024;
+    }
+# elif defined(__sparc__)
+    start = 0x40000000ul;
+# elif defined(__s390x__)
+    start = 0x90000000ul;
+# endif
+
+    buf = mmap((void *)start, code_gen_buffer_size,
+               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    return buf == MAP_FAILED ? NULL : buf;
+}
+#else
+static inline void *alloc_code_gen_buffer(void)
+{
+    void *buf = g_malloc(code_gen_buffer_size);
+
+    if (buf) {
+        map_exec(buf, code_gen_buffer_size);
+    }
+    return buf;
+}
+#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+
+static inline void code_gen_alloc(size_t tb_size)
+{
+    code_gen_buffer_size = size_code_gen_buffer(tb_size);
+    code_gen_buffer = alloc_code_gen_buffer();
+    if (code_gen_buffer == NULL) {
+        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
+        exit(1);
+    }
+
+    qemu_madvise(code_gen_buffer, code_gen_buffer_size, QEMU_MADV_HUGEPAGE);
+
+    /* Steal room for the prologue at the end of the buffer.  This ensures
+       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
+       from TB's to the prologue are going to be in range.  It also means
+       that we don't need to mark (additional) portions of the data segment
+       as executable.  */
+    code_gen_prologue = code_gen_buffer + code_gen_buffer_size - 1024;
+    code_gen_buffer_size -= 1024;
+
+    code_gen_buffer_max_size = code_gen_buffer_size -
+        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
+    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
+    tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
+}
+
+/* Must be called before using the QEMU cpus. 'tb_size' is the size
+   (in bytes) allocated to the translation buffer. Zero means default
+   size. */
+void tcg_exec_init(unsigned long tb_size)
+{
+    cpu_gen_init();
+    code_gen_alloc(tb_size);
+    code_gen_ptr = code_gen_buffer;
+    tcg_register_jit(code_gen_buffer, code_gen_buffer_size);
+    page_init();
+#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    /* There's no guest base to take into account, so go ahead and
+       initialize the prologue now.  */
+    tcg_prologue_init(&tcg_ctx);
+#endif
+}
+
+bool tcg_enabled(void)
+{
+    return code_gen_buffer != NULL;
+}
+
+/* Allocate a new translation block. Flush the translation buffer if
+   too many translation blocks or too much generated code. */
+static TranslationBlock *tb_alloc(target_ulong pc)
+{
+    TranslationBlock *tb;
+
+    if (nb_tbs >= code_gen_max_blocks ||
+        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size) {
+        return NULL;
+    }
+    tb = &tbs[nb_tbs++];
+    tb->pc = pc;
+    tb->cflags = 0;
+    return tb;
+}
+
+void tb_free(TranslationBlock *tb)
+{
+    /* In practice this is mostly used for single use temporary TB
+       Ignore the hard cases and just back up if this TB happens to
+       be the last one generated.  */
+    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
+        code_gen_ptr = tb->tc_ptr;
+        nb_tbs--;
+    }
+}
+
+static inline void invalidate_page_bitmap(PageDesc *p)
+{
+    if (p->code_bitmap) {
+        g_free(p->code_bitmap);
+        p->code_bitmap = NULL;
+    }
+    p->code_write_count = 0;
+}
+
+/* Set to NULL all the 'first_tb' fields in all PageDescs. */
+static void page_flush_tb_1(int level, void **lp)
+{
+    int i;
+
+    if (*lp == NULL) {
+        return;
+    }
+    if (level == 0) {
+        PageDesc *pd = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            pd[i].first_tb = NULL;
+            invalidate_page_bitmap(pd + i);
+        }
+    } else {
+        void **pp = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            page_flush_tb_1(level - 1, pp + i);
+        }
+    }
+}
+
+static void page_flush_tb(void)
+{
+    int i;
+
+    for (i = 0; i < V_L1_SIZE; i++) {
+        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+    }
+}
+
+/* flush all the translation blocks */
+/* XXX: tb_flush is currently not thread safe */
+void tb_flush(CPUArchState *env1)
+{
+    CPUArchState *env;
+
+#if defined(DEBUG_FLUSH)
+    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
+           (unsigned long)(code_gen_ptr - code_gen_buffer),
+           nb_tbs, nb_tbs > 0 ?
+           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
+#endif
+    if ((unsigned long)(code_gen_ptr - code_gen_buffer)
+        > code_gen_buffer_size) {
+        cpu_abort(env1, "Internal error: code buffer overflow\n");
+    }
+    nb_tbs = 0;
+
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
+    }
+
+    memset(tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof(void *));
+    page_flush_tb();
+
+    code_gen_ptr = code_gen_buffer;
+    /* XXX: flush processor icache at this point if cache flush is
+       expensive */
+    tb_flush_count++;
+}
+
+#ifdef DEBUG_TB_CHECK
+
+static void tb_invalidate_check(target_ulong address)
+{
+    TranslationBlock *tb;
+    int i;
+
+    address &= TARGET_PAGE_MASK;
+    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
+        for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
+            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
+                  address >= tb->pc + tb->size)) {
+                printf("ERROR invalidate: address=" TARGET_FMT_lx
+                       " PC=%08lx size=%04x\n",
+                       address, (long)tb->pc, tb->size);
+            }
+        }
+    }
+}
+
+/* verify that all the pages have correct rights for code */
+static void tb_page_check(void)
+{
+    TranslationBlock *tb;
+    int i, flags1, flags2;
+
+    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
+        for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
+            flags1 = page_get_flags(tb->pc);
+            flags2 = page_get_flags(tb->pc + tb->size - 1);
+            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
+                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
+                       (long)tb->pc, tb->size, flags1, flags2);
+            }
+        }
+    }
+}
+
+#endif
+
+static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+
+    for (;;) {
+        tb1 = *ptb;
+        if (tb1 == tb) {
+            *ptb = tb1->phys_hash_next;
+            break;
+        }
+        ptb = &tb1->phys_hash_next;
+    }
+}
+
+static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+    unsigned int n1;
+
+    for (;;) {
+        tb1 = *ptb;
+        n1 = (uintptr_t)tb1 & 3;
+        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+        if (tb1 == tb) {
+            *ptb = tb1->page_next[n1];
+            break;
+        }
+        ptb = &tb1->page_next[n1];
+    }
+}
+
+static inline void tb_jmp_remove(TranslationBlock *tb, int n)
+{
+    TranslationBlock *tb1, **ptb;
+    unsigned int n1;
+
+    ptb = &tb->jmp_next[n];
+    tb1 = *ptb;
+    if (tb1) {
+        /* find tb(n) in circular list */
+        for (;;) {
+            tb1 = *ptb;
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == n && tb1 == tb) {
+                break;
+            }
+            if (n1 == 2) {
+                ptb = &tb1->jmp_first;
+            } else {
+                ptb = &tb1->jmp_next[n1];
+            }
+        }
+        /* now we can suppress tb(n) from the list */
+        *ptb = tb->jmp_next[n];
+
+        tb->jmp_next[n] = NULL;
+    }
+}
+
+/* reset the jump entry 'n' of a TB so that it is not chained to
+   another TB */
+static inline void tb_reset_jump(TranslationBlock *tb, int n)
+{
+    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
+}
+
+/* invalidate one TB */
+void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
+{
+    CPUArchState *env;
+    PageDesc *p;
+    unsigned int h, n1;
+    tb_page_addr_t phys_pc;
+    TranslationBlock *tb1, *tb2;
+
+    /* remove the TB from the hash list */
+    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
+    h = tb_phys_hash_func(phys_pc);
+    tb_hash_remove(&tb_phys_hash[h], tb);
+
+    /* remove the TB from the page list */
+    if (tb->page_addr[0] != page_addr) {
+        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
+        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+
+    tb_invalidated_flag = 1;
+
+    /* remove the TB from the hash list */
+    h = tb_jmp_cache_hash_func(tb->pc);
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        if (env->tb_jmp_cache[h] == tb) {
+            env->tb_jmp_cache[h] = NULL;
+        }
+    }
+
+    /* suppress this TB from the two jump lists */
+    tb_jmp_remove(tb, 0);
+    tb_jmp_remove(tb, 1);
+
+    /* suppress any remaining jumps to this TB */
+    tb1 = tb->jmp_first;
+    for (;;) {
+        n1 = (uintptr_t)tb1 & 3;
+        if (n1 == 2) {
+            break;
+        }
+        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+        tb2 = tb1->jmp_next[n1];
+        tb_reset_jump(tb1, n1);
+        tb1->jmp_next[n1] = NULL;
+        tb1 = tb2;
+    }
+    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
+
+    tb_phys_invalidate_count++;
+}
+
+static inline void set_bits(uint8_t *tab, int start, int len)
+{
+    int end, mask, end1;
+
+    end = start + len;
+    tab += start >> 3;
+    mask = 0xff << (start & 7);
+    if ((start & ~7) == (end & ~7)) {
+        if (start < end) {
+            mask &= ~(0xff << (end & 7));
+            *tab |= mask;
+        }
+    } else {
+        *tab++ |= mask;
+        start = (start + 8) & ~7;
+        end1 = end & ~7;
+        while (start < end1) {
+            *tab++ = 0xff;
+            start += 8;
+        }
+        if (start < end) {
+            mask = ~(0xff << (end & 7));
+            *tab |= mask;
+        }
+    }
+}
+
+static void build_page_bitmap(PageDesc *p)
+{
+    int n, tb_start, tb_end;
+    TranslationBlock *tb;
+
+    p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
+
+    tb = p->first_tb;
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+        /* NOTE: this is subtle as a TB may span two physical pages */
+        if (n == 0) {
+            /* NOTE: tb_end may be after the end of the page, but
+               it is not a problem */
+            tb_start = tb->pc & ~TARGET_PAGE_MASK;
+            tb_end = tb_start + tb->size;
+            if (tb_end > TARGET_PAGE_SIZE) {
+                tb_end = TARGET_PAGE_SIZE;
+            }
+        } else {
+            tb_start = 0;
+            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
+        }
+        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
+        tb = tb->page_next[n];
+    }
+}
+
+TranslationBlock *tb_gen_code(CPUArchState *env,
+                              target_ulong pc, target_ulong cs_base,
+                              int flags, int cflags)
+{
+    TranslationBlock *tb;
+    uint8_t *tc_ptr;
+    tb_page_addr_t phys_pc, phys_page2;
+    target_ulong virt_page2;
+    int code_gen_size;
+
+    phys_pc = get_page_addr_code(env, pc);
+    tb = tb_alloc(pc);
+    if (!tb) {
+        /* flush must be done */
+        tb_flush(env);
+        /* cannot fail at this point */
+        tb = tb_alloc(pc);
+        /* Don't forget to invalidate previous TB info.  */
+        tb_invalidated_flag = 1;
+    }
+    tc_ptr = code_gen_ptr;
+    tb->tc_ptr = tc_ptr;
+    tb->cs_base = cs_base;
+    tb->flags = flags;
+    tb->cflags = cflags;
+    cpu_gen_code(env, tb, &code_gen_size);
+    code_gen_ptr = (void *)(((uintptr_t)code_gen_ptr + code_gen_size +
+                             CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+    /* check next page if needed */
+    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
+    phys_page2 = -1;
+    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
+        phys_page2 = get_page_addr_code(env, virt_page2);
+    }
+    tb_link_page(tb, phys_pc, phys_page2);
+    return tb;
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical address range
+ * [start;end[. NOTE: start and end may refer to *different* physical pages.
+ * 'is_cpu_write_access' should be true if called from a real cpu write
+ * access: the virtual CPU will exit the current TB if code is modified inside
+ * this TB.
+ */
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
+                              int is_cpu_write_access)
+{
+    while (start < end) {
+        tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
+        start &= TARGET_PAGE_MASK;
+        start += TARGET_PAGE_SIZE;
+    }
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical address range
+ * [start;end[. NOTE: start and end must refer to the *same* physical page.
+ * 'is_cpu_write_access' should be true if called from a real cpu write
+ * access: the virtual CPU will exit the current TB if code is modified inside
+ * this TB.
+ */
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
+                                   int is_cpu_write_access)
+{
+    TranslationBlock *tb, *tb_next, *saved_tb;
+    CPUArchState *env = cpu_single_env;
+    tb_page_addr_t tb_start, tb_end;
+    PageDesc *p;
+    int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+    int current_tb_not_found = is_cpu_write_access;
+    TranslationBlock *current_tb = NULL;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
+#endif /* TARGET_HAS_PRECISE_SMC */
+
+    p = page_find(start >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    if (!p->code_bitmap &&
+        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
+        is_cpu_write_access) {
+        /* build code bitmap */
+        build_page_bitmap(p);
+    }
+
+    /* we remove all the TBs in the range [start, end[ */
+    /* XXX: see if in some cases it could be faster to invalidate all
+       the code */
+    tb = p->first_tb;
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+        tb_next = tb->page_next[n];
+        /* NOTE: this is subtle as a TB may span two physical pages */
+        if (n == 0) {
+            /* NOTE: tb_end may be after the end of the page, but
+               it is not a problem */
+            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
+            tb_end = tb_start + tb->size;
+        } else {
+            tb_start = tb->page_addr[1];
+            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
+        }
+        if (!(tb_end <= start || tb_start >= end)) {
+#ifdef TARGET_HAS_PRECISE_SMC
+            if (current_tb_not_found) {
+                current_tb_not_found = 0;
+                current_tb = NULL;
+                if (env->mem_io_pc) {
+                    /* now we have a real cpu fault */
+                    current_tb = tb_find_pc(env->mem_io_pc);
+                }
+            }
+            if (current_tb == tb &&
+                (current_tb->cflags & CF_COUNT_MASK) != 1) {
+                /* If we are modifying the current TB, we must stop
+                its execution. We could be more precise by checking
+                that the modification is after the current PC, but it
+                would require a specialized function to partially
+                restore the CPU state */
+
+                current_tb_modified = 1;
+                cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
+                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                     &current_flags);
+            }
+#endif /* TARGET_HAS_PRECISE_SMC */
+            /* we need to do that to handle the case where a signal
+               occurs while doing tb_phys_invalidate() */
+            saved_tb = NULL;
+            if (env) {
+                saved_tb = env->current_tb;
+                env->current_tb = NULL;
+            }
+            tb_phys_invalidate(tb, -1);
+            if (env) {
+                env->current_tb = saved_tb;
+                if (env->interrupt_request && env->current_tb) {
+                    cpu_interrupt(env, env->interrupt_request);
+                }
+            }
+        }
+        tb = tb_next;
+    }
+#if !defined(CONFIG_USER_ONLY)
+    /* if no code remaining, no need to continue to use slow writes */
+    if (!p->first_tb) {
+        invalidate_page_bitmap(p);
+        if (is_cpu_write_access) {
+            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
+        }
+    }
+#endif
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (current_tb_modified) {
+        /* we generate a block containing just the instruction
+           modifying the memory. It will ensure that it cannot modify
+           itself */
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, NULL);
+    }
+#endif
+}
+
+/* len must be <= 8 and start must be a multiple of len */
+void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
+{
+    PageDesc *p;
+    int offset, b;
+
+#if 0
+    if (1) {
+        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
+                  cpu_single_env->mem_io_vaddr, len,
+                  cpu_single_env->eip,
+                  cpu_single_env->eip +
+                  (intptr_t)cpu_single_env->segs[R_CS].base);
+    }
+#endif
+    p = page_find(start >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    if (p->code_bitmap) {
+        offset = start & ~TARGET_PAGE_MASK;
+        b = p->code_bitmap[offset >> 3] >> (offset & 7);
+        if (b & ((1 << len) - 1)) {
+            goto do_invalidate;
+        }
+    } else {
+    do_invalidate:
+        tb_invalidate_phys_page_range(start, start + len, 1);
+    }
+}
+
+#if !defined(CONFIG_SOFTMMU)
+static void tb_invalidate_phys_page(tb_page_addr_t addr,
+                                    uintptr_t pc, void *puc)
+{
+    TranslationBlock *tb;
+    PageDesc *p;
+    int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+    TranslationBlock *current_tb = NULL;
+    CPUArchState *env = cpu_single_env;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
+#endif
+
+    addr &= TARGET_PAGE_MASK;
+    p = page_find(addr >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    tb = p->first_tb;
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (tb && pc != 0) {
+        current_tb = tb_find_pc(pc);
+    }
+#endif
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+#ifdef TARGET_HAS_PRECISE_SMC
+        if (current_tb == tb &&
+            (current_tb->cflags & CF_COUNT_MASK) != 1) {
+                /* If we are modifying the current TB, we must stop
+                   its execution. We could be more precise by checking
+                   that the modification is after the current PC, but it
+                   would require a specialized function to partially
+                   restore the CPU state */
+
+            current_tb_modified = 1;
+            cpu_restore_state_from_tb(current_tb, env, pc);
+            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                 &current_flags);
+        }
+#endif /* TARGET_HAS_PRECISE_SMC */
+        tb_phys_invalidate(tb, addr);
+        tb = tb->page_next[n];
+    }
+    p->first_tb = NULL;
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (current_tb_modified) {
+        /* we generate a block containing just the instruction
+           modifying the memory. It will ensure that it cannot modify
+           itself */
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, puc);
+    }
+#endif
+}
+#endif
+
+/* add the tb in the target page and protect it if necessary */
+static inline void tb_alloc_page(TranslationBlock *tb,
+                                 unsigned int n, tb_page_addr_t page_addr)
+{
+    PageDesc *p;
+#ifndef CONFIG_USER_ONLY
+    bool page_already_protected;
+#endif
+
+    tb->page_addr[n] = page_addr;
+    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
+    tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+    page_already_protected = p->first_tb != NULL;
+#endif
+    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
+    invalidate_page_bitmap(p);
+
+#if defined(TARGET_HAS_SMC) || 1
+
+#if defined(CONFIG_USER_ONLY)
+    if (p->flags & PAGE_WRITE) {
+        target_ulong addr;
+        PageDesc *p2;
+        int prot;
+
+        /* force the host page as non writable (writes will have a
+           page fault + mprotect overhead) */
+        page_addr &= qemu_host_page_mask;
+        prot = 0;
+        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
+            addr += TARGET_PAGE_SIZE) {
+
+            p2 = page_find(addr >> TARGET_PAGE_BITS);
+            if (!p2) {
+                continue;
+            }
+            prot |= p2->flags;
+            p2->flags &= ~PAGE_WRITE;
+          }
+        mprotect(g2h(page_addr), qemu_host_page_size,
+                 (prot & PAGE_BITS) & ~PAGE_WRITE);
+#ifdef DEBUG_TB_INVALIDATE
+        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
+               page_addr);
+#endif
+    }
+#else
+    /* if some code is already present, then the pages are already
+       protected. So we handle the case where only the first TB is
+       allocated in a physical page */
+    if (!page_already_protected) {
+        tlb_protect_code(page_addr);
+    }
+#endif
+
+#endif /* TARGET_HAS_SMC */
+}
+
+/* add a new TB and link it to the physical page tables. phys_page2 is
+   (-1) to indicate that only one page contains the TB. */
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2)
+{
+    unsigned int h;
+    TranslationBlock **ptb;
+
+    /* Grab the mmap lock to stop another thread invalidating this TB
+       before we are done.  */
+    mmap_lock();
+    /* add in the physical hash table */
+    h = tb_phys_hash_func(phys_pc);
+    ptb = &tb_phys_hash[h];
+    tb->phys_hash_next = *ptb;
+    *ptb = tb;
+
+    /* add in the page list */
+    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
+    if (phys_page2 != -1) {
+        tb_alloc_page(tb, 1, phys_page2);
+    } else {
+        tb->page_addr[1] = -1;
+    }
+
+    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
+    tb->jmp_next[0] = NULL;
+    tb->jmp_next[1] = NULL;
+
+    /* init original jump addresses */
+    if (tb->tb_next_offset[0] != 0xffff) {
+        tb_reset_jump(tb, 0);
+    }
+    if (tb->tb_next_offset[1] != 0xffff) {
+        tb_reset_jump(tb, 1);
+    }
+
+#ifdef DEBUG_TB_CHECK
+    tb_page_check();
+#endif
+    mmap_unlock();
+}
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* check whether the given addr is in TCG generated code buffer or not */
+bool is_tcg_gen_code(uintptr_t tc_ptr)
+{
+    /* This can be called during code generation, code_gen_buffer_max_size
+       is used instead of code_gen_ptr for upper boundary checking */
+    return (tc_ptr >= (uintptr_t)code_gen_buffer &&
+            tc_ptr < (uintptr_t)(code_gen_buffer + code_gen_buffer_max_size));
+}
+#endif
+
+/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
+   tb[1].tc_ptr. Return NULL if not found */
+static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
+{
+    int m_min, m_max, m;
+    uintptr_t v;
+    TranslationBlock *tb;
+
+    if (nb_tbs <= 0) {
+        return NULL;
+    }
+    if (tc_ptr < (uintptr_t)code_gen_buffer ||
+        tc_ptr >= (uintptr_t)code_gen_ptr) {
+        return NULL;
+    }
+    /* binary search (cf Knuth) */
+    m_min = 0;
+    m_max = nb_tbs - 1;
+    while (m_min <= m_max) {
+        m = (m_min + m_max) >> 1;
+        tb = &tbs[m];
+        v = (uintptr_t)tb->tc_ptr;
+        if (v == tc_ptr) {
+            return tb;
+        } else if (tc_ptr < v) {
+            m_max = m - 1;
+        } else {
+            m_min = m + 1;
+        }
+    }
+    return &tbs[m_max];
+}
+
+static void tb_reset_jump_recursive(TranslationBlock *tb);
+
+static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
+{
+    TranslationBlock *tb1, *tb_next, **ptb;
+    unsigned int n1;
+
+    tb1 = tb->jmp_next[n];
+    if (tb1 != NULL) {
+        /* find head of list */
+        for (;;) {
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == 2) {
+                break;
+            }
+            tb1 = tb1->jmp_next[n1];
+        }
+        /* we are now sure now that tb jumps to tb1 */
+        tb_next = tb1;
+
+        /* remove tb from the jmp_first list */
+        ptb = &tb_next->jmp_first;
+        for (;;) {
+            tb1 = *ptb;
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == n && tb1 == tb) {
+                break;
+            }
+            ptb = &tb1->jmp_next[n1];
+        }
+        *ptb = tb->jmp_next[n];
+        tb->jmp_next[n] = NULL;
+
+        /* suppress the jump to next tb in generated code */
+        tb_reset_jump(tb, n);
+
+        /* suppress jumps in the tb on which we could have jumped */
+        tb_reset_jump_recursive(tb_next);
+    }
+}
+
+static void tb_reset_jump_recursive(TranslationBlock *tb)
+{
+    tb_reset_jump_recursive2(tb, 0);
+    tb_reset_jump_recursive2(tb, 1);
+}
+
+#if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
+void tb_invalidate_phys_addr(hwaddr addr)
+{
+    ram_addr_t ram_addr;
+    MemoryRegionSection *section;
+
+    section = phys_page_find(address_space_memory.dispatch,
+                             addr >> TARGET_PAGE_BITS);
+    if (!(memory_region_is_ram(section->mr)
+          || (section->mr->rom_device && section->mr->readable))) {
+        return;
+    }
+    ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
+        + memory_region_section_addr(section, addr);
+    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
+}
+#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
+
+void cpu_unlink_tb(CPUArchState *env)
+{
+    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
+       problem and hope the cpu will stop of its own accord.  For userspace
+       emulation this often isn't actually as bad as it sounds.  Often
+       signals are used primarily to interrupt blocking syscalls.  */
+    TranslationBlock *tb;
+    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
+
+    spin_lock(&interrupt_lock);
+    tb = env->current_tb;
+    /* if the cpu is currently executing code, we must unlink it and
+       all the potentially executing TB */
+    if (tb) {
+        env->current_tb = NULL;
+        tb_reset_jump_recursive(tb);
+    }
+    spin_unlock(&interrupt_lock);
+}
+
+void tb_check_watchpoint(CPUArchState *env)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_pc(env->mem_io_pc);
+    if (!tb) {
+        cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
+                  (void *)env->mem_io_pc);
+    }
+    cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
+    tb_phys_invalidate(tb, -1);
+}
+
+#ifndef CONFIG_USER_ONLY
+/* mask must never be zero, except for A20 change call */
+static void tcg_handle_interrupt(CPUArchState *env, int mask)
+{
+    CPUState *cpu = ENV_GET_CPU(env);
+    int old_mask;
+
+    old_mask = env->interrupt_request;
+    env->interrupt_request |= mask;
+
+    /*
+     * If called from iothread context, wake the target cpu in
+     * case its halted.
+     */
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
+        return;
+    }
+
+    if (use_icount) {
+        env->icount_decr.u16.high = 0xffff;
+        if (!can_do_io(env)
+            && (mask & ~old_mask) != 0) {
+            cpu_abort(env, "Raised interrupt while not in I/O function");
+        }
+    } else {
+        cpu_unlink_tb(env);
+    }
+}
+
+CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
+
+/* in deterministic execution mode, instructions doing device I/Os
+   must be at the end of the TB */
+void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
+{
+    TranslationBlock *tb;
+    uint32_t n, cflags;
+    target_ulong pc, cs_base;
+    uint64_t flags;
+
+    tb = tb_find_pc(retaddr);
+    if (!tb) {
+        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
+                  (void *)retaddr);
+    }
+    n = env->icount_decr.u16.low + tb->icount;
+    cpu_restore_state_from_tb(tb, env, retaddr);
+    /* Calculate how many instructions had been executed before the fault
+       occurred.  */
+    n = n - env->icount_decr.u16.low;
+    /* Generate a new TB ending on the I/O insn.  */
+    n++;
+    /* On MIPS and SH, delay slot instructions can only be restarted if
+       they were already the first instruction in the TB.  If this is not
+       the first instruction in a TB then re-execute the preceding
+       branch.  */
+#if defined(TARGET_MIPS)
+    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
+        env->active_tc.PC -= 4;
+        env->icount_decr.u16.low++;
+        env->hflags &= ~MIPS_HFLAG_BMASK;
+    }
+#elif defined(TARGET_SH4)
+    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
+            && n > 1) {
+        env->pc -= 2;
+        env->icount_decr.u16.low++;
+        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
+    }
+#endif
+    /* This should never happen.  */
+    if (n > CF_COUNT_MASK) {
+        cpu_abort(env, "TB too big during recompile");
+    }
+
+    cflags = n | CF_LAST_IO;
+    pc = tb->pc;
+    cs_base = tb->cs_base;
+    flags = tb->flags;
+    tb_phys_invalidate(tb, -1);
+    /* FIXME: In theory this could raise an exception.  In practice
+       we have already translated the block once so it's probably ok.  */
+    tb_gen_code(env, pc, cs_base, flags, cflags);
+    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
+       the first in the TB) then we end up generating a whole new TB and
+       repeating the fault, which is horribly inefficient.
+       Better would be to execute just this insn uncached, or generate a
+       second new TB.  */
+    cpu_resume_from_signal(env, NULL);
+}
+
+void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
+{
+    unsigned int i;
+
+    /* Discard jump cache entries for any tb which might potentially
+       overlap the flushed page.  */
+    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
+    memset(&env->tb_jmp_cache[i], 0,
+           TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
+
+    i = tb_jmp_cache_hash_page(addr);
+    memset(&env->tb_jmp_cache[i], 0,
+           TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
+}
+
+void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
+{
+    int i, target_code_size, max_target_code_size;
+    int direct_jmp_count, direct_jmp2_count, cross_page;
+    TranslationBlock *tb;
+
+    target_code_size = 0;
+    max_target_code_size = 0;
+    cross_page = 0;
+    direct_jmp_count = 0;
+    direct_jmp2_count = 0;
+    for (i = 0; i < nb_tbs; i++) {
+        tb = &tbs[i];
+        target_code_size += tb->size;
+        if (tb->size > max_target_code_size) {
+            max_target_code_size = tb->size;
+        }
+        if (tb->page_addr[1] != -1) {
+            cross_page++;
+        }
+        if (tb->tb_next_offset[0] != 0xffff) {
+            direct_jmp_count++;
+            if (tb->tb_next_offset[1] != 0xffff) {
+                direct_jmp2_count++;
+            }
+        }
+    }
+    /* XXX: avoid using doubles ? */
+    cpu_fprintf(f, "Translation buffer state:\n");
+    cpu_fprintf(f, "gen code size       %td/%zd\n",
+                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
+    cpu_fprintf(f, "TB count            %d/%d\n",
+                nb_tbs, code_gen_max_blocks);
+    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
+                nb_tbs ? target_code_size / nb_tbs : 0,
+                max_target_code_size);
+    cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
+                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
+                target_code_size ? (double) (code_gen_ptr - code_gen_buffer)
+                / target_code_size : 0);
+    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
+            cross_page,
+            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
+    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
+                direct_jmp_count,
+                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
+                direct_jmp2_count,
+                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
+    cpu_fprintf(f, "\nStatistics:\n");
+    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
+    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
+    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
+    tcg_dump_info(f, cpu_fprintf);
+}
+
+#else /* CONFIG_USER_ONLY */
+
+void cpu_interrupt(CPUArchState *env, int mask)
+{
+    env->interrupt_request |= mask;
+    cpu_unlink_tb(env);
+}
+
+/*
+ * Walks guest process memory "regions" one by one
+ * and calls callback function 'fn' for each region.
+ */
+struct walk_memory_regions_data {
+    walk_memory_regions_fn fn;
+    void *priv;
+    uintptr_t start;
+    int prot;
+};
+
+static int walk_memory_regions_end(struct walk_memory_regions_data *data,
+                                   abi_ulong end, int new_prot)
+{
+    if (data->start != -1ul) {
+        int rc = data->fn(data->priv, data->start, end, data->prot);
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    data->start = (new_prot ? end : -1ul);
+    data->prot = new_prot;
+
+    return 0;
+}
+
+static int walk_memory_regions_1(struct walk_memory_regions_data *data,
+                                 abi_ulong base, int level, void **lp)
+{
+    abi_ulong pa;
+    int i, rc;
+
+    if (*lp == NULL) {
+        return walk_memory_regions_end(data, base, 0);
+    }
+
+    if (level == 0) {
+        PageDesc *pd = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            int prot = pd[i].flags;
+
+            pa = base | (i << TARGET_PAGE_BITS);
+            if (prot != data->prot) {
+                rc = walk_memory_regions_end(data, pa, prot);
+                if (rc != 0) {
+                    return rc;
+                }
+            }
+        }
+    } else {
+        void **pp = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            pa = base | ((abi_ulong)i <<
+                (TARGET_PAGE_BITS + L2_BITS * level));
+            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
+            if (rc != 0) {
+                return rc;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
+{
+    struct walk_memory_regions_data data;
+    uintptr_t i;
+
+    data.fn = fn;
+    data.priv = priv;
+    data.start = -1ul;
+    data.prot = 0;
+
+    for (i = 0; i < V_L1_SIZE; i++) {
+        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
+                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    return walk_memory_regions_end(&data, 0, 0);
+}
+
+static int dump_region(void *priv, abi_ulong start,
+    abi_ulong end, unsigned long prot)
+{
+    FILE *f = (FILE *)priv;
+
+    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
+        " "TARGET_ABI_FMT_lx" %c%c%c\n",
+        start, end, end - start,
+        ((prot & PAGE_READ) ? 'r' : '-'),
+        ((prot & PAGE_WRITE) ? 'w' : '-'),
+        ((prot & PAGE_EXEC) ? 'x' : '-'));
+
+    return 0;
+}
+
+/* dump memory mappings */
+void page_dump(FILE *f)
+{
+    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
+            "start", "end", "size", "prot");
+    walk_memory_regions(f, dump_region);
+}
+
+int page_get_flags(target_ulong address)
+{
+    PageDesc *p;
+
+    p = page_find(address >> TARGET_PAGE_BITS);
+    if (!p) {
+        return 0;
+    }
+    return p->flags;
+}
+
+/* Modify the flags of a page and invalidate the code if necessary.
+   The flag PAGE_WRITE_ORG is positioned automatically depending
+   on PAGE_WRITE.  The mmap_lock should already be held.  */
+void page_set_flags(target_ulong start, target_ulong end, int flags)
+{
+    target_ulong addr, len;
+
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+    assert(start < end);
+
+    start = start & TARGET_PAGE_MASK;
+    end = TARGET_PAGE_ALIGN(end);
+
+    if (flags & PAGE_WRITE) {
+        flags |= PAGE_WRITE_ORG;
+    }
+
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
+
+        /* If the write protection bit is set, then we invalidate
+           the code inside.  */
+        if (!(p->flags & PAGE_WRITE) &&
+            (flags & PAGE_WRITE) &&
+            p->first_tb) {
+            tb_invalidate_phys_page(addr, 0, NULL);
+        }
+        p->flags = flags;
+    }
+}
+
+int page_check_range(target_ulong start, target_ulong len, int flags)
+{
+    PageDesc *p;
+    target_ulong end;
+    target_ulong addr;
+
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+
+    if (len == 0) {
+        return 0;
+    }
+    if (start + len - 1 < start) {
+        /* We've wrapped around.  */
+        return -1;
+    }
+
+    /* must do before we loose bits in the next step */
+    end = TARGET_PAGE_ALIGN(start + len);
+    start = start & TARGET_PAGE_MASK;
+
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+        p = page_find(addr >> TARGET_PAGE_BITS);
+        if (!p) {
+            return -1;
+        }
+        if (!(p->flags & PAGE_VALID)) {
+            return -1;
+        }
+
+        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
+            return -1;
+        }
+        if (flags & PAGE_WRITE) {
+            if (!(p->flags & PAGE_WRITE_ORG)) {
+                return -1;
+            }
+            /* unprotect the page if it was put read-only because it
+               contains translated code */
+            if (!(p->flags & PAGE_WRITE)) {
+                if (!page_unprotect(addr, 0, NULL)) {
+                    return -1;
+                }
+            }
+            return 0;
+        }
+    }
+    return 0;
+}
+
+/* called from signal handler: invalidate the code and unprotect the
+   page. Return TRUE if the fault was successfully handled. */
+int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
+{
+    unsigned int prot;
+    PageDesc *p;
+    target_ulong host_start, host_end, addr;
+
+    /* Technically this isn't safe inside a signal handler.  However we
+       know this only ever happens in a synchronous SEGV handler, so in
+       practice it seems to be ok.  */
+    mmap_lock();
+
+    p = page_find(address >> TARGET_PAGE_BITS);
+    if (!p) {
+        mmap_unlock();
+        return 0;
+    }
+
+    /* if the page was really writable, then we change its
+       protection back to writable */
+    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
+        host_start = address & qemu_host_page_mask;
+        host_end = host_start + qemu_host_page_size;
+
+        prot = 0;
+        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
+            p = page_find(addr >> TARGET_PAGE_BITS);
+            p->flags |= PAGE_WRITE;
+            prot |= p->flags;
+
+            /* and since the content will be modified, we must invalidate
+               the corresponding translated code. */
+            tb_invalidate_phys_page(addr, pc, puc);
+#ifdef DEBUG_TB_CHECK
+            tb_invalidate_check(addr);
+#endif
+        }
+        mprotect((void *)g2h(host_start), qemu_host_page_size,
+                 prot & PAGE_BITS);
+
+        mmap_unlock();
+        return 1;
+    }
+    mmap_unlock();
+    return 0;
+}
+#endif /* CONFIG_USER_ONLY */
diff --git a/target-ppc/mpic_helper.c b/translate-all.h
index 2c6a4d30a9c..b181fb48ad2 100644
--- a/target-ppc/mpic_helper.c
+++ b/translate-all.h
@@ -1,7 +1,7 @@
 /*
- *  PowerPC emulation helpers for QEMU.
+ *  Translated block handling
  *
- *  Copyright (c) 2003-2007 Jocelyn Mayer
+ *  Copyright (c) 2003 Fabrice Bellard
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -16,20 +16,19 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "cpu.h"
-#include "helper.h"
+#ifndef TRANSLATE_ALL_H
+#define TRANSLATE_ALL_H
 
-/*****************************************************************************/
-/* SPR accesses */
+/* Size of the L2 (and L3, etc) page tables.  */
+#define L2_BITS 10
+#define L2_SIZE (1 << L2_BITS)
 
-#if !defined(CONFIG_USER_ONLY)
-/*
- * This is an ugly helper for EPR, which is basically the same as accessing
- * the IACK (PIAC) register on the MPIC. Because we model the MPIC as a device
- * that can only talk to the CPU through MMIO, let's access it that way!
- */
-target_ulong helper_load_epr(CPUPPCState *env)
-{
-    return ldl_phys(env->mpic_cpu_base + 0xA0);
-}
-#endif
+#define P_L2_LEVELS \
+    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
+
+/* translate-all.c */
+void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len);
+void cpu_unlink_tb(CPUArchState *env);
+void tb_check_watchpoint(CPUArchState *env);
+
+#endif /* TRANSLATE_ALL_H */
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index adc07be7611..d9db0735843 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -4,11 +4,16 @@ vnc-obj-y += vnc-enc-tight.o vnc-palette.o
 vnc-obj-y += vnc-enc-zrle.o
 vnc-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
 vnc-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
+vnc-obj-$(CONFIG_VNC_WS) += vnc-ws.o
 vnc-obj-y += vnc-jobs.o
 
-common-obj-y += keymaps.o
+common-obj-y += keymaps.o console.o cursor.o input.o qemu-pixman.o
 common-obj-$(CONFIG_SPICE) += spice-core.o spice-input.o spice-display.o
 common-obj-$(CONFIG_SDL) += sdl.o sdl_zoom.o x_keymap.o
 common-obj-$(CONFIG_COCOA) += cocoa.o
 common-obj-$(CONFIG_CURSES) += curses.o
 common-obj-$(CONFIG_VNC) += $(vnc-obj-y)
+
+$(obj)/sdl.o $(obj)/sdl_zoom.o: QEMU_CFLAGS += $(SDL_CFLAGS) 
+
+$(obj)/cocoa.o: $(SRC_PATH)/$(obj)/cocoa.m
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 2383646dc04..3bf1c6e8901 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -26,8 +26,8 @@
 #include <crt_externs.h>
 
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 #ifndef MAC_OS_X_VERSION_10_4
 #define MAC_OS_X_VERSION_10_4 1040
@@ -1017,8 +1017,8 @@ void cocoa_display_init(DisplayState *ds, int full_screen)
     dcl = g_malloc0(sizeof(DisplayChangeListener));
 
     // register vga output callbacks
-    dcl->dpy_update = cocoa_update;
-    dcl->dpy_resize = cocoa_resize;
+    dcl->dpy_gfx_update = cocoa_update;
+    dcl->dpy_gfx_resize = cocoa_resize;
     dcl->dpy_refresh = cocoa_refresh;
 
 	register_displaychangelistener(ds, dcl);
diff --git a/console.c b/ui/console.c
index 3b5cabba3a6..d880ebff07e 100644
--- a/console.c
+++ b/ui/console.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
+#include "qmp-commands.h"
+#include "char/char.h"
 
 //#define DEBUG_CONSOLE
 #define DEFAULT_BACKSCROLL 512
@@ -113,20 +115,20 @@ typedef enum {
     TEXT_CONSOLE_FIXED_SIZE
 } console_type_t;
 
-/* ??? This is mis-named.
-   It is used for both text and graphical consoles.  */
-struct TextConsole {
+struct QemuConsole {
     int index;
     console_type_t console_type;
     DisplayState *ds;
+
     /* Graphic console state.  */
     vga_hw_update_ptr hw_update;
     vga_hw_invalidate_ptr hw_invalidate;
     vga_hw_screen_dump_ptr hw_screen_dump;
     vga_hw_text_update_ptr hw_text_update;
     void *hw;
-
     int g_width, g_height;
+
+    /* Text console state */
     int width;
     int height;
     int total_height;
@@ -160,8 +162,8 @@ struct TextConsole {
 };
 
 static DisplayState *display_state;
-static TextConsole *active_console;
-static TextConsole *consoles[MAX_CONSOLES];
+static QemuConsole *active_console;
+static QemuConsole *consoles[MAX_CONSOLES];
 static int nb_consoles = 0;
 
 void vga_hw_update(void)
@@ -176,9 +178,9 @@ void vga_hw_invalidate(void)
         active_console->hw_invalidate(active_console->hw);
 }
 
-void vga_hw_screen_dump(const char *filename)
+void qmp_screendump(const char *filename, Error **errp)
 {
-    TextConsole *previous_active_console;
+    QemuConsole *previous_active_console;
     bool cswitch;
 
     previous_active_console = active_console;
@@ -190,9 +192,9 @@ void vga_hw_screen_dump(const char *filename)
         console_select(0);
     }
     if (consoles[0] && consoles[0]->hw_screen_dump) {
-        consoles[0]->hw_screen_dump(consoles[0]->hw, filename, cswitch);
+        consoles[0]->hw_screen_dump(consoles[0]->hw, filename, cswitch, errp);
     } else {
-        error_report("screen dump not implemented");
+        error_setg(errp, "device doesn't support screendump\n");
     }
 
     if (cswitch) {
@@ -520,7 +522,7 @@ static void vga_putcharxy(DisplayState *ds, int x, int y, int ch,
     }
 }
 
-static void text_console_resize(TextConsole *s)
+static void text_console_resize(QemuConsole *s)
 {
     TextCell *cells, *c, *c1;
     int w1, x, y, last_width;
@@ -552,7 +554,7 @@ static void text_console_resize(TextConsole *s)
     s->cells = cells;
 }
 
-static inline void text_update_xy(TextConsole *s, int x, int y)
+static inline void text_update_xy(QemuConsole *s, int x, int y)
 {
     s->text_x[0] = MIN(s->text_x[0], x);
     s->text_x[1] = MAX(s->text_x[1], x);
@@ -560,7 +562,7 @@ static inline void text_update_xy(TextConsole *s, int x, int y)
     s->text_y[1] = MAX(s->text_y[1], y);
 }
 
-static void invalidate_xy(TextConsole *s, int x, int y)
+static void invalidate_xy(QemuConsole *s, int x, int y)
 {
     if (s->update_x0 > x * FONT_WIDTH)
         s->update_x0 = x * FONT_WIDTH;
@@ -572,7 +574,7 @@ static void invalidate_xy(TextConsole *s, int x, int y)
         s->update_y1 = (y + 1) * FONT_HEIGHT;
 }
 
-static void update_xy(TextConsole *s, int x, int y)
+static void update_xy(QemuConsole *s, int x, int y)
 {
     TextCell *c;
     int y1, y2;
@@ -596,7 +598,7 @@ static void update_xy(TextConsole *s, int x, int y)
     }
 }
 
-static void console_show_cursor(TextConsole *s, int show)
+static void console_show_cursor(QemuConsole *s, int show)
 {
     TextCell *c;
     int y, y1;
@@ -630,42 +632,45 @@ static void console_show_cursor(TextConsole *s, int show)
     }
 }
 
-static void console_refresh(TextConsole *s)
+static void console_refresh(QemuConsole *s)
 {
     TextCell *c;
     int x, y, y1;
 
     if (s != active_console)
         return;
-    if (!ds_get_bits_per_pixel(s->ds)) {
+
+    if (s->ds->have_text) {
         s->text_x[0] = 0;
         s->text_y[0] = 0;
         s->text_x[1] = s->width - 1;
         s->text_y[1] = s->height - 1;
         s->cursor_invalidate = 1;
-        return;
     }
 
-    vga_fill_rect(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds),
-                  color_table[0][COLOR_BLACK]);
-    y1 = s->y_displayed;
-    for(y = 0; y < s->height; y++) {
-        c = s->cells + y1 * s->width;
-        for(x = 0; x < s->width; x++) {
-            vga_putcharxy(s->ds, x, y, c->ch,
-                          &(c->t_attrib));
-            c++;
+    if (s->ds->have_gfx) {
+        vga_fill_rect(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds),
+                      color_table[0][COLOR_BLACK]);
+        y1 = s->y_displayed;
+        for (y = 0; y < s->height; y++) {
+            c = s->cells + y1 * s->width;
+            for (x = 0; x < s->width; x++) {
+                vga_putcharxy(s->ds, x, y, c->ch,
+                              &(c->t_attrib));
+                c++;
+            }
+            if (++y1 == s->total_height) {
+                y1 = 0;
+            }
         }
-        if (++y1 == s->total_height)
-            y1 = 0;
+        console_show_cursor(s, 1);
+        dpy_gfx_update(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds));
     }
-    console_show_cursor(s, 1);
-    dpy_update(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds));
 }
 
 static void console_scroll(int ydelta)
 {
-    TextConsole *s;
+    QemuConsole *s;
     int i, y1;
 
     s = active_console;
@@ -697,7 +702,7 @@ static void console_scroll(int ydelta)
     console_refresh(s);
 }
 
-static void console_put_lf(TextConsole *s)
+static void console_put_lf(QemuConsole *s)
 {
     TextCell *c;
     int x, y1;
@@ -748,7 +753,7 @@ static void console_put_lf(TextConsole *s)
  * NOTE: I know this code is not very efficient (checking every color for it
  * self) but it is more readable and better maintainable.
  */
-static void console_handle_escape(TextConsole *s)
+static void console_handle_escape(QemuConsole *s)
 {
     int i;
 
@@ -841,7 +846,7 @@ static void console_handle_escape(TextConsole *s)
     }
 }
 
-static void console_clear_xy(TextConsole *s, int x, int y)
+static void console_clear_xy(QemuConsole *s, int x, int y)
 {
     int y1 = (s->y_base + y) % s->total_height;
     TextCell *c = &s->cells[y1 * s->width + x];
@@ -851,7 +856,7 @@ static void console_clear_xy(TextConsole *s, int x, int y)
 }
 
 /* set cursor, checking bounds */
-static void set_cursor(TextConsole *s, int x, int y)
+static void set_cursor(QemuConsole *s, int x, int y)
 {
     if (x < 0) {
         x = 0;
@@ -870,7 +875,7 @@ static void set_cursor(TextConsole *s, int x, int y)
     s->y = y;
 }
 
-static void console_putchar(TextConsole *s, int ch)
+static void console_putchar(QemuConsole *s, int ch)
 {
     TextCell *c;
     int y1, i;
@@ -937,8 +942,11 @@ static void console_putchar(TextConsole *s, int ch)
     case TTY_STATE_CSI: /* handle escape sequence parameters */
         if (ch >= '0' && ch <= '9') {
             if (s->nb_esc_params < MAX_ESC_PARAMS) {
-                s->esc_params[s->nb_esc_params] =
-                    s->esc_params[s->nb_esc_params] * 10 + ch - '0';
+                int *param = &s->esc_params[s->nb_esc_params];
+                int digit = (ch - '0');
+
+                *param = (*param <= (INT_MAX - digit) / 10) ?
+                         *param * 10 + digit : INT_MAX;
             }
         } else {
             if (s->nb_esc_params < MAX_ESC_PARAMS)
@@ -1074,7 +1082,7 @@ static void console_putchar(TextConsole *s, int ch)
 
 void console_select(unsigned int index)
 {
-    TextConsole *s;
+    QemuConsole *s;
 
     if (index >= MAX_CONSOLES)
         return;
@@ -1090,24 +1098,24 @@ void console_select(unsigned int index)
             qemu_del_timer(active_console->cursor_timer);
         }
         active_console = s;
-        if (ds_get_bits_per_pixel(s->ds)) {
+        if (ds->have_gfx) {
             ds->surface = qemu_resize_displaysurface(ds, s->g_width, s->g_height);
-        } else {
-            s->ds->surface->width = s->width;
-            s->ds->surface->height = s->height;
+            dpy_gfx_resize(ds);
+        }
+        if (ds->have_text) {
+            dpy_text_resize(ds, s->width, s->height);
         }
         if (s->cursor_timer) {
             qemu_mod_timer(s->cursor_timer,
                    qemu_get_clock_ms(rt_clock) + CONSOLE_CURSOR_PERIOD / 2);
         }
-        dpy_resize(s->ds);
         vga_hw_invalidate();
     }
 }
 
 static int console_puts(CharDriverState *chr, const uint8_t *buf, int len)
 {
-    TextConsole *s = chr->opaque;
+    QemuConsole *s = chr->opaque;
     int i;
 
     s->update_x0 = s->width * FONT_WIDTH;
@@ -1119,17 +1127,17 @@ static int console_puts(CharDriverState *chr, const uint8_t *buf, int len)
         console_putchar(s, buf[i]);
     }
     console_show_cursor(s, 1);
-    if (ds_get_bits_per_pixel(s->ds) && s->update_x0 < s->update_x1) {
-        dpy_update(s->ds, s->update_x0, s->update_y0,
-                   s->update_x1 - s->update_x0,
-                   s->update_y1 - s->update_y0);
+    if (s->ds->have_gfx && s->update_x0 < s->update_x1) {
+        dpy_gfx_update(s->ds, s->update_x0, s->update_y0,
+                       s->update_x1 - s->update_x0,
+                       s->update_y1 - s->update_y0);
     }
     return len;
 }
 
 static void kbd_send_chars(void *opaque)
 {
-    TextConsole *s = opaque;
+    QemuConsole *s = opaque;
     int len;
     uint8_t buf[16];
 
@@ -1152,7 +1160,7 @@ static void kbd_send_chars(void *opaque)
 /* called when an ascii key is pressed */
 void kbd_put_keysym(int keysym)
 {
-    TextConsole *s;
+    QemuConsole *s;
     uint8_t buf[16], *q;
     int c;
 
@@ -1207,7 +1215,7 @@ void kbd_put_keysym(int keysym)
 
 static void text_console_invalidate(void *opaque)
 {
-    TextConsole *s = (TextConsole *) opaque;
+    QemuConsole *s = (QemuConsole *) opaque;
     if (!ds_get_bits_per_pixel(s->ds) && s->console_type == TEXT_CONSOLE) {
         s->g_width = ds_get_width(s->ds);
         s->g_height = ds_get_height(s->ds);
@@ -1218,7 +1226,7 @@ static void text_console_invalidate(void *opaque)
 
 static void text_console_update(void *opaque, console_ch_t *chardata)
 {
-    TextConsole *s = (TextConsole *) opaque;
+    QemuConsole *s = (QemuConsole *) opaque;
     int i, j, src;
 
     if (s->text_x[0] <= s->text_x[1]) {
@@ -1230,23 +1238,23 @@ static void text_console_update(void *opaque, console_ch_t *chardata)
                                 (s->cells[src].t_attrib.fgcol << 12) |
                                 (s->cells[src].t_attrib.bgcol << 8) |
                                 (s->cells[src].t_attrib.bold << 21));
-        dpy_update(s->ds, s->text_x[0], s->text_y[0],
-                   s->text_x[1] - s->text_x[0], i - s->text_y[0]);
+        dpy_text_update(s->ds, s->text_x[0], s->text_y[0],
+                        s->text_x[1] - s->text_x[0], i - s->text_y[0]);
         s->text_x[0] = s->width;
         s->text_y[0] = s->height;
         s->text_x[1] = 0;
         s->text_y[1] = 0;
     }
     if (s->cursor_invalidate) {
-        dpy_cursor(s->ds, s->x, s->y);
+        dpy_text_cursor(s->ds, s->x, s->y);
         s->cursor_invalidate = 0;
     }
 }
 
-static TextConsole *get_graphic_console(DisplayState *ds)
+static QemuConsole *get_graphic_console(DisplayState *ds)
 {
     int i;
-    TextConsole *s;
+    QemuConsole *s;
     for (i = 0; i < nb_consoles; i++) {
         s = consoles[i];
         if (s->console_type == GRAPHIC_CONSOLE && s->ds == ds)
@@ -1255,14 +1263,14 @@ static TextConsole *get_graphic_console(DisplayState *ds)
     return NULL;
 }
 
-static TextConsole *new_console(DisplayState *ds, console_type_t console_type)
+static QemuConsole *new_console(DisplayState *ds, console_type_t console_type)
 {
-    TextConsole *s;
+    QemuConsole *s;
     int i;
 
     if (nb_consoles >= MAX_CONSOLES)
         return NULL;
-    s = g_malloc0(sizeof(TextConsole));
+    s = g_malloc0(sizeof(QemuConsole));
     if (!active_console || ((active_console->console_type != GRAPHIC_CONSOLE) &&
         (console_type == GRAPHIC_CONSOLE))) {
         active_console = s;
@@ -1287,85 +1295,86 @@ static TextConsole *new_console(DisplayState *ds, console_type_t console_type)
     return s;
 }
 
-static DisplaySurface* defaultallocator_create_displaysurface(int width, int height)
+static void qemu_alloc_display(DisplaySurface *surface, int width, int height,
+                               int linesize, PixelFormat pf, int newflags)
 {
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
+    surface->pf = pf;
 
-    int linesize = width * 4;
-    qemu_alloc_display(surface, width, height, linesize,
-                       qemu_default_pixelformat(32), 0);
-    return surface;
+    qemu_pixman_image_unref(surface->image);
+    surface->image = NULL;
+
+    surface->format = qemu_pixman_get_format(&pf);
+    assert(surface->format != 0);
+    surface->image = pixman_image_create_bits(surface->format,
+                                              width, height,
+                                              NULL, linesize);
+    assert(surface->image != NULL);
+
+    surface->flags = newflags | QEMU_ALLOCATED_FLAG;
+#ifdef HOST_WORDS_BIGENDIAN
+    surface->flags |= QEMU_BIG_ENDIAN_FLAG;
+#endif
 }
 
-static DisplaySurface* defaultallocator_resize_displaysurface(DisplaySurface *surface,
-                                          int width, int height)
+DisplaySurface *qemu_create_displaysurface(DisplayState *ds,
+                                           int width, int height)
 {
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
+
     int linesize = width * 4;
     qemu_alloc_display(surface, width, height, linesize,
                        qemu_default_pixelformat(32), 0);
     return surface;
 }
 
-void qemu_alloc_display(DisplaySurface *surface, int width, int height,
-                        int linesize, PixelFormat pf, int newflags)
+DisplaySurface *qemu_resize_displaysurface(DisplayState *ds,
+                                           int width, int height)
 {
-    void *data;
-    surface->width = width;
-    surface->height = height;
-    surface->linesize = linesize;
-    surface->pf = pf;
-    if (surface->flags & QEMU_ALLOCATED_FLAG) {
-        data = g_realloc(surface->data,
-                            surface->linesize * surface->height);
-    } else {
-        data = g_malloc(surface->linesize * surface->height);
-    }
-    surface->data = (uint8_t *)data;
-    surface->flags = newflags | QEMU_ALLOCATED_FLAG;
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags |= QEMU_BIG_ENDIAN_FLAG;
-#endif
+    int linesize = width * 4;
+
+    trace_displaysurface_resize(ds, ds->surface, width, height);
+    qemu_alloc_display(ds->surface, width, height, linesize,
+                       qemu_default_pixelformat(32), 0);
+    return ds->surface;
 }
 
-DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
-                                              int linesize, uint8_t *data)
+DisplaySurface *qemu_create_displaysurface_from(int width, int height, int bpp,
+                                                int linesize, uint8_t *data)
 {
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
 
-    surface->width = width;
-    surface->height = height;
-    surface->linesize = linesize;
     surface->pf = qemu_default_pixelformat(bpp);
+
+    surface->format = qemu_pixman_get_format(&surface->pf);
+    assert(surface->format != 0);
+    surface->image = pixman_image_create_bits(surface->format,
+                                              width, height,
+                                              (void *)data, linesize);
+    assert(surface->image != NULL);
+
 #ifdef HOST_WORDS_BIGENDIAN
     surface->flags = QEMU_BIG_ENDIAN_FLAG;
 #endif
-    surface->data = data;
 
     return surface;
 }
 
-static void defaultallocator_free_displaysurface(DisplaySurface *surface)
+void qemu_free_displaysurface(DisplayState *ds)
 {
-    if (surface == NULL)
+    trace_displaysurface_free(ds, ds->surface);
+    if (ds->surface == NULL) {
         return;
-    if (surface->flags & QEMU_ALLOCATED_FLAG)
-        g_free(surface->data);
-    g_free(surface);
+    }
+    qemu_pixman_image_unref(ds->surface->image);
+    g_free(ds->surface);
 }
 
-static struct DisplayAllocator default_allocator = {
-    defaultallocator_create_displaysurface,
-    defaultallocator_resize_displaysurface,
-    defaultallocator_free_displaysurface
-};
-
 static void dumb_display_init(void)
 {
     DisplayState *ds = g_malloc0(sizeof(DisplayState));
     int width = 640;
     int height = 480;
 
-    ds->allocator = &default_allocator;
     if (is_fixedsize_console()) {
         width = active_console->g_width;
         height = active_console->g_height;
@@ -1395,29 +1404,16 @@ DisplayState *get_displaystate(void)
     return display_state;
 }
 
-DisplayAllocator *register_displayallocator(DisplayState *ds, DisplayAllocator *da)
-{
-    if(ds->allocator ==  &default_allocator) {
-        DisplaySurface *surf;
-        surf = da->create_displaysurface(ds_get_width(ds), ds_get_height(ds));
-        defaultallocator_free_displaysurface(ds->surface);
-        ds->surface = surf;
-        ds->allocator = da;
-    }
-    return ds->allocator;
-}
-
 DisplayState *graphic_console_init(vga_hw_update_ptr update,
                                    vga_hw_invalidate_ptr invalidate,
                                    vga_hw_screen_dump_ptr screen_dump,
                                    vga_hw_text_update_ptr text_update,
                                    void *opaque)
 {
-    TextConsole *s;
+    QemuConsole *s;
     DisplayState *ds;
 
     ds = (DisplayState *) g_malloc0(sizeof(DisplayState));
-    ds->allocator = &default_allocator; 
     ds->surface = qemu_create_displaysurface(ds, 640, 480);
 
     s = new_console(ds, GRAPHIC_CONSOLE);
@@ -1459,14 +1455,14 @@ void console_color_init(DisplayState *ds)
 
 static void text_console_set_echo(CharDriverState *chr, bool echo)
 {
-    TextConsole *s = chr->opaque;
+    QemuConsole *s = chr->opaque;
 
     s->echo = echo;
 }
 
 static void text_console_update_cursor(void *opaque)
 {
-    TextConsole *s = opaque;
+    QemuConsole *s = opaque;
 
     s->cursor_visible_phase = !s->cursor_visible_phase;
     vga_hw_invalidate();
@@ -1476,7 +1472,7 @@ static void text_console_update_cursor(void *opaque)
 
 static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
 {
-    TextConsole *s;
+    QemuConsole *s;
     static int color_inited;
 
     s = chr->opaque;
@@ -1539,7 +1535,7 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
 CharDriverState *text_console_init(QemuOpts *opts)
 {
     CharDriverState *chr;
-    TextConsole *s;
+    QemuConsole *s;
     unsigned width;
     unsigned height;
 
@@ -1585,14 +1581,14 @@ void text_consoles_set_display(DisplayState *ds)
 
 void qemu_console_resize(DisplayState *ds, int width, int height)
 {
-    TextConsole *s = get_graphic_console(ds);
+    QemuConsole *s = get_graphic_console(ds);
     if (!s) return;
 
     s->g_width = width;
     s->g_height = height;
     if (is_graphic_console()) {
         ds->surface = qemu_resize_displaysurface(ds, width, height);
-        dpy_resize(ds);
+        dpy_gfx_resize(ds);
     }
 }
 
@@ -1600,7 +1596,7 @@ void qemu_console_copy(DisplayState *ds, int src_x, int src_y,
                        int dst_x, int dst_y, int w, int h)
 {
     if (is_graphic_console()) {
-        dpy_copy(ds, src_x, src_y, dst_x, dst_y, w, h);
+        dpy_gfx_copy(ds, src_x, src_y, dst_x, dst_y, w, h);
     }
 }
 
@@ -1611,7 +1607,7 @@ PixelFormat qemu_different_endianness_pixelformat(int bpp)
     memset(&pf, 0x00, sizeof(PixelFormat));
 
     pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = bpp / 8;
+    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
@@ -1660,13 +1656,12 @@ PixelFormat qemu_default_pixelformat(int bpp)
     memset(&pf, 0x00, sizeof(PixelFormat));
 
     pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = bpp / 8;
+    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
         case 15:
             pf.bits_per_pixel = 16;
-            pf.bytes_per_pixel = 2;
             pf.rmask = 0x00007c00;
             pf.gmask = 0x000003E0;
             pf.bmask = 0x0000001F;
@@ -1712,18 +1707,15 @@ PixelFormat qemu_default_pixelformat(int bpp)
             pf.rmask = 0x00FF0000;
             pf.gmask = 0x0000FF00;
             pf.bmask = 0x000000FF;
-            pf.amax = 255;
             pf.rmax = 255;
             pf.gmax = 255;
             pf.bmax = 255;
-            pf.ashift = 24;
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;
             pf.rbits = 8;
             pf.gbits = 8;
             pf.bbits = 8;
-            pf.abits = 8;
             break;
         default:
             break;
diff --git a/ui/curses.c b/ui/curses.c
index c2be2c641a1..d78e3784400 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -28,13 +28,9 @@
 #include <termios.h>
 #endif
 
-#ifdef __OpenBSD__
-#define resize_term resizeterm
-#endif
-
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 #define FONT_HEIGHT 16
 #define FONT_WIDTH 8
@@ -95,17 +91,16 @@ static void curses_calc_pad(void)
     }
 }
 
-static void curses_resize(DisplayState *ds)
+static void curses_resize(DisplayState *ds, int width, int height)
 {
-    if (ds_get_width(ds) == gwidth && ds_get_height(ds) == gheight)
+    if (width == gwidth && height == gheight) {
         return;
+    }
 
-    gwidth = ds_get_width(ds);
-    gheight = ds_get_height(ds);
+    gwidth = width;
+    gheight = height;
 
     curses_calc_pad();
-    ds->surface->width = width * FONT_WIDTH;
-    ds->surface->height = height * FONT_HEIGHT;
 }
 
 #ifndef _WIN32
@@ -167,8 +162,6 @@ static void curses_refresh(DisplayState *ds)
         clear();
         refresh();
         curses_calc_pad();
-        ds->surface->width = FONT_WIDTH * width;
-        ds->surface->height = FONT_HEIGHT * height;
         vga_hw_invalidate();
         invalidate = 0;
     }
@@ -195,8 +188,6 @@ static void curses_refresh(DisplayState *ds)
             refresh();
             curses_calc_pad();
             curses_update(ds, 0, 0, width, height);
-            ds->surface->width = FONT_WIDTH * width;
-            ds->surface->height = FONT_HEIGHT * height;
             continue;
         }
 #endif
@@ -355,13 +346,11 @@ void curses_display_init(DisplayState *ds, int full_screen)
 #endif
 
     dcl = (DisplayChangeListener *) g_malloc0(sizeof(DisplayChangeListener));
-    dcl->dpy_update = curses_update;
-    dcl->dpy_resize = curses_resize;
+    dcl->dpy_text_update = curses_update;
+    dcl->dpy_text_resize = curses_resize;
     dcl->dpy_refresh = curses_refresh;
     dcl->dpy_text_cursor = curses_cursor_position;
     register_displaychangelistener(ds, dcl);
-    qemu_free_displaysurface(ds);
-    ds->surface = qemu_create_displaysurface_from(640, 400, 0, 0, (uint8_t*) screen);
 
     invalidate = 1;
 }
diff --git a/ui/curses_keys.h b/ui/curses_keys.h
index c0d5eb452fa..18ce6dceeed 100644
--- a/ui/curses_keys.h
+++ b/ui/curses_keys.h
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#ifndef QEMU_CURSES_KEYS_H
+#define QEMU_CURSES_KEYS_H 1
+
 #include <curses.h>
 #include "keymaps.h"
 
@@ -507,3 +510,5 @@ static const name2keysym_t name2keysym[] = {
 
     { NULL, 0 },
 };
+
+#endif
diff --git a/cursor.c b/ui/cursor.c
index 76e262caf7b..2b8dd3fa50a 100644
--- a/cursor.c
+++ b/ui/cursor.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 
 #include "cursor_hidden.xpm"
 #include "cursor_left_ptr.xpm"
diff --git a/cursor_hidden.xpm b/ui/cursor_hidden.xpm
index 354e7a9391e..354e7a9391e 100644
--- a/cursor_hidden.xpm
+++ b/ui/cursor_hidden.xpm
diff --git a/cursor_left_ptr.xpm b/ui/cursor_left_ptr.xpm
index 6c9ada9020a..6c9ada9020a 100644
--- a/cursor_left_ptr.xpm
+++ b/ui/cursor_left_ptr.xpm
diff --git a/ui/d3des.h b/ui/d3des.h
index 78d546f7d8f..70cb6b57ead 100644
--- a/ui/d3des.h
+++ b/ui/d3des.h
@@ -9,6 +9,8 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  */
+#ifndef D3DES_H
+#define D3DES_H 1
 
 /* d3des.h -
  *
@@ -49,3 +51,5 @@ void des(unsigned char *, unsigned char *);
 
 /* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery
  ********************************************************************/
+
+#endif
diff --git a/input.c b/ui/input.c
index 6968b31781d..259fd1808db 100644
--- a/input.c
+++ b/ui/input.c
@@ -22,12 +22,12 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
-#include "error.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "qapi/error.h"
 #include "qmp-commands.h"
+#include "qapi-types.h"
 
 static QEMUPutKBDEvent *qemu_put_kbd_event;
 static void *qemu_put_kbd_event_opaque;
@@ -37,6 +37,257 @@ static QTAILQ_HEAD(, QEMUPutMouseEntry) mouse_handlers =
 static NotifierList mouse_mode_notifiers = 
     NOTIFIER_LIST_INITIALIZER(mouse_mode_notifiers);
 
+static const int key_defs[] = {
+    [Q_KEY_CODE_SHIFT] = 0x2a,
+    [Q_KEY_CODE_SHIFT_R] = 0x36,
+
+    [Q_KEY_CODE_ALT] = 0x38,
+    [Q_KEY_CODE_ALT_R] = 0xb8,
+    [Q_KEY_CODE_ALTGR] = 0x64,
+    [Q_KEY_CODE_ALTGR_R] = 0xe4,
+    [Q_KEY_CODE_CTRL] = 0x1d,
+    [Q_KEY_CODE_CTRL_R] = 0x9d,
+
+    [Q_KEY_CODE_MENU] = 0xdd,
+
+    [Q_KEY_CODE_ESC] = 0x01,
+
+    [Q_KEY_CODE_1] = 0x02,
+    [Q_KEY_CODE_2] = 0x03,
+    [Q_KEY_CODE_3] = 0x04,
+    [Q_KEY_CODE_4] = 0x05,
+    [Q_KEY_CODE_5] = 0x06,
+    [Q_KEY_CODE_6] = 0x07,
+    [Q_KEY_CODE_7] = 0x08,
+    [Q_KEY_CODE_8] = 0x09,
+    [Q_KEY_CODE_9] = 0x0a,
+    [Q_KEY_CODE_0] = 0x0b,
+    [Q_KEY_CODE_MINUS] = 0x0c,
+    [Q_KEY_CODE_EQUAL] = 0x0d,
+    [Q_KEY_CODE_BACKSPACE] = 0x0e,
+
+    [Q_KEY_CODE_TAB] = 0x0f,
+    [Q_KEY_CODE_Q] = 0x10,
+    [Q_KEY_CODE_W] = 0x11,
+    [Q_KEY_CODE_E] = 0x12,
+    [Q_KEY_CODE_R] = 0x13,
+    [Q_KEY_CODE_T] = 0x14,
+    [Q_KEY_CODE_Y] = 0x15,
+    [Q_KEY_CODE_U] = 0x16,
+    [Q_KEY_CODE_I] = 0x17,
+    [Q_KEY_CODE_O] = 0x18,
+    [Q_KEY_CODE_P] = 0x19,
+    [Q_KEY_CODE_BRACKET_LEFT] = 0x1a,
+    [Q_KEY_CODE_BRACKET_RIGHT] = 0x1b,
+    [Q_KEY_CODE_RET] = 0x1c,
+
+    [Q_KEY_CODE_A] = 0x1e,
+    [Q_KEY_CODE_S] = 0x1f,
+    [Q_KEY_CODE_D] = 0x20,
+    [Q_KEY_CODE_F] = 0x21,
+    [Q_KEY_CODE_G] = 0x22,
+    [Q_KEY_CODE_H] = 0x23,
+    [Q_KEY_CODE_J] = 0x24,
+    [Q_KEY_CODE_K] = 0x25,
+    [Q_KEY_CODE_L] = 0x26,
+    [Q_KEY_CODE_SEMICOLON] = 0x27,
+    [Q_KEY_CODE_APOSTROPHE] = 0x28,
+    [Q_KEY_CODE_GRAVE_ACCENT] = 0x29,
+
+    [Q_KEY_CODE_BACKSLASH] = 0x2b,
+    [Q_KEY_CODE_Z] = 0x2c,
+    [Q_KEY_CODE_X] = 0x2d,
+    [Q_KEY_CODE_C] = 0x2e,
+    [Q_KEY_CODE_V] = 0x2f,
+    [Q_KEY_CODE_B] = 0x30,
+    [Q_KEY_CODE_N] = 0x31,
+    [Q_KEY_CODE_M] = 0x32,
+    [Q_KEY_CODE_COMMA] = 0x33,
+    [Q_KEY_CODE_DOT] = 0x34,
+    [Q_KEY_CODE_SLASH] = 0x35,
+
+    [Q_KEY_CODE_ASTERISK] = 0x37,
+
+    [Q_KEY_CODE_SPC] = 0x39,
+    [Q_KEY_CODE_CAPS_LOCK] = 0x3a,
+    [Q_KEY_CODE_F1] = 0x3b,
+    [Q_KEY_CODE_F2] = 0x3c,
+    [Q_KEY_CODE_F3] = 0x3d,
+    [Q_KEY_CODE_F4] = 0x3e,
+    [Q_KEY_CODE_F5] = 0x3f,
+    [Q_KEY_CODE_F6] = 0x40,
+    [Q_KEY_CODE_F7] = 0x41,
+    [Q_KEY_CODE_F8] = 0x42,
+    [Q_KEY_CODE_F9] = 0x43,
+    [Q_KEY_CODE_F10] = 0x44,
+    [Q_KEY_CODE_NUM_LOCK] = 0x45,
+    [Q_KEY_CODE_SCROLL_LOCK] = 0x46,
+
+    [Q_KEY_CODE_KP_DIVIDE] = 0xb5,
+    [Q_KEY_CODE_KP_MULTIPLY] = 0x37,
+    [Q_KEY_CODE_KP_SUBTRACT] = 0x4a,
+    [Q_KEY_CODE_KP_ADD] = 0x4e,
+    [Q_KEY_CODE_KP_ENTER] = 0x9c,
+    [Q_KEY_CODE_KP_DECIMAL] = 0x53,
+    [Q_KEY_CODE_SYSRQ] = 0x54,
+
+    [Q_KEY_CODE_KP_0] = 0x52,
+    [Q_KEY_CODE_KP_1] = 0x4f,
+    [Q_KEY_CODE_KP_2] = 0x50,
+    [Q_KEY_CODE_KP_3] = 0x51,
+    [Q_KEY_CODE_KP_4] = 0x4b,
+    [Q_KEY_CODE_KP_5] = 0x4c,
+    [Q_KEY_CODE_KP_6] = 0x4d,
+    [Q_KEY_CODE_KP_7] = 0x47,
+    [Q_KEY_CODE_KP_8] = 0x48,
+    [Q_KEY_CODE_KP_9] = 0x49,
+
+    [Q_KEY_CODE_LESS] = 0x56,
+
+    [Q_KEY_CODE_F11] = 0x57,
+    [Q_KEY_CODE_F12] = 0x58,
+
+    [Q_KEY_CODE_PRINT] = 0xb7,
+
+    [Q_KEY_CODE_HOME] = 0xc7,
+    [Q_KEY_CODE_PGUP] = 0xc9,
+    [Q_KEY_CODE_PGDN] = 0xd1,
+    [Q_KEY_CODE_END] = 0xcf,
+
+    [Q_KEY_CODE_LEFT] = 0xcb,
+    [Q_KEY_CODE_UP] = 0xc8,
+    [Q_KEY_CODE_DOWN] = 0xd0,
+    [Q_KEY_CODE_RIGHT] = 0xcd,
+
+    [Q_KEY_CODE_INSERT] = 0xd2,
+    [Q_KEY_CODE_DELETE] = 0xd3,
+#ifdef NEED_CPU_H
+#if defined(TARGET_SPARC) && !defined(TARGET_SPARC64)
+    [Q_KEY_CODE_STOP] = 0xf0,
+    [Q_KEY_CODE_AGAIN] = 0xf1,
+    [Q_KEY_CODE_PROPS] = 0xf2,
+    [Q_KEY_CODE_UNDO] = 0xf3,
+    [Q_KEY_CODE_FRONT] = 0xf4,
+    [Q_KEY_CODE_COPY] = 0xf5,
+    [Q_KEY_CODE_OPEN] = 0xf6,
+    [Q_KEY_CODE_PASTE] = 0xf7,
+    [Q_KEY_CODE_FIND] = 0xf8,
+    [Q_KEY_CODE_CUT] = 0xf9,
+    [Q_KEY_CODE_LF] = 0xfa,
+    [Q_KEY_CODE_HELP] = 0xfb,
+    [Q_KEY_CODE_META_L] = 0xfc,
+    [Q_KEY_CODE_META_R] = 0xfd,
+    [Q_KEY_CODE_COMPOSE] = 0xfe,
+#endif
+#endif
+    [Q_KEY_CODE_MAX] = 0,
+};
+
+int index_from_key(const char *key)
+{
+    int i;
+
+    for (i = 0; QKeyCode_lookup[i] != NULL; i++) {
+        if (!strcmp(key, QKeyCode_lookup[i])) {
+            break;
+        }
+    }
+
+    /* Return Q_KEY_CODE_MAX if the key is invalid */
+    return i;
+}
+
+int index_from_keycode(int code)
+{
+    int i;
+
+    for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+        if (key_defs[i] == code) {
+            break;
+        }
+    }
+
+    /* Return Q_KEY_CODE_MAX if the code is invalid */
+    return i;
+}
+
+static int *keycodes;
+static int keycodes_size;
+static QEMUTimer *key_timer;
+
+static int keycode_from_keyvalue(const KeyValue *value)
+{
+    if (value->kind == KEY_VALUE_KIND_QCODE) {
+        return key_defs[value->qcode];
+    } else {
+        assert(value->kind == KEY_VALUE_KIND_NUMBER);
+        return value->number;
+    }
+}
+
+static void free_keycodes(void)
+{
+    g_free(keycodes);
+    keycodes = NULL;
+    keycodes_size = 0;
+}
+
+static void release_keys(void *opaque)
+{
+    int i;
+
+    for (i = 0; i < keycodes_size; i++) {
+        if (keycodes[i] & 0x80) {
+            kbd_put_keycode(0xe0);
+        }
+        kbd_put_keycode(keycodes[i]| 0x80);
+    }
+
+    free_keycodes();
+}
+
+void qmp_send_key(KeyValueList *keys, bool has_hold_time, int64_t hold_time,
+                  Error **errp)
+{
+    int keycode;
+    KeyValueList *p;
+
+    if (!key_timer) {
+        key_timer = qemu_new_timer_ns(vm_clock, release_keys, NULL);
+    }
+
+    if (keycodes != NULL) {
+        qemu_del_timer(key_timer);
+        release_keys(NULL);
+    }
+
+    if (!has_hold_time) {
+        hold_time = 100;
+    }
+
+    for (p = keys; p != NULL; p = p->next) {
+        /* key down events */
+        keycode = keycode_from_keyvalue(p->value);
+        if (keycode < 0x01 || keycode > 0xff) {
+            error_setg(errp, "invalid hex keycode 0x%x\n", keycode);
+            free_keycodes();
+            return;
+        }
+
+        if (keycode & 0x80) {
+            kbd_put_keycode(0xe0);
+        }
+        kbd_put_keycode(keycode & 0x7f);
+
+        keycodes = g_realloc(keycodes, sizeof(int) * (keycodes_size + 1));
+        keycodes[keycodes_size++] = keycode;
+    }
+
+    /* delayed key up events */
+    qemu_mod_timer(key_timer, qemu_get_clock_ns(vm_clock) +
+                   muldiv64(get_ticks_per_sec(), hold_time, 1000));
+}
+
 void qemu_add_kbd_event_handler(QEMUPutKBDEvent *func, void *opaque)
 {
     qemu_put_kbd_event_opaque = opaque;
diff --git a/ui/keymaps.c b/ui/keymaps.c
index f55a2aa4645..f373cc53d9f 100644
--- a/ui/keymaps.c
+++ b/ui/keymaps.c
@@ -23,7 +23,7 @@
  */
 
 #include "keymaps.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 static int get_keysym(const name2keysym_t *table,
 		      const char *name)
@@ -127,25 +127,27 @@ static kbd_layout_t *parse_keyboard_layout(const name2keysym_t *table,
                     //		    fprintf(stderr, "Warning: unknown keysym %s\n", line);
 		} else {
 		    const char *rest = end_of_keysym + 1;
-		    char *rest2;
-		    int keycode = strtol(rest, &rest2, 0);
+                    int keycode = strtol(rest, NULL, 0);
 
-		    if (rest && strstr(rest, "numlock")) {
+                    if (strstr(rest, "numlock")) {
 			add_to_key_range(&k->keypad_range, keycode);
 			add_to_key_range(&k->numlock_range, keysym);
 			//fprintf(stderr, "keypad keysym %04x keycode %d\n", keysym, keycode);
 		    }
 
-		    if (rest && strstr(rest, "shift"))
+                    if (strstr(rest, "shift")) {
 			keycode |= SCANCODE_SHIFT;
-		    if (rest && strstr(rest, "altgr"))
+                    }
+                    if (strstr(rest, "altgr")) {
 			keycode |= SCANCODE_ALTGR;
-		    if (rest && strstr(rest, "ctrl"))
+                    }
+                    if (strstr(rest, "ctrl")) {
 			keycode |= SCANCODE_CTRL;
+                    }
 
 		    add_keysym(line, keysym, keycode, k);
 
-		    if (rest && strstr(rest, "addupper")) {
+                    if (strstr(rest, "addupper")) {
 			char *c;
 			for (c = line; *c; c++)
 			    *c = qemu_toupper(*c);
diff --git a/ui/qemu-pixman.c b/ui/qemu-pixman.c
new file mode 100644
index 00000000000..609335ab115
--- /dev/null
+++ b/ui/qemu-pixman.c
@@ -0,0 +1,80 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "ui/qemu-pixman.h"
+
+int qemu_pixman_get_type(int rshift, int gshift, int bshift)
+{
+    int type = PIXMAN_TYPE_OTHER;
+
+    if (rshift > gshift && gshift > bshift) {
+        if (bshift == 0) {
+            type = PIXMAN_TYPE_ARGB;
+        } else {
+#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 21, 8)
+            type = PIXMAN_TYPE_RGBA;
+#endif
+        }
+    } else if (rshift < gshift && gshift < bshift) {
+        if (rshift == 0) {
+            type = PIXMAN_TYPE_ABGR;
+        } else {
+#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 16, 0)
+            type = PIXMAN_TYPE_BGRA;
+#endif
+        }
+    }
+    return type;
+}
+
+pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf)
+{
+    pixman_format_code_t format;
+    int type;
+
+    type = qemu_pixman_get_type(pf->rshift, pf->gshift, pf->bshift);
+    format = PIXMAN_FORMAT(pf->bits_per_pixel, type,
+                           pf->abits, pf->rbits, pf->gbits, pf->bbits);
+    if (!pixman_format_supported_source(format)) {
+        return 0;
+    }
+    return format;
+}
+
+pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
+                                           int width)
+{
+    pixman_image_t *image = pixman_image_create_bits(format, width, 1, NULL, 0);
+    assert(image != NULL);
+    return image;
+}
+
+void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
+                              int width, int x, int y)
+{
+    pixman_image_composite(PIXMAN_OP_SRC, fb, NULL, linebuf,
+                           x, y, 0, 0, 0, 0, width, 1);
+}
+
+pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
+                                          pixman_image_t *image)
+{
+    pixman_image_t *mirror;
+
+    mirror = pixman_image_create_bits(format,
+                                      pixman_image_get_width(image),
+                                      pixman_image_get_height(image),
+                                      NULL,
+                                      pixman_image_get_stride(image));
+    return mirror;
+}
+
+void qemu_pixman_image_unref(pixman_image_t *image)
+{
+    if (image == NULL) {
+        return;
+    }
+    pixman_image_unref(image);
+}
diff --git a/qemu-x509.h b/ui/qemu-x509.h
index 095aec161dc..095aec161dc 100644
--- a/qemu-x509.h
+++ b/ui/qemu-x509.h
diff --git a/ui/sdl.c b/ui/sdl.c
index f6f711c1bba..1657848e9f5 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -29,8 +29,8 @@
 #include <SDL_syswm.h>
 
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 #include "x_keymap.h"
 #include "sdl_zoom.h"
 
@@ -55,7 +55,6 @@ static int absolute_enabled = 0;
 static int guest_cursor = 0;
 static int guest_x, guest_y;
 static SDL_Cursor *guest_sprite = NULL;
-static uint8_t allocator;
 static SDL_PixelFormat host_format;
 static int scaling_active = 0;
 static Notifier mouse_mode_notifier;
@@ -117,108 +116,13 @@ static void do_sdl_resize(int width, int height, int bpp)
 
 static void sdl_resize(DisplayState *ds)
 {
-    if  (!allocator) {
-        if (!scaling_active)
-            do_sdl_resize(ds_get_width(ds), ds_get_height(ds), 0);
-        else if (real_screen->format->BitsPerPixel != ds_get_bits_per_pixel(ds))
-            do_sdl_resize(real_screen->w, real_screen->h, ds_get_bits_per_pixel(ds));
-        sdl_setdata(ds);
-    } else {
-        if (guest_screen != NULL) {
-            SDL_FreeSurface(guest_screen);
-            guest_screen = NULL;
-        }
-    }
-}
-
-static PixelFormat sdl_to_qemu_pixelformat(SDL_PixelFormat *sdl_pf)
-{
-    PixelFormat qemu_pf;
-
-    memset(&qemu_pf, 0x00, sizeof(PixelFormat));
-
-    qemu_pf.bits_per_pixel = sdl_pf->BitsPerPixel;
-    qemu_pf.bytes_per_pixel = sdl_pf->BytesPerPixel;
-    qemu_pf.depth = (qemu_pf.bits_per_pixel) == 32 ? 24 : (qemu_pf.bits_per_pixel);
-
-    qemu_pf.rmask = sdl_pf->Rmask;
-    qemu_pf.gmask = sdl_pf->Gmask;
-    qemu_pf.bmask = sdl_pf->Bmask;
-    qemu_pf.amask = sdl_pf->Amask;
-
-    qemu_pf.rshift = sdl_pf->Rshift;
-    qemu_pf.gshift = sdl_pf->Gshift;
-    qemu_pf.bshift = sdl_pf->Bshift;
-    qemu_pf.ashift = sdl_pf->Ashift;
-
-    qemu_pf.rbits = 8 - sdl_pf->Rloss;
-    qemu_pf.gbits = 8 - sdl_pf->Gloss;
-    qemu_pf.bbits = 8 - sdl_pf->Bloss;
-    qemu_pf.abits = 8 - sdl_pf->Aloss;
-
-    qemu_pf.rmax = ((1 << qemu_pf.rbits) - 1);
-    qemu_pf.gmax = ((1 << qemu_pf.gbits) - 1);
-    qemu_pf.bmax = ((1 << qemu_pf.bbits) - 1);
-    qemu_pf.amax = ((1 << qemu_pf.abits) - 1);
-
-    return qemu_pf;
-}
-
-static DisplaySurface* sdl_create_displaysurface(int width, int height)
-{
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
-
-    surface->width = width;
-    surface->height = height;
-
-    if (scaling_active) {
-        int linesize;
-        PixelFormat pf;
-        if (host_format.BytesPerPixel != 2 && host_format.BytesPerPixel != 4) {
-            linesize = width * 4;
-            pf = qemu_default_pixelformat(32);
-        } else {
-            linesize = width * host_format.BytesPerPixel;
-            pf = sdl_to_qemu_pixelformat(&host_format);
-        }
-        qemu_alloc_display(surface, width, height, linesize, pf, 0);
-        return surface;
+    if (!scaling_active) {
+        do_sdl_resize(ds_get_width(ds), ds_get_height(ds), 0);
+    } else if (real_screen->format->BitsPerPixel != ds_get_bits_per_pixel(ds)) {
+        do_sdl_resize(real_screen->w, real_screen->h,
+                      ds_get_bits_per_pixel(ds));
     }
-
-    if (host_format.BitsPerPixel == 16)
-        do_sdl_resize(width, height, 16);
-    else
-        do_sdl_resize(width, height, 32);
-
-    surface->pf = sdl_to_qemu_pixelformat(real_screen->format);
-    surface->linesize = real_screen->pitch;
-    surface->data = real_screen->pixels;
-
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags = QEMU_REALPIXELS_FLAG | QEMU_BIG_ENDIAN_FLAG;
-#else
-    surface->flags = QEMU_REALPIXELS_FLAG;
-#endif
-    allocator = 1;
-
-    return surface;
-}
-
-static void sdl_free_displaysurface(DisplaySurface *surface)
-{
-    allocator = 0;
-    if (surface == NULL)
-        return;
-
-    if (surface->flags & QEMU_ALLOCATED_FLAG)
-        g_free(surface->data);
-    g_free(surface);
-}
-
-static DisplaySurface* sdl_resize_displaysurface(DisplaySurface *surface, int width, int height)
-{
-    sdl_free_displaysurface(surface);
-    return sdl_create_displaysurface(width, height);
+    sdl_setdata(ds);
 }
 
 /* generic keyboard conversion */
@@ -553,7 +457,7 @@ static void sdl_scale(DisplayState *ds, int width, int height)
     if (!is_buffer_shared(ds->surface)) {
         ds->surface = qemu_resize_displaysurface(ds, ds_get_width(ds),
                                                  ds_get_height(ds));
-        dpy_resize(ds);
+        dpy_gfx_resize(ds);
     }
 }
 
@@ -899,13 +803,7 @@ static void sdl_refresh(DisplayState *ds)
     }
 }
 
-static void sdl_fill(DisplayState *ds, int x, int y, int w, int h, uint32_t c)
-{
-    SDL_Rect dst = { x, y, w, h };
-    SDL_FillRect(real_screen, &dst, c);
-}
-
-static void sdl_mouse_warp(int x, int y, int on)
+static void sdl_mouse_warp(DisplayState *ds, int x, int y, int on)
 {
     if (on) {
         if (!guest_cursor)
@@ -921,7 +819,7 @@ static void sdl_mouse_warp(int x, int y, int on)
     guest_x = x, guest_y = y;
 }
 
-static void sdl_mouse_define(QEMUCursor *c)
+static void sdl_mouse_define(DisplayState *ds, QEMUCursor *c)
 {
     uint8_t *image, *mask;
     int bpl;
@@ -955,7 +853,6 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
 {
     int flags;
     uint8_t data = 0;
-    DisplayAllocator *da;
     const SDL_VideoInfo *vi;
     char *filename;
 
@@ -1020,23 +917,14 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
     }
 
     dcl = g_malloc0(sizeof(DisplayChangeListener));
-    dcl->dpy_update = sdl_update;
-    dcl->dpy_resize = sdl_resize;
+    dcl->dpy_gfx_update = sdl_update;
+    dcl->dpy_gfx_resize = sdl_resize;
     dcl->dpy_refresh = sdl_refresh;
-    dcl->dpy_setdata = sdl_setdata;
-    dcl->dpy_fill = sdl_fill;
-    ds->mouse_set = sdl_mouse_warp;
-    ds->cursor_define = sdl_mouse_define;
+    dcl->dpy_gfx_setdata = sdl_setdata;
+    dcl->dpy_mouse_set = sdl_mouse_warp;
+    dcl->dpy_cursor_define = sdl_mouse_define;
     register_displaychangelistener(ds, dcl);
 
-    da = g_malloc0(sizeof(DisplayAllocator));
-    da->create_displaysurface = sdl_create_displaysurface;
-    da->resize_displaysurface = sdl_resize_displaysurface;
-    da->free_displaysurface = sdl_free_displaysurface;
-    if (register_displayallocator(ds, da) == da) {
-        dpy_resize(ds);
-    }
-
     mouse_mode_notifier.notify = sdl_mouse_mode_change;
     qemu_add_mouse_mode_change_notifier(&mouse_mode_notifier);
 
diff --git a/ui/sdl_zoom.c b/ui/sdl_zoom.c
index a986c7c14c6..2625c4557ec 100644
--- a/ui/sdl_zoom.c
+++ b/ui/sdl_zoom.c
@@ -12,14 +12,15 @@
  */
 
 #include "sdl_zoom.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
+#include <glib.h>
 #include <stdint.h>
 #include <stdio.h>
 
-static int sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
-                          SDL_Rect *dst_rect);
-static int sdl_zoom_rgb32(SDL_Surface *src, SDL_Surface *dst, int smooth,
-                          SDL_Rect *dst_rect);
+static void sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
+                           SDL_Rect *dst_rect);
+static void sdl_zoom_rgb32(SDL_Surface *src, SDL_Surface *dst, int smooth,
+                           SDL_Rect *dst_rect);
 
 #define BPP 32
 #include  "sdl_zoom_template.h"
diff --git a/ui/sdl_zoom_template.h b/ui/sdl_zoom_template.h
index 64bbca849bd..3bb508b51e4 100644
--- a/ui/sdl_zoom_template.h
+++ b/ui/sdl_zoom_template.h
@@ -51,7 +51,7 @@
               (((a) & (dpf->Amask >> dpf->Ashift)) << dpf->Ashift); \
 } while (0);
 
-static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smooth,
+static void glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smooth,
                                    SDL_Rect *dst_rect)
 {
     int x, y, sx, sy, *sax, *say, *csax, *csay, csx, csy, ex, ey, t1, t2, sstep, sstep_jump;
@@ -71,13 +71,8 @@ static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smoot
         sy = (int) (65536.0 * (float) src->h / (float) dst->h);
     }
 
-    if ((sax = (int *) malloc((dst->w + 1) * sizeof(Uint32))) == NULL) {
-        return (-1);
-    }
-    if ((say = (int *) malloc((dst->h + 1) * sizeof(Uint32))) == NULL) {
-        free(sax);
-        return (-1);
-    }
+    sax = g_new(int, dst->w + 1);
+    say = g_new(int, dst->h + 1);
 
     sp = csp = (SDL_TYPE *) src->pixels;
     dp = (SDL_TYPE *) (dst->pixels + dst_rect->y * dst->pitch +
@@ -216,9 +211,8 @@ static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smoot
         }
     }
 
-    free(sax);
-    free(say);
-    return (0);
+    g_free(sax);
+    g_free(say);
 }
 
 #undef SDL_TYPE
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 4fc48f89026..3f2c5650cd8 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -19,24 +19,25 @@
 #include <spice-experimental.h>
 
 #include <netdb.h>
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "qemu-thread.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
+#include "ui/qemu-spice.h"
+#include "qemu/thread.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
 #include "qemu-x509.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "qmp-commands.h"
-#include "qint.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qjson.h"
-#include "notify.h"
-#include "migration.h"
-#include "monitor.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qjson.h"
+#include "qemu/notify.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
 #include "hw/hw.h"
+#include "ui/spice-display.h"
 
 /* core bits */
 
@@ -45,6 +46,7 @@ static Notifier migration_state;
 static const char *auth = "spice";
 static char *auth_passwd;
 static time_t auth_expires = TIME_MAX;
+static int spice_migration_completed;
 int using_spice = 0;
 
 static QemuThread me;
@@ -221,7 +223,6 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     client = qdict_new();
     server = qdict_new();
 
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
     if (info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
         add_addr_info(client, (struct sockaddr *)&info->paddr_ext,
                       info->plen_ext);
@@ -230,12 +231,7 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     } else {
         error_report("spice: %s, extended address is expected",
                      __func__);
-#endif
-        add_addr_info(client, &info->paddr, info->plen);
-        add_addr_info(server, &info->laddr, info->llen);
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
     }
-#endif
 
     if (event == SPICE_CHANNEL_EVENT_INITIALIZED) {
         qdict_put(server, "auth", qstring_from_str(auth));
@@ -274,7 +270,6 @@ static SpiceCoreInterface core_interface = {
     .channel_event      = channel_event,
 };
 
-#ifdef SPICE_INTERFACE_MIGRATION
 typedef struct SpiceMigration {
     SpiceMigrateInstance sin;
     struct {
@@ -284,6 +279,7 @@ typedef struct SpiceMigration {
 } SpiceMigration;
 
 static void migrate_connect_complete_cb(SpiceMigrateInstance *sin);
+static void migrate_end_complete_cb(SpiceMigrateInstance *sin);
 
 static const SpiceMigrateInterface migrate_interface = {
     .base.type = SPICE_INTERFACE_MIGRATION,
@@ -291,7 +287,7 @@ static const SpiceMigrateInterface migrate_interface = {
     .base.major_version = SPICE_INTERFACE_MIGRATION_MAJOR,
     .base.minor_version = SPICE_INTERFACE_MIGRATION_MINOR,
     .migrate_connect_complete = migrate_connect_complete_cb,
-    .migrate_end_complete = NULL,
+    .migrate_end_complete = migrate_end_complete_cb,
 };
 
 static SpiceMigration spice_migrate;
@@ -304,7 +300,12 @@ static void migrate_connect_complete_cb(SpiceMigrateInstance *sin)
     }
     sm->connect_complete.cb = NULL;
 }
-#endif
+
+static void migrate_end_complete_cb(SpiceMigrateInstance *sin)
+{
+    monitor_protocol_event(QEVENT_SPICE_MIGRATE_COMPLETED, NULL);
+    spice_migration_completed = true;
+}
 
 /* config string parsing */
 
@@ -344,7 +345,8 @@ static const char *stream_video_names[] = {
     [ SPICE_STREAM_VIDEO_FILTER ] = "filter",
 };
 #define parse_stream_video(_name) \
-    name2enum(_name, stream_video_names, ARRAY_SIZE(stream_video_names))
+    parse_name(_name, "stream video control", \
+               stream_video_names, ARRAY_SIZE(stream_video_names))
 
 static const char *compression_names[] = {
     [ SPICE_IMAGE_COMPRESS_OFF ]      = "off",
@@ -383,17 +385,13 @@ static SpiceChannelList *qmp_query_spice_channels(void)
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
 
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
         if (item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
             paddr = (struct sockaddr *)&item->info->paddr_ext;
             plen = item->info->plen_ext;
         } else {
-#endif
             paddr = &item->info->paddr;
             plen = item->info->plen;
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
         }
-#endif
 
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
@@ -419,6 +417,90 @@ static SpiceChannelList *qmp_query_spice_channels(void)
     return head;
 }
 
+static QemuOptsList qemu_spice_opts = {
+    .name = "spice",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_spice_opts.head),
+    .desc = {
+        {
+            .name = "port",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "tls-port",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "addr",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "password",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "disable-ticketing",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "disable-copy-paste",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "sasl",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "x509-dir",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-key-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-key-password",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-cert-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-cacert-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-dh-key-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "tls-ciphers",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "tls-channel",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "plaintext-channel",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "image-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "jpeg-wan-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "zlib-glz-wan-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "streaming-video",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "agent-mouse",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "playback-compression",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "seamless-migration",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
 SpiceInfo *qmp_query_spice(Error **errp)
 {
     QemuOpts *opts = QTAILQ_FIRST(&qemu_spice_opts.head);
@@ -435,6 +517,7 @@ SpiceInfo *qmp_query_spice(Error **errp)
     }
 
     info->enabled = true;
+    info->migrated = spice_migration_completed;
 
     addr = qemu_opt_get(opts, "addr");
     port = qemu_opt_get_number(opts, "port", 0);
@@ -462,13 +545,10 @@ SpiceInfo *qmp_query_spice(Error **errp)
         info->tls_port = tls_port;
     }
 
-#if SPICE_SERVER_VERSION >= 0x000a03 /* 0.10.3 */
     info->mouse_mode = spice_server_is_server_mouse(spice_server) ?
                        SPICE_QUERY_MOUSE_MODE_SERVER :
                        SPICE_QUERY_MOUSE_MODE_CLIENT;
-#else
-    info->mouse_mode = SPICE_QUERY_MOUSE_MODE_UNKNOWN;
-#endif
+
     /* for compatibility with the original command */
     info->has_channels = true;
     info->channels = qmp_query_spice_channels();
@@ -481,17 +561,11 @@ static void migration_state_notifier(Notifier *notifier, void *data)
     MigrationState *s = data;
 
     if (migration_is_active(s)) {
-#ifdef SPICE_INTERFACE_MIGRATION
         spice_server_migrate_start(spice_server);
-#endif
     } else if (migration_has_finished(s)) {
-#ifndef SPICE_INTERFACE_MIGRATION
-        spice_server_migrate_switch(spice_server);
-#else
         spice_server_migrate_end(spice_server, true);
     } else if (migration_has_failed(s)) {
         spice_server_migrate_end(spice_server, false);
-#endif
     }
 }
 
@@ -500,16 +574,11 @@ int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
                             MonitorCompletion *cb, void *opaque)
 {
     int ret;
-#ifdef SPICE_INTERFACE_MIGRATION
+
     spice_migrate.connect_complete.cb = cb;
     spice_migrate.connect_complete.opaque = opaque;
     ret = spice_server_migrate_connect(spice_server, hostname,
                                        port, tls_port, subject);
-#else
-    ret = spice_server_migrate_info(spice_server, hostname,
-                                    port, tls_port, subject);
-    cb(opaque, NULL);
-#endif
     return ret;
 }
 
@@ -545,6 +614,18 @@ static int add_channel(const char *name, const char *value, void *opaque)
     return 0;
 }
 
+static void vm_change_state_handler(void *opaque, int running,
+                                    RunState state)
+{
+    if (running) {
+        qemu_spice_display_start();
+        spice_server_vm_start(spice_server);
+    } else {
+        spice_server_vm_stop(spice_server);
+        qemu_spice_display_stop();
+    }
+}
+
 void qemu_spice_init(void)
 {
     QemuOpts *opts = QTAILQ_FIRST(&qemu_spice_opts.head);
@@ -558,6 +639,7 @@ void qemu_spice_init(void)
     int port, tls_port, len, addr_flags;
     spice_image_compression_t compression;
     spice_wan_compression_t wan_compr;
+    bool seamless_migration;
 
     qemu_thread_get_self(&me);
 
@@ -612,7 +694,7 @@ void qemu_spice_init(void)
         }
 
         x509_key_password = qemu_opt_get(opts, "x509-key-password");
-        x509_dh_file = qemu_opt_get(opts, "x509-dh-file");
+        x509_dh_file = qemu_opt_get(opts, "x509-dh-key-file");
         tls_ciphers = qemu_opt_get(opts, "tls-ciphers");
     }
 
@@ -642,16 +724,11 @@ void qemu_spice_init(void)
         spice_server_set_ticket(spice_server, password, 0, 0, 0);
     }
     if (qemu_opt_get_bool(opts, "sasl", 0)) {
-#if SPICE_SERVER_VERSION >= 0x000900 /* 0.9.0 */
         if (spice_server_set_sasl_appname(spice_server, "qemu") == -1 ||
             spice_server_set_sasl(spice_server, 1) == -1) {
             error_report("spice: failed to enable sasl");
             exit(1);
         }
-#else
-        error_report("spice: sasl is not available (spice >= 0.9 required)");
-        exit(1);
-#endif
     }
     if (qemu_opt_get_bool(opts, "disable-ticketing", 0)) {
         auth = "none";
@@ -696,11 +773,11 @@ void qemu_spice_init(void)
 
     qemu_opt_foreach(opts, add_channel, &tls_port, 0);
 
-#if SPICE_SERVER_VERSION >= 0x000a02 /* 0.10.2 */
     spice_server_set_name(spice_server, qemu_name);
     spice_server_set_uuid(spice_server, qemu_uuid);
-#endif
 
+    seamless_migration = qemu_opt_get_bool(opts, "seamless-migration", 0);
+    spice_server_set_seamless_migration(spice_server, seamless_migration);
     if (0 != spice_server_init(spice_server, &core_interface)) {
         error_report("failed to initialize spice server");
         exit(1);
@@ -709,18 +786,22 @@ void qemu_spice_init(void)
 
     migration_state.notify = migration_state_notifier;
     add_migration_state_change_notifier(&migration_state);
-#ifdef SPICE_INTERFACE_MIGRATION
     spice_migrate.sin.base.sif = &migrate_interface.base;
     spice_migrate.connect_complete.cb = NULL;
     qemu_spice_add_interface(&spice_migrate.sin.base);
-#endif
 
     qemu_spice_input_init();
     qemu_spice_audio_init();
 
+    qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
+
     g_free(x509_key_file);
     g_free(x509_cert_file);
     g_free(x509_cacert_file);
+
+#if SPICE_SERVER_VERSION >= 0x000c02
+    qemu_spice_register_ports();
+#endif
 }
 
 int qemu_spice_add_interface(SpiceBaseInstance *sin)
@@ -739,7 +820,9 @@ int qemu_spice_add_interface(SpiceBaseInstance *sin)
          */
         spice_server = spice_server_new();
         spice_server_init(spice_server, &core_interface);
+        qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
     }
+
     return spice_server_add_interface(spice_server, sin);
 }
 
@@ -778,15 +861,11 @@ int qemu_spice_set_pw_expire(time_t expires)
 
 int qemu_spice_display_add_client(int csock, int skipauth, int tls)
 {
-#if SPICE_SERVER_VERSION >= 0x000a01
     if (tls) {
         return spice_server_add_ssl_client(spice_server, csock, skipauth);
     } else {
         return spice_server_add_client(spice_server, csock, skipauth);
     }
-#else
-    return -1;
-#endif
 }
 
 static void spice_register_config(void)
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 3e8f0b3ad56..dc7e58d0edd 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -16,15 +16,15 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/qemu-spice.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
-#include "spice-display.h"
+#include "ui/spice-display.h"
 
 static int debug = 0;
 
@@ -126,46 +126,48 @@ void qemu_spice_wakeup(SimpleSpiceDisplay *ssd)
     ssd->worker->wakeup(ssd->worker);
 }
 
-void qemu_spice_start(SimpleSpiceDisplay *ssd)
+static int spice_display_is_running;
+
+void qemu_spice_display_start(void)
+{
+    spice_display_is_running = true;
+}
+
+void qemu_spice_display_stop(void)
 {
-    trace_qemu_spice_start(ssd->qxl.id);
-    ssd->worker->start(ssd->worker);
+    spice_display_is_running = false;
 }
 
-void qemu_spice_stop(SimpleSpiceDisplay *ssd)
+int qemu_spice_display_is_running(SimpleSpiceDisplay *ssd)
 {
-    trace_qemu_spice_stop(ssd->qxl.id);
-    ssd->worker->stop(ssd->worker);
+    return spice_display_is_running;
 }
 
-static SimpleSpiceUpdate *qemu_spice_create_update(SimpleSpiceDisplay *ssd)
+static void qemu_spice_create_one_update(SimpleSpiceDisplay *ssd,
+                                         QXLRect *rect)
 {
     SimpleSpiceUpdate *update;
     QXLDrawable *drawable;
     QXLImage *image;
     QXLCommand *cmd;
-    uint8_t *src, *dst;
-    int by, bw, bh;
+    int bw, bh;
     struct timespec time_space;
-
-    if (qemu_spice_rect_is_empty(&ssd->dirty)) {
-        return NULL;
-    };
+    pixman_image_t *dest;
 
     trace_qemu_spice_create_update(
-           ssd->dirty.left, ssd->dirty.right,
-           ssd->dirty.top, ssd->dirty.bottom);
+           rect->left, rect->right,
+           rect->top, rect->bottom);
 
     update   = g_malloc0(sizeof(*update));
     drawable = &update->drawable;
     image    = &update->image;
     cmd      = &update->ext.cmd;
 
-    bw       = ssd->dirty.right - ssd->dirty.left;
-    bh       = ssd->dirty.bottom - ssd->dirty.top;
+    bw       = rect->right - rect->left;
+    bh       = rect->bottom - rect->top;
     update->bitmap = g_malloc(bw * bh * 4);
 
-    drawable->bbox            = ssd->dirty;
+    drawable->bbox            = *rect;
     drawable->clip.type       = SPICE_CLIP_TYPE_NONE;
     drawable->effect          = QXL_EFFECT_OPAQUE;
     drawable->release_info.id = (uintptr_t)update;
@@ -193,31 +195,94 @@ static SimpleSpiceUpdate *qemu_spice_create_update(SimpleSpiceDisplay *ssd)
     image->bitmap.palette = 0;
     image->bitmap.format = SPICE_BITMAP_FMT_32BIT;
 
-    if (ssd->conv == NULL) {
-        PixelFormat dst = qemu_default_pixelformat(32);
-        ssd->conv = qemu_pf_conv_get(&dst, &ssd->ds->surface->pf);
-        assert(ssd->conv);
+    dest = pixman_image_create_bits(PIXMAN_x8r8g8b8, bw, bh,
+                                    (void *)update->bitmap, bw * 4);
+    pixman_image_composite(PIXMAN_OP_SRC, ssd->surface, NULL, ssd->mirror,
+                           rect->left, rect->top, 0, 0,
+                           rect->left, rect->top, bw, bh);
+    pixman_image_composite(PIXMAN_OP_SRC, ssd->mirror, NULL, dest,
+                           rect->left, rect->top, 0, 0,
+                           0, 0, bw, bh);
+    pixman_image_unref(dest);
+
+    cmd->type = QXL_CMD_DRAW;
+    cmd->data = (uintptr_t)drawable;
+
+    QTAILQ_INSERT_TAIL(&ssd->updates, update, next);
+}
+
+static void qemu_spice_create_update(SimpleSpiceDisplay *ssd)
+{
+    static const int blksize = 32;
+    int blocks = (ds_get_width(ssd->ds) + blksize - 1) / blksize;
+    int dirty_top[blocks];
+    int y, yoff, x, xoff, blk, bw;
+    int bpp = ds_get_bytes_per_pixel(ssd->ds);
+    uint8_t *guest, *mirror;
+
+    if (qemu_spice_rect_is_empty(&ssd->dirty)) {
+        return;
+    };
+
+    if (ssd->surface == NULL) {
+        ssd->surface = pixman_image_ref(ds_get_image(ssd->ds));
+        ssd->mirror  = qemu_pixman_mirror_create(ds_get_format(ssd->ds),
+                                                 ds_get_image(ssd->ds));
     }
 
-    src = ds_get_data(ssd->ds) +
-        ssd->dirty.top * ds_get_linesize(ssd->ds) +
-        ssd->dirty.left * ds_get_bytes_per_pixel(ssd->ds);
-    dst = update->bitmap;
-    for (by = 0; by < bh; by++) {
-        qemu_pf_conv_run(ssd->conv, dst, src, bw);
-        src += ds_get_linesize(ssd->ds);
-        dst += image->bitmap.stride;
+    for (blk = 0; blk < blocks; blk++) {
+        dirty_top[blk] = -1;
     }
 
-    cmd->type = QXL_CMD_DRAW;
-    cmd->data = (uintptr_t)drawable;
+    guest = ds_get_data(ssd->ds);
+    mirror = (void *)pixman_image_get_data(ssd->mirror);
+    for (y = ssd->dirty.top; y < ssd->dirty.bottom; y++) {
+        yoff = y * ds_get_linesize(ssd->ds);
+        for (x = ssd->dirty.left; x < ssd->dirty.right; x += blksize) {
+            xoff = x * bpp;
+            blk = x / blksize;
+            bw = MIN(blksize, ssd->dirty.right - x);
+            if (memcmp(guest + yoff + xoff,
+                       mirror + yoff + xoff,
+                       bw * bpp) == 0) {
+                if (dirty_top[blk] != -1) {
+                    QXLRect update = {
+                        .top    = dirty_top[blk],
+                        .bottom = y,
+                        .left   = x,
+                        .right  = x + bw,
+                    };
+                    qemu_spice_create_one_update(ssd, &update);
+                    dirty_top[blk] = -1;
+                }
+            } else {
+                if (dirty_top[blk] == -1) {
+                    dirty_top[blk] = y;
+                }
+            }
+        }
+    }
+
+    for (x = ssd->dirty.left; x < ssd->dirty.right; x += blksize) {
+        blk = x / blksize;
+        bw = MIN(blksize, ssd->dirty.right - x);
+        if (dirty_top[blk] != -1) {
+            QXLRect update = {
+                .top    = dirty_top[blk],
+                .bottom = ssd->dirty.bottom,
+                .left   = x,
+                .right  = x + bw,
+            };
+            qemu_spice_create_one_update(ssd, &update);
+            dirty_top[blk] = -1;
+        }
+    }
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
-    return update;
 }
 
 /*
- * Called from spice server thread context (via interface_release_ressource)
+ * Called from spice server thread context (via interface_release_resource)
  * We do *not* hold the global qemu mutex here, so extra care is needed
  * when calling qemu functions.  QEMU interfaces used:
  *    - g_free (underlying glibc free is re-entrant).
@@ -269,26 +334,16 @@ void qemu_spice_destroy_host_primary(SimpleSpiceDisplay *ssd)
     qemu_spice_destroy_primary_surface(ssd, 0, QXL_SYNC);
 }
 
-void qemu_spice_vm_change_state_handler(void *opaque, int running,
-                                        RunState state)
-{
-    SimpleSpiceDisplay *ssd = opaque;
-
-    if (running) {
-        ssd->running = true;
-        qemu_spice_start(ssd);
-    } else {
-        qemu_spice_stop(ssd);
-        ssd->running = false;
-    }
-}
-
 void qemu_spice_display_init_common(SimpleSpiceDisplay *ssd, DisplayState *ds)
 {
     ssd->ds = ds;
     qemu_mutex_init(&ssd->lock);
+    QTAILQ_INIT(&ssd->updates);
     ssd->mouse_x = -1;
     ssd->mouse_y = -1;
+    if (ssd->num_surfaces == 0) {
+        ssd->num_surfaces = 1024;
+    }
     ssd->bufsize = (16 * 1024 * 1024);
     ssd->buf = g_malloc(ssd->bufsize);
 }
@@ -314,16 +369,22 @@ void qemu_spice_display_update(SimpleSpiceDisplay *ssd,
 
 void qemu_spice_display_resize(SimpleSpiceDisplay *ssd)
 {
+    SimpleSpiceUpdate *update;
+
     dprint(1, "%s:\n", __FUNCTION__);
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
-    qemu_pf_conv_put(ssd->conv);
-    ssd->conv = NULL;
+    if (ssd->surface) {
+        pixman_image_unref(ssd->surface);
+        ssd->surface = NULL;
+        pixman_image_unref(ssd->mirror);
+        ssd->mirror = NULL;
+    }
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update != NULL) {
-        qemu_spice_destroy_update(ssd, ssd->update);
-        ssd->update = NULL;
+    while ((update = QTAILQ_FIRST(&ssd->updates)) != NULL) {
+        QTAILQ_REMOVE(&ssd->updates, update, next);
+        qemu_spice_destroy_update(ssd, update);
     }
     qemu_mutex_unlock(&ssd->lock);
     qemu_spice_destroy_host_primary(ssd);
@@ -336,12 +397,12 @@ void qemu_spice_display_resize(SimpleSpiceDisplay *ssd)
 void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
 {
     if (ssd->cursor) {
-        ssd->ds->cursor_define(ssd->cursor);
+        dpy_cursor_define(ssd->ds, ssd->cursor);
         cursor_put(ssd->cursor);
         ssd->cursor = NULL;
     }
     if (ssd->mouse_x != -1 && ssd->mouse_y != -1) {
-        ssd->ds->mouse_set(ssd->mouse_x, ssd->mouse_y, 1);
+        dpy_mouse_set(ssd->ds, ssd->mouse_x, ssd->mouse_y, 1);
         ssd->mouse_x = -1;
         ssd->mouse_y = -1;
     }
@@ -353,8 +414,8 @@ void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd)
     vga_hw_update();
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update == NULL) {
-        ssd->update = qemu_spice_create_update(ssd);
+    if (QTAILQ_EMPTY(&ssd->updates)) {
+        qemu_spice_create_update(ssd);
         ssd->notify++;
     }
     qemu_spice_cursor_refresh_unlocked(ssd);
@@ -399,7 +460,7 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
     info->qxl_ram_size = ssd->bufsize;
-    info->n_surfaces = NUM_SURFACES;
+    info->n_surfaces = ssd->num_surfaces;
 }
 
 static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
@@ -411,9 +472,9 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
     dprint(3, "%s:\n", __FUNCTION__);
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update != NULL) {
-        update = ssd->update;
-        ssd->update = NULL;
+    update = QTAILQ_FIRST(&ssd->updates);
+    if (update != NULL) {
+        QTAILQ_REMOVE(&ssd->updates, update, next);
         *ext = update->ext;
         ret = true;
     }
@@ -464,6 +525,37 @@ static int interface_flush_resources(QXLInstance *sin)
     return 0;
 }
 
+static void interface_update_area_complete(QXLInstance *sin,
+        uint32_t surface_id,
+        QXLRect *dirty, uint32_t num_updated_rects)
+{
+    /* should never be called, used in qxl native mode only */
+    fprintf(stderr, "%s: abort()\n", __func__);
+    abort();
+}
+
+/* called from spice server thread context only */
+static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
+{
+    /* should never be called, used in qxl native mode only */
+    fprintf(stderr, "%s: abort()\n", __func__);
+    abort();
+}
+
+static void interface_set_client_capabilities(QXLInstance *sin,
+                                              uint8_t client_present,
+                                              uint8_t caps[58])
+{
+    dprint(3, "%s:\n", __func__);
+}
+
+static int interface_client_monitors_config(QXLInstance *sin,
+                                        VDAgentMonitorsConfig *monitors_config)
+{
+    dprint(3, "%s:\n", __func__);
+    return 0; /* == not supported by guest */
+}
+
 static const QXLInterface dpy_interface = {
     .base.type               = SPICE_INTERFACE_QXL,
     .base.description        = "qemu simple display",
@@ -483,6 +575,10 @@ static const QXLInterface dpy_interface = {
     .req_cursor_notification = interface_req_cursor_notification,
     .notify_update           = interface_notify_update,
     .flush_resources         = interface_flush_resources,
+    .async_complete          = interface_async_complete,
+    .update_area_complete    = interface_update_area_complete,
+    .set_client_capabilities = interface_set_client_capabilities,
+    .client_monitors_config  = interface_client_monitors_config,
 };
 
 static SimpleSpiceDisplay sdpy;
@@ -503,8 +599,8 @@ static void display_refresh(struct DisplayState *ds)
 }
 
 static DisplayChangeListener display_listener = {
-    .dpy_update  = display_update,
-    .dpy_resize  = display_resize,
+    .dpy_gfx_update  = display_update,
+    .dpy_gfx_resize  = display_resize,
     .dpy_refresh = display_refresh,
 };
 
@@ -512,13 +608,12 @@ void qemu_spice_display_init(DisplayState *ds)
 {
     assert(sdpy.ds == NULL);
     qemu_spice_display_init_common(&sdpy, ds);
-    register_displaychangelistener(ds, &display_listener);
 
     sdpy.qxl.base.sif = &dpy_interface.base;
     qemu_spice_add_interface(&sdpy.qxl.base);
     assert(sdpy.worker);
 
-    qemu_add_vm_change_state_handler(qemu_spice_vm_change_state_handler, &sdpy);
     qemu_spice_create_host_memslot(&sdpy);
     qemu_spice_create_host_primary(&sdpy);
+    register_displaychangelistener(ds, &display_listener);
 }
diff --git a/ui/spice-input.c b/ui/spice-input.c
index af4223d4426..3beb8deadb5 100644
--- a/ui/spice-input.c
+++ b/ui/spice-input.c
@@ -24,8 +24,8 @@
 #include <spice/enums.h>
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "console.h"
+#include "ui/qemu-spice.h"
+#include "ui/console.h"
 
 /* keyboard bits */
 
diff --git a/vgafont.h b/ui/vgafont.h
index 3606dd7338b..3606dd7338b 100644
--- a/vgafont.h
+++ b/ui/vgafont.h
diff --git a/ui/vnc-auth-sasl.c b/ui/vnc-auth-sasl.c
index 8fba7702c2a..f3ad75d52bb 100644
--- a/ui/vnc-auth-sasl.c
+++ b/ui/vnc-auth-sasl.c
@@ -432,9 +432,7 @@ static int protocol_client_auth_sasl_start_len(VncState *vs, uint8_t *data, size
 
 static int protocol_client_auth_sasl_mechname(VncState *vs, uint8_t *data, size_t len)
 {
-    char *mechname = g_malloc(len + 1);
-    strncpy(mechname, (char*)data, len);
-    mechname[len] = '\0';
+    char *mechname = g_strndup((const char *) data, len);
     VNC_DEBUG("Got client mechname '%s' check against '%s'\n",
               mechname, vs->sasl.mechlist);
 
@@ -619,7 +617,6 @@ void start_auth_sasl(VncState *vs)
 
  authabort:
     vnc_client_error(vs);
-    return;
 }
 
 
diff --git a/ui/vnc-auth-sasl.h b/ui/vnc-auth-sasl.h
index ee243a9d64c..8091d689cbc 100644
--- a/ui/vnc-auth-sasl.h
+++ b/ui/vnc-auth-sasl.h
@@ -32,7 +32,7 @@
 typedef struct VncStateSASL VncStateSASL;
 typedef struct VncDisplaySASL VncDisplaySASL;
 
-#include "acl.h"
+#include "qemu/acl.h"
 
 struct VncStateSASL {
     sasl_conn_t *conn;
diff --git a/ui/vnc-enc-hextile-template.h b/ui/vnc-enc-hextile-template.h
index a7310e19475..d868d75720b 100644
--- a/ui/vnc-enc-hextile-template.h
+++ b/ui/vnc-enc-hextile-template.h
@@ -14,7 +14,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
                                              int *has_bg, int *has_fg)
 {
     VncDisplay *vd = vs->vd;
-    uint8_t *row = vd->server->data + y * ds_get_linesize(vs->ds) + x * ds_get_bytes_per_pixel(vs->ds);
+    uint8_t *row = vnc_server_fb_ptr(vd, x, y);
     pixel_t *irow = (pixel_t *)row;
     int j, i;
     pixel_t *last_bg = (pixel_t *)last_bg_;
@@ -25,7 +25,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
     int bg_count = 0;
     int fg_count = 0;
     int flags = 0;
-    uint8_t data[(vs->clientds.pf.bytes_per_pixel + 2) * 16 * 16];
+    uint8_t data[(vs->client_pf.bytes_per_pixel + 2) * 16 * 16];
     int n_data = 0;
     int n_subtiles = 0;
 
@@ -58,7 +58,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 	}
 	if (n_colors > 2)
 	    break;
-	irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
     }
 
     if (n_colors > 1 && fg_count > bg_count) {
@@ -106,7 +106,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		n_data += 2;
 		n_subtiles++;
 	    }
-	    irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	    irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
 	}
 	break;
     case 3:
@@ -133,7 +133,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		    has_color = 0;
 #ifdef GENERIC
                     vnc_convert_pixel(vs, data + n_data, color);
-                    n_data += vs->clientds.pf.bytes_per_pixel;
+                    n_data += vs->client_pf.bytes_per_pixel;
 #else
 		    memcpy(data + n_data, &color, sizeof(color));
                     n_data += sizeof(pixel_t);
@@ -153,7 +153,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 	    if (has_color) {
 #ifdef GENERIC
                 vnc_convert_pixel(vs, data + n_data, color);
-                n_data += vs->clientds.pf.bytes_per_pixel;
+                n_data += vs->client_pf.bytes_per_pixel;
 #else
                 memcpy(data + n_data, &color, sizeof(color));
                 n_data += sizeof(pixel_t);
@@ -162,7 +162,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		n_data += 2;
 		n_subtiles++;
 	    }
-	    irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	    irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
 	}
 
 	/* A SubrectsColoured subtile invalidates the foreground color */
@@ -190,18 +190,17 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
     vnc_write_u8(vs, flags);
     if (n_colors < 4) {
 	if (flags & 0x02)
-	    vs->write_pixels(vs, &vd->server->pf, last_bg, sizeof(pixel_t));
+	    vs->write_pixels(vs, last_bg, sizeof(pixel_t));
 	if (flags & 0x04)
-	    vs->write_pixels(vs, &vd->server->pf, last_fg, sizeof(pixel_t));
+	    vs->write_pixels(vs, last_fg, sizeof(pixel_t));
 	if (n_subtiles) {
 	    vnc_write_u8(vs, n_subtiles);
 	    vnc_write(vs, data, n_data);
 	}
     } else {
 	for (j = 0; j < h; j++) {
-	    vs->write_pixels(vs, &vd->server->pf, row,
-                             w * ds_get_bytes_per_pixel(vs->ds));
-	    row += ds_get_linesize(vs->ds);
+	    vs->write_pixels(vs, row, w * 4);
+	    row += vnc_server_fb_stride(vd);
 	}
     }
 }
diff --git a/ui/vnc-enc-hextile.c b/ui/vnc-enc-hextile.c
index c860dbb2e56..2e768fd8998 100644
--- a/ui/vnc-enc-hextile.c
+++ b/ui/vnc-enc-hextile.c
@@ -32,31 +32,11 @@ static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
     ptr[1] = (((w - 1) & 0x0F) << 4) | ((h - 1) & 0x0F);
 }
 
-#define BPP 8
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-
-#define BPP 16
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-
 #define BPP 32
 #include "vnc-enc-hextile-template.h"
 #undef BPP
 
 #define GENERIC
-#define BPP 8
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
-#define BPP 16
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
 #define BPP 32
 #include "vnc-enc-hextile-template.h"
 #undef BPP
@@ -68,10 +48,9 @@ int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
     int i, j;
     int has_fg, has_bg;
     uint8_t *last_fg, *last_bg;
-    VncDisplay *vd = vs->vd;
 
-    last_fg = (uint8_t *) g_malloc(vd->server->pf.bytes_per_pixel);
-    last_bg = (uint8_t *) g_malloc(vd->server->pf.bytes_per_pixel);
+    last_fg = (uint8_t *) g_malloc(VNC_SERVER_FB_BYTES);
+    last_bg = (uint8_t *) g_malloc(VNC_SERVER_FB_BYTES);
     has_fg = has_bg = 0;
     for (j = y; j < (y + h); j += 16) {
         for (i = x; i < (x + w); i += 16) {
@@ -89,28 +68,16 @@ int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
 void vnc_hextile_set_pixel_conversion(VncState *vs, int generic)
 {
     if (!generic) {
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->hextile.send_tile = send_hextile_tile_8;
-                break;
-            case 16:
-                vs->hextile.send_tile = send_hextile_tile_16;
-                break;
-            case 32:
-                vs->hextile.send_tile = send_hextile_tile_32;
-                break;
+        switch (VNC_SERVER_FB_BITS) {
+        case 32:
+            vs->hextile.send_tile = send_hextile_tile_32;
+            break;
         }
     } else {
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->hextile.send_tile = send_hextile_tile_generic_8;
-                break;
-            case 16:
-                vs->hextile.send_tile = send_hextile_tile_generic_16;
-                break;
-            case 32:
-                vs->hextile.send_tile = send_hextile_tile_generic_32;
-                break;
+        switch (VNC_SERVER_FB_BITS) {
+        case 32:
+            vs->hextile.send_tile = send_hextile_tile_generic_32;
+            break;
         }
     }
 }
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 5d492abe920..4ddea7d4f55 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -44,8 +44,8 @@
 #include <jpeglib.h>
 #endif
 
-#include "bswap.h"
-#include "qint.h"
+#include "qemu/bswap.h"
+#include "qapi/qmp/qint.h"
 #include "vnc.h"
 #include "vnc-enc-tight.h"
 #include "vnc-palette.h"
@@ -124,7 +124,7 @@ static bool tight_can_send_png_rect(VncState *vs, int w, int h)
     }
 
     if (ds_get_bytes_per_pixel(vs->ds) == 1 ||
-        vs->clientds.pf.bytes_per_pixel == 1) {
+        vs->client_pf.bytes_per_pixel == 1) {
         return false;
     }
 
@@ -153,7 +153,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
      * If client is big-endian, color samples begin from the second
      * byte (offset 1) of a 32-bit pixel value.
      */
-    off = !!(vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG);
+    off = vs->client_be;
 
     memset(stats, 0, sizeof (stats));
 
@@ -216,16 +216,16 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
         unsigned int errors;                                            \
         unsigned char *buf = vs->tight.tight.buffer;                    \
                                                                         \
-        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \
-                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \
+        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
+                      (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
                                                                         \
                                                                         \
-        max[0] = vs->clientds.pf.rmax;                                  \
-        max[1] = vs->clientds.pf.gmax;                                  \
-        max[2] = vs->clientds.pf.bmax;                                  \
-        shift[0] = vs->clientds.pf.rshift;                              \
-        shift[1] = vs->clientds.pf.gshift;                              \
-        shift[2] = vs->clientds.pf.bshift;                              \
+        max[0] = vs->client_pf.rmax;                                  \
+        max[1] = vs->client_pf.gmax;                                  \
+        max[2] = vs->client_pf.bmax;                                  \
+        shift[0] = vs->client_pf.rshift;                              \
+        shift[1] = vs->client_pf.gshift;                              \
+        shift[2] = vs->client_pf.bshift;                              \
                                                                         \
         memset(stats, 0, sizeof(stats));                                \
                                                                         \
@@ -302,7 +302,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
     }
 
     if (ds_get_bytes_per_pixel(vs->ds) == 1 ||
-        vs->clientds.pf.bytes_per_pixel == 1 ||
+        vs->client_pf.bytes_per_pixel == 1 ||
         w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) {
         return 0;
     }
@@ -317,7 +317,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
         }
     }
 
-    if (vs->clientds.pf.bytes_per_pixel == 4) {
+    if (vs->client_pf.bytes_per_pixel == 4) {
         if (vs->tight.pixel24) {
             errors = tight_detect_smooth_image24(vs, w, h);
             if (vs->tight.quality != (uint8_t)-1) {
@@ -430,7 +430,7 @@ static int tight_fill_palette(VncState *vs, int x, int y,
         max = 256;
     }
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
         return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette);
     case 2:
@@ -557,15 +557,15 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
     buf32 = (uint32_t *)buf;
     memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
 
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)) {
-        shift[0] = vs->clientds.pf.rshift;
-        shift[1] = vs->clientds.pf.gshift;
-        shift[2] = vs->clientds.pf.bshift;
+    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
+             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+        shift[0] = vs->client_pf.rshift;
+        shift[1] = vs->client_pf.gshift;
+        shift[2] = vs->client_pf.bshift;
     } else {
-        shift[0] = 24 - vs->clientds.pf.rshift;
-        shift[1] = 24 - vs->clientds.pf.gshift;
-        shift[2] = 24 - vs->clientds.pf.bshift;
+        shift[0] = 24 - vs->client_pf.rshift;
+        shift[1] = 24 - vs->client_pf.gshift;
+        shift[2] = 24 - vs->client_pf.bshift;
     }
 
     for (y = 0; y < h; y++) {
@@ -615,15 +615,15 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
                                                                         \
         memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));     \
                                                                         \
-        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \
-                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \
+        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
+                       (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
                                                                         \
-        max[0] = vs->clientds.pf.rmax;                                  \
-        max[1] = vs->clientds.pf.gmax;                                  \
-        max[2] = vs->clientds.pf.bmax;                                  \
-        shift[0] = vs->clientds.pf.rshift;                              \
-        shift[1] = vs->clientds.pf.gshift;                              \
-        shift[2] = vs->clientds.pf.bshift;                              \
+        max[0] = vs->client_pf.rmax;                                    \
+        max[1] = vs->client_pf.gmax;                                    \
+        max[2] = vs->client_pf.bmax;                                    \
+        shift[0] = vs->client_pf.rshift;                                \
+        shift[1] = vs->client_pf.gshift;                                \
+        shift[2] = vs->client_pf.bshift;                                \
                                                                         \
         for (y = 0; y < h; y++) {                                       \
             for (c = 0; c < 3; c++) {                                   \
@@ -671,56 +671,42 @@ DEFINE_GRADIENT_FILTER_FUNCTION(32)
  * that case new color will be stored in *colorPtr.
  */
 
-#define DEFINE_CHECK_SOLID_FUNCTION(bpp)                                \
-                                                                        \
-    static bool                                                         \
-    check_solid_tile##bpp(VncState *vs, int x, int y, int w, int h,     \
-                          uint32_t* color, bool samecolor)              \
-    {                                                                   \
-        VncDisplay *vd = vs->vd;                                        \
-        uint##bpp##_t *fbptr;                                           \
-        uint##bpp##_t c;                                                \
-        int dx, dy;                                                     \
-                                                                        \
-        fbptr = (uint##bpp##_t *)                                       \
-            (vd->server->data + y * ds_get_linesize(vs->ds) +           \
-             x * ds_get_bytes_per_pixel(vs->ds));                       \
-                                                                        \
-        c = *fbptr;                                                     \
-        if (samecolor && (uint32_t)c != *color) {                       \
-            return false;                                               \
-        }                                                               \
-                                                                        \
-        for (dy = 0; dy < h; dy++) {                                    \
-            for (dx = 0; dx < w; dx++) {                                \
-                if (c != fbptr[dx]) {                                   \
-                    return false;                                       \
-                }                                                       \
-            }                                                           \
-            fbptr = (uint##bpp##_t *)                                   \
-                ((uint8_t *)fbptr + ds_get_linesize(vs->ds));           \
-        }                                                               \
-                                                                        \
-        *color = (uint32_t)c;                                           \
-        return true;                                                    \
+static bool
+check_solid_tile32(VncState *vs, int x, int y, int w, int h,
+                   uint32_t *color, bool samecolor)
+{
+    VncDisplay *vd = vs->vd;
+    uint32_t *fbptr;
+    uint32_t c;
+    int dx, dy;
+
+    fbptr = vnc_server_fb_ptr(vd, x, y);
+
+    c = *fbptr;
+    if (samecolor && (uint32_t)c != *color) {
+        return false;
+    }
+
+    for (dy = 0; dy < h; dy++) {
+        for (dx = 0; dx < w; dx++) {
+            if (c != fbptr[dx]) {
+                return false;
+            }
+        }
+        fbptr = (uint32_t *)
+            ((uint8_t *)fbptr + vnc_server_fb_stride(vd));
     }
 
-DEFINE_CHECK_SOLID_FUNCTION(32)
-DEFINE_CHECK_SOLID_FUNCTION(16)
-DEFINE_CHECK_SOLID_FUNCTION(8)
+    *color = (uint32_t)c;
+    return true;
+}
 
 static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,
                              uint32_t* color, bool samecolor)
 {
-    VncDisplay *vd = vs->vd;
-
-    switch(vd->server->pf.bytes_per_pixel) {
+    switch (VNC_SERVER_FB_BYTES) {
     case 4:
         return check_solid_tile32(vs, x, y, w, h, color, samecolor);
-    case 2:
-        return check_solid_tile16(vs, x, y, w, h, color, samecolor);
-    default:
-        return check_solid_tile8(vs, x, y, w, h, color, samecolor);
     }
 }
 
@@ -906,15 +892,15 @@ static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
 
     buf32 = (uint32_t *)buf;
 
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)) {
-        rshift = vs->clientds.pf.rshift;
-        gshift = vs->clientds.pf.gshift;
-        bshift = vs->clientds.pf.bshift;
+    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
+             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+        rshift = vs->client_pf.rshift;
+        gshift = vs->client_pf.gshift;
+        bshift = vs->client_pf.bshift;
     } else {
-        rshift = 24 - vs->clientds.pf.rshift;
-        gshift = 24 - vs->clientds.pf.gshift;
-        bshift = 24 - vs->clientds.pf.bshift;
+        rshift = 24 - vs->client_pf.rshift;
+        gshift = 24 - vs->client_pf.gshift;
+        bshift = 24 - vs->client_pf.bshift;
     }
 
     if (ret) {
@@ -946,7 +932,7 @@ static int send_full_color_rect(VncState *vs, int x, int y, int w, int h)
         tight_pack24(vs, vs->tight.tight.buffer, w * h, &vs->tight.tight.offset);
         bytes = 3;
     } else {
-        bytes = vs->clientds.pf.bytes_per_pixel;
+        bytes = vs->client_pf.bytes_per_pixel;
     }
 
     bytes = tight_compress_data(vs, stream, w * h * bytes,
@@ -966,7 +952,7 @@ static int send_solid_rect(VncState *vs)
         tight_pack24(vs, vs->tight.tight.buffer, 1, &vs->tight.tight.offset);
         bytes = 3;
     } else {
-        bytes = vs->clientds.pf.bytes_per_pixel;
+        bytes = vs->client_pf.bytes_per_pixel;
     }
 
     vnc_write(vs, vs->tight.tight.buffer, bytes);
@@ -983,7 +969,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
 #ifdef CONFIG_VNC_PNG
     if (tight_can_send_png_rect(vs, w, h)) {
         int ret;
-        int bpp = vs->clientds.pf.bytes_per_pixel * 8;
+        int bpp = vs->client_pf.bytes_per_pixel * 8;
         VncPalette *palette = palette_new(2, bpp);
 
         palette_put(palette, bg);
@@ -1000,7 +986,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
     vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
     vnc_write_u8(vs, 1);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
     {
         uint32_t buf[2] = {bg, fg};
@@ -1043,7 +1029,7 @@ static void write_palette(int idx, uint32_t color, void *opaque)
 {
     struct palette_cb_priv *priv = opaque;
     VncState *vs = priv->vs;
-    uint32_t bytes = vs->clientds.pf.bytes_per_pixel;
+    uint32_t bytes = vs->client_pf.bytes_per_pixel;
 
     if (bytes == 4) {
         ((uint32_t*)priv->header)[idx] = color;
@@ -1058,8 +1044,9 @@ static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
     int level = tight_conf[vs->tight.compression].gradient_zlib_level;
     ssize_t bytes;
 
-    if (vs->clientds.pf.bytes_per_pixel == 1)
+    if (vs->client_pf.bytes_per_pixel == 1) {
         return send_full_color_rect(vs, x, y, w, h);
+    }
 
     vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
     vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT);
@@ -1069,7 +1056,7 @@ static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
     if (vs->tight.pixel24) {
         tight_filter_gradient24(vs, vs->tight.tight.buffer, w, h);
         bytes = 3;
-    } else if (vs->clientds.pf.bytes_per_pixel == 4) {
+    } else if (vs->client_pf.bytes_per_pixel == 4) {
         tight_filter_gradient32(vs, (uint32_t *)vs->tight.tight.buffer, w, h);
         bytes = 4;
     } else {
@@ -1107,7 +1094,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
     vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
     vnc_write_u8(vs, colors - 1);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
     {
         size_t old_offset, offset;
@@ -1148,79 +1135,6 @@ static int send_palette_rect(VncState *vs, int x, int y,
     return (bytes >= 0);
 }
 
-#if defined(CONFIG_VNC_JPEG) || defined(CONFIG_VNC_PNG)
-static void rgb_prepare_row24(VncState *vs, uint8_t *dst, int x, int y,
-                              int count)
-{
-    VncDisplay *vd = vs->vd;
-    uint32_t *fbptr;
-    uint32_t pix;
-
-    fbptr = (uint32_t *)(vd->server->data + y * ds_get_linesize(vs->ds) +
-                         x * ds_get_bytes_per_pixel(vs->ds));
-
-    while (count--) {
-        pix = *fbptr++;
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.rshift);
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.gshift);
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.bshift);
-    }
-}
-
-#define DEFINE_RGB_GET_ROW_FUNCTION(bpp)                                \
-                                                                        \
-    static void                                                         \
-    rgb_prepare_row##bpp(VncState *vs, uint8_t *dst,                    \
-                         int x, int y, int count)                       \
-    {                                                                   \
-        VncDisplay *vd = vs->vd;                                        \
-        uint##bpp##_t *fbptr;                                           \
-        uint##bpp##_t pix;                                              \
-        int r, g, b;                                                    \
-                                                                        \
-        fbptr = (uint##bpp##_t *)                                       \
-            (vd->server->data + y * ds_get_linesize(vs->ds) +           \
-             x * ds_get_bytes_per_pixel(vs->ds));                       \
-                                                                        \
-        while (count--) {                                               \
-            pix = *fbptr++;                                             \
-                                                                        \
-            r = (int)((pix >> vs->ds->surface->pf.rshift)               \
-                      & vs->ds->surface->pf.rmax);                      \
-            g = (int)((pix >> vs->ds->surface->pf.gshift)               \
-                      & vs->ds->surface->pf.gmax);                      \
-            b = (int)((pix >> vs->ds->surface->pf.bshift)               \
-                      & vs->ds->surface->pf.bmax);                      \
-                                                                        \
-            *dst++ = (uint8_t)((r * 255 + vs->ds->surface->pf.rmax / 2) \
-                               / vs->ds->surface->pf.rmax);             \
-            *dst++ = (uint8_t)((g * 255 + vs->ds->surface->pf.gmax / 2) \
-                               / vs->ds->surface->pf.gmax);             \
-            *dst++ = (uint8_t)((b * 255 + vs->ds->surface->pf.bmax / 2) \
-                               / vs->ds->surface->pf.bmax);             \
-        }                                                               \
-    }
-
-DEFINE_RGB_GET_ROW_FUNCTION(16)
-DEFINE_RGB_GET_ROW_FUNCTION(32)
-
-static void rgb_prepare_row(VncState *vs, uint8_t *dst, int x, int y,
-                            int count)
-{
-    if (ds_get_bytes_per_pixel(vs->ds) == 4) {
-        if (vs->ds->surface->pf.rmax == 0xFF &&
-            vs->ds->surface->pf.gmax == 0xFF &&
-            vs->ds->surface->pf.bmax == 0xFF) {
-            rgb_prepare_row24(vs, dst, x, y, count);
-        } else {
-            rgb_prepare_row32(vs, dst, x, y, count);
-        }
-    } else {
-        rgb_prepare_row16(vs, dst, x, y, count);
-    }
-}
-#endif /* CONFIG_VNC_JPEG or CONFIG_VNC_PNG */
-
 /*
  * JPEG compression stuff.
  */
@@ -1265,6 +1179,7 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
     struct jpeg_compress_struct cinfo;
     struct jpeg_error_mgr jerr;
     struct jpeg_destination_mgr manager;
+    pixman_image_t *linebuf;
     JSAMPROW row[1];
     uint8_t *buf;
     int dy;
@@ -1293,13 +1208,14 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
 
     jpeg_start_compress(&cinfo, true);
 
-    buf = g_malloc(w * 3);
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
+    buf = (uint8_t *)pixman_image_get_data(linebuf);
     row[0] = buf;
     for (dy = 0; dy < h; dy++) {
-        rgb_prepare_row(vs, buf, x, y + dy, w);
+        qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
         jpeg_write_scanlines(&cinfo, row, 1);
     }
-    g_free(buf);
+    qemu_pixman_image_unref(linebuf);
 
     jpeg_finish_compress(&cinfo);
     jpeg_destroy_compress(&cinfo);
@@ -1326,23 +1242,23 @@ static void write_png_palette(int idx, uint32_t pix, void *opaque)
 
     if (vs->tight.pixel24)
     {
-        color->red = (pix >> vs->clientds.pf.rshift) & vs->clientds.pf.rmax;
-        color->green = (pix >> vs->clientds.pf.gshift) & vs->clientds.pf.gmax;
-        color->blue = (pix >> vs->clientds.pf.bshift) & vs->clientds.pf.bmax;
+        color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
+        color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
+        color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
     }
     else
     {
         int red, green, blue;
 
-        red = (pix >> vs->clientds.pf.rshift) & vs->clientds.pf.rmax;
-        green = (pix >> vs->clientds.pf.gshift) & vs->clientds.pf.gmax;
-        blue = (pix >> vs->clientds.pf.bshift) & vs->clientds.pf.bmax;
-        color->red = ((red * 255 + vs->clientds.pf.rmax / 2) /
-                      vs->clientds.pf.rmax);
-        color->green = ((green * 255 + vs->clientds.pf.gmax / 2) /
-                        vs->clientds.pf.gmax);
-        color->blue = ((blue * 255 + vs->clientds.pf.bmax / 2) /
-                       vs->clientds.pf.bmax);
+        red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
+        green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
+        blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
+        color->red = ((red * 255 + vs->client_pf.rmax / 2) /
+                      vs->client_pf.rmax);
+        color->green = ((green * 255 + vs->client_pf.gmax / 2) /
+                        vs->client_pf.gmax);
+        color->blue = ((blue * 255 + vs->client_pf.bmax / 2) /
+                       vs->client_pf.bmax);
     }
 }
 
@@ -1378,6 +1294,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
     png_structp png_ptr;
     png_infop info_ptr;
     png_colorp png_palette = NULL;
+    pixman_image_t *linebuf;
     int level = tight_png_conf[vs->tight.compression].png_zlib_level;
     int filters = tight_png_conf[vs->tight.compression].png_filters;
     uint8_t *buf;
@@ -1422,7 +1339,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
 
         png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
 
-        if (vs->clientds.pf.bytes_per_pixel == 4) {
+        if (vs->client_pf.bytes_per_pixel == 4) {
             tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
         } else {
             tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
@@ -1432,17 +1349,18 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
     png_write_info(png_ptr, info_ptr);
 
     buffer_reserve(&vs->tight.png, 2048);
-    buf = g_malloc(w * 3);
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
+    buf = (uint8_t *)pixman_image_get_data(linebuf);
     for (dy = 0; dy < h; dy++)
     {
         if (color_type == PNG_COLOR_TYPE_PALETTE) {
             memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
         } else {
-            rgb_prepare_row(vs, buf, x, y + dy, w);
+            qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
         }
         png_write_row(png_ptr, buf);
     }
-    g_free(buf);
+    qemu_pixman_image_unref(linebuf);
 
     png_write_end(png_ptr, NULL);
 
@@ -1713,8 +1631,8 @@ static int tight_send_framebuffer_update(VncState *vs, int x, int y,
 {
     int max_rows;
 
-    if (vs->clientds.pf.bytes_per_pixel == 4 && vs->clientds.pf.rmax == 0xFF &&
-        vs->clientds.pf.bmax == 0xFF && vs->clientds.pf.gmax == 0xFF) {
+    if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF &&
+        vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) {
         vs->tight.pixel24 = true;
     } else {
         vs->tight.pixel24 = false;
diff --git a/ui/vnc-enc-zrle.c b/ui/vnc-enc-zrle.c
index 917d384f57f..ed3b48465d7 100644
--- a/ui/vnc-enc-zrle.c
+++ b/ui/vnc-enc-zrle.c
@@ -255,7 +255,7 @@ static void zrle_write_u8(VncState *vs, uint8_t value)
 static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
                                         int w, int h)
 {
-    bool be = !!(vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG);
+    bool be = vs->client_be;
     size_t bytes;
     int zywrle_level;
 
@@ -277,13 +277,13 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
 
     vnc_zrle_start(vs);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 1:
         zrle_encode_8ne(vs, x, y, w, h, zywrle_level);
         break;
 
     case 2:
-        if (vs->clientds.pf.gmax > 0x1F) {
+        if (vs->client_pf.gmax > 0x1F) {
             if (be) {
                 zrle_encode_16be(vs, x, y, w, h, zywrle_level);
             } else {
@@ -304,13 +304,13 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
         bool fits_in_ms3bytes;
 
         fits_in_ls3bytes =
-            ((vs->clientds.pf.rmax << vs->clientds.pf.rshift) < (1 << 24) &&
-             (vs->clientds.pf.gmax << vs->clientds.pf.gshift) < (1 << 24) &&
-             (vs->clientds.pf.bmax << vs->clientds.pf.bshift) < (1 << 24));
+            ((vs->client_pf.rmax << vs->client_pf.rshift) < (1 << 24) &&
+             (vs->client_pf.gmax << vs->client_pf.gshift) < (1 << 24) &&
+             (vs->client_pf.bmax << vs->client_pf.bshift) < (1 << 24));
 
-        fits_in_ms3bytes = (vs->clientds.pf.rshift > 7 &&
-                            vs->clientds.pf.gshift > 7 &&
-                            vs->clientds.pf.bshift > 7);
+        fits_in_ms3bytes = (vs->client_pf.rshift > 7 &&
+                            vs->client_pf.gshift > 7 &&
+                            vs->client_pf.bshift > 7);
 
         if ((fits_in_ls3bytes && !be) || (fits_in_ms3bytes && be)) {
             if (be) {
diff --git a/ui/vnc-jobs.c b/ui/vnc-jobs.c
index 087b84d319e..0bfc0c5485b 100644
--- a/ui/vnc-jobs.c
+++ b/ui/vnc-jobs.c
@@ -28,26 +28,26 @@
 
 #include "vnc.h"
 #include "vnc-jobs.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 /*
  * Locking:
  *
- * There is three levels of locking:
+ * There are three levels of locking:
  * - jobs queue lock: for each operation on the queue (push, pop, isEmpty?)
  * - VncDisplay global lock: mainly used for framebuffer updates to avoid
  *                      screen corruption if the framebuffer is updated
- *			while the worker is doing something.
+ *                      while the worker is doing something.
  * - VncState::output lock: used to make sure the output buffer is not corrupted
- * 		   	 if two threads try to write on it at the same time
+ *                          if two threads try to write on it at the same time
  *
- * While the VNC worker thread is working, the VncDisplay global lock is hold
- * to avoid screen corruptions (this does not block vnc_refresh() because it
- * uses trylock()) but the output lock is not hold because the thread work on
+ * While the VNC worker thread is working, the VncDisplay global lock is held
+ * to avoid screen corruption (this does not block vnc_refresh() because it
+ * uses trylock()) but the output lock is not held because the thread works on
  * its own output buffer.
  * When the encoding job is done, the worker thread will hold the output lock
  * and copy its output buffer in vs->output.
-*/
+ */
 
 struct VncJobQueue {
     QemuCond cond;
@@ -62,7 +62,7 @@ typedef struct VncJobQueue VncJobQueue;
 
 /*
  * We use a single global queue, but most of the functions are
- * already reetrant, so we can easilly add more than one encoding thread
+ * already reentrant, so we can easily add more than one encoding thread
  */
 static VncJobQueue *queue;
 
@@ -187,7 +187,8 @@ static void vnc_async_encoding_start(VncState *orig, VncState *local)
     local->vd = orig->vd;
     local->lossy_rect = orig->lossy_rect;
     local->write_pixels = orig->write_pixels;
-    local->clientds = orig->clientds;
+    local->client_pf = orig->client_pf;
+    local->client_be = orig->client_be;
     local->tight = orig->tight;
     local->zlib = orig->zlib;
     local->hextile = orig->hextile;
@@ -320,6 +321,11 @@ static void *vnc_worker_thread(void *arg)
     return NULL;
 }
 
+static bool vnc_worker_thread_running(void)
+{
+    return queue; /* Check global queue */
+}
+
 void vnc_start_worker_thread(void)
 {
     VncJobQueue *q;
@@ -332,11 +338,6 @@ void vnc_start_worker_thread(void)
     queue = q; /* Set global queue */
 }
 
-bool vnc_worker_thread_running(void)
-{
-    return queue; /* Check global queue */
-}
-
 void vnc_stop_worker_thread(void)
 {
     if (!vnc_worker_thread_running())
diff --git a/ui/vnc-jobs.h b/ui/vnc-jobs.h
index 86e6d888c6f..31da103facb 100644
--- a/ui/vnc-jobs.h
+++ b/ui/vnc-jobs.h
@@ -40,7 +40,6 @@ void vnc_jobs_join(VncState *vs);
 
 void vnc_jobs_consume_buffer(VncState *vs);
 void vnc_start_worker_thread(void);
-bool vnc_worker_thread_running(void);
 void vnc_stop_worker_thread(void);
 
 /* Locks */
diff --git a/ui/vnc-palette.c b/ui/vnc-palette.c
index 63d5f64917d..c130deee9d4 100644
--- a/ui/vnc-palette.c
+++ b/ui/vnc-palette.c
@@ -27,6 +27,8 @@
  */
 
 #include "vnc-palette.h"
+#include <glib.h>
+#include <string.h>
 
 static VncPaletteEntry *palette_find(const VncPalette *palette,
                                      uint32_t color, unsigned int hash)
diff --git a/ui/vnc-palette.h b/ui/vnc-palette.h
index 3260885ff0d..d02f0236c1c 100644
--- a/ui/vnc-palette.h
+++ b/ui/vnc-palette.h
@@ -29,9 +29,10 @@
 #ifndef VNC_PALETTE_H
 #define VNC_PALETTE_H
 
-#include "qlist.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qlist.h"
+#include "qemu/queue.h"
 #include <stdint.h>
+#include <stdbool.h>
 
 #define VNC_PALETTE_HASH_SIZE 256
 #define VNC_PALETTE_MAX_SIZE  256
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
index 3aaa93928ad..56292636d7c 100644
--- a/ui/vnc-tls.c
+++ b/ui/vnc-tls.c
@@ -26,7 +26,7 @@
 
 #include "qemu-x509.h"
 #include "vnc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
 /* Very verbose, so only enabled for _VNC_DEBUG >= 2 */
@@ -49,7 +49,7 @@ static int vnc_tls_initialize(void)
     if (gnutls_global_init () < 0)
         return 0;
 
-    /* XXX ought to re-generate diffie-hellmen params periodically */
+    /* XXX ought to re-generate diffie-hellman params periodically */
     if (gnutls_dh_params_init (&dh_params) < 0)
         return 0;
     if (gnutls_dh_params_generate2 (dh_params, DH_BITS) < 0)
diff --git a/ui/vnc-tls.h b/ui/vnc-tls.h
index 2b93633896b..36a2227fece 100644
--- a/ui/vnc-tls.h
+++ b/ui/vnc-tls.h
@@ -31,7 +31,7 @@
 #include <gnutls/gnutls.h>
 #include <gnutls/x509.h>
 
-#include "acl.h"
+#include "qemu/acl.h"
 
 enum {
     VNC_WIREMODE_CLEAR,
diff --git a/ui/vnc-ws.c b/ui/vnc-ws.c
new file mode 100644
index 00000000000..9ccdc1971cd
--- /dev/null
+++ b/ui/vnc-ws.c
@@ -0,0 +1,284 @@
+/*
+ * QEMU VNC display driver: Websockets support
+ *
+ * Copyright (C) 2010 Joel Martin
+ * Copyright (C) 2012 Tim Hardeck
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnc.h"
+
+void vncws_handshake_read(void *opaque)
+{
+    VncState *vs = opaque;
+    uint8_t *handshake_end;
+    long ret;
+    buffer_reserve(&vs->ws_input, 4096);
+    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
+
+    if (!ret) {
+        if (vs->csock == -1) {
+            vnc_disconnect_finish(vs);
+        }
+        return;
+    }
+    vs->ws_input.offset += ret;
+
+    handshake_end = (uint8_t *)g_strstr_len((char *)vs->ws_input.buffer,
+            vs->ws_input.offset, WS_HANDSHAKE_END);
+    if (handshake_end) {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+        vncws_process_handshake(vs, vs->ws_input.buffer, vs->ws_input.offset);
+        buffer_advance(&vs->ws_input, handshake_end - vs->ws_input.buffer +
+                strlen(WS_HANDSHAKE_END));
+    }
+}
+
+
+long vnc_client_read_ws(VncState *vs)
+{
+    int ret, err;
+    uint8_t *payload;
+    size_t payload_size, frame_size;
+    VNC_DEBUG("Read websocket %p size %zd offset %zd\n", vs->ws_input.buffer,
+            vs->ws_input.capacity, vs->ws_input.offset);
+    buffer_reserve(&vs->ws_input, 4096);
+    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
+    if (!ret) {
+        return 0;
+    }
+    vs->ws_input.offset += ret;
+
+    /* make sure that nothing is left in the ws_input buffer */
+    do {
+        err = vncws_decode_frame(&vs->ws_input, &payload,
+                              &payload_size, &frame_size);
+        if (err <= 0) {
+            return err;
+        }
+
+        buffer_reserve(&vs->input, payload_size);
+        buffer_append(&vs->input, payload, payload_size);
+
+        buffer_advance(&vs->ws_input, frame_size);
+    } while (vs->ws_input.offset > 0);
+
+    return ret;
+}
+
+long vnc_client_write_ws(VncState *vs)
+{
+    long ret;
+    VNC_DEBUG("Write WS: Pending output %p size %zd offset %zd\n",
+              vs->output.buffer, vs->output.capacity, vs->output.offset);
+    vncws_encode_frame(&vs->ws_output, vs->output.buffer, vs->output.offset);
+    buffer_reset(&vs->output);
+    ret = vnc_client_write_buf(vs, vs->ws_output.buffer, vs->ws_output.offset);
+    if (!ret) {
+        return 0;
+    }
+
+    buffer_advance(&vs->ws_output, ret);
+
+    if (vs->ws_output.offset == 0) {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    }
+
+    return ret;
+}
+
+static char *vncws_extract_handshake_entry(const char *handshake,
+        size_t handshake_len, const char *name)
+{
+    char *begin, *end, *ret = NULL;
+    char *line = g_strdup_printf("%s%s: ", WS_HANDSHAKE_DELIM, name);
+    begin = g_strstr_len(handshake, handshake_len, line);
+    if (begin != NULL) {
+        begin += strlen(line);
+        end = g_strstr_len(begin, handshake_len - (begin - handshake),
+                WS_HANDSHAKE_DELIM);
+        if (end != NULL) {
+            ret = g_strndup(begin, end - begin);
+        }
+    }
+    g_free(line);
+    return ret;
+}
+
+static void vncws_send_handshake_response(VncState *vs, const char* key)
+{
+    char combined_key[WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1];
+    char hash[SHA1_DIGEST_LEN];
+    size_t hash_size = SHA1_DIGEST_LEN;
+    char *accept = NULL, *response = NULL;
+    gnutls_datum_t in;
+
+    g_strlcpy(combined_key, key, WS_CLIENT_KEY_LEN + 1);
+    g_strlcat(combined_key, WS_GUID, WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1);
+
+    /* hash and encode it */
+    in.data = (void *)combined_key;
+    in.size = WS_CLIENT_KEY_LEN + WS_GUID_LEN;
+    if (gnutls_fingerprint(GNUTLS_DIG_SHA1, &in, hash, &hash_size)
+            == GNUTLS_E_SUCCESS) {
+        accept = g_base64_encode((guchar *)hash, SHA1_DIGEST_LEN);
+    }
+    if (accept == NULL) {
+        VNC_DEBUG("Hashing Websocket combined key failed\n");
+        vnc_client_error(vs);
+        return;
+    }
+
+    response = g_strdup_printf(WS_HANDSHAKE, accept);
+    vnc_write(vs, response, strlen(response));
+    vnc_flush(vs);
+
+    g_free(accept);
+    g_free(response);
+
+    vs->encode_ws = 1;
+    vnc_init_state(vs);
+}
+
+void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size)
+{
+    char *protocols = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Protocol");
+    char *version = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Version");
+    char *key = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Key");
+
+    if (protocols && version && key
+            && g_strrstr(protocols, "binary")
+            && !strcmp(version, WS_SUPPORTED_VERSION)
+            && strlen(key) == WS_CLIENT_KEY_LEN) {
+        vncws_send_handshake_response(vs, key);
+    } else {
+        VNC_DEBUG("Defective Websockets header or unsupported protocol\n");
+        vnc_client_error(vs);
+    }
+
+    g_free(protocols);
+    g_free(version);
+    g_free(key);
+}
+
+void vncws_encode_frame(Buffer *output, const void *payload,
+        const size_t payload_size)
+{
+    size_t header_size = 0;
+    unsigned char opcode = WS_OPCODE_BINARY_FRAME;
+    union {
+        char buf[WS_HEAD_MAX_LEN];
+        WsHeader ws;
+    } header;
+
+    if (!payload_size) {
+        return;
+    }
+
+    header.ws.b0 = 0x80 | (opcode & 0x0f);
+    if (payload_size <= 125) {
+        header.ws.b1 = (uint8_t)payload_size;
+        header_size = 2;
+    } else if (payload_size < 65536) {
+        header.ws.b1 = 0x7e;
+        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)payload_size);
+        header_size = 4;
+    } else {
+        header.ws.b1 = 0x7f;
+        header.ws.u.s64.l64 = cpu_to_be64(payload_size);
+        header_size = 10;
+    }
+
+    buffer_reserve(output, header_size + payload_size);
+    buffer_append(output, header.buf, header_size);
+    buffer_append(output, payload, payload_size);
+}
+
+int vncws_decode_frame(Buffer *input, uint8_t **payload,
+                           size_t *payload_size, size_t *frame_size)
+{
+    unsigned char opcode = 0, fin = 0, has_mask = 0;
+    size_t header_size = 0;
+    uint32_t *payload32;
+    WsHeader *header = (WsHeader *)input->buffer;
+    WsMask mask;
+    int i;
+
+    if (input->offset < WS_HEAD_MIN_LEN + 4) {
+        /* header not complete */
+        return 0;
+    }
+
+    fin = (header->b0 & 0x80) >> 7;
+    opcode = header->b0 & 0x0f;
+    has_mask = (header->b1 & 0x80) >> 7;
+    *payload_size = header->b1 & 0x7f;
+
+    if (opcode == WS_OPCODE_CLOSE) {
+        /* disconnect */
+        return -1;
+    }
+
+    /* Websocket frame sanity check:
+     * * Websocket fragmentation is not supported.
+     * * All  websockets frames sent by a client have to be masked.
+     * * Only binary encoding is supported.
+     */
+    if (!fin || !has_mask || opcode != WS_OPCODE_BINARY_FRAME) {
+        VNC_DEBUG("Received faulty/unsupported Websocket frame\n");
+        return -2;
+    }
+
+    if (*payload_size < 126) {
+        header_size = 6;
+        mask = header->u.m;
+    } else if (*payload_size == 126 && input->offset >= 8) {
+        *payload_size = be16_to_cpu(header->u.s16.l16);
+        header_size = 8;
+        mask = header->u.s16.m16;
+    } else if (*payload_size == 127 && input->offset >= 14) {
+        *payload_size = be64_to_cpu(header->u.s64.l64);
+        header_size = 14;
+        mask = header->u.s64.m64;
+    } else {
+        /* header not complete */
+        return 0;
+    }
+
+    *frame_size = header_size + *payload_size;
+
+    if (input->offset < *frame_size) {
+        /* frame not complete */
+        return 0;
+    }
+
+    *payload = input->buffer + header_size;
+
+    /* unmask frame */
+    /* process 1 frame (32 bit op) */
+    payload32 = (uint32_t *)(*payload);
+    for (i = 0; i < *payload_size / 4; i++) {
+        payload32[i] ^= mask.u;
+    }
+    /* process the remaining bytes (if any) */
+    for (i *= 4; i < *payload_size; i++) {
+        (*payload)[i] ^= mask.c[i % 4];
+    }
+
+    return 1;
+}
diff --git a/ui/vnc-ws.h b/ui/vnc-ws.h
new file mode 100644
index 00000000000..039a58765ca
--- /dev/null
+++ b/ui/vnc-ws.h
@@ -0,0 +1,86 @@
+/*
+ * QEMU VNC display driver: Websockets support
+ *
+ * Copyright (C) 2010 Joel Martin
+ * Copyright (C) 2012 Tim Hardeck
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __QEMU_UI_VNC_WS_H
+#define __QEMU_UI_VNC_WS_H
+
+#include <gnutls/gnutls.h>
+
+#define B64LEN(__x) (((__x + 2) / 3) * 12 / 3)
+#define SHA1_DIGEST_LEN 20
+
+#define WS_ACCEPT_LEN (B64LEN(SHA1_DIGEST_LEN) + 1)
+#define WS_CLIENT_KEY_LEN 24
+#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+#define WS_GUID_LEN strlen(WS_GUID)
+
+#define WS_HANDSHAKE "HTTP/1.1 101 Switching Protocols\r\n\
+Upgrade: websocket\r\n\
+Connection: Upgrade\r\n\
+Sec-WebSocket-Accept: %s\r\n\
+Sec-WebSocket-Protocol: binary\r\n\
+\r\n"
+#define WS_HANDSHAKE_DELIM "\r\n"
+#define WS_HANDSHAKE_END "\r\n\r\n"
+#define WS_SUPPORTED_VERSION "13"
+
+#define WS_HEAD_MIN_LEN sizeof(uint16_t)
+#define WS_HEAD_MAX_LEN (WS_HEAD_MIN_LEN + sizeof(uint64_t) + sizeof(uint32_t))
+
+typedef union WsMask {
+    char c[4];
+    uint32_t u;
+} WsMask;
+
+typedef struct QEMU_PACKED WsHeader {
+    unsigned char b0;
+    unsigned char b1;
+    union {
+        struct QEMU_PACKED {
+            uint16_t l16;
+            WsMask m16;
+        } s16;
+        struct QEMU_PACKED {
+            uint64_t l64;
+            WsMask m64;
+        } s64;
+        WsMask m;
+    } u;
+} WsHeader;
+
+enum {
+    WS_OPCODE_CONTINUATION = 0x0,
+    WS_OPCODE_TEXT_FRAME = 0x1,
+    WS_OPCODE_BINARY_FRAME = 0x2,
+    WS_OPCODE_CLOSE = 0x8,
+    WS_OPCODE_PING = 0x9,
+    WS_OPCODE_PONG = 0xA
+};
+
+void vncws_handshake_read(void *opaque);
+long vnc_client_write_ws(VncState *vs);
+long vnc_client_read_ws(VncState *vs);
+void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size);
+void vncws_encode_frame(Buffer *output, const void *payload,
+            const size_t payload_size);
+int vncws_decode_frame(Buffer *input, uint8_t **payload,
+                               size_t *payload_size, size_t *frame_size);
+
+#endif /* __QEMU_UI_VNC_WS_H */
diff --git a/ui/vnc.c b/ui/vnc.c
index 385e345c312..ff4e2ae5860 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -26,13 +26,13 @@
 
 #include "vnc.h"
 #include "vnc-jobs.h"
-#include "sysemu.h"
-#include "qemu_socket.h"
-#include "qemu-timer.h"
-#include "acl.h"
-#include "qemu-objects.h"
+#include "sysemu/sysemu.h"
+#include "qemu/sockets.h"
+#include "qemu/timer.h"
+#include "qemu/acl.h"
+#include "qapi/qmp/types.h"
 #include "qmp-commands.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 #define VNC_REFRESH_INTERVAL_BASE 30
 #define VNC_REFRESH_INTERVAL_INC  50
@@ -372,6 +372,10 @@ VncInfo *qmp_query_vnc(Error **errp)
             }
         }
 
+        if (vnc_display->lsock == -1) {
+            return info;
+        }
+
         if (getsockname(vnc_display->lsock, (struct sockaddr *)&sa,
                         &salen) == -1) {
             error_set(errp, QERR_UNDEFINED_ERROR);
@@ -416,7 +420,6 @@ out_error:
 static int vnc_update_client(VncState *vs, int has_dirty);
 static int vnc_update_client_sync(VncState *vs, int has_dirty);
 static void vnc_disconnect_start(VncState *vs);
-static void vnc_disconnect_finish(VncState *vs);
 static void vnc_init_timer(VncDisplay *vd);
 static void vnc_remove_timer(VncDisplay *vd);
 
@@ -432,6 +435,8 @@ static void vnc_dpy_update(DisplayState *ds, int x, int y, int w, int h)
     int i;
     VncDisplay *vd = ds->opaque;
     struct VncSurface *s = &vd->guest;
+    int width = ds_get_width(ds);
+    int height = ds_get_height(ds);
 
     h += y;
 
@@ -442,10 +447,10 @@ static void vnc_dpy_update(DisplayState *ds, int x, int y, int w, int h)
     w += (x % 16);
     x -= (x % 16);
 
-    x = MIN(x, s->ds->width);
-    y = MIN(y, s->ds->height);
-    w = MIN(x + w, s->ds->width) - x;
-    h = MIN(h, s->ds->height);
+    x = MIN(x, width);
+    y = MIN(y, height);
+    w = MIN(x + w, width) - x;
+    h = MIN(h, height);
 
     for (; y < h; y++)
         for (i = 0; i < w; i += 16)
@@ -475,7 +480,7 @@ void buffer_reserve(Buffer *buffer, size_t len)
     }
 }
 
-int buffer_empty(Buffer *buffer)
+static int buffer_empty(Buffer *buffer)
 {
     return buffer->offset == 0;
 }
@@ -504,6 +509,13 @@ void buffer_append(Buffer *buffer, const void *data, size_t len)
     buffer->offset += len;
 }
 
+void buffer_advance(Buffer *buf, size_t len)
+{
+    memmove(buf->buffer, buf->buffer + len,
+            (buf->offset - len));
+    buf->offset -= len;
+}
+
 static void vnc_desktop_resize(VncState *vs)
 {
     DisplayState *ds = vs->ds;
@@ -546,6 +558,21 @@ static void vnc_abort_display_jobs(VncDisplay *vd)
     }
 }
 
+int vnc_server_fb_stride(VncDisplay *vd)
+{
+    return pixman_image_get_stride(vd->server);
+}
+
+void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y)
+{
+    uint8_t *ptr;
+
+    ptr  = (uint8_t *)pixman_image_get_data(vd->server);
+    ptr += y * vnc_server_fb_stride(vd);
+    ptr += x * VNC_SERVER_FB_BYTES;
+    return ptr;
+}
+
 static void vnc_dpy_resize(DisplayState *ds)
 {
     VncDisplay *vd = ds->opaque;
@@ -554,20 +581,20 @@ static void vnc_dpy_resize(DisplayState *ds)
     vnc_abort_display_jobs(vd);
 
     /* server surface */
-    if (!vd->server)
-        vd->server = g_malloc0(sizeof(*vd->server));
-    if (vd->server->data)
-        g_free(vd->server->data);
-    *(vd->server) = *(ds->surface);
-    vd->server->data = g_malloc0(vd->server->linesize *
-                                    vd->server->height);
+    qemu_pixman_image_unref(vd->server);
+    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
+                                          ds_get_width(ds),
+                                          ds_get_height(ds),
+                                          NULL, 0);
 
     /* guest surface */
-    if (!vd->guest.ds)
-        vd->guest.ds = g_malloc0(sizeof(*vd->guest.ds));
+#if 0 /* FIXME */
     if (ds_get_bytes_per_pixel(ds) != vd->guest.ds->pf.bytes_per_pixel)
         console_color_init(ds);
-    *(vd->guest.ds) = *(ds->surface);
+#endif
+    qemu_pixman_image_unref(vd->guest.fb);
+    vd->guest.fb = pixman_image_ref(ds->surface->image);
+    vd->guest.format = ds->surface->format;
     memset(vd->guest.dirty, 0xFF, sizeof(vd->guest.dirty));
 
     QTAILQ_FOREACH(vs, &vd->clients, next) {
@@ -581,7 +608,7 @@ static void vnc_dpy_resize(DisplayState *ds)
 }
 
 /* fastest code */
-static void vnc_write_pixels_copy(VncState *vs, struct PixelFormat *pf,
+static void vnc_write_pixels_copy(VncState *vs,
                                   void *pixels, int size)
 {
     vnc_write(vs, pixels, size);
@@ -591,23 +618,23 @@ static void vnc_write_pixels_copy(VncState *vs, struct PixelFormat *pf,
 void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 {
     uint8_t r, g, b;
-    VncDisplay *vd = vs->vd;
 
-    r = ((((v & vd->server->pf.rmask) >> vd->server->pf.rshift) << vs->clientds.pf.rbits) >>
-        vd->server->pf.rbits);
-    g = ((((v & vd->server->pf.gmask) >> vd->server->pf.gshift) << vs->clientds.pf.gbits) >>
-        vd->server->pf.gbits);
-    b = ((((v & vd->server->pf.bmask) >> vd->server->pf.bshift) << vs->clientds.pf.bbits) >>
-        vd->server->pf.bbits);
-    v = (r << vs->clientds.pf.rshift) |
-        (g << vs->clientds.pf.gshift) |
-        (b << vs->clientds.pf.bshift);
-    switch(vs->clientds.pf.bytes_per_pixel) {
+#if VNC_SERVER_FB_FORMAT == PIXMAN_FORMAT(32, PIXMAN_TYPE_ARGB, 0, 8, 8, 8)
+    r = (((v & 0x00ff0000) >> 16) << vs->client_pf.rbits) >> 8;
+    g = (((v & 0x0000ff00) >>  8) << vs->client_pf.gbits) >> 8;
+    b = (((v & 0x000000ff) >>  0) << vs->client_pf.bbits) >> 8;
+#else
+# error need some bits here if you change VNC_SERVER_FB_FORMAT
+#endif
+    v = (r << vs->client_pf.rshift) |
+        (g << vs->client_pf.gshift) |
+        (b << vs->client_pf.bshift);
+    switch (vs->client_pf.bytes_per_pixel) {
     case 1:
         buf[0] = v;
         break;
     case 2:
-        if (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) {
+        if (vs->client_be) {
             buf[0] = v >> 8;
             buf[1] = v;
         } else {
@@ -617,7 +644,7 @@ void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
         break;
     default:
     case 4:
-        if (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) {
+        if (vs->client_be) {
             buf[0] = v >> 24;
             buf[1] = v >> 16;
             buf[2] = v >> 8;
@@ -632,37 +659,19 @@ void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
     }
 }
 
-static void vnc_write_pixels_generic(VncState *vs, struct PixelFormat *pf,
+static void vnc_write_pixels_generic(VncState *vs,
                                      void *pixels1, int size)
 {
     uint8_t buf[4];
 
-    if (pf->bytes_per_pixel == 4) {
+    if (VNC_SERVER_FB_BYTES == 4) {
         uint32_t *pixels = pixels1;
         int n, i;
         n = size >> 2;
-        for(i = 0; i < n; i++) {
+        for (i = 0; i < n; i++) {
             vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
+            vnc_write(vs, buf, vs->client_pf.bytes_per_pixel);
         }
-    } else if (pf->bytes_per_pixel == 2) {
-        uint16_t *pixels = pixels1;
-        int n, i;
-        n = size >> 1;
-        for(i = 0; i < n; i++) {
-            vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
-        }
-    } else if (pf->bytes_per_pixel == 1) {
-        uint8_t *pixels = pixels1;
-        int n, i;
-        n = size;
-        for(i = 0; i < n; i++) {
-            vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
-        }
-    } else {
-        fprintf(stderr, "vnc_write_pixels_generic: VncState color depth not supported\n");
     }
 }
 
@@ -672,10 +681,10 @@ int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
     uint8_t *row;
     VncDisplay *vd = vs->vd;
 
-    row = vd->server->data + y * ds_get_linesize(vs->ds) + x * ds_get_bytes_per_pixel(vs->ds);
+    row = vnc_server_fb_ptr(vd, x, y);
     for (i = 0; i < h; i++) {
-        vs->write_pixels(vs, &vd->server->pf, row, w * ds_get_bytes_per_pixel(vs->ds));
-        row += ds_get_linesize(vs->ds);
+        vs->write_pixels(vs, row, w * VNC_SERVER_FB_BYTES);
+        row += vnc_server_fb_stride(vd);
     }
     return 1;
 }
@@ -732,7 +741,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     VncState *vs, *vn;
     uint8_t *src_row;
     uint8_t *dst_row;
-    int i,x,y,pitch,depth,inc,w_lim,s;
+    int i, x, y, pitch, inc, w_lim, s;
     int cmp_bytes;
 
     vnc_refresh_server_surface(vd);
@@ -745,10 +754,9 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     }
 
     /* do bitblit op on the local surface too */
-    pitch = ds_get_linesize(vd->ds);
-    depth = ds_get_bytes_per_pixel(vd->ds);
-    src_row = vd->server->data + pitch * src_y + depth * src_x;
-    dst_row = vd->server->data + pitch * dst_y + depth * dst_x;
+    pitch = vnc_server_fb_stride(vd);
+    src_row = vnc_server_fb_ptr(vd, src_x, src_y);
+    dst_row = vnc_server_fb_ptr(vd, dst_x, dst_y);
     y = dst_y;
     inc = 1;
     if (dst_y > src_y) {
@@ -776,7 +784,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
             } else {
                 s = 16;
             }
-            cmp_bytes = s * depth;
+            cmp_bytes = s * VNC_SERVER_FB_BYTES;
             if (memcmp(src_row, dst_row, cmp_bytes) == 0)
                 continue;
             memmove(dst_row, src_row, cmp_bytes);
@@ -786,8 +794,8 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
                 }
             }
         }
-        src_row += pitch - w * depth;
-        dst_row += pitch - w * depth;
+        src_row += pitch - w * VNC_SERVER_FB_BYTES;
+        dst_row += pitch - w * VNC_SERVER_FB_BYTES;
         y += inc;
     }
 
@@ -798,7 +806,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     }
 }
 
-static void vnc_mouse_set(int x, int y, int visible)
+static void vnc_mouse_set(DisplayState *ds, int x, int y, int visible)
 {
     /* can we ask the client(s) to move the pointer ??? */
 }
@@ -806,7 +814,6 @@ static void vnc_mouse_set(int x, int y, int visible)
 static int vnc_cursor_define(VncState *vs)
 {
     QEMUCursor *c = vs->vd->cursor;
-    PixelFormat pf = qemu_default_pixelformat(32);
     int isize;
 
     if (vnc_has_feature(vs, VNC_FEATURE_RICH_CURSOR)) {
@@ -816,8 +823,8 @@ static int vnc_cursor_define(VncState *vs)
         vnc_write_u16(vs, 1);  /*  # of rects  */
         vnc_framebuffer_update(vs, c->hot_x, c->hot_y, c->width, c->height,
                                VNC_ENCODING_RICH_CURSOR);
-        isize = c->width * c->height * vs->clientds.pf.bytes_per_pixel;
-        vnc_write_pixels_generic(vs, &pf, c->data, isize);
+        isize = c->width * c->height * vs->client_pf.bytes_per_pixel;
+        vnc_write_pixels_generic(vs, c->data, isize);
         vnc_write(vs, vs->vd->cursor_mask, vs->vd->cursor_msize);
         vnc_unlock_output(vs);
         return 0;
@@ -825,7 +832,7 @@ static int vnc_cursor_define(VncState *vs)
     return -1;
 }
 
-static void vnc_dpy_cursor_define(QEMUCursor *c)
+static void vnc_dpy_cursor_define(DisplayState *ds, QEMUCursor *c)
 {
     VncDisplay *vd = vnc_display;
     VncState *vs;
@@ -894,8 +901,8 @@ static int vnc_update_client(VncState *vs, int has_dirty)
          */
         job = vnc_job_new(vs);
 
-        width = MIN(vd->server->width, vs->client_width);
-        height = MIN(vd->server->height, vs->client_height);
+        width = MIN(pixman_image_get_width(vd->server), vs->client_width);
+        height = MIN(pixman_image_get_height(vd->server), vs->client_height);
 
         for (y = 0; y < height; y++) {
             int x;
@@ -1015,7 +1022,7 @@ static void vnc_disconnect_start(VncState *vs)
     vs->csock = -1;
 }
 
-static void vnc_disconnect_finish(VncState *vs)
+void vnc_disconnect_finish(VncState *vs)
 {
     int i;
 
@@ -1026,6 +1033,10 @@ static void vnc_disconnect_finish(VncState *vs)
 
     buffer_free(&vs->input);
     buffer_free(&vs->output);
+#ifdef CONFIG_VNC_WS
+    buffer_free(&vs->ws_input);
+    buffer_free(&vs->ws_output);
+#endif /* CONFIG_VNC_WS */
 
     qobject_decref(vs->info);
 
@@ -1042,20 +1053,24 @@ static void vnc_disconnect_finish(VncState *vs)
     audio_del(vs);
     vnc_release_modifiers(vs);
 
-    QTAILQ_REMOVE(&vs->vd->clients, vs, next);
+    if (vs->initialized) {
+        QTAILQ_REMOVE(&vs->vd->clients, vs, next);
+        qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+    }
 
     if (QTAILQ_EMPTY(&vs->vd->clients)) {
         dcl->idle = 1;
     }
 
-    qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
     vnc_remove_timer(vs->vd);
     if (vs->vd->lock_key_sync)
         qemu_remove_led_event_handler(vs->led);
     vnc_unlock_output(vs);
 
     qemu_mutex_destroy(&vs->output_mutex);
-    qemu_bh_delete(vs->bh);
+    if (vs->bh != NULL) {
+        qemu_bh_delete(vs->bh);
+    }
     buffer_free(&vs->jobs_buffer);
 
     for (i = 0; i < VNC_STAT_ROWS; ++i) {
@@ -1165,8 +1180,7 @@ static long vnc_client_write_plain(VncState *vs)
     if (!ret)
         return 0;
 
-    memmove(vs->output.buffer, vs->output.buffer + ret, (vs->output.offset - ret));
-    vs->output.offset -= ret;
+    buffer_advance(&vs->output, ret);
 
     if (vs->output.offset == 0) {
         qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
@@ -1192,7 +1206,16 @@ static void vnc_client_write_locked(void *opaque)
         vnc_client_write_sasl(vs);
     } else
 #endif /* CONFIG_VNC_SASL */
-        vnc_client_write_plain(vs);
+    {
+#ifdef CONFIG_VNC_WS
+        if (vs->encode_ws) {
+            vnc_client_write_ws(vs);
+        } else
+#endif /* CONFIG_VNC_WS */
+        {
+            vnc_client_write_plain(vs);
+        }
+    }
 }
 
 void vnc_client_write(void *opaque)
@@ -1200,7 +1223,11 @@ void vnc_client_write(void *opaque)
     VncState *vs = opaque;
 
     vnc_lock_output(vs);
-    if (vs->output.offset) {
+    if (vs->output.offset
+#ifdef CONFIG_VNC_WS
+            || vs->ws_output.offset
+#endif
+            ) {
         vnc_client_write_locked(opaque);
     } else if (vs->csock != -1) {
         qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
@@ -1294,7 +1321,21 @@ void vnc_client_read(void *opaque)
         ret = vnc_client_read_sasl(vs);
     else
 #endif /* CONFIG_VNC_SASL */
+#ifdef CONFIG_VNC_WS
+        if (vs->encode_ws) {
+            ret = vnc_client_read_ws(vs);
+            if (ret == -1) {
+                vnc_disconnect_start(vs);
+                return;
+            } else if (ret == -2) {
+                vnc_client_error(vs);
+                return;
+            }
+        } else
+#endif /* CONFIG_VNC_WS */
+        {
         ret = vnc_client_read_plain(vs);
+        }
     if (!ret) {
         if (vs->csock == -1)
             vnc_disconnect_finish(vs);
@@ -1312,8 +1353,7 @@ void vnc_client_read(void *opaque)
         }
 
         if (!ret) {
-            memmove(vs->input.buffer, vs->input.buffer + len, (vs->input.offset - len));
-            vs->input.offset -= len;
+            buffer_advance(&vs->input, len);
         } else {
             vs->read_handler_expect = ret;
         }
@@ -1366,23 +1406,27 @@ void vnc_write_u8(VncState *vs, uint8_t value)
 void vnc_flush(VncState *vs)
 {
     vnc_lock_output(vs);
-    if (vs->csock != -1 && vs->output.offset) {
+    if (vs->csock != -1 && (vs->output.offset
+#ifdef CONFIG_VNC_WS
+                || vs->ws_output.offset
+#endif
+                )) {
         vnc_client_write_locked(vs);
     }
     vnc_unlock_output(vs);
 }
 
-uint8_t read_u8(uint8_t *data, size_t offset)
+static uint8_t read_u8(uint8_t *data, size_t offset)
 {
     return data[offset];
 }
 
-uint16_t read_u16(uint8_t *data, size_t offset)
+static uint16_t read_u16(uint8_t *data, size_t offset)
 {
     return ((data[offset] & 0xFF) << 8) | (data[offset + 1] & 0xFF);
 }
 
-int32_t read_s32(uint8_t *data, size_t offset)
+static int32_t read_s32(uint8_t *data, size_t offset)
 {
     return (int32_t)((data[offset] << 24) | (data[offset + 1] << 16) |
                      (data[offset + 2] << 8) | data[offset + 3]);
@@ -1802,10 +1846,12 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
             vs->features |= VNC_FEATURE_TIGHT_MASK;
             vs->vnc_encoding = enc;
             break;
+#ifdef CONFIG_VNC_PNG
         case VNC_ENCODING_TIGHT_PNG:
             vs->features |= VNC_FEATURE_TIGHT_PNG_MASK;
             vs->vnc_encoding = enc;
             break;
+#endif
         case VNC_ENCODING_ZLIB:
             vs->features |= VNC_FEATURE_ZLIB_MASK;
             vs->vnc_encoding = enc;
@@ -1855,9 +1901,9 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
 
 static void set_pixel_conversion(VncState *vs)
 {
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) && 
-        !memcmp(&(vs->clientds.pf), &(vs->ds->surface->pf), sizeof(PixelFormat))) {
+    pixman_format_code_t fmt = qemu_pixman_get_format(&vs->client_pf);
+
+    if (fmt == VNC_SERVER_FB_FORMAT) {
         vs->write_pixels = vnc_write_pixels_copy;
         vnc_hextile_set_pixel_conversion(vs, 0);
     } else {
@@ -1877,23 +1923,22 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
-    vs->clientds = *(vs->vd->guest.ds);
-    vs->clientds.pf.rmax = red_max;
-    vs->clientds.pf.rbits = hweight_long(red_max);
-    vs->clientds.pf.rshift = red_shift;
-    vs->clientds.pf.rmask = red_max << red_shift;
-    vs->clientds.pf.gmax = green_max;
-    vs->clientds.pf.gbits = hweight_long(green_max);
-    vs->clientds.pf.gshift = green_shift;
-    vs->clientds.pf.gmask = green_max << green_shift;
-    vs->clientds.pf.bmax = blue_max;
-    vs->clientds.pf.bbits = hweight_long(blue_max);
-    vs->clientds.pf.bshift = blue_shift;
-    vs->clientds.pf.bmask = blue_max << blue_shift;
-    vs->clientds.pf.bits_per_pixel = bits_per_pixel;
-    vs->clientds.pf.bytes_per_pixel = bits_per_pixel / 8;
-    vs->clientds.pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
-    vs->clientds.flags = big_endian_flag ? QEMU_BIG_ENDIAN_FLAG : 0x00;
+    vs->client_pf.rmax = red_max;
+    vs->client_pf.rbits = hweight_long(red_max);
+    vs->client_pf.rshift = red_shift;
+    vs->client_pf.rmask = red_max << red_shift;
+    vs->client_pf.gmax = green_max;
+    vs->client_pf.gbits = hweight_long(green_max);
+    vs->client_pf.gshift = green_shift;
+    vs->client_pf.gmask = green_max << green_shift;
+    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bbits = hweight_long(blue_max);
+    vs->client_pf.bshift = blue_shift;
+    vs->client_pf.bmask = blue_max << blue_shift;
+    vs->client_pf.bits_per_pixel = bits_per_pixel;
+    vs->client_pf.bytes_per_pixel = bits_per_pixel / 8;
+    vs->client_pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
+    vs->client_be = big_endian_flag;
 
     set_pixel_conversion(vs);
 
@@ -1904,8 +1949,10 @@ static void set_pixel_format(VncState *vs,
 static void pixel_format_message (VncState *vs) {
     char pad[3] = { 0, 0, 0 };
 
-    vnc_write_u8(vs, vs->ds->surface->pf.bits_per_pixel); /* bits-per-pixel */
-    vnc_write_u8(vs, vs->ds->surface->pf.depth); /* depth */
+    vs->client_pf = qemu_default_pixelformat(32);
+
+    vnc_write_u8(vs, vs->client_pf.bits_per_pixel); /* bits-per-pixel */
+    vnc_write_u8(vs, vs->client_pf.depth); /* depth */
 
 #ifdef HOST_WORDS_BIGENDIAN
     vnc_write_u8(vs, 1);             /* big-endian-flag */
@@ -1913,27 +1960,25 @@ static void pixel_format_message (VncState *vs) {
     vnc_write_u8(vs, 0);             /* big-endian-flag */
 #endif
     vnc_write_u8(vs, 1);             /* true-color-flag */
-    vnc_write_u16(vs, vs->ds->surface->pf.rmax);     /* red-max */
-    vnc_write_u16(vs, vs->ds->surface->pf.gmax);     /* green-max */
-    vnc_write_u16(vs, vs->ds->surface->pf.bmax);     /* blue-max */
-    vnc_write_u8(vs, vs->ds->surface->pf.rshift);    /* red-shift */
-    vnc_write_u8(vs, vs->ds->surface->pf.gshift);    /* green-shift */
-    vnc_write_u8(vs, vs->ds->surface->pf.bshift);    /* blue-shift */
+    vnc_write_u16(vs, vs->client_pf.rmax);     /* red-max */
+    vnc_write_u16(vs, vs->client_pf.gmax);     /* green-max */
+    vnc_write_u16(vs, vs->client_pf.bmax);     /* blue-max */
+    vnc_write_u8(vs, vs->client_pf.rshift);    /* red-shift */
+    vnc_write_u8(vs, vs->client_pf.gshift);    /* green-shift */
+    vnc_write_u8(vs, vs->client_pf.bshift);    /* blue-shift */
+    vnc_write(vs, pad, 3);           /* padding */
 
     vnc_hextile_set_pixel_conversion(vs, 0);
-
-    vs->clientds = *(vs->ds->surface);
-    vs->clientds.flags &= ~QEMU_ALLOCATED_FLAG;
     vs->write_pixels = vnc_write_pixels_copy;
-
-    vnc_write(vs, pad, 3);           /* padding */
 }
 
 static void vnc_dpy_setdata(DisplayState *ds)
 {
     VncDisplay *vd = ds->opaque;
 
-    *(vd->guest.ds) = *(ds->surface);
+    qemu_pixman_image_unref(vd->guest.fb);
+    vd->guest.fb = pixman_image_ref(ds->surface->image);
+    vd->guest.format = ds->surface->format;
     vnc_dpy_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
 }
 
@@ -2437,12 +2482,14 @@ static int vnc_refresh_lossy_rect(VncDisplay *vd, int x, int y)
 
 static int vnc_update_stats(VncDisplay *vd,  struct timeval * tv)
 {
+    int width = pixman_image_get_width(vd->guest.fb);
+    int height = pixman_image_get_height(vd->guest.fb);
     int x, y;
     struct timeval res;
     int has_dirty = 0;
 
-    for (y = 0; y < vd->guest.ds->height; y += VNC_STAT_RECT) {
-        for (x = 0; x < vd->guest.ds->width; x += VNC_STAT_RECT) {
+    for (y = 0; y < height; y += VNC_STAT_RECT) {
+        for (x = 0; x < width; x += VNC_STAT_RECT) {
             VncRectStat *rect = vnc_stat_rect(vd, x, y);
 
             rect->updated = false;
@@ -2456,8 +2503,8 @@ static int vnc_update_stats(VncDisplay *vd,  struct timeval * tv)
     }
     vd->guest.last_freq_check = *tv;
 
-    for (y = 0; y < vd->guest.ds->height; y += VNC_STAT_RECT) {
-        for (x = 0; x < vd->guest.ds->width; x += VNC_STAT_RECT) {
+    for (y = 0; y < height; y += VNC_STAT_RECT) {
+        for (x = 0; x < width; x += VNC_STAT_RECT) {
             VncRectStat *rect= vnc_stat_rect(vd, x, y);
             int count = ARRAY_SIZE(rect->times);
             struct timeval min, max;
@@ -2526,12 +2573,15 @@ static void vnc_rect_updated(VncDisplay *vd, int x, int y, struct timeval * tv)
 
 static int vnc_refresh_server_surface(VncDisplay *vd)
 {
+    int width = pixman_image_get_width(vd->guest.fb);
+    int height = pixman_image_get_height(vd->guest.fb);
     int y;
     uint8_t *guest_row;
     uint8_t *server_row;
     int cmp_bytes;
     VncState *vs;
     int has_dirty = 0;
+    pixman_image_t *tmpbuf = NULL;
 
     struct timeval tv = { 0, 0 };
 
@@ -2545,22 +2595,31 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
      * Check and copy modified bits from guest to server surface.
      * Update server dirty map.
      */
-    cmp_bytes = 16 * ds_get_bytes_per_pixel(vd->ds);
-    if (cmp_bytes > vd->ds->surface->linesize) {
-        cmp_bytes = vd->ds->surface->linesize;
+    cmp_bytes = 64;
+    if (cmp_bytes > vnc_server_fb_stride(vd)) {
+        cmp_bytes = vnc_server_fb_stride(vd);
     }
-    guest_row  = vd->guest.ds->data;
-    server_row = vd->server->data;
-    for (y = 0; y < vd->guest.ds->height; y++) {
+    if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
+        int width = pixman_image_get_width(vd->server);
+        tmpbuf = qemu_pixman_linebuf_create(VNC_SERVER_FB_FORMAT, width);
+    }
+    guest_row = (uint8_t *)pixman_image_get_data(vd->guest.fb);
+    server_row = (uint8_t *)pixman_image_get_data(vd->server);
+    for (y = 0; y < height; y++) {
         if (!bitmap_empty(vd->guest.dirty[y], VNC_DIRTY_BITS)) {
             int x;
             uint8_t *guest_ptr;
             uint8_t *server_ptr;
 
-            guest_ptr  = guest_row;
+            if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
+                qemu_pixman_linebuf_fill(tmpbuf, vd->guest.fb, width, 0, y);
+                guest_ptr = (uint8_t *)pixman_image_get_data(tmpbuf);
+            } else {
+                guest_ptr = guest_row;
+            }
             server_ptr = server_row;
 
-            for (x = 0; x + 15 < vd->guest.ds->width;
+            for (x = 0; x + 15 < width;
                     x += 16, guest_ptr += cmp_bytes, server_ptr += cmp_bytes) {
                 if (!test_and_clear_bit((x / 16), vd->guest.dirty[y]))
                     continue;
@@ -2575,9 +2634,10 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
                 has_dirty++;
             }
         }
-        guest_row  += ds_get_linesize(vd->ds);
-        server_row += ds_get_linesize(vd->ds);
+        guest_row  += pixman_image_get_stride(vd->guest.fb);
+        server_row += pixman_image_get_stride(vd->server);
     }
+    qemu_pixman_image_unref(tmpbuf);
     return has_dirty;
 }
 
@@ -2640,7 +2700,7 @@ static void vnc_remove_timer(VncDisplay *vd)
     }
 }
 
-static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
+static void vnc_connect(VncDisplay *vd, int csock, int skipauth, bool websocket)
 {
     VncState *vs = g_malloc0(sizeof(VncState));
     int i;
@@ -2667,13 +2727,35 @@ static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
     VNC_DEBUG("New client on socket %d\n", csock);
     dcl->idle = 0;
     socket_set_nonblock(vs->csock);
-    qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+#ifdef CONFIG_VNC_WS
+    if (websocket) {
+        vs->websocket = 1;
+        qemu_set_fd_handler2(vs->csock, NULL, vncws_handshake_read, NULL, vs);
+    } else
+#endif /* CONFIG_VNC_WS */
+    {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    }
 
     vnc_client_cache_addr(vs);
     vnc_qmp_event(vs, QEVENT_VNC_CONNECTED);
     vnc_set_share_mode(vs, VNC_SHARE_MODE_CONNECTING);
 
     vs->vd = vd;
+
+#ifdef CONFIG_VNC_WS
+    if (!vs->websocket)
+#endif
+    {
+        vnc_init_state(vs);
+    }
+}
+
+void vnc_init_state(VncState *vs)
+{
+    vs->initialized = true;
+    VncDisplay *vd = vs->vd;
+
     vs->ds = vd->ds;
     vs->last_x = -1;
     vs->last_y = -1;
@@ -2705,21 +2787,41 @@ static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
     /* vs might be free()ed here */
 }
 
-static void vnc_listen_read(void *opaque)
+static void vnc_listen_read(void *opaque, bool websocket)
 {
     VncDisplay *vs = opaque;
     struct sockaddr_in addr;
     socklen_t addrlen = sizeof(addr);
+    int csock;
 
     /* Catch-up */
     vga_hw_update();
+#ifdef CONFIG_VNC_WS
+    if (websocket) {
+        csock = qemu_accept(vs->lwebsock, (struct sockaddr *)&addr, &addrlen);
+    } else
+#endif /* CONFIG_VNC_WS */
+    {
+        csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
+    }
 
-    int csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
     if (csock != -1) {
-        vnc_connect(vs, csock, 0);
+        vnc_connect(vs, csock, 0, websocket);
     }
 }
 
+static void vnc_listen_regular_read(void *opaque)
+{
+    vnc_listen_read(opaque, 0);
+}
+
+#ifdef CONFIG_VNC_WS
+static void vnc_listen_websocket_read(void *opaque)
+{
+    vnc_listen_read(opaque, 1);
+}
+#endif /* CONFIG_VNC_WS */
+
 void vnc_display_init(DisplayState *ds)
 {
     VncDisplay *vs = g_malloc0(sizeof(*vs));
@@ -2731,6 +2833,9 @@ void vnc_display_init(DisplayState *ds)
     vnc_display = vs;
 
     vs->lsock = -1;
+#ifdef CONFIG_VNC_WS
+    vs->lwebsock = -1;
+#endif
 
     vs->ds = ds;
     QTAILQ_INIT(&vs->clients);
@@ -2747,17 +2852,17 @@ void vnc_display_init(DisplayState *ds)
     qemu_mutex_init(&vs->mutex);
     vnc_start_worker_thread();
 
-    dcl->dpy_copy = vnc_dpy_copy;
-    dcl->dpy_update = vnc_dpy_update;
-    dcl->dpy_resize = vnc_dpy_resize;
-    dcl->dpy_setdata = vnc_dpy_setdata;
+    dcl->dpy_gfx_copy = vnc_dpy_copy;
+    dcl->dpy_gfx_update = vnc_dpy_update;
+    dcl->dpy_gfx_resize = vnc_dpy_resize;
+    dcl->dpy_gfx_setdata = vnc_dpy_setdata;
+    dcl->dpy_mouse_set = vnc_mouse_set;
+    dcl->dpy_cursor_define = vnc_dpy_cursor_define;
     register_displaychangelistener(ds, dcl);
-    ds->mouse_set = vnc_mouse_set;
-    ds->cursor_define = vnc_dpy_cursor_define;
 }
 
 
-void vnc_display_close(DisplayState *ds)
+static void vnc_display_close(DisplayState *ds)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
@@ -2772,6 +2877,15 @@ void vnc_display_close(DisplayState *ds)
         close(vs->lsock);
         vs->lsock = -1;
     }
+#ifdef CONFIG_VNC_WS
+    g_free(vs->ws_display);
+    vs->ws_display = NULL;
+    if (vs->lwebsock != -1) {
+        qemu_set_fd_handler2(vs->lwebsock, NULL, NULL, NULL, NULL);
+        close(vs->lwebsock);
+        vs->lwebsock = -1;
+    }
+#endif /* CONFIG_VNC_WS */
     vs->auth = VNC_AUTH_INVALID;
 #ifdef CONFIG_VNC_TLS
     vs->subauth = VNC_AUTH_INVALID;
@@ -2779,7 +2893,7 @@ void vnc_display_close(DisplayState *ds)
 #endif
 }
 
-int vnc_display_disable_login(DisplayState *ds)
+static int vnc_display_disable_login(DisplayState *ds)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
@@ -2844,7 +2958,7 @@ char *vnc_display_local_addr(DisplayState *ds)
     return vnc_socket_local_addr("%s:%s", vs->lsock);
 }
 
-int vnc_display_open(DisplayState *ds, const char *display)
+void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
     const char *options;
@@ -2862,14 +2976,15 @@ int vnc_display_open(DisplayState *ds, const char *display)
 #endif
     int lock_key_sync = 1;
 
-    if (!vnc_display)
-        return -1;
+    if (!vnc_display) {
+        error_setg(errp, "VNC display not active");
+        return;
+    }
     vnc_display_close(ds);
     if (strcmp(display, "none") == 0)
-        return 0;
+        return;
 
-    if (!(vs->display = strdup(display)))
-        return -1;
+    vs->display = g_strdup(display);
     vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
 
     options = display;
@@ -2877,13 +2992,11 @@ int vnc_display_open(DisplayState *ds, const char *display)
         options++;
         if (strncmp(options, "password", 8) == 0) {
             if (fips_get_state()) {
-                fprintf(stderr,
-                        "VNC password auth disabled due to FIPS mode, "
-                        "consider using the VeNCrypt or SASL authentication "
-                        "methods as an alternative\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp,
+                           "VNC password auth disabled due to FIPS mode, "
+                           "consider using the VeNCrypt or SASL authentication "
+                           "methods as an alternative");
+                goto fail;
             }
             password = 1; /* Require password auth */
         } else if (strncmp(options, "reverse", 7) == 0) {
@@ -2894,6 +3007,36 @@ int vnc_display_open(DisplayState *ds, const char *display)
         } else if (strncmp(options, "sasl", 4) == 0) {
             sasl = 1; /* Require SASL auth */
 #endif
+#ifdef CONFIG_VNC_WS
+        } else if (strncmp(options, "websocket", 9) == 0) {
+            char *start, *end;
+            vs->websocket = 1;
+
+            /* Check for 'websocket=<port>' */
+            start = strchr(options, '=');
+            end = strchr(options, ',');
+            if (start && (!end || (start < end))) {
+                int len = end ? end-(start+1) : strlen(start+1);
+                if (len < 6) {
+                    /* extract the host specification from display */
+                    char  *host = NULL, *port = NULL, *host_end = NULL;
+                    port = g_strndup(start + 1, len);
+
+                    /* ipv6 hosts have colons */
+                    end = strchr(display, ',');
+                    host_end = g_strrstr_len(display, end - display, ":");
+
+                    if (host_end) {
+                        host = g_strndup(display, host_end - display + 1);
+                    } else {
+                        host = g_strndup(":", 1);
+                    }
+                    vs->ws_display = g_strconcat(host, port, NULL);
+                    g_free(host);
+                    g_free(port);
+                }
+            }
+#endif /* CONFIG_VNC_WS */
 #ifdef CONFIG_VNC_TLS
         } else if (strncmp(options, "tls", 3) == 0) {
             tls = 1; /* Require TLS */
@@ -2913,18 +3056,14 @@ int vnc_display_open(DisplayState *ds, const char *display)
 
                 VNC_DEBUG("Trying certificate path '%s'\n", path);
                 if (vnc_tls_set_x509_creds_dir(vs, path) < 0) {
-                    fprintf(stderr, "Failed to find x509 certificates/keys in %s\n", path);
+                    error_setg(errp, "Failed to find x509 certificates/keys in %s", path);
                     g_free(path);
-                    g_free(vs->display);
-                    vs->display = NULL;
-                    return -1;
+                    goto fail;
                 }
                 g_free(path);
             } else {
-                fprintf(stderr, "No certificate path provided\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp, "No certificate path provided");
+                goto fail;
             }
 #endif
 #if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
@@ -2933,7 +3072,7 @@ int vnc_display_open(DisplayState *ds, const char *display)
 #endif
         } else if (strncmp(options, "lossy", 5) == 0) {
             vs->lossy = true;
-        } else if (strncmp(options, "non-adapative", 13) == 0) {
+        } else if (strncmp(options, "non-adaptive", 12) == 0) {
             vs->non_adaptive = true;
         } else if (strncmp(options, "share=", 6) == 0) {
             if (strncmp(options+6, "ignore", 6) == 0) {
@@ -2943,10 +3082,8 @@ int vnc_display_open(DisplayState *ds, const char *display)
             } else if (strncmp(options+6, "force-shared", 12) == 0) {
                 vs->share_policy = VNC_SHARE_POLICY_FORCE_SHARED;
             } else {
-                fprintf(stderr, "unknown vnc share= option\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp, "unknown vnc share= option");
+                goto fail;
             }
         }
     }
@@ -3047,57 +3184,89 @@ int vnc_display_open(DisplayState *ds, const char *display)
 
 #ifdef CONFIG_VNC_SASL
     if ((saslErr = sasl_server_init(NULL, "qemu")) != SASL_OK) {
-        fprintf(stderr, "Failed to initialize SASL auth %s",
-                sasl_errstring(saslErr, NULL, NULL));
-        g_free(vs->display);
-        vs->display = NULL;
-        return -1;
+        error_setg(errp, "Failed to initialize SASL auth: %s",
+                   sasl_errstring(saslErr, NULL, NULL));
+        goto fail;
     }
 #endif
     vs->lock_key_sync = lock_key_sync;
 
     if (reverse) {
         /* connect to viewer */
-        if (strncmp(display, "unix:", 5) == 0)
-            vs->lsock = unix_connect(display+5);
-        else
-            vs->lsock = inet_connect(display, true, NULL, NULL);
-        if (-1 == vs->lsock) {
-            g_free(vs->display);
-            vs->display = NULL;
-            return -1;
+        int csock;
+        vs->lsock = -1;
+#ifdef CONFIG_VNC_WS
+        vs->lwebsock = -1;
+#endif
+        if (strncmp(display, "unix:", 5) == 0) {
+            csock = unix_connect(display+5, errp);
         } else {
-            int csock = vs->lsock;
-            vs->lsock = -1;
-            vnc_connect(vs, csock, 0);
+            csock = inet_connect(display, errp);
         }
-        return 0;
-
+        if (csock < 0) {
+            goto fail;
+        }
+        vnc_connect(vs, csock, 0, 0);
     } else {
         /* listen for connects */
         char *dpy;
         dpy = g_malloc(256);
         if (strncmp(display, "unix:", 5) == 0) {
             pstrcpy(dpy, 256, "unix:");
-            vs->lsock = unix_listen(display+5, dpy+5, 256-5);
+            vs->lsock = unix_listen(display+5, dpy+5, 256-5, errp);
         } else {
             vs->lsock = inet_listen(display, dpy, 256,
-                                    SOCK_STREAM, 5900, NULL);
+                                    SOCK_STREAM, 5900, errp);
+            if (vs->lsock < 0) {
+                g_free(dpy);
+                goto fail;
+            }
+#ifdef CONFIG_VNC_WS
+            if (vs->websocket) {
+                if (vs->ws_display) {
+                    vs->lwebsock = inet_listen(vs->ws_display, NULL, 256,
+                        SOCK_STREAM, 0, errp);
+                } else {
+                    vs->lwebsock = inet_listen(vs->display, NULL, 256,
+                        SOCK_STREAM, 5700, errp);
+                }
+
+                if (vs->lwebsock < 0) {
+                    if (vs->lsock) {
+                        close(vs->lsock);
+                        vs->lsock = -1;
+                    }
+                    g_free(dpy);
+                    goto fail;
+                }
+            }
+#endif /* CONFIG_VNC_WS */
         }
-        if (-1 == vs->lsock) {
-            g_free(dpy);
-            return -1;
-        } else {
-            g_free(vs->display);
-            vs->display = dpy;
+        g_free(vs->display);
+        vs->display = dpy;
+        qemu_set_fd_handler2(vs->lsock, NULL,
+                vnc_listen_regular_read, NULL, vs);
+#ifdef CONFIG_VNC_WS
+        if (vs->websocket) {
+            qemu_set_fd_handler2(vs->lwebsock, NULL,
+                    vnc_listen_websocket_read, NULL, vs);
         }
+#endif /* CONFIG_VNC_WS */
     }
-    return qemu_set_fd_handler2(vs->lsock, NULL, vnc_listen_read, NULL, vs);
+    return;
+
+fail:
+    g_free(vs->display);
+    vs->display = NULL;
+#ifdef CONFIG_VNC_WS
+    g_free(vs->ws_display);
+    vs->ws_display = NULL;
+#endif /* CONFIG_VNC_WS */
 }
 
 void vnc_display_add_client(DisplayState *ds, int csock, int skipauth)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
-    vnc_connect(vs, csock, skipauth);
+    vnc_connect(vs, csock, skipauth, 0);
 }
diff --git a/ui/vnc.h b/ui/vnc.h
index 068c2fcda52..45d7686843b 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -28,12 +28,12 @@
 #define __QEMU_VNC_H
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-thread.h"
-#include "console.h"
-#include "monitor.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "ui/console.h"
+#include "monitor/monitor.h"
 #include "audio/audio.h"
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 #include <zlib.h>
 #include <stdbool.h>
 
@@ -69,7 +69,7 @@ typedef struct VncRectEntry VncRectEntry;
 
 typedef int VncReadEvent(VncState *vs, uint8_t *data, size_t len);
 
-typedef void VncWritePixels(VncState *vs, struct PixelFormat *pf, void *data, int size);
+typedef void VncWritePixels(VncState *vs, void *data, int size);
 
 typedef void VncSendHextileTile(VncState *vs,
                                 int x, int y, int w, int h,
@@ -99,6 +99,9 @@ typedef struct VncDisplay VncDisplay;
 #ifdef CONFIG_VNC_SASL
 #include "vnc-auth-sasl.h"
 #endif
+#ifdef CONFIG_VNC_WS
+#include "vnc-ws.h"
+#endif
 
 struct VncRectStat
 {
@@ -117,7 +120,8 @@ struct VncSurface
     struct timeval last_freq_check;
     DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT], VNC_MAX_WIDTH / 16);
     VncRectStat stats[VNC_STAT_ROWS][VNC_STAT_COLS];
-    DisplaySurface *ds;
+    pixman_image_t *fb;
+    pixman_format_code_t format;
 };
 
 typedef enum VncShareMode {
@@ -141,6 +145,11 @@ struct VncDisplay
     QEMUTimer *timer;
     int timer_interval;
     int lsock;
+#ifdef CONFIG_VNC_WS
+    int lwebsock;
+    bool websocket;
+    char *ws_display;
+#endif
     DisplayState *ds;
     kbd_layout_t *kbd_layout;
     int lock_key_sync;
@@ -151,7 +160,7 @@ struct VncDisplay
     uint8_t *cursor_mask;
 
     struct VncSurface guest;   /* guest visible surface (aka ds->surface) */
-    DisplaySurface *server;  /* vnc server surface */
+    pixman_image_t *server;    /* vnc server surface */
 
     char *display;
     char *password;
@@ -268,14 +277,24 @@ struct VncState
 #ifdef CONFIG_VNC_SASL
     VncStateSASL sasl;
 #endif
+#ifdef CONFIG_VNC_WS
+    bool encode_ws;
+    bool websocket;
+#endif
 
     QObject *info;
 
     Buffer output;
     Buffer input;
+#ifdef CONFIG_VNC_WS
+    Buffer ws_input;
+    Buffer ws_output;
+#endif
     /* current output mode information */
     VncWritePixels *write_pixels;
-    DisplaySurface clientds;
+    PixelFormat client_pf;
+    pixman_format_code_t client_format;
+    bool client_be;
 
     CaptureVoiceOut *audio_cap;
     struct audsettings as;
@@ -287,6 +306,7 @@ struct VncState
     QEMUPutLEDEntry *led;
 
     bool abort;
+    bool initialized;
     QemuMutex output_mutex;
     QEMUBH *bh;
     Buffer jobs_buffer;
@@ -490,12 +510,11 @@ void vnc_write_u16(VncState *vs, uint16_t value);
 void vnc_write_u8(VncState *vs, uint8_t value);
 void vnc_flush(VncState *vs);
 void vnc_read_when(VncState *vs, VncReadEvent *func, size_t expecting);
+void vnc_disconnect_finish(VncState *vs);
+void vnc_init_state(VncState *vs);
 
 
 /* Buffer I/O functions */
-uint8_t read_u8(uint8_t *data, size_t offset);
-uint16_t read_u16(uint8_t *data, size_t offset);
-int32_t read_s32(uint8_t *data, size_t offset);
 uint32_t read_u32(uint8_t *data, size_t offset);
 
 /* Protocol stage functions */
@@ -507,11 +526,11 @@ void start_auth_vnc(VncState *vs);
 
 /* Buffer management */
 void buffer_reserve(Buffer *buffer, size_t len);
-int buffer_empty(Buffer *buffer);
-uint8_t *buffer_end(Buffer *buffer);
 void buffer_reset(Buffer *buffer);
 void buffer_free(Buffer *buffer);
 void buffer_append(Buffer *buffer, const void *data, size_t len);
+void buffer_advance(Buffer *buf, size_t len);
+uint8_t *buffer_end(Buffer *buffer);
 
 
 /* Misc helpers */
@@ -527,6 +546,14 @@ static inline uint32_t vnc_has_feature(VncState *vs, int feature) {
 void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
                             int32_t encoding);
 
+/* server fb is in PIXMAN_x8r8g8b8 */
+#define VNC_SERVER_FB_FORMAT PIXMAN_FORMAT(32, PIXMAN_TYPE_ARGB, 0, 8, 8, 8)
+#define VNC_SERVER_FB_BITS   (PIXMAN_FORMAT_BPP(VNC_SERVER_FB_FORMAT))
+#define VNC_SERVER_FB_BYTES  ((VNC_SERVER_FB_BITS+7)/8)
+
+void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y);
+int vnc_server_fb_stride(VncDisplay *vd);
+
 void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v);
 double vnc_update_freq(VncState *vs, int x, int y, int w, int h);
 void vnc_sent_lossy_rect(VncState *vs, int x, int y, int w, int h);
diff --git a/user-exec.c b/user-exec.c
index b9ea9dd32f0..c71acbc5031 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -18,10 +18,7 @@
  */
 #include "config.h"
 #include "cpu.h"
-#ifndef CONFIG_TCG_PASS_AREG0
-#include "dyngen-exec.h"
-#endif
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
 
 #undef EAX
@@ -60,12 +57,6 @@ void cpu_resume_from_signal(CPUArchState *env1, void *puc)
     struct sigcontext *uc = puc;
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-    env = env1;
-
-    /* XXX: restore cpu registers saved in host registers */
-#endif
-
     if (puc) {
         /* XXX: use siglongjmp ? */
 #ifdef __linux__
@@ -90,14 +81,8 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
                                     int is_write, sigset_t *old_set,
                                     void *puc)
 {
-    TranslationBlock *tb;
     int ret;
 
-#ifndef CONFIG_TCG_PASS_AREG0
-    if (cpu_single_env) {
-        env = cpu_single_env; /* XXX: find a correct solution for multithread */
-    }
-#endif
 #if defined(DEBUG_SIGNAL)
     qemu_printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
                 pc, address, is_write, *(unsigned long *)old_set);
@@ -118,12 +103,7 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
         return 1; /* the MMU fault was handled without causing real CPU fault */
     }
     /* now we have a real cpu fault */
-    tb = tb_find_pc(pc);
-    if (tb) {
-        /* the PC is inside the translated code. It means that we have
-           a virtual CPU fault */
-        cpu_restore_state(tb, cpu_single_env, pc);
-    }
+    cpu_restore_state(cpu_single_env, pc);
 
     /* we restore the process signal mask as the sigreturn should
        do it (XXX: use sigsetjmp) */
@@ -456,7 +436,7 @@ int cpu_signal_handler(int host_signum, void *pinfo,
     unsigned long pc;
     int is_write;
 
-#if (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
+#if defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
     pc = uc->uc_mcontext.gregs[R15];
 #else
     pc = uc->uc_mcontext.arm_pc;
diff --git a/util/Makefile.objs b/util/Makefile.objs
new file mode 100644
index 00000000000..495a178557b
--- /dev/null
+++ b/util/Makefile.objs
@@ -0,0 +1,10 @@
+util-obj-y = osdep.o cutils.o qemu-timer-common.o
+util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
+util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o
+util-obj-y += envlist.o path.o host-utils.o cache-utils.o module.o
+util-obj-y += bitmap.o bitops.o hbitmap.o
+util-obj-y += acl.o
+util-obj-y += error.o qemu-error.o
+util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-y += iov.o aes.o qemu-config.o qemu-sockets.o uri.o notify.o
+util-obj-y += qemu-option.o qemu-progress.o
diff --git a/acl.c b/util/acl.c
index e840b9b6339..a7f33ff7bb0 100644
--- a/acl.c
+++ b/util/acl.c
@@ -24,7 +24,7 @@
 
 
 #include "qemu-common.h"
-#include "acl.h"
+#include "qemu/acl.h"
 
 #ifdef CONFIG_FNMATCH
 #include <fnmatch.h>
@@ -103,8 +103,8 @@ void qemu_acl_reset(qemu_acl *acl)
     acl->defaultDeny = 1;
     QTAILQ_FOREACH_SAFE(entry, &acl->entries, next, next_entry) {
         QTAILQ_REMOVE(&acl->entries, entry, next);
-        free(entry->match);
-        free(entry);
+        g_free(entry->match);
+        g_free(entry);
     }
     acl->nentries = 0;
 }
@@ -168,6 +168,9 @@ int qemu_acl_remove(qemu_acl *acl,
         i++;
         if (strcmp(entry->match, match) == 0) {
             QTAILQ_REMOVE(&acl->entries, entry, next);
+            acl->nentries--;
+            g_free(entry->match);
+            g_free(entry);
             return i;
         }
     }
diff --git a/aes.c b/util/aes.c
index eb37adbed86..1da7bff1c96 100644
--- a/aes.c
+++ b/util/aes.c
@@ -28,7 +28,7 @@
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "qemu-common.h"
-#include "aes.h"
+#include "block/aes.h"
 
 #ifndef NDEBUG
 #define NDEBUG
diff --git a/bitmap.c b/util/bitmap.c
index a62c8ba6814..687841dcec0 100644
--- a/bitmap.c
+++ b/util/bitmap.c
@@ -9,8 +9,8 @@
  * Version 2.
  */
 
-#include "bitops.h"
-#include "bitmap.h"
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
 
 /*
  * bitmaps provide an array of bits, implemented using an an
diff --git a/bitops.c b/util/bitops.c
index d9de71f7e88..4c3a836a014 100644
--- a/bitops.c
+++ b/util/bitops.c
@@ -11,7 +11,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
 
diff --git a/cache-utils.c b/util/cache-utils.c
index 2db5af2db14..b94013a8cb9 100644
--- a/cache-utils.c
+++ b/util/cache-utils.c
@@ -1,4 +1,4 @@
-#include "cache-utils.h"
+#include "qemu/cache-utils.h"
 
 #if defined(_ARCH_PPC)
 struct qemu_cache_conf qemu_cache_conf = {
diff --git a/compatfd.c b/util/compatfd.c
index 42f81cafe49..9cf3f2834d5 100644
--- a/compatfd.c
+++ b/util/compatfd.c
@@ -14,7 +14,7 @@
  */
 
 #include "qemu-common.h"
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 
 #include <sys/syscall.h>
 #include <pthread.h>
diff --git a/cutils.c b/util/cutils.c
index 8ef648f4b92..80bb1dcbf76 100644
--- a/cutils.c
+++ b/util/cutils.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include <math.h>
 
-#include "qemu_socket.h"
-#include "iov.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
 
 void strpadcpy(char *buf, int buf_size, const char *str, char pad)
 {
@@ -115,7 +115,7 @@ time_t mktimegm(struct tm *tm)
         m += 12;
         y--;
     }
-    t = 86400 * (d + (153 * m - 457) / 5 + 365 * y + y / 4 - y / 100 + 
+    t = 86400ULL * (d + (153 * m - 457) / 5 + 365 * y + y / 4 - y / 100 + 
                  y / 400 - 719469);
     t += 3600 * tm->tm_hour + 60 * tm->tm_min + tm->tm_sec;
     return t;
@@ -142,109 +142,6 @@ int qemu_fdatasync(int fd)
 #endif
 }
 
-/* io vectors */
-
-void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
-{
-    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
-    qiov->niov = 0;
-    qiov->nalloc = alloc_hint;
-    qiov->size = 0;
-}
-
-void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov)
-{
-    int i;
-
-    qiov->iov = iov;
-    qiov->niov = niov;
-    qiov->nalloc = -1;
-    qiov->size = 0;
-    for (i = 0; i < niov; i++)
-        qiov->size += iov[i].iov_len;
-}
-
-void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
-{
-    assert(qiov->nalloc != -1);
-
-    if (qiov->niov == qiov->nalloc) {
-        qiov->nalloc = 2 * qiov->nalloc + 1;
-        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
-    }
-    qiov->iov[qiov->niov].iov_base = base;
-    qiov->iov[qiov->niov].iov_len = len;
-    qiov->size += len;
-    ++qiov->niov;
-}
-
-/*
- * Concatenates (partial) iovecs from src to the end of dst.
- * It starts copying after skipping `soffset' bytes at the
- * beginning of src and adds individual vectors from src to
- * dst copies up to `sbytes' bytes total, or up to the end
- * of src if it comes first.  This way, it is okay to specify
- * very large value for `sbytes' to indicate "up to the end
- * of src".
- * Only vector pointers are processed, not the actual data buffers.
- */
-void qemu_iovec_concat(QEMUIOVector *dst,
-                       QEMUIOVector *src, size_t soffset, size_t sbytes)
-{
-    int i;
-    size_t done;
-    struct iovec *siov = src->iov;
-    assert(dst->nalloc != -1);
-    assert(src->size >= soffset);
-    for (i = 0, done = 0; done < sbytes && i < src->niov; i++) {
-        if (soffset < siov[i].iov_len) {
-            size_t len = MIN(siov[i].iov_len - soffset, sbytes - done);
-            qemu_iovec_add(dst, siov[i].iov_base + soffset, len);
-            done += len;
-            soffset = 0;
-        } else {
-            soffset -= siov[i].iov_len;
-        }
-    }
-    /* return done; */
-}
-
-void qemu_iovec_destroy(QEMUIOVector *qiov)
-{
-    assert(qiov->nalloc != -1);
-
-    qemu_iovec_reset(qiov);
-    g_free(qiov->iov);
-    qiov->nalloc = 0;
-    qiov->iov = NULL;
-}
-
-void qemu_iovec_reset(QEMUIOVector *qiov)
-{
-    assert(qiov->nalloc != -1);
-
-    qiov->niov = 0;
-    qiov->size = 0;
-}
-
-size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
-                         void *buf, size_t bytes)
-{
-    return iov_to_buf(qiov->iov, qiov->niov, offset, buf, bytes);
-}
-
-size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
-                           const void *buf, size_t bytes)
-{
-    return iov_from_buf(qiov->iov, qiov->niov, offset, buf, bytes);
-}
-
-size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
-                         int fillc, size_t bytes)
-{
-    return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
-}
-
 /*
  * Checks if a buffer is all zeroes
  *
@@ -317,12 +214,13 @@ static int64_t suffix_mul(char suffix, int64_t unit)
 /*
  * Convert string to bytes, allowing either B/b for bytes, K/k for KB,
  * M/m for MB, G/g for GB or T/t for TB. End pointer will be returned
- * in *end, if not NULL. Return -1 on error.
+ * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
+ * other error.
  */
 int64_t strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit)
 {
-    int64_t retval = -1;
+    int64_t retval = -EINVAL;
     char *endptr;
     unsigned char c;
     int mul_required = 0;
@@ -349,6 +247,7 @@ int64_t strtosz_suffix_unit(const char *nptr, char **end,
         goto fail;
     }
     if ((val * mul >= INT64_MAX) || val < 0) {
+        retval = -ERANGE;
         goto fail;
     }
     retval = val * mul;
@@ -383,11 +282,6 @@ int qemu_parse_fd(const char *param)
     return fd;
 }
 
-int qemu_parse_fdset(const char *param)
-{
-    return qemu_parse_fd(param);
-}
-
 /* round down to the nearest power of 2*/
 int64_t pow2floor(int64_t value)
 {
diff --git a/envlist.c b/util/envlist.c
index f2303cdd79d..ff99fc44e97 100644
--- a/envlist.c
+++ b/util/envlist.c
@@ -4,8 +4,8 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "qemu-queue.h"
-#include "envlist.h"
+#include "qemu/queue.h"
+#include "qemu/envlist.h"
 
 struct envlist_entry {
 	const char *ev_var;			/* actual env value */
diff --git a/error.c b/util/error.c
index 1f05fc466e7..519f6b6ce00 100644
--- a/error.c
+++ b/util/error.c
@@ -11,11 +11,11 @@
  */
 
 #include "qemu-common.h"
-#include "error.h"
-#include "qjson.h"
-#include "qdict.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qdict.h"
 #include "qapi-types.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 struct Error
 {
@@ -43,6 +43,34 @@ void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
     *errp = err;
 }
 
+void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
+                     const char *fmt, ...)
+{
+    Error *err;
+    char *msg1;
+    va_list ap;
+
+    if (errp == NULL) {
+        return;
+    }
+    assert(*errp == NULL);
+
+    err = g_malloc0(sizeof(*err));
+
+    va_start(ap, fmt);
+    msg1 = g_strdup_vprintf(fmt, ap);
+    if (os_errno != 0) {
+        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
+        g_free(msg1);
+    } else {
+        err->msg = msg1;
+    }
+    va_end(ap);
+    err->err_class = err_class;
+
+    *errp = err;
+}
+
 Error *error_copy(const Error *err)
 {
     Error *err_new;
diff --git a/util/event_notifier-posix.c b/util/event_notifier-posix.c
new file mode 100644
index 00000000000..713d7560d0a
--- /dev/null
+++ b/util/event_notifier-posix.c
@@ -0,0 +1,121 @@
+/*
+ * event notifier support
+ *
+ * Copyright Red Hat, Inc. 2010
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "qemu/event_notifier.h"
+#include "char/char.h"
+#include "qemu/main-loop.h"
+
+#ifdef CONFIG_EVENTFD
+#include <sys/eventfd.h>
+#endif
+
+void event_notifier_init_fd(EventNotifier *e, int fd)
+{
+    e->rfd = fd;
+    e->wfd = fd;
+}
+
+int event_notifier_init(EventNotifier *e, int active)
+{
+    int fds[2];
+    int ret;
+
+#ifdef CONFIG_EVENTFD
+    ret = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+#else
+    ret = -1;
+    errno = ENOSYS;
+#endif
+    if (ret >= 0) {
+        e->rfd = e->wfd = ret;
+    } else {
+        if (errno != ENOSYS) {
+            return -errno;
+        }
+        if (qemu_pipe(fds) < 0) {
+            return -errno;
+        }
+        ret = fcntl_setfl(fds[0], O_NONBLOCK);
+        if (ret < 0) {
+            ret = -errno;
+            goto fail;
+        }
+        ret = fcntl_setfl(fds[1], O_NONBLOCK);
+        if (ret < 0) {
+            ret = -errno;
+            goto fail;
+        }
+        e->rfd = fds[0];
+        e->wfd = fds[1];
+    }
+    if (active) {
+        event_notifier_set(e);
+    }
+    return 0;
+
+fail:
+    close(fds[0]);
+    close(fds[1]);
+    return ret;
+}
+
+void event_notifier_cleanup(EventNotifier *e)
+{
+    if (e->rfd != e->wfd) {
+        close(e->rfd);
+    }
+    close(e->wfd);
+}
+
+int event_notifier_get_fd(EventNotifier *e)
+{
+    return e->rfd;
+}
+
+int event_notifier_set_handler(EventNotifier *e,
+                               EventNotifierHandler *handler)
+{
+    return qemu_set_fd_handler(e->rfd, (IOHandler *)handler, NULL, e);
+}
+
+int event_notifier_set(EventNotifier *e)
+{
+    static const uint64_t value = 1;
+    ssize_t ret;
+
+    do {
+        ret = write(e->wfd, &value, sizeof(value));
+    } while (ret < 0 && errno == EINTR);
+
+    /* EAGAIN is fine, a read must be pending.  */
+    if (ret < 0 && errno != EAGAIN) {
+        return -errno;
+    }
+    return 0;
+}
+
+int event_notifier_test_and_clear(EventNotifier *e)
+{
+    int value;
+    ssize_t len;
+    char buffer[512];
+
+    /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
+    value = 0;
+    do {
+        len = read(e->rfd, buffer, sizeof(buffer));
+        value |= (len > 0);
+    } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
+
+    return value;
+}
diff --git a/util/event_notifier-win32.c b/util/event_notifier-win32.c
new file mode 100644
index 00000000000..6dbb530cfae
--- /dev/null
+++ b/util/event_notifier-win32.c
@@ -0,0 +1,59 @@
+/*
+ * event notifier support
+ *
+ * Copyright Red Hat, Inc. 2010
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "qemu/event_notifier.h"
+#include "qemu/main-loop.h"
+
+int event_notifier_init(EventNotifier *e, int active)
+{
+    e->event = CreateEvent(NULL, TRUE, FALSE, NULL);
+    assert(e->event);
+    return 0;
+}
+
+void event_notifier_cleanup(EventNotifier *e)
+{
+    CloseHandle(e->event);
+}
+
+HANDLE event_notifier_get_handle(EventNotifier *e)
+{
+    return e->event;
+}
+
+int event_notifier_set_handler(EventNotifier *e,
+                               EventNotifierHandler *handler)
+{
+    if (handler) {
+        return qemu_add_wait_object(e->event, (IOHandler *)handler, e);
+    } else {
+        qemu_del_wait_object(e->event, (IOHandler *)handler, e);
+        return 0;
+    }
+}
+
+int event_notifier_set(EventNotifier *e)
+{
+    SetEvent(e->event);
+    return 0;
+}
+
+int event_notifier_test_and_clear(EventNotifier *e)
+{
+    int ret = WaitForSingleObject(e->event, 0);
+    if (ret == WAIT_OBJECT_0) {
+        ResetEvent(e->event);
+        return true;
+    }
+    return false;
+}
diff --git a/util/hbitmap.c b/util/hbitmap.c
new file mode 100644
index 00000000000..2aa487db74b
--- /dev/null
+++ b/util/hbitmap.c
@@ -0,0 +1,401 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include <string.h>
+#include <glib.h>
+#include <assert.h>
+#include "qemu/osdep.h"
+#include "qemu/hbitmap.h"
+#include "qemu/host-utils.h"
+#include "trace.h"
+
+/* HBitmaps provides an array of bits.  The bits are stored as usual in an
+ * array of unsigned longs, but HBitmap is also optimized to provide fast
+ * iteration over set bits; going from one bit to the next is O(logB n)
+ * worst case, with B = sizeof(long) * CHAR_BIT: the result is low enough
+ * that the number of levels is in fact fixed.
+ *
+ * In order to do this, it stacks multiple bitmaps with progressively coarser
+ * granularity; in all levels except the last, bit N is set iff the N-th
+ * unsigned long is nonzero in the immediately next level.  When iteration
+ * completes on the last level it can examine the 2nd-last level to quickly
+ * skip entire words, and even do so recursively to skip blocks of 64 words or
+ * powers thereof (32 on 32-bit machines).
+ *
+ * Given an index in the bitmap, it can be split in group of bits like
+ * this (for the 64-bit case):
+ *
+ *   bits 0-57 => word in the last bitmap     | bits 58-63 => bit in the word
+ *   bits 0-51 => word in the 2nd-last bitmap | bits 52-57 => bit in the word
+ *   bits 0-45 => word in the 3rd-last bitmap | bits 46-51 => bit in the word
+ *
+ * So it is easy to move up simply by shifting the index right by
+ * log2(BITS_PER_LONG) bits.  To move down, you shift the index left
+ * similarly, and add the word index within the group.  Iteration uses
+ * ffs (find first set bit) to find the next word to examine; this
+ * operation can be done in constant time in most current architectures.
+ *
+ * Setting or clearing a range of m bits on all levels, the work to perform
+ * is O(m + m/W + m/W^2 + ...), which is O(m) like on a regular bitmap.
+ *
+ * When iterating on a bitmap, each bit (on any level) is only visited
+ * once.  Hence, The total cost of visiting a bitmap with m bits in it is
+ * the number of bits that are set in all bitmaps.  Unless the bitmap is
+ * extremely sparse, this is also O(m + m/W + m/W^2 + ...), so the amortized
+ * cost of advancing from one bit to the next is usually constant (worst case
+ * O(logB n) as in the non-amortized complexity).
+ */
+
+struct HBitmap {
+    /* Number of total bits in the bottom level.  */
+    uint64_t size;
+
+    /* Number of set bits in the bottom level.  */
+    uint64_t count;
+
+    /* A scaling factor.  Given a granularity of G, each bit in the bitmap will
+     * will actually represent a group of 2^G elements.  Each operation on a
+     * range of bits first rounds the bits to determine which group they land
+     * in, and then affect the entire page; iteration will only visit the first
+     * bit of each group.  Here is an example of operations in a size-16,
+     * granularity-1 HBitmap:
+     *
+     *    initial state            00000000
+     *    set(start=0, count=9)    11111000 (iter: 0, 2, 4, 6, 8)
+     *    reset(start=1, count=3)  00111000 (iter: 4, 6, 8)
+     *    set(start=9, count=2)    00111100 (iter: 4, 6, 8, 10)
+     *    reset(start=5, count=5)  00000000
+     *
+     * From an implementation point of view, when setting or resetting bits,
+     * the bitmap will scale bit numbers right by this amount of bits.  When
+     * iterating, the bitmap will scale bit numbers left by this amount of
+     * bits.
+     */
+    int granularity;
+
+    /* A number of progressively less coarse bitmaps (i.e. level 0 is the
+     * coarsest).  Each bit in level N represents a word in level N+1 that
+     * has a set bit, except the last level where each bit represents the
+     * actual bitmap.
+     *
+     * Note that all bitmaps have the same number of levels.  Even a 1-bit
+     * bitmap will still allocate HBITMAP_LEVELS arrays.
+     */
+    unsigned long *levels[HBITMAP_LEVELS];
+};
+
+static inline int popcountl(unsigned long l)
+{
+    return BITS_PER_LONG == 32 ? ctpop32(l) : ctpop64(l);
+}
+
+/* Advance hbi to the next nonzero word and return it.  hbi->pos
+ * is updated.  Returns zero if we reach the end of the bitmap.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
+{
+    size_t pos = hbi->pos;
+    const HBitmap *hb = hbi->hb;
+    unsigned i = HBITMAP_LEVELS - 1;
+
+    unsigned long cur;
+    do {
+        cur = hbi->cur[--i];
+        pos >>= BITS_PER_LEVEL;
+    } while (cur == 0);
+
+    /* Check for end of iteration.  We always use fewer than BITS_PER_LONG
+     * bits in the level 0 bitmap; thus we can repurpose the most significant
+     * bit as a sentinel.  The sentinel is set in hbitmap_alloc and ensures
+     * that the above loop ends even without an explicit check on i.
+     */
+
+    if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) {
+        return 0;
+    }
+    for (; i < HBITMAP_LEVELS - 1; i++) {
+        /* Shift back pos to the left, matching the right shifts above.
+         * The index of this word's least significant set bit provides
+         * the low-order bits.
+         */
+        pos = (pos << BITS_PER_LEVEL) + ffsl(cur) - 1;
+        hbi->cur[i] = cur & (cur - 1);
+
+        /* Set up next level for iteration.  */
+        cur = hb->levels[i + 1][pos];
+    }
+
+    hbi->pos = pos;
+    trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur);
+
+    assert(cur);
+    return cur;
+}
+
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first)
+{
+    unsigned i, bit;
+    uint64_t pos;
+
+    hbi->hb = hb;
+    pos = first >> hb->granularity;
+    assert(pos < hb->size);
+    hbi->pos = pos >> BITS_PER_LEVEL;
+    hbi->granularity = hb->granularity;
+
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        bit = pos & (BITS_PER_LONG - 1);
+        pos >>= BITS_PER_LEVEL;
+
+        /* Drop bits representing items before first.  */
+        hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1);
+
+        /* We have already added level i+1, so the lowest set bit has
+         * been processed.  Clear it.
+         */
+        if (i != HBITMAP_LEVELS - 1) {
+            hbi->cur[i] &= ~(1UL << bit);
+        }
+    }
+}
+
+bool hbitmap_empty(const HBitmap *hb)
+{
+    return hb->count == 0;
+}
+
+int hbitmap_granularity(const HBitmap *hb)
+{
+    return hb->granularity;
+}
+
+uint64_t hbitmap_count(const HBitmap *hb)
+{
+    return hb->count << hb->granularity;
+}
+
+/* Count the number of set bits between start and end, not accounting for
+ * the granularity.  Also an example of how to use hbitmap_iter_next_word.
+ */
+static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last)
+{
+    HBitmapIter hbi;
+    uint64_t count = 0;
+    uint64_t end = last + 1;
+    unsigned long cur;
+    size_t pos;
+
+    hbitmap_iter_init(&hbi, hb, start << hb->granularity);
+    for (;;) {
+        pos = hbitmap_iter_next_word(&hbi, &cur);
+        if (pos >= (end >> BITS_PER_LEVEL)) {
+            break;
+        }
+        count += popcountl(cur);
+    }
+
+    if (pos == (end >> BITS_PER_LEVEL)) {
+        /* Drop bits representing the END-th and subsequent items.  */
+        int bit = end & (BITS_PER_LONG - 1);
+        cur &= (1UL << bit) - 1;
+        count += popcountl(cur);
+    }
+
+    return count;
+}
+
+/* Setting starts at the last layer and propagates up if an element
+ * changes from zero to non-zero.
+ */
+static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool changed;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    changed = (*elem == 0);
+    *elem |= mask;
+    return changed;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_set_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+        changed |= hb_set_elem(&hb->levels[level][i], start, next - 1);
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] == 0);
+            hb->levels[level][i] = ~0UL;
+        }
+    }
+    changed |= hb_set_elem(&hb->levels[level][i], start, last);
+
+    /* If there was any change in this layer, we may have to update
+     * the one above.
+     */
+    if (level > 0 && changed) {
+        hb_set_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_set(hb, start, count,
+                      start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+    count = last - start + 1;
+
+    hb->count += count - hb_count_between(hb, start, last);
+    hb_set_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+/* Resetting works the other way round: propagate up if the new
+ * value is zero.
+ */
+static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool blanked;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    blanked = *elem != 0 && ((*elem & ~mask) == 0);
+    *elem &= ~mask;
+    return blanked;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_reset_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+
+        /* Here we need a more complex test than when setting bits.  Even if
+         * something was changed, we must not blank bits in the upper level
+         * unless the lower-level word became entirely zero.  So, remove pos
+         * from the upper-level range if bits remain set.
+         */
+        if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) {
+            changed = true;
+        } else {
+            pos++;
+        }
+
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] != 0);
+            hb->levels[level][i] = 0UL;
+        }
+    }
+
+    /* Same as above, this time for lastpos.  */
+    if (hb_reset_elem(&hb->levels[level][i], start, last)) {
+        changed = true;
+    } else {
+        lastpos--;
+    }
+
+    if (level > 0 && changed) {
+        hb_reset_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_reset(hb, start, count,
+                        start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+
+    hb->count -= hb_count_between(hb, start, last);
+    hb_reset_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+bool hbitmap_get(const HBitmap *hb, uint64_t item)
+{
+    /* Compute position and bit in the last layer.  */
+    uint64_t pos = item >> hb->granularity;
+    unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
+
+    return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
+}
+
+void hbitmap_free(HBitmap *hb)
+{
+    unsigned i;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        g_free(hb->levels[i]);
+    }
+    g_free(hb);
+}
+
+HBitmap *hbitmap_alloc(uint64_t size, int granularity)
+{
+    HBitmap *hb = g_malloc0(sizeof (struct HBitmap));
+    unsigned i;
+
+    assert(granularity >= 0 && granularity < 64);
+    size = (size + (1ULL << granularity) - 1) >> granularity;
+    assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
+
+    hb->size = size;
+    hb->granularity = granularity;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
+        hb->levels[i] = g_malloc0(size * sizeof(unsigned long));
+    }
+
+    /* We necessarily have free bits in level 0 due to the definition
+     * of HBITMAP_LEVELS, so use one for a sentinel.  This speeds up
+     * hbitmap_iter_skip_words.
+     */
+    assert(size == 1);
+    hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
+    return hb;
+}
diff --git a/host-utils.c b/util/host-utils.c
index dc9612387b3..5e3915abba4 100644
--- a/host-utils.c
+++ b/util/host-utils.c
@@ -25,7 +25,7 @@
 
 #include <stdlib.h>
 #include <stdint.h>
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 //#define DEBUG_MULDIV
 
diff --git a/iov.c b/util/iov.c
index 60705c73ab1..c0f5c566185 100644
--- a/iov.c
+++ b/util/iov.c
@@ -16,7 +16,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
 
 #ifdef _WIN32
 # include <windows.h>
@@ -26,7 +26,7 @@
 # include <sys/socket.h>
 #endif
 
-size_t iov_from_buf(struct iovec *iov, unsigned int iov_cnt,
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
                     size_t offset, const void *buf, size_t bytes)
 {
     size_t done;
@@ -228,3 +228,195 @@ void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
         fprintf(fp, "\n");
     }
 }
+
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes)
+{
+    size_t len;
+    unsigned int i, j;
+    for (i = 0, j = 0; i < iov_cnt && j < dst_iov_cnt && bytes; i++) {
+        if (offset >= iov[i].iov_len) {
+            offset -= iov[i].iov_len;
+            continue;
+        }
+        len = MIN(bytes, iov[i].iov_len - offset);
+
+        dst_iov[j].iov_base = iov[i].iov_base + offset;
+        dst_iov[j].iov_len = len;
+        j++;
+        bytes -= len;
+        offset = 0;
+    }
+    assert(offset == 0);
+    return j;
+}
+
+/* io vectors */
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
+{
+    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
+    qiov->niov = 0;
+    qiov->nalloc = alloc_hint;
+    qiov->size = 0;
+}
+
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov)
+{
+    int i;
+
+    qiov->iov = iov;
+    qiov->niov = niov;
+    qiov->nalloc = -1;
+    qiov->size = 0;
+    for (i = 0; i < niov; i++)
+        qiov->size += iov[i].iov_len;
+}
+
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
+{
+    assert(qiov->nalloc != -1);
+
+    if (qiov->niov == qiov->nalloc) {
+        qiov->nalloc = 2 * qiov->nalloc + 1;
+        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
+    }
+    qiov->iov[qiov->niov].iov_base = base;
+    qiov->iov[qiov->niov].iov_len = len;
+    qiov->size += len;
+    ++qiov->niov;
+}
+
+/*
+ * Concatenates (partial) iovecs from src_iov to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src_iov if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes)
+{
+    int i;
+    size_t done;
+    assert(dst->nalloc != -1);
+    for (i = 0, done = 0; done < sbytes && i < src_cnt; i++) {
+        if (soffset < src_iov[i].iov_len) {
+            size_t len = MIN(src_iov[i].iov_len - soffset, sbytes - done);
+            qemu_iovec_add(dst, src_iov[i].iov_base + soffset, len);
+            done += len;
+            soffset = 0;
+        } else {
+            soffset -= src_iov[i].iov_len;
+        }
+    }
+    assert(soffset == 0); /* offset beyond end of src */
+}
+
+/*
+ * Concatenates (partial) iovecs from src to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes)
+{
+    qemu_iovec_concat_iov(dst, src->iov, src->niov, soffset, sbytes);
+}
+
+void qemu_iovec_destroy(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qemu_iovec_reset(qiov);
+    g_free(qiov->iov);
+    qiov->nalloc = 0;
+    qiov->iov = NULL;
+}
+
+void qemu_iovec_reset(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qiov->niov = 0;
+    qiov->size = 0;
+}
+
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes)
+{
+    return iov_to_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes)
+{
+    return iov_from_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes)
+{
+    return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
+}
+
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    for (cur = *iov; *iov_cnt > 0; cur++) {
+        if (cur->iov_len > bytes) {
+            cur->iov_base += bytes;
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        *iov_cnt -= 1;
+    }
+
+    *iov = cur;
+    return total;
+}
+
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    if (*iov_cnt == 0) {
+        return 0;
+    }
+
+    cur = iov + (*iov_cnt - 1);
+
+    while (*iov_cnt > 0) {
+        if (cur->iov_len > bytes) {
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        cur--;
+        *iov_cnt -= 1;
+    }
+
+    return total;
+}
diff --git a/module.c b/util/module.c
index c3a6da7a867..7acc33d076a 100644
--- a/module.c
+++ b/util/module.c
@@ -14,8 +14,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "module.h"
+#include "qemu/queue.h"
+#include "qemu/module.h"
 
 typedef struct ModuleEntry
 {
diff --git a/notify.c b/util/notify.c
index 12282a6745c..7b7692acb2c 100644
--- a/notify.c
+++ b/util/notify.c
@@ -14,7 +14,7 @@
  */
 
 #include "qemu-common.h"
-#include "notify.h"
+#include "qemu/notify.h"
 
 void notifier_list_init(NotifierList *list)
 {
diff --git a/osdep.c b/util/osdep.c
index 3b25297a25b..5b51a0322e5 100644
--- a/osdep.c
+++ b/util/osdep.c
@@ -47,8 +47,8 @@ extern int madvise(caddr_t, size_t, int);
 
 #include "qemu-common.h"
 #include "trace.h"
-#include "qemu_socket.h"
-#include "monitor.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
 
 static bool fips_enabled = false;
 
@@ -88,7 +88,6 @@ static int qemu_dup_flags(int fd, int flags)
     int ret;
     int serrno;
     int dup_flags;
-    int setfl_flags;
 
 #ifdef F_DUPFD_CLOEXEC
     ret = fcntl(fd, F_DUPFD_CLOEXEC, 0);
@@ -113,16 +112,7 @@ static int qemu_dup_flags(int fd, int flags)
     }
 
     /* Set/unset flags that we can with fcntl */
-    setfl_flags = O_APPEND | O_ASYNC | O_NONBLOCK;
-#ifdef O_NOATIME
-    setfl_flags |= O_NOATIME;
-#endif
-#ifdef O_DIRECT
-    setfl_flags |= O_DIRECT;
-#endif
-    dup_flags &= ~setfl_flags;
-    dup_flags |= (flags & setfl_flags);
-    if (fcntl(ret, F_SETFL, dup_flags) == -1) {
+    if (fcntl(ret, F_SETFL, flags) == -1) {
         goto fail;
     }
 
@@ -144,6 +134,11 @@ fail:
     errno = serrno;
     return -1;
 }
+
+static int qemu_parse_fdset(const char *param)
+{
+    return qemu_parse_fd(param);
+}
 #endif
 
 /*
@@ -404,3 +399,4 @@ bool fips_get_state(void)
 {
     return fips_enabled;
 }
+
diff --git a/oslib-posix.c b/util/oslib-posix.c
index dbeb6272b8d..b4152fb33cd 100644
--- a/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -49,9 +49,9 @@ extern int daemon(int, int);
 #endif
 
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #if defined(CONFIG_VALGRIND)
 static int running_on_valgrind = -1;
@@ -61,9 +61,6 @@ static int running_on_valgrind = -1;
 #ifdef CONFIG_LINUX
 #include <sys/syscall.h>
 #endif
-#ifdef CONFIG_EVENTFD
-#include <sys/eventfd.h>
-#endif
 
 int qemu_get_thread_id(void)
 {
@@ -108,8 +105,6 @@ void *qemu_memalign(size_t alignment, size_t size)
     return ptr;
 }
 
-/* conflicts with qemu_vmalloc in bsd-user/mmap.c */
-#if !defined(CONFIG_BSD_USER)
 /* alloc shared memory pages */
 void *qemu_vmalloc(size_t size)
 {
@@ -132,7 +127,6 @@ void *qemu_vmalloc(size_t size)
     trace_qemu_vmalloc(size, ptr);
     return ptr;
 }
-#endif
 
 void qemu_vfree(void *ptr)
 {
@@ -183,34 +177,6 @@ int qemu_pipe(int pipefd[2])
     return ret;
 }
 
-/*
- * Creates an eventfd that looks like a pipe and has EFD_CLOEXEC set.
- */
-int qemu_eventfd(int fds[2])
-{
-#ifdef CONFIG_EVENTFD
-    int ret;
-
-    ret = eventfd(0, 0);
-    if (ret >= 0) {
-        fds[0] = ret;
-        fds[1] = dup(ret);
-        if (fds[1] == -1) {
-            close(ret);
-            return -1;
-        }
-        qemu_set_cloexec(ret);
-        qemu_set_cloexec(fds[1]);
-        return 0;
-    }
-    if (errno != ENOSYS) {
-        return -1;
-    }
-#endif
-
-    return qemu_pipe(fds);
-}
-
 int qemu_utimens(const char *path, const struct timespec *times)
 {
     struct timeval tv[2], tv_now;
diff --git a/oslib-win32.c b/util/oslib-win32.c
index ffbc6d0c9fb..640194c0cf7 100644
--- a/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -27,10 +27,10 @@
  */
 #include <windows.h>
 #include "config-host.h"
-#include "sysemu.h"
-#include "main-loop.h"
+#include "sysemu/sysemu.h"
+#include "qemu/main-loop.h"
 #include "trace.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 void *qemu_oom_check(void *ptr)
 {
@@ -71,7 +71,33 @@ void *qemu_vmalloc(size_t size)
 void qemu_vfree(void *ptr)
 {
     trace_qemu_vfree(ptr);
-    VirtualFree(ptr, 0, MEM_RELEASE);
+    if (ptr) {
+        VirtualFree(ptr, 0, MEM_RELEASE);
+    }
+}
+
+/* FIXME: add proper locking */
+struct tm *gmtime_r(const time_t *timep, struct tm *result)
+{
+    struct tm *p = gmtime(timep);
+    memset(result, 0, sizeof(*result));
+    if (p) {
+        *result = *p;
+        p = result;
+    }
+    return p;
+}
+
+/* FIXME: add proper locking */
+struct tm *localtime_r(const time_t *timep, struct tm *result)
+{
+    struct tm *p = localtime(timep);
+    memset(result, 0, sizeof(*result));
+    if (p) {
+        *result = *p;
+        p = result;
+    }
+    return p;
 }
 
 void socket_set_block(int fd)
diff --git a/path.c b/util/path.c
index ef3f277f17a..f0c69627c7e 100644
--- a/path.c
+++ b/util/path.c
@@ -47,10 +47,7 @@ static struct pathelem *new_entry(const char *root,
 {
     struct pathelem *new = malloc(sizeof(*new));
     new->name = strdup(name);
-    if (asprintf(&new->pathname, "%s/%s", root, name) == -1) {
-        printf("Cannot allocate memory\n");
-        exit(1);
-    }
+    new->pathname = g_strdup_printf("%s/%s", root, name);
     new->num_entries = 0;
     return new;
 }
@@ -58,9 +55,10 @@ static struct pathelem *new_entry(const char *root,
 #define streq(a,b) (strcmp((a), (b)) == 0)
 
 /* Not all systems provide this feature */
-#if defined(DT_DIR) && defined(DT_UNKNOWN)
+#if defined(DT_DIR) && defined(DT_UNKNOWN) && defined(DT_LNK)
 # define dirent_type(dirent) ((dirent)->d_type)
-# define is_dir_maybe(type)  ((type) == DT_DIR || (type) == DT_UNKNOWN)
+# define is_dir_maybe(type) \
+    ((type) == DT_DIR || (type) == DT_UNKNOWN || (type) == DT_LNK)
 #else
 # define dirent_type(dirent) (1)
 # define is_dir_maybe(type)  (type)
diff --git a/util/qemu-config.c b/util/qemu-config.c
new file mode 100644
index 00000000000..47c81f72d37
--- /dev/null
+++ b/util/qemu-config.c
@@ -0,0 +1,215 @@
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "hw/qdev.h"
+#include "qapi/error.h"
+
+static QemuOptsList *vm_config_groups[32];
+
+static QemuOptsList *find_list(QemuOptsList **lists, const char *group,
+                               Error **errp)
+{
+    int i;
+
+    for (i = 0; lists[i] != NULL; i++) {
+        if (strcmp(lists[i]->name, group) == 0)
+            break;
+    }
+    if (lists[i] == NULL) {
+        error_set(errp, QERR_INVALID_OPTION_GROUP, group);
+    }
+    return lists[i];
+}
+
+QemuOptsList *qemu_find_opts(const char *group)
+{
+    QemuOptsList *ret;
+    Error *local_err = NULL;
+
+    ret = find_list(vm_config_groups, group, &local_err);
+    if (error_is_set(&local_err)) {
+        error_report("%s\n", error_get_pretty(local_err));
+        error_free(local_err);
+    }
+
+    return ret;
+}
+
+QemuOptsList *qemu_find_opts_err(const char *group, Error **errp)
+{
+    return find_list(vm_config_groups, group, errp);
+}
+
+void qemu_add_opts(QemuOptsList *list)
+{
+    int entries, i;
+
+    entries = ARRAY_SIZE(vm_config_groups);
+    entries--; /* keep list NULL terminated */
+    for (i = 0; i < entries; i++) {
+        if (vm_config_groups[i] == NULL) {
+            vm_config_groups[i] = list;
+            return;
+        }
+    }
+    fprintf(stderr, "ran out of space in vm_config_groups");
+    abort();
+}
+
+int qemu_set_option(const char *str)
+{
+    char group[64], id[64], arg[64];
+    QemuOptsList *list;
+    QemuOpts *opts;
+    int rc, offset;
+
+    rc = sscanf(str, "%63[^.].%63[^.].%63[^=]%n", group, id, arg, &offset);
+    if (rc < 3 || str[offset] != '=') {
+        error_report("can't parse: \"%s\"", str);
+        return -1;
+    }
+
+    list = qemu_find_opts(group);
+    if (list == NULL) {
+        return -1;
+    }
+
+    opts = qemu_opts_find(list, id);
+    if (!opts) {
+        error_report("there is no %s \"%s\" defined",
+                     list->name, id);
+        return -1;
+    }
+
+    if (qemu_opt_set(opts, arg, str+offset+1) == -1) {
+        return -1;
+    }
+    return 0;
+}
+
+struct ConfigWriteData {
+    QemuOptsList *list;
+    FILE *fp;
+};
+
+static int config_write_opt(const char *name, const char *value, void *opaque)
+{
+    struct ConfigWriteData *data = opaque;
+
+    fprintf(data->fp, "  %s = \"%s\"\n", name, value);
+    return 0;
+}
+
+static int config_write_opts(QemuOpts *opts, void *opaque)
+{
+    struct ConfigWriteData *data = opaque;
+    const char *id = qemu_opts_id(opts);
+
+    if (id) {
+        fprintf(data->fp, "[%s \"%s\"]\n", data->list->name, id);
+    } else {
+        fprintf(data->fp, "[%s]\n", data->list->name);
+    }
+    qemu_opt_foreach(opts, config_write_opt, data, 0);
+    fprintf(data->fp, "\n");
+    return 0;
+}
+
+void qemu_config_write(FILE *fp)
+{
+    struct ConfigWriteData data = { .fp = fp };
+    QemuOptsList **lists = vm_config_groups;
+    int i;
+
+    fprintf(fp, "# qemu config file\n\n");
+    for (i = 0; lists[i] != NULL; i++) {
+        data.list = lists[i];
+        qemu_opts_foreach(data.list, config_write_opts, &data, 0);
+    }
+}
+
+int qemu_config_parse(FILE *fp, QemuOptsList **lists, const char *fname)
+{
+    char line[1024], group[64], id[64], arg[64], value[1024];
+    Location loc;
+    QemuOptsList *list = NULL;
+    Error *local_err = NULL;
+    QemuOpts *opts = NULL;
+    int res = -1, lno = 0;
+
+    loc_push_none(&loc);
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        loc_set_file(fname, ++lno);
+        if (line[0] == '\n') {
+            /* skip empty lines */
+            continue;
+        }
+        if (line[0] == '#') {
+            /* comment */
+            continue;
+        }
+        if (sscanf(line, "[%63s \"%63[^\"]\"]", group, id) == 2) {
+            /* group with id */
+            list = find_list(lists, group, &local_err);
+            if (error_is_set(&local_err)) {
+                error_report("%s\n", error_get_pretty(local_err));
+                error_free(local_err);
+                goto out;
+            }
+            opts = qemu_opts_create(list, id, 1, NULL);
+            continue;
+        }
+        if (sscanf(line, "[%63[^]]]", group) == 1) {
+            /* group without id */
+            list = find_list(lists, group, &local_err);
+            if (error_is_set(&local_err)) {
+                error_report("%s\n", error_get_pretty(local_err));
+                error_free(local_err);
+                goto out;
+            }
+            opts = qemu_opts_create_nofail(list);
+            continue;
+        }
+        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2) {
+            /* arg = value */
+            if (opts == NULL) {
+                error_report("no group defined");
+                goto out;
+            }
+            if (qemu_opt_set(opts, arg, value) != 0) {
+                goto out;
+            }
+            continue;
+        }
+        error_report("parse error");
+        goto out;
+    }
+    if (ferror(fp)) {
+        error_report("error reading file");
+        goto out;
+    }
+    res = 0;
+out:
+    loc_pop(&loc);
+    return res;
+}
+
+int qemu_read_config_file(const char *filename)
+{
+    FILE *f = fopen(filename, "r");
+    int ret;
+
+    if (f == NULL) {
+        return -errno;
+    }
+
+    ret = qemu_config_parse(f, vm_config_groups, filename);
+    fclose(f);
+
+    if (ret == 0) {
+        return 0;
+    } else {
+        return -EINVAL;
+    }
+}
diff --git a/qemu-error.c b/util/qemu-error.c
index 7cd5ffe1e9a..08a36f480c8 100644
--- a/qemu-error.c
+++ b/util/qemu-error.c
@@ -11,7 +11,7 @@
  */
 
 #include <stdio.h>
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 /*
  * Print to current monitor if we have one, else to stderr.
diff --git a/qemu-option.c b/util/qemu-option.c
index 27891e74e7c..f532b765a0b 100644
--- a/qemu-option.c
+++ b/util/qemu-option.c
@@ -27,11 +27,11 @@
 #include <string.h>
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-objects.h"
-#include "error.h"
-#include "qerror.h"
-#include "qemu-option-internal.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/types.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/option_int.h"
 
 /*
  * Extracts the name of an option from the parameter string (p points at the
@@ -602,26 +602,36 @@ static void qemu_opt_del(QemuOpt *opt)
     g_free(opt);
 }
 
-static void opt_set(QemuOpts *opts, const char *name, const char *value,
-                    bool prepend, Error **errp)
+static bool opts_accepts_any(const QemuOpts *opts)
+{
+    return opts->list->desc[0].name == NULL;
+}
+
+static const QemuOptDesc *find_desc_by_name(const QemuOptDesc *desc,
+                                            const char *name)
 {
-    QemuOpt *opt;
-    const QemuOptDesc *desc = opts->list->desc;
-    Error *local_err = NULL;
     int i;
 
     for (i = 0; desc[i].name != NULL; i++) {
         if (strcmp(desc[i].name, name) == 0) {
-            break;
+            return &desc[i];
         }
     }
-    if (desc[i].name == NULL) {
-        if (i == 0) {
-            /* empty list -> allow any */;
-        } else {
-            error_set(errp, QERR_INVALID_PARAMETER, name);
-            return;
-        }
+
+    return NULL;
+}
+
+static void opt_set(QemuOpts *opts, const char *name, const char *value,
+                    bool prepend, Error **errp)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc;
+    Error *local_err = NULL;
+
+    desc = find_desc_by_name(opts->list->desc, name);
+    if (!desc && !opts_accepts_any(opts)) {
+        error_set(errp, QERR_INVALID_PARAMETER, name);
+        return;
     }
 
     opt = g_malloc0(sizeof(*opt));
@@ -632,9 +642,7 @@ static void opt_set(QemuOpts *opts, const char *name, const char *value,
     } else {
         QTAILQ_INSERT_TAIL(&opts->head, opt, next);
     }
-    if (desc[i].name != NULL) {
-        opt->desc = desc+i;
-    }
+    opt->desc = desc;
     if (value) {
         opt->str = g_strdup(value);
     }
@@ -669,30 +677,43 @@ int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val)
 {
     QemuOpt *opt;
     const QemuOptDesc *desc = opts->list->desc;
-    int i;
 
-    for (i = 0; desc[i].name != NULL; i++) {
-        if (strcmp(desc[i].name, name) == 0) {
-            break;
-        }
-    }
-    if (desc[i].name == NULL) {
-        if (i == 0) {
-            /* empty list -> allow any */;
-        } else {
-            qerror_report(QERR_INVALID_PARAMETER, name);
-            return -1;
-        }
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
     }
 
-    opt = g_malloc0(sizeof(*opt));
     opt->name = g_strdup(name);
     opt->opts = opts;
+    opt->value.boolean = !!val;
+    opt->str = g_strdup(val ? "on" : "off");
     QTAILQ_INSERT_TAIL(&opts->head, opt, next);
-    if (desc[i].name != NULL) {
-        opt->desc = desc+i;
+
+    return 0;
+}
+
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc = opts->list->desc;
+
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
     }
-    opt->value.boolean = !!val;
+
+    opt->name = g_strdup(name);
+    opt->opts = opts;
+    opt->value.uint = val;
+    opt->str = g_strdup_printf("%" PRId64, val);
+    QTAILQ_INSERT_TAIL(&opts->head, opt, next);
+
     return 0;
 }
 
@@ -781,6 +802,15 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
     return opts;
 }
 
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list)
+{
+    QemuOpts *opts;
+    Error *errp = NULL;
+    opts = qemu_opts_create(list, NULL, 0, &errp);
+    assert_no_error(errp);
+    return opts;
+}
+
 void qemu_opts_reset(QemuOptsList *list)
 {
     QemuOpts *opts, *next_opts;
@@ -1068,23 +1098,15 @@ void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp)
     QemuOpt *opt;
     Error *local_err = NULL;
 
-    assert(opts->list->desc[0].name == NULL);
+    assert(opts_accepts_any(opts));
 
     QTAILQ_FOREACH(opt, &opts->head, next) {
-        int i;
-
-        for (i = 0; desc[i].name != NULL; i++) {
-            if (strcmp(desc[i].name, opt->name) == 0) {
-                break;
-            }
-        }
-        if (desc[i].name == NULL) {
+        opt->desc = find_desc_by_name(desc, opt->name);
+        if (!opt->desc) {
             error_set(errp, QERR_INVALID_PARAMETER, opt->name);
             return;
         }
 
-        opt->desc = &desc[i];
-
         qemu_opt_parse(opt, &local_err);
         if (error_is_set(&local_err)) {
             error_propagate(errp, local_err);
diff --git a/qemu-progress.c b/util/qemu-progress.c
index 5f1b8dfb97a..9a3f96cd470 100644
--- a/qemu-progress.c
+++ b/util/qemu-progress.c
@@ -23,8 +23,8 @@
  */
 
 #include "qemu-common.h"
-#include "osdep.h"
-#include "sysemu.h"
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
 #include <stdio.h>
 
 struct progress_state {
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
new file mode 100644
index 00000000000..3537bf3d451
--- /dev/null
+++ b/util/qemu-sockets.c
@@ -0,0 +1,970 @@
+/*
+ *  inet and unix socket functions for qemu
+ *
+ *  (c) 2008 Gerd Hoffmann <kraxel@redhat.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include "monitor/monitor.h"
+#include "qemu/sockets.h"
+#include "qemu-common.h" /* for qemu_isdigit */
+#include "qemu/main-loop.h"
+
+#ifndef AI_ADDRCONFIG
+# define AI_ADDRCONFIG 0
+#endif
+
+static const int on=1, off=0;
+
+/* used temporarely until all users are converted to QemuOpts */
+static QemuOptsList dummy_opts = {
+    .name = "dummy",
+    .head = QTAILQ_HEAD_INITIALIZER(dummy_opts.head),
+    .desc = {
+        {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "host",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "port",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "to",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end if list */ }
+    },
+};
+
+static int inet_getport(struct addrinfo *e)
+{
+    struct sockaddr_in *i4;
+    struct sockaddr_in6 *i6;
+
+    switch (e->ai_family) {
+    case PF_INET6:
+        i6 = (void*)e->ai_addr;
+        return ntohs(i6->sin6_port);
+    case PF_INET:
+        i4 = (void*)e->ai_addr;
+        return ntohs(i4->sin_port);
+    default:
+        return 0;
+    }
+}
+
+static void inet_setport(struct addrinfo *e, int port)
+{
+    struct sockaddr_in *i4;
+    struct sockaddr_in6 *i6;
+
+    switch (e->ai_family) {
+    case PF_INET6:
+        i6 = (void*)e->ai_addr;
+        i6->sin6_port = htons(port);
+        break;
+    case PF_INET:
+        i4 = (void*)e->ai_addr;
+        i4->sin_port = htons(port);
+        break;
+    }
+}
+
+const char *inet_strfamily(int family)
+{
+    switch (family) {
+    case PF_INET6: return "ipv6";
+    case PF_INET:  return "ipv4";
+    case PF_UNIX:  return "unix";
+    }
+    return "unknown";
+}
+
+int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
+{
+    struct addrinfo ai,*res,*e;
+    const char *addr;
+    char port[33];
+    char uaddr[INET6_ADDRSTRLEN+1];
+    char uport[33];
+    int slisten, rc, to, port_min, port_max, p;
+
+    memset(&ai,0, sizeof(ai));
+    ai.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
+    ai.ai_family = PF_UNSPEC;
+    ai.ai_socktype = SOCK_STREAM;
+
+    if ((qemu_opt_get(opts, "host") == NULL) ||
+        (qemu_opt_get(opts, "port") == NULL)) {
+        error_setg(errp, "host and/or port not specified");
+        return -1;
+    }
+    pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
+    addr = qemu_opt_get(opts, "host");
+
+    to = qemu_opt_get_number(opts, "to", 0);
+    if (qemu_opt_get_bool(opts, "ipv4", 0))
+        ai.ai_family = PF_INET;
+    if (qemu_opt_get_bool(opts, "ipv6", 0))
+        ai.ai_family = PF_INET6;
+
+    /* lookup */
+    if (port_offset)
+        snprintf(port, sizeof(port), "%d", atoi(port) + port_offset);
+    rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
+    if (rc != 0) {
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+        return -1;
+    }
+
+    /* create socket + bind */
+    for (e = res; e != NULL; e = e->ai_next) {
+        getnameinfo((struct sockaddr*)e->ai_addr,e->ai_addrlen,
+		        uaddr,INET6_ADDRSTRLEN,uport,32,
+		        NI_NUMERICHOST | NI_NUMERICSERV);
+        slisten = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
+        if (slisten < 0) {
+            if (!e->ai_next) {
+                error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+            }
+            continue;
+        }
+
+        setsockopt(slisten,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
+#ifdef IPV6_V6ONLY
+        if (e->ai_family == PF_INET6) {
+            /* listen on both ipv4 and ipv6 */
+            setsockopt(slisten,IPPROTO_IPV6,IPV6_V6ONLY,(void*)&off,
+                sizeof(off));
+        }
+#endif
+
+        port_min = inet_getport(e);
+        port_max = to ? to + port_offset : port_min;
+        for (p = port_min; p <= port_max; p++) {
+            inet_setport(e, p);
+            if (bind(slisten, e->ai_addr, e->ai_addrlen) == 0) {
+                goto listen;
+            }
+            if (p == port_max) {
+                if (!e->ai_next) {
+                    error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+                }
+            }
+        }
+        closesocket(slisten);
+    }
+    freeaddrinfo(res);
+    return -1;
+
+listen:
+    if (listen(slisten,1) != 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
+        closesocket(slisten);
+        freeaddrinfo(res);
+        return -1;
+    }
+    snprintf(uport, sizeof(uport), "%d", inet_getport(e) - port_offset);
+    qemu_opt_set(opts, "host", uaddr);
+    qemu_opt_set(opts, "port", uport);
+    qemu_opt_set(opts, "ipv6", (e->ai_family == PF_INET6) ? "on" : "off");
+    qemu_opt_set(opts, "ipv4", (e->ai_family != PF_INET6) ? "on" : "off");
+    freeaddrinfo(res);
+    return slisten;
+}
+
+#ifdef _WIN32
+#define QEMU_SOCKET_RC_INPROGRESS(rc) \
+    ((rc) == -EINPROGRESS || (rc) == -EWOULDBLOCK || (rc) == -WSAEALREADY)
+#else
+#define QEMU_SOCKET_RC_INPROGRESS(rc) \
+    ((rc) == -EINPROGRESS)
+#endif
+
+/* Struct to store connect state for non blocking connect */
+typedef struct ConnectState {
+    int fd;
+    struct addrinfo *addr_list;
+    struct addrinfo *current_addr;
+    NonBlockingConnectHandler *callback;
+    void *opaque;
+} ConnectState;
+
+static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
+                             ConnectState *connect_state, Error **errp);
+
+static void wait_for_connect(void *opaque)
+{
+    ConnectState *s = opaque;
+    int val = 0, rc = 0;
+    socklen_t valsize = sizeof(val);
+    bool in_progress;
+
+    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+
+    do {
+        rc = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
+    } while (rc == -1 && socket_error() == EINTR);
+
+    /* update rc to contain error */
+    if (!rc && val) {
+        rc = -1;
+    }
+
+    /* connect error */
+    if (rc < 0) {
+        closesocket(s->fd);
+        s->fd = rc;
+    }
+
+    /* try to connect to the next address on the list */
+    if (s->current_addr) {
+        while (s->current_addr->ai_next != NULL && s->fd < 0) {
+            s->current_addr = s->current_addr->ai_next;
+            s->fd = inet_connect_addr(s->current_addr, &in_progress, s, NULL);
+            /* connect in progress */
+            if (in_progress) {
+                return;
+            }
+        }
+
+        freeaddrinfo(s->addr_list);
+    }
+
+    if (s->callback) {
+        s->callback(s->fd, s->opaque);
+    }
+    g_free(s);
+}
+
+static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
+                             ConnectState *connect_state, Error **errp)
+{
+    int sock, rc;
+
+    *in_progress = false;
+
+    sock = qemu_socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
+    if (sock < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        return -1;
+    }
+    qemu_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+    if (connect_state != NULL) {
+        socket_set_nonblock(sock);
+    }
+    /* connect to peer */
+    do {
+        rc = 0;
+        if (connect(sock, addr->ai_addr, addr->ai_addrlen) < 0) {
+            rc = -socket_error();
+        }
+    } while (rc == -EINTR);
+
+    if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
+        connect_state->fd = sock;
+        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
+                             connect_state);
+        *in_progress = true;
+    } else if (rc < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
+        closesocket(sock);
+        return -1;
+    }
+    return sock;
+}
+
+static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
+{
+    struct addrinfo ai, *res;
+    int rc;
+    const char *addr;
+    const char *port;
+
+    memset(&ai, 0, sizeof(ai));
+
+    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
+    ai.ai_family = PF_UNSPEC;
+    ai.ai_socktype = SOCK_STREAM;
+
+    addr = qemu_opt_get(opts, "host");
+    port = qemu_opt_get(opts, "port");
+    if (addr == NULL || port == NULL) {
+        error_setg(errp, "host and/or port not specified");
+        return NULL;
+    }
+
+    if (qemu_opt_get_bool(opts, "ipv4", 0)) {
+        ai.ai_family = PF_INET;
+    }
+    if (qemu_opt_get_bool(opts, "ipv6", 0)) {
+        ai.ai_family = PF_INET6;
+    }
+
+    /* lookup */
+    rc = getaddrinfo(addr, port, &ai, &res);
+    if (rc != 0) {
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+        return NULL;
+    }
+    return res;
+}
+
+/**
+ * Create a socket and connect it to an address.
+ *
+ * @opts: QEMU options, recognized parameters strings "host" and "port",
+ *        bools "ipv4" and "ipv6".
+ * @errp: set on error
+ * @callback: callback function for non-blocking connect
+ * @opaque: opaque for callback function
+ *
+ * Returns: -1 on error, file descriptor on success.
+ *
+ * If @callback is non-null, the connect is non-blocking.  If this
+ * function succeeds, callback will be called when the connection
+ * completes, with the file descriptor on success, or -1 on error.
+ */
+int inet_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
+{
+    struct addrinfo *res, *e;
+    int sock = -1;
+    bool in_progress;
+    ConnectState *connect_state = NULL;
+
+    res = inet_parse_connect_opts(opts, errp);
+    if (!res) {
+        return -1;
+    }
+
+    if (callback != NULL) {
+        connect_state = g_malloc0(sizeof(*connect_state));
+        connect_state->addr_list = res;
+        connect_state->callback = callback;
+        connect_state->opaque = opaque;
+    }
+
+    for (e = res; e != NULL; e = e->ai_next) {
+        if (connect_state != NULL) {
+            connect_state->current_addr = e;
+        }
+        sock = inet_connect_addr(e, &in_progress, connect_state, errp);
+        if (in_progress) {
+            return sock;
+        } else if (sock >= 0) {
+            /* non blocking socket immediate success, call callback */
+            if (callback != NULL) {
+                callback(sock, opaque);
+            }
+            break;
+        }
+    }
+    g_free(connect_state);
+    freeaddrinfo(res);
+    return sock;
+}
+
+int inet_dgram_opts(QemuOpts *opts, Error **errp)
+{
+    struct addrinfo ai, *peer = NULL, *local = NULL;
+    const char *addr;
+    const char *port;
+    int sock = -1, rc;
+
+    /* lookup peer addr */
+    memset(&ai,0, sizeof(ai));
+    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
+    ai.ai_family = PF_UNSPEC;
+    ai.ai_socktype = SOCK_DGRAM;
+
+    addr = qemu_opt_get(opts, "host");
+    port = qemu_opt_get(opts, "port");
+    if (addr == NULL || strlen(addr) == 0) {
+        addr = "localhost";
+    }
+    if (port == NULL || strlen(port) == 0) {
+        error_setg(errp, "remote port not specified");
+        return -1;
+    }
+
+    if (qemu_opt_get_bool(opts, "ipv4", 0))
+        ai.ai_family = PF_INET;
+    if (qemu_opt_get_bool(opts, "ipv6", 0))
+        ai.ai_family = PF_INET6;
+
+    if (0 != (rc = getaddrinfo(addr, port, &ai, &peer))) {
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+	return -1;
+    }
+
+    /* lookup local addr */
+    memset(&ai,0, sizeof(ai));
+    ai.ai_flags = AI_PASSIVE;
+    ai.ai_family = peer->ai_family;
+    ai.ai_socktype = SOCK_DGRAM;
+
+    addr = qemu_opt_get(opts, "localaddr");
+    port = qemu_opt_get(opts, "localport");
+    if (addr == NULL || strlen(addr) == 0) {
+        addr = NULL;
+    }
+    if (!port || strlen(port) == 0)
+        port = "0";
+
+    if (0 != (rc = getaddrinfo(addr, port, &ai, &local))) {
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+        goto err;
+    }
+
+    /* create socket */
+    sock = qemu_socket(peer->ai_family, peer->ai_socktype, peer->ai_protocol);
+    if (sock < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        goto err;
+    }
+    setsockopt(sock,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
+
+    /* bind socket */
+    if (bind(sock, local->ai_addr, local->ai_addrlen) < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+        goto err;
+    }
+
+    /* connect to peer */
+    if (connect(sock,peer->ai_addr,peer->ai_addrlen) < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
+        goto err;
+    }
+
+    freeaddrinfo(local);
+    freeaddrinfo(peer);
+    return sock;
+
+err:
+    if (-1 != sock)
+        closesocket(sock);
+    if (local)
+        freeaddrinfo(local);
+    if (peer)
+        freeaddrinfo(peer);
+    return -1;
+}
+
+/* compatibility wrapper */
+static InetSocketAddress *inet_parse(const char *str, Error **errp)
+{
+    InetSocketAddress *addr;
+    const char *optstr, *h;
+    char host[64];
+    char port[33];
+    int to;
+    int pos;
+
+    addr = g_new0(InetSocketAddress, 1);
+
+    /* parse address */
+    if (str[0] == ':') {
+        /* no host given */
+        host[0] = '\0';
+        if (1 != sscanf(str, ":%32[^,]%n", port, &pos)) {
+            error_setg(errp, "error parsing port in address '%s'", str);
+            goto fail;
+        }
+    } else if (str[0] == '[') {
+        /* IPv6 addr */
+        if (2 != sscanf(str, "[%64[^]]]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing IPv6 address '%s'", str);
+            goto fail;
+        }
+        addr->ipv6 = addr->has_ipv6 = true;
+    } else if (qemu_isdigit(str[0])) {
+        /* IPv4 addr */
+        if (2 != sscanf(str, "%64[0-9.]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing IPv4 address '%s'", str);
+            goto fail;
+        }
+        addr->ipv4 = addr->has_ipv4 = true;
+    } else {
+        /* hostname */
+        if (2 != sscanf(str, "%64[^:]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing address '%s'", str);
+            goto fail;
+        }
+    }
+
+    addr->host = g_strdup(host);
+    addr->port = g_strdup(port);
+
+    /* parse options */
+    optstr = str + pos;
+    h = strstr(optstr, ",to=");
+    if (h) {
+        h += 4;
+        if (sscanf(h, "%d%n", &to, &pos) != 1 ||
+            (h[pos] != '\0' && h[pos] != ',')) {
+            error_setg(errp, "error parsing to= argument");
+            goto fail;
+        }
+        addr->has_to = true;
+        addr->to = to;
+    }
+    if (strstr(optstr, ",ipv4")) {
+        addr->ipv4 = addr->has_ipv4 = true;
+    }
+    if (strstr(optstr, ",ipv6")) {
+        addr->ipv6 = addr->has_ipv6 = true;
+    }
+    return addr;
+
+fail:
+    qapi_free_InetSocketAddress(addr);
+    return NULL;
+}
+
+static void inet_addr_to_opts(QemuOpts *opts, InetSocketAddress *addr)
+{
+    bool ipv4 = addr->ipv4 || !addr->has_ipv4;
+    bool ipv6 = addr->ipv6 || !addr->has_ipv6;
+
+    if (!ipv4 || !ipv6) {
+        qemu_opt_set_bool(opts, "ipv4", ipv4);
+        qemu_opt_set_bool(opts, "ipv6", ipv6);
+    }
+    if (addr->has_to) {
+        char to[20];
+        snprintf(to, sizeof(to), "%d", addr->to);
+        qemu_opt_set(opts, "to", to);
+    }
+    qemu_opt_set(opts, "host", addr->host);
+    qemu_opt_set(opts, "port", addr->port);
+}
+
+int inet_listen(const char *str, char *ostr, int olen,
+                int socktype, int port_offset, Error **errp)
+{
+    QemuOpts *opts;
+    char *optstr;
+    int sock = -1;
+    InetSocketAddress *addr;
+
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create_nofail(&dummy_opts);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
+        sock = inet_listen_opts(opts, port_offset, errp);
+        if (sock != -1 && ostr) {
+            optstr = strchr(str, ',');
+            if (qemu_opt_get_bool(opts, "ipv6", 0)) {
+                snprintf(ostr, olen, "[%s]:%s%s",
+                         qemu_opt_get(opts, "host"),
+                         qemu_opt_get(opts, "port"),
+                         optstr ? optstr : "");
+            } else {
+                snprintf(ostr, olen, "%s:%s%s",
+                         qemu_opt_get(opts, "host"),
+                         qemu_opt_get(opts, "port"),
+                         optstr ? optstr : "");
+            }
+        }
+        qemu_opts_del(opts);
+    }
+    return sock;
+}
+
+/**
+ * Create a blocking socket and connect it to an address.
+ *
+ * @str: address string
+ * @errp: set in case of an error
+ *
+ * Returns -1 in case of error, file descriptor on success
+ **/
+int inet_connect(const char *str, Error **errp)
+{
+    QemuOpts *opts;
+    int sock = -1;
+    InetSocketAddress *addr;
+
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create_nofail(&dummy_opts);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
+        sock = inet_connect_opts(opts, errp, NULL, NULL);
+        qemu_opts_del(opts);
+    }
+    return sock;
+}
+
+/**
+ * Create a non-blocking socket and connect it to an address.
+ * Calls the callback function with fd in case of success or -1 in case of
+ * error.
+ *
+ * @str: address string
+ * @callback: callback function that is called when connect completes,
+ *            cannot be NULL.
+ * @opaque: opaque for callback function
+ * @errp: set in case of an error
+ *
+ * Returns: -1 on immediate error, file descriptor on success.
+ **/
+int inet_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp)
+{
+    QemuOpts *opts;
+    int sock = -1;
+    InetSocketAddress *addr;
+
+    g_assert(callback != NULL);
+
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create_nofail(&dummy_opts);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
+        sock = inet_connect_opts(opts, errp, callback, opaque);
+        qemu_opts_del(opts);
+    }
+    return sock;
+}
+
+#ifndef _WIN32
+
+int unix_listen_opts(QemuOpts *opts, Error **errp)
+{
+    struct sockaddr_un un;
+    const char *path = qemu_opt_get(opts, "path");
+    int sock, fd;
+
+    sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
+    if (sock < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        return -1;
+    }
+
+    memset(&un, 0, sizeof(un));
+    un.sun_family = AF_UNIX;
+    if (path && strlen(path)) {
+        snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+    } else {
+        char *tmpdir = getenv("TMPDIR");
+        snprintf(un.sun_path, sizeof(un.sun_path), "%s/qemu-socket-XXXXXX",
+                 tmpdir ? tmpdir : "/tmp");
+        /*
+         * This dummy fd usage silences the mktemp() unsecure warning.
+         * Using mkstemp() doesn't make things more secure here
+         * though.  bind() complains about existing files, so we have
+         * to unlink first and thus re-open the race window.  The
+         * worst case possible is bind() failing, i.e. a DoS attack.
+         */
+        fd = mkstemp(un.sun_path); close(fd);
+        qemu_opt_set(opts, "path", un.sun_path);
+    }
+
+    unlink(un.sun_path);
+    if (bind(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
+        goto err;
+    }
+    if (listen(sock, 1) < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
+        goto err;
+    }
+
+    return sock;
+
+err:
+    closesocket(sock);
+    return -1;
+}
+
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
+{
+    struct sockaddr_un un;
+    const char *path = qemu_opt_get(opts, "path");
+    ConnectState *connect_state = NULL;
+    int sock, rc;
+
+    if (NULL == path) {
+        error_setg(errp, "unix connect: no path specified\n");
+        return -1;
+    }
+
+    sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
+    if (sock < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        return -1;
+    }
+    if (callback != NULL) {
+        connect_state = g_malloc0(sizeof(*connect_state));
+        connect_state->callback = callback;
+        connect_state->opaque = opaque;
+        socket_set_nonblock(sock);
+    }
+
+    memset(&un, 0, sizeof(un));
+    un.sun_family = AF_UNIX;
+    snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+
+    /* connect to peer */
+    do {
+        rc = 0;
+        if (connect(sock, (struct sockaddr *) &un, sizeof(un)) < 0) {
+            rc = -socket_error();
+        }
+    } while (rc == -EINTR);
+
+    if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
+        connect_state->fd = sock;
+        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
+                             connect_state);
+        return sock;
+    } else if (rc >= 0) {
+        /* non blocking socket immediate success, call callback */
+        if (callback != NULL) {
+            callback(sock, opaque);
+        }
+    }
+
+    if (rc < 0) {
+        error_set_errno(errp, -rc, QERR_SOCKET_CONNECT_FAILED);
+        close(sock);
+        sock = -1;
+    }
+
+    g_free(connect_state);
+    return sock;
+}
+
+#else
+
+int unix_listen_opts(QemuOpts *opts, Error **errp)
+{
+    error_setg(errp, "unix sockets are not available on windows");
+    errno = ENOTSUP;
+    return -1;
+}
+
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
+{
+    error_setg(errp, "unix sockets are not available on windows");
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
+
+/* compatibility wrapper */
+int unix_listen(const char *str, char *ostr, int olen, Error **errp)
+{
+    QemuOpts *opts;
+    char *path, *optstr;
+    int sock, len;
+
+    opts = qemu_opts_create_nofail(&dummy_opts);
+
+    optstr = strchr(str, ',');
+    if (optstr) {
+        len = optstr - str;
+        if (len) {
+            path = g_malloc(len+1);
+            snprintf(path, len+1, "%.*s", len, str);
+            qemu_opt_set(opts, "path", path);
+            g_free(path);
+        }
+    } else {
+        qemu_opt_set(opts, "path", str);
+    }
+
+    sock = unix_listen_opts(opts, errp);
+
+    if (sock != -1 && ostr)
+        snprintf(ostr, olen, "%s%s", qemu_opt_get(opts, "path"), optstr ? optstr : "");
+    qemu_opts_del(opts);
+    return sock;
+}
+
+int unix_connect(const char *path, Error **errp)
+{
+    QemuOpts *opts;
+    int sock;
+
+    opts = qemu_opts_create_nofail(&dummy_opts);
+    qemu_opt_set(opts, "path", path);
+    sock = unix_connect_opts(opts, errp, NULL, NULL);
+    qemu_opts_del(opts);
+    return sock;
+}
+
+
+int unix_nonblocking_connect(const char *path,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp)
+{
+    QemuOpts *opts;
+    int sock = -1;
+
+    g_assert(callback != NULL);
+
+    opts = qemu_opts_create_nofail(&dummy_opts);
+    qemu_opt_set(opts, "path", path);
+    sock = unix_connect_opts(opts, errp, callback, opaque);
+    qemu_opts_del(opts);
+    return sock;
+}
+
+SocketAddress *socket_parse(const char *str, Error **errp)
+{
+    SocketAddress *addr = NULL;
+
+    addr = g_new(SocketAddress, 1);
+    if (strstart(str, "unix:", NULL)) {
+        if (str[5] == '\0') {
+            error_setg(errp, "invalid Unix socket address\n");
+            goto fail;
+        } else {
+            addr->kind = SOCKET_ADDRESS_KIND_UNIX;
+            addr->q_unix = g_new(UnixSocketAddress, 1);
+            addr->q_unix->path = g_strdup(str + 5);
+        }
+    } else if (strstart(str, "fd:", NULL)) {
+        if (str[3] == '\0') {
+            error_setg(errp, "invalid file descriptor address\n");
+            goto fail;
+        } else {
+            addr->kind = SOCKET_ADDRESS_KIND_FD;
+            addr->fd = g_new(String, 1);
+            addr->fd->str = g_strdup(str + 3);
+        }
+    } else {
+        addr->kind = SOCKET_ADDRESS_KIND_INET;
+        addr->inet = g_new(InetSocketAddress, 1);
+        addr->inet = inet_parse(str, errp);
+        if (addr->inet == NULL) {
+            goto fail;
+        }
+    }
+    return addr;
+
+fail:
+    qapi_free_SocketAddress(addr);
+    return NULL;
+}
+
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque)
+{
+    QemuOpts *opts;
+    int fd;
+
+    opts = qemu_opts_create_nofail(&dummy_opts);
+    switch (addr->kind) {
+    case SOCKET_ADDRESS_KIND_INET:
+        inet_addr_to_opts(opts, addr->inet);
+        fd = inet_connect_opts(opts, errp, callback, opaque);
+        break;
+
+    case SOCKET_ADDRESS_KIND_UNIX:
+        qemu_opt_set(opts, "path", addr->q_unix->path);
+        fd = unix_connect_opts(opts, errp, callback, opaque);
+        break;
+
+    case SOCKET_ADDRESS_KIND_FD:
+        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        if (callback) {
+            callback(fd, opaque);
+        }
+        break;
+
+    default:
+        abort();
+    }
+    qemu_opts_del(opts);
+    return fd;
+}
+
+int socket_listen(SocketAddress *addr, Error **errp)
+{
+    QemuOpts *opts;
+    int fd;
+
+    opts = qemu_opts_create_nofail(&dummy_opts);
+    switch (addr->kind) {
+    case SOCKET_ADDRESS_KIND_INET:
+        inet_addr_to_opts(opts, addr->inet);
+        fd = inet_listen_opts(opts, 0, errp);
+        break;
+
+    case SOCKET_ADDRESS_KIND_UNIX:
+        qemu_opt_set(opts, "path", addr->q_unix->path);
+        fd = unix_listen_opts(opts, errp);
+        break;
+
+    case SOCKET_ADDRESS_KIND_FD:
+        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        break;
+
+    default:
+        abort();
+    }
+    qemu_opts_del(opts);
+    return fd;
+}
+
+#ifdef _WIN32
+static void socket_cleanup(void)
+{
+    WSACleanup();
+}
+#endif
+
+int socket_init(void)
+{
+#ifdef _WIN32
+    WSADATA Data;
+    int ret, err;
+
+    ret = WSAStartup(MAKEWORD(2,2), &Data);
+    if (ret != 0) {
+        err = WSAGetLastError();
+        fprintf(stderr, "WSAStartup: %d\n", err);
+        return -1;
+    }
+    atexit(socket_cleanup);
+#endif
+    return 0;
+}
diff --git a/qemu-thread-posix.c b/util/qemu-thread-posix.c
index 8fbabdac364..4489abf1d85 100644
--- a/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -17,7 +17,10 @@
 #include <signal.h>
 #include <stdint.h>
 #include <string.h>
-#include "qemu-thread.h"
+#include <limits.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include "qemu/thread.h"
 
 static void error_exit(int err, const char *msg)
 {
@@ -115,6 +118,156 @@ void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
         error_exit(err, __func__);
 }
 
+void qemu_sem_init(QemuSemaphore *sem, int init)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_mutex_init(&sem->lock, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_cond_init(&sem->cond, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    if (init < 0) {
+        error_exit(EINVAL, __func__);
+    }
+    sem->count = init;
+#else
+    rc = sem_init(&sem->sem, 0, init);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_destroy(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_cond_destroy(&sem->cond);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_mutex_destroy(&sem->lock);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_destroy(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_post(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    if (sem->count == INT_MAX) {
+        rc = EINVAL;
+    } else if (sem->count++ < 0) {
+        rc = pthread_cond_signal(&sem->cond);
+    } else {
+        rc = 0;
+    }
+    pthread_mutex_unlock(&sem->lock);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_post(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+static void compute_abs_deadline(struct timespec *ts, int ms)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    ts->tv_nsec = tv.tv_usec * 1000 + (ms % 1000) * 1000000;
+    ts->tv_sec = tv.tv_sec + ms / 1000;
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_sec++;
+        ts->tv_nsec -= 1000000000;
+    }
+}
+
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
+{
+    int rc;
+    struct timespec ts;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    compute_abs_deadline(&ts, ms);
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        rc = pthread_cond_timedwait(&sem->cond, &sem->lock, &ts);
+        if (rc == ETIMEDOUT) {
+            ++sem->count;
+            break;
+        }
+        if (rc != 0) {
+            error_exit(rc, __func__);
+        }
+    }
+    pthread_mutex_unlock(&sem->lock);
+    return (rc == ETIMEDOUT ? -1 : 0);
+#else
+    if (ms <= 0) {
+        /* This is cheaper than sem_timedwait.  */
+        do {
+            rc = sem_trywait(&sem->sem);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == EAGAIN) {
+            return -1;
+        }
+    } else {
+        compute_abs_deadline(&ts, ms);
+        do {
+            rc = sem_timedwait(&sem->sem, &ts);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == ETIMEDOUT) {
+            return -1;
+        }
+    }
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+    return 0;
+#endif
+}
+
+void qemu_sem_wait(QemuSemaphore *sem)
+{
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        pthread_cond_wait(&sem->cond, &sem->lock);
+    }
+    pthread_mutex_unlock(&sem->lock);
+#else
+    int rc;
+
+    do {
+        rc = sem_wait(&sem->sem);
+    } while (rc == -1 && errno == EINTR);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
 void qemu_thread_create(QemuThread *thread,
                        void *(*start_routine)(void*),
                        void *arg, int mode)
diff --git a/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 177b398cc4f..517878dcc17 100644
--- a/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -11,7 +11,7 @@
  *
  */
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 #include <process.h>
 #include <assert.h>
 #include <limits.h>
@@ -192,6 +192,41 @@ void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
     qemu_mutex_lock(mutex);
 }
 
+void qemu_sem_init(QemuSemaphore *sem, int init)
+{
+    /* Manual reset.  */
+    sem->sema = CreateSemaphore(NULL, init, LONG_MAX, NULL);
+}
+
+void qemu_sem_destroy(QemuSemaphore *sem)
+{
+    CloseHandle(sem->sema);
+}
+
+void qemu_sem_post(QemuSemaphore *sem)
+{
+    ReleaseSemaphore(sem->sema, 1, NULL);
+}
+
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
+{
+    int rc = WaitForSingleObject(sem->sema, ms);
+    if (rc == WAIT_OBJECT_0) {
+        return 0;
+    }
+    if (rc != WAIT_TIMEOUT) {
+        error_exit(GetLastError(), __func__);
+    }
+    return -1;
+}
+
+void qemu_sem_wait(QemuSemaphore *sem)
+{
+    if (WaitForSingleObject(sem->sema, INFINITE) != WAIT_OBJECT_0) {
+        error_exit(GetLastError(), __func__);
+    }
+}
+
 struct QemuThreadData {
     /* Passed to win32_start_routine.  */
     void             *(*start_routine)(void *);
@@ -204,7 +239,7 @@ struct QemuThreadData {
     CRITICAL_SECTION  cs;
 };
 
-static int qemu_thread_tls_index = TLS_OUT_OF_INDEXES;
+static __thread QemuThreadData *qemu_thread_data;
 
 static unsigned __stdcall win32_start_routine(void *arg)
 {
@@ -216,14 +251,15 @@ static unsigned __stdcall win32_start_routine(void *arg)
         g_free(data);
         data = NULL;
     }
-    TlsSetValue(qemu_thread_tls_index, data);
+    qemu_thread_data = data;
     qemu_thread_exit(start_routine(thread_arg));
     abort();
 }
 
 void qemu_thread_exit(void *arg)
 {
-    QemuThreadData *data = TlsGetValue(qemu_thread_tls_index);
+    QemuThreadData *data = qemu_thread_data;
+
     if (data) {
         assert(data->mode != QEMU_THREAD_DETACHED);
         data->ret = arg;
@@ -263,25 +299,13 @@ void *qemu_thread_join(QemuThread *thread)
     return ret;
 }
 
-static inline void qemu_thread_init(void)
-{
-    if (qemu_thread_tls_index == TLS_OUT_OF_INDEXES) {
-        qemu_thread_tls_index = TlsAlloc();
-        if (qemu_thread_tls_index == TLS_OUT_OF_INDEXES) {
-            error_exit(ERROR_NO_SYSTEM_RESOURCES, __func__);
-        }
-    }
-}
-
-
 void qemu_thread_create(QemuThread *thread,
                        void *(*start_routine)(void *),
                        void *arg, int mode)
 {
     HANDLE hThread;
-
     struct QemuThreadData *data;
-    qemu_thread_init();
+
     data = g_malloc(sizeof *data);
     data->start_routine = start_routine;
     data->arg = arg;
@@ -303,8 +327,7 @@ void qemu_thread_create(QemuThread *thread,
 
 void qemu_thread_get_self(QemuThread *thread)
 {
-    qemu_thread_init();
-    thread->data = TlsGetValue(qemu_thread_tls_index);
+    thread->data = qemu_thread_data;
     thread->tid = GetCurrentThreadId();
 }
 
diff --git a/qemu-timer-common.c b/util/qemu-timer-common.c
index 755e300bc9f..16f5e758b21 100644
--- a/qemu-timer-common.c
+++ b/util/qemu-timer-common.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /***********************************************************/
 /* real time host monotonic timer */
diff --git a/util/uri.c b/util/uri.c
new file mode 100644
index 00000000000..4238729b83f
--- /dev/null
+++ b/util/uri.c
@@ -0,0 +1,2249 @@
+/**
+ * uri.c: set of generic URI related routines
+ *
+ * Reference: RFCs 3986, 2732 and 2373
+ *
+ * Copyright (C) 1998-2003 Daniel Veillard.  All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * DANIEL VEILLARD BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Except as contained in this notice, the name of Daniel Veillard shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization from him.
+ *
+ * daniel@veillard.com
+ *
+ **
+ *
+ * Copyright (C) 2007, 2009-2010 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *    Richard W.M. Jones <rjones@redhat.com>
+ *
+ */
+
+#include <glib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "qemu/uri.h"
+
+static void uri_clean(URI *uri);
+
+/*
+ * Old rule from 2396 used in legacy handling code
+ * alpha    = lowalpha | upalpha
+ */
+#define IS_ALPHA(x) (IS_LOWALPHA(x) || IS_UPALPHA(x))
+
+
+/*
+ * lowalpha = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" |
+ *            "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" |
+ *            "u" | "v" | "w" | "x" | "y" | "z"
+ */
+
+#define IS_LOWALPHA(x) (((x) >= 'a') && ((x) <= 'z'))
+
+/*
+ * upalpha = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" |
+ *           "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" |
+ *           "U" | "V" | "W" | "X" | "Y" | "Z"
+ */
+#define IS_UPALPHA(x) (((x) >= 'A') && ((x) <= 'Z'))
+
+#ifdef IS_DIGIT
+#undef IS_DIGIT
+#endif
+/*
+ * digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+ */
+#define IS_DIGIT(x) (((x) >= '0') && ((x) <= '9'))
+
+/*
+ * alphanum = alpha | digit
+ */
+
+#define IS_ALPHANUM(x) (IS_ALPHA(x) || IS_DIGIT(x))
+
+/*
+ * mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
+ */
+
+#define IS_MARK(x) (((x) == '-') || ((x) == '_') || ((x) == '.') ||     \
+    ((x) == '!') || ((x) == '~') || ((x) == '*') || ((x) == '\'') ||    \
+    ((x) == '(') || ((x) == ')'))
+
+/*
+ * unwise = "{" | "}" | "|" | "\" | "^" | "`"
+ */
+
+#define IS_UNWISE(p)                                                    \
+      (((*(p) == '{')) || ((*(p) == '}')) || ((*(p) == '|')) ||         \
+       ((*(p) == '\\')) || ((*(p) == '^')) || ((*(p) == '[')) ||        \
+       ((*(p) == ']')) || ((*(p) == '`')))
+/*
+ * reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," |
+ *            "[" | "]"
+ */
+
+#define IS_RESERVED(x) (((x) == ';') || ((x) == '/') || ((x) == '?') || \
+        ((x) == ':') || ((x) == '@') || ((x) == '&') || ((x) == '=') || \
+        ((x) == '+') || ((x) == '$') || ((x) == ',') || ((x) == '[') || \
+        ((x) == ']'))
+
+/*
+ * unreserved = alphanum | mark
+ */
+
+#define IS_UNRESERVED(x) (IS_ALPHANUM(x) || IS_MARK(x))
+
+/*
+ * Skip to next pointer char, handle escaped sequences
+ */
+
+#define NEXT(p) ((*p == '%')? p += 3 : p++)
+
+/*
+ * Productions from the spec.
+ *
+ *    authority     = server | reg_name
+ *    reg_name      = 1*( unreserved | escaped | "$" | "," |
+ *                        ";" | ":" | "@" | "&" | "=" | "+" )
+ *
+ * path          = [ abs_path | opaque_part ]
+ */
+
+
+/************************************************************************
+ *									*
+ *                         RFC 3986 parser				*
+ *									*
+ ************************************************************************/
+
+#define ISA_DIGIT(p) ((*(p) >= '0') && (*(p) <= '9'))
+#define ISA_ALPHA(p) (((*(p) >= 'a') && (*(p) <= 'z')) ||		\
+                      ((*(p) >= 'A') && (*(p) <= 'Z')))
+#define ISA_HEXDIG(p)							\
+       (ISA_DIGIT(p) || ((*(p) >= 'a') && (*(p) <= 'f')) ||		\
+        ((*(p) >= 'A') && (*(p) <= 'F')))
+
+/*
+ *    sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
+ *                     / "*" / "+" / "," / ";" / "="
+ */
+#define ISA_SUB_DELIM(p)						\
+      (((*(p) == '!')) || ((*(p) == '$')) || ((*(p) == '&')) ||		\
+       ((*(p) == '(')) || ((*(p) == ')')) || ((*(p) == '*')) ||		\
+       ((*(p) == '+')) || ((*(p) == ',')) || ((*(p) == ';')) ||		\
+       ((*(p) == '=')) || ((*(p) == '\'')))
+
+/*
+ *    gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+ */
+#define ISA_GEN_DELIM(p)						\
+      (((*(p) == ':')) || ((*(p) == '/')) || ((*(p) == '?')) ||         \
+       ((*(p) == '#')) || ((*(p) == '[')) || ((*(p) == ']')) ||         \
+       ((*(p) == '@')))
+
+/*
+ *    reserved      = gen-delims / sub-delims
+ */
+#define ISA_RESERVED(p) (ISA_GEN_DELIM(p) || (ISA_SUB_DELIM(p)))
+
+/*
+ *    unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ */
+#define ISA_UNRESERVED(p)						\
+      ((ISA_ALPHA(p)) || (ISA_DIGIT(p)) || ((*(p) == '-')) ||		\
+       ((*(p) == '.')) || ((*(p) == '_')) || ((*(p) == '~')))
+
+/*
+ *    pct-encoded   = "%" HEXDIG HEXDIG
+ */
+#define ISA_PCT_ENCODED(p)						\
+     ((*(p) == '%') && (ISA_HEXDIG(p + 1)) && (ISA_HEXDIG(p + 2)))
+
+/*
+ *    pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
+ */
+#define ISA_PCHAR(p)							\
+     (ISA_UNRESERVED(p) || ISA_PCT_ENCODED(p) || ISA_SUB_DELIM(p) ||	\
+      ((*(p) == ':')) || ((*(p) == '@')))
+
+/**
+ * rfc3986_parse_scheme:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse an URI scheme
+ *
+ * ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_scheme(URI *uri, const char **str) {
+    const char *cur;
+
+    if (str == NULL)
+	return(-1);
+
+    cur = *str;
+    if (!ISA_ALPHA(cur))
+	return(2);
+    cur++;
+    while (ISA_ALPHA(cur) || ISA_DIGIT(cur) ||
+           (*cur == '+') || (*cur == '-') || (*cur == '.')) cur++;
+    if (uri != NULL) {
+	if (uri->scheme != NULL) g_free(uri->scheme);
+	uri->scheme = g_strndup(*str, cur - *str);
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_fragment:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse the query part of an URI
+ *
+ * fragment      = *( pchar / "/" / "?" )
+ * NOTE: the strict syntax as defined by 3986 does not allow '[' and ']'
+ *       in the fragment identifier but this is used very broadly for
+ *       xpointer scheme selection, so we are allowing it here to not break
+ *       for example all the DocBook processing chains.
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_fragment(URI *uri, const char **str)
+{
+    const char *cur;
+
+    if (str == NULL)
+        return (-1);
+
+    cur = *str;
+
+    while ((ISA_PCHAR(cur)) || (*cur == '/') || (*cur == '?') ||
+           (*cur == '[') || (*cur == ']') ||
+           ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
+        NEXT(cur);
+    if (uri != NULL) {
+        if (uri->fragment != NULL)
+            g_free(uri->fragment);
+	if (uri->cleanup & 2)
+	    uri->fragment = g_strndup(*str, cur - *str);
+	else
+	    uri->fragment = uri_string_unescape(*str, cur - *str, NULL);
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_query:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse the query part of an URI
+ *
+ * query = *uric
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_query(URI *uri, const char **str)
+{
+    const char *cur;
+
+    if (str == NULL)
+        return (-1);
+
+    cur = *str;
+
+    while ((ISA_PCHAR(cur)) || (*cur == '/') || (*cur == '?') ||
+           ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
+        NEXT(cur);
+    if (uri != NULL) {
+	if (uri->query != NULL)
+	    g_free (uri->query);
+	uri->query = g_strndup (*str, cur - *str);
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_port:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse a port  part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * port          = *DIGIT
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_port(URI *uri, const char **str)
+{
+    const char *cur = *str;
+
+    if (ISA_DIGIT(cur)) {
+	if (uri != NULL)
+	    uri->port = 0;
+	while (ISA_DIGIT(cur)) {
+	    if (uri != NULL)
+		uri->port = uri->port * 10 + (*cur - '0');
+	    cur++;
+	}
+	*str = cur;
+	return(0);
+    }
+    return(1);
+}
+
+/**
+ * rfc3986_parse_user_info:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an user informations part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * userinfo      = *( unreserved / pct-encoded / sub-delims / ":" )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_user_info(URI *uri, const char **str)
+{
+    const char *cur;
+
+    cur = *str;
+    while (ISA_UNRESERVED(cur) || ISA_PCT_ENCODED(cur) ||
+           ISA_SUB_DELIM(cur) || (*cur == ':'))
+	NEXT(cur);
+    if (*cur == '@') {
+	if (uri != NULL) {
+	    if (uri->user != NULL) g_free(uri->user);
+	    if (uri->cleanup & 2)
+		uri->user = g_strndup(*str, cur - *str);
+	    else
+		uri->user = uri_string_unescape(*str, cur - *str, NULL);
+	}
+	*str = cur;
+	return(0);
+    }
+    return(1);
+}
+
+/**
+ * rfc3986_parse_dec_octet:
+ * @str:  the string to analyze
+ *
+ *    dec-octet     = DIGIT                 ; 0-9
+ *                  / %x31-39 DIGIT         ; 10-99
+ *                  / "1" 2DIGIT            ; 100-199
+ *                  / "2" %x30-34 DIGIT     ; 200-249
+ *                  / "25" %x30-35          ; 250-255
+ *
+ * Skip a dec-octet.
+ *
+ * Returns 0 if found and skipped, 1 otherwise
+ */
+static int
+rfc3986_parse_dec_octet(const char **str) {
+    const char *cur = *str;
+
+    if (!(ISA_DIGIT(cur)))
+        return(1);
+    if (!ISA_DIGIT(cur+1))
+	cur++;
+    else if ((*cur != '0') && (ISA_DIGIT(cur + 1)) && (!ISA_DIGIT(cur+2)))
+	cur += 2;
+    else if ((*cur == '1') && (ISA_DIGIT(cur + 1)) && (ISA_DIGIT(cur + 2)))
+	cur += 3;
+    else if ((*cur == '2') && (*(cur + 1) >= '0') &&
+	     (*(cur + 1) <= '4') && (ISA_DIGIT(cur + 2)))
+	cur += 3;
+    else if ((*cur == '2') && (*(cur + 1) == '5') &&
+	     (*(cur + 2) >= '0') && (*(cur + 1) <= '5'))
+	cur += 3;
+    else
+        return(1);
+    *str = cur;
+    return(0);
+}
+/**
+ * rfc3986_parse_host:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an host part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * host          = IP-literal / IPv4address / reg-name
+ * IP-literal    = "[" ( IPv6address / IPvFuture  ) "]"
+ * IPv4address   = dec-octet "." dec-octet "." dec-octet "." dec-octet
+ * reg-name      = *( unreserved / pct-encoded / sub-delims )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_host(URI *uri, const char **str)
+{
+    const char *cur = *str;
+    const char *host;
+
+    host = cur;
+    /*
+     * IPv6 and future addressing scheme are enclosed between brackets
+     */
+    if (*cur == '[') {
+        cur++;
+	while ((*cur != ']') && (*cur != 0))
+	    cur++;
+	if (*cur != ']')
+	    return(1);
+	cur++;
+	goto found;
+    }
+    /*
+     * try to parse an IPv4
+     */
+    if (ISA_DIGIT(cur)) {
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+	cur++;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	goto found;
+not_ipv4:
+        cur = *str;
+    }
+    /*
+     * then this should be a hostname which can be empty
+     */
+    while (ISA_UNRESERVED(cur) || ISA_PCT_ENCODED(cur) || ISA_SUB_DELIM(cur))
+        NEXT(cur);
+found:
+    if (uri != NULL) {
+	if (uri->authority != NULL) g_free(uri->authority);
+	uri->authority = NULL;
+	if (uri->server != NULL) g_free(uri->server);
+	if (cur != host) {
+	    if (uri->cleanup & 2)
+		uri->server = g_strndup(host, cur - host);
+	    else
+		uri->server = uri_string_unescape(host, cur - host, NULL);
+	} else
+	    uri->server = NULL;
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_authority:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an authority part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * authority     = [ userinfo "@" ] host [ ":" port ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_authority(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+    /*
+     * try to parse an userinfo and check for the trailing @
+     */
+    ret = rfc3986_parse_user_info(uri, &cur);
+    if ((ret != 0) || (*cur != '@'))
+        cur = *str;
+    else
+        cur++;
+    ret = rfc3986_parse_host(uri, &cur);
+    if (ret != 0) return(ret);
+    if (*cur == ':') {
+        cur++;
+        ret = rfc3986_parse_port(uri, &cur);
+	if (ret != 0) return(ret);
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_segment:
+ * @str:  the string to analyze
+ * @forbid: an optional forbidden character
+ * @empty: allow an empty segment
+ *
+ * Parse a segment and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * segment       = *pchar
+ * segment-nz    = 1*pchar
+ * segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
+ *               ; non-zero-length segment without any colon ":"
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_segment(const char **str, char forbid, int empty)
+{
+    const char *cur;
+
+    cur = *str;
+    if (!ISA_PCHAR(cur)) {
+        if (empty)
+	    return(0);
+	return(1);
+    }
+    while (ISA_PCHAR(cur) && (*cur != forbid))
+        NEXT(cur);
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_ab_empty:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path absolute or empty and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-abempty  = *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_ab_empty(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (*str != cur) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_absolute:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path absolute and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-absolute = "/" [ segment-nz *( "/" segment ) ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_absolute(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    if (*cur != '/')
+        return(1);
+    cur++;
+    ret = rfc3986_parse_segment(&cur, 0, 0);
+    if (ret == 0) {
+	while (*cur == '/') {
+	    cur++;
+	    ret = rfc3986_parse_segment(&cur, 0, 1);
+	    if (ret != 0) return(ret);
+	}
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_rootless:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path without root and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-rootless = segment-nz *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_rootless(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    ret = rfc3986_parse_segment(&cur, 0, 0);
+    if (ret != 0) return(ret);
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_no_scheme:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path which is not a scheme and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-noscheme = segment-nz-nc *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_no_scheme(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    ret = rfc3986_parse_segment(&cur, ':', 0);
+    if (ret != 0) return(ret);
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_hier_part:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an hierarchical part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * hier-part     = "//" authority path-abempty
+ *                / path-absolute
+ *                / path-rootless
+ *                / path-empty
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_hier_part(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    if ((*cur == '/') && (*(cur + 1) == '/')) {
+        cur += 2;
+	ret = rfc3986_parse_authority(uri, &cur);
+	if (ret != 0) return(ret);
+	ret = rfc3986_parse_path_ab_empty(uri, &cur);
+	if (ret != 0) return(ret);
+	*str = cur;
+	return(0);
+    } else if (*cur == '/') {
+        ret = rfc3986_parse_path_absolute(uri, &cur);
+	if (ret != 0) return(ret);
+    } else if (ISA_PCHAR(cur)) {
+        ret = rfc3986_parse_path_rootless(uri, &cur);
+	if (ret != 0) return(ret);
+    } else {
+	/* path-empty is effectively empty */
+	if (uri != NULL) {
+	    if (uri->path != NULL) g_free(uri->path);
+	    uri->path = NULL;
+	}
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_relative_ref:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * relative-ref  = relative-part [ "?" query ] [ "#" fragment ]
+ * relative-part = "//" authority path-abempty
+ *               / path-absolute
+ *               / path-noscheme
+ *               / path-empty
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_relative_ref(URI *uri, const char *str) {
+    int ret;
+
+    if ((*str == '/') && (*(str + 1) == '/')) {
+        str += 2;
+	ret = rfc3986_parse_authority(uri, &str);
+	if (ret != 0) return(ret);
+	ret = rfc3986_parse_path_ab_empty(uri, &str);
+	if (ret != 0) return(ret);
+    } else if (*str == '/') {
+	ret = rfc3986_parse_path_absolute(uri, &str);
+	if (ret != 0) return(ret);
+    } else if (ISA_PCHAR(str)) {
+        ret = rfc3986_parse_path_no_scheme(uri, &str);
+	if (ret != 0) return(ret);
+    } else {
+	/* path-empty is effectively empty */
+	if (uri != NULL) {
+	    if (uri->path != NULL) g_free(uri->path);
+	    uri->path = NULL;
+	}
+    }
+
+    if (*str == '?') {
+	str++;
+	ret = rfc3986_parse_query(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str == '#') {
+	str++;
+	ret = rfc3986_parse_fragment(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str != 0) {
+	uri_clean(uri);
+	return(1);
+    }
+    return(0);
+}
+
+
+/**
+ * rfc3986_parse:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * scheme ":" hier-part [ "?" query ] [ "#" fragment ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse(URI *uri, const char *str) {
+    int ret;
+
+    ret = rfc3986_parse_scheme(uri, &str);
+    if (ret != 0) return(ret);
+    if (*str != ':') {
+	return(1);
+    }
+    str++;
+    ret = rfc3986_parse_hier_part(uri, &str);
+    if (ret != 0) return(ret);
+    if (*str == '?') {
+	str++;
+	ret = rfc3986_parse_query(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str == '#') {
+	str++;
+	ret = rfc3986_parse_fragment(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str != 0) {
+	uri_clean(uri);
+	return(1);
+    }
+    return(0);
+}
+
+/**
+ * rfc3986_parse_uri_reference:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI reference string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_uri_reference(URI *uri, const char *str) {
+    int ret;
+
+    if (str == NULL)
+	return(-1);
+    uri_clean(uri);
+
+    /*
+     * Try first to parse absolute refs, then fallback to relative if
+     * it fails.
+     */
+    ret = rfc3986_parse(uri, str);
+    if (ret != 0) {
+	uri_clean(uri);
+        ret = rfc3986_parse_relative_ref(uri, str);
+	if (ret != 0) {
+	    uri_clean(uri);
+	    return(ret);
+	}
+    }
+    return(0);
+}
+
+/**
+ * uri_parse:
+ * @str:  the URI string to analyze
+ *
+ * Parse an URI based on RFC 3986
+ *
+ * URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ]
+ *
+ * Returns a newly built URI or NULL in case of error
+ */
+URI *
+uri_parse(const char *str) {
+    URI *uri;
+    int ret;
+
+    if (str == NULL)
+	return(NULL);
+    uri = uri_new();
+    if (uri != NULL) {
+	ret = rfc3986_parse_uri_reference(uri, str);
+        if (ret) {
+	    uri_free(uri);
+	    return(NULL);
+	}
+    }
+    return(uri);
+}
+
+/**
+ * uri_parse_into:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI reference string based on RFC 3986 and fills in the
+ * appropriate fields of the @uri structure
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns 0 or the error code
+ */
+int
+uri_parse_into(URI *uri, const char *str) {
+    return(rfc3986_parse_uri_reference(uri, str));
+}
+
+/**
+ * uri_parse_raw:
+ * @str:  the URI string to analyze
+ * @raw:  if 1 unescaping of URI pieces are disabled
+ *
+ * Parse an URI but allows to keep intact the original fragments.
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns a newly built URI or NULL in case of error
+ */
+URI *
+uri_parse_raw(const char *str, int raw) {
+    URI *uri;
+    int ret;
+
+    if (str == NULL)
+	return(NULL);
+    uri = uri_new();
+    if (uri != NULL) {
+        if (raw) {
+	    uri->cleanup |= 2;
+	}
+	ret = uri_parse_into(uri, str);
+        if (ret) {
+	    uri_free(uri);
+	    return(NULL);
+	}
+    }
+    return(uri);
+}
+
+/************************************************************************
+ *									*
+ *			Generic URI structure functions			*
+ *									*
+ ************************************************************************/
+
+/**
+ * uri_new:
+ *
+ * Simply creates an empty URI
+ *
+ * Returns the new structure or NULL in case of error
+ */
+URI *
+uri_new(void) {
+    URI *ret;
+
+    ret = (URI *) g_malloc(sizeof(URI));
+    memset(ret, 0, sizeof(URI));
+    return(ret);
+}
+
+/**
+ * realloc2n:
+ *
+ * Function to handle properly a reallocation when saving an URI
+ * Also imposes some limit on the length of an URI string output
+ */
+static char *
+realloc2n(char *ret, int *max) {
+    char *temp;
+    int tmp;
+
+    tmp = *max * 2;
+    temp = g_realloc(ret, (tmp + 1));
+    *max = tmp;
+    return(temp);
+}
+
+/**
+ * uri_to_string:
+ * @uri:  pointer to an URI
+ *
+ * Save the URI as an escaped string
+ *
+ * Returns a new string (to be deallocated by caller)
+ */
+char *
+uri_to_string(URI *uri) {
+    char *ret = NULL;
+    char *temp;
+    const char *p;
+    int len;
+    int max;
+
+    if (uri == NULL) return(NULL);
+
+
+    max = 80;
+    ret = g_malloc(max + 1);
+    len = 0;
+
+    if (uri->scheme != NULL) {
+	p = uri->scheme;
+	while (*p != 0) {
+	    if (len >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+		ret = temp;
+	    }
+	    ret[len++] = *p++;
+	}
+	if (len >= max) {
+            temp = realloc2n(ret, &max);
+            if (temp == NULL) goto mem_error;
+            ret = temp;
+	}
+	ret[len++] = ':';
+    }
+    if (uri->opaque != NULL) {
+	p = uri->opaque;
+	while (*p != 0) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    if (IS_RESERVED(*(p)) || IS_UNRESERVED(*(p)))
+		ret[len++] = *p++;
+	    else {
+		int val = *(unsigned char *)p++;
+		int hi = val / 0x10, lo = val % 0x10;
+		ret[len++] = '%';
+		ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+	    }
+	}
+    } else {
+	if (uri->server != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	    if (uri->user != NULL) {
+		p = uri->user;
+		while (*p != 0) {
+		    if (len + 3 >= max) {
+                        temp = realloc2n(ret, &max);
+                        if (temp == NULL) goto mem_error;
+                        ret = temp;
+		    }
+		    if ((IS_UNRESERVED(*(p))) ||
+			((*(p) == ';')) || ((*(p) == ':')) ||
+			((*(p) == '&')) || ((*(p) == '=')) ||
+			((*(p) == '+')) || ((*(p) == '$')) ||
+			((*(p) == ',')))
+			ret[len++] = *p++;
+		    else {
+			int val = *(unsigned char *)p++;
+			int hi = val / 0x10, lo = val % 0x10;
+			ret[len++] = '%';
+			ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+			ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		    }
+		}
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = '@';
+	    }
+	    p = uri->server;
+	    while (*p != 0) {
+		if (len >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+	    }
+	    if (uri->port > 0) {
+		if (len + 10 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		len += snprintf(&ret[len], max - len, ":%d", uri->port);
+	    }
+	} else if (uri->authority != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	    p = uri->authority;
+	    while (*p != 0) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		if ((IS_UNRESERVED(*(p))) ||
+                    ((*(p) == '$')) || ((*(p) == ',')) || ((*(p) == ';')) ||
+                    ((*(p) == ':')) || ((*(p) == '@')) || ((*(p) == '&')) ||
+                    ((*(p) == '=')) || ((*(p) == '+')))
+		    ret[len++] = *p++;
+		else {
+		    int val = *(unsigned char *)p++;
+		    int hi = val / 0x10, lo = val % 0x10;
+		    ret[len++] = '%';
+		    ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		    ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		}
+	    }
+	} else if (uri->scheme != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	}
+	if (uri->path != NULL) {
+	    p = uri->path;
+	    /*
+	     * the colon in file:///d: should not be escaped or
+	     * Windows accesses fail later.
+	     */
+	    if ((uri->scheme != NULL) &&
+		(p[0] == '/') &&
+		(((p[1] >= 'a') && (p[1] <= 'z')) ||
+		 ((p[1] >= 'A') && (p[1] <= 'Z'))) &&
+		(p[2] == ':') &&
+	        (!strcmp(uri->scheme, "file"))) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+		ret[len++] = *p++;
+		ret[len++] = *p++;
+	    }
+	    while (*p != 0) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		if ((IS_UNRESERVED(*(p))) || ((*(p) == '/')) ||
+                    ((*(p) == ';')) || ((*(p) == '@')) || ((*(p) == '&')) ||
+	            ((*(p) == '=')) || ((*(p) == '+')) || ((*(p) == '$')) ||
+	            ((*(p) == ',')))
+		    ret[len++] = *p++;
+		else {
+		    int val = *(unsigned char *)p++;
+		    int hi = val / 0x10, lo = val % 0x10;
+		    ret[len++] = '%';
+		    ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		    ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		}
+	    }
+	}
+	if (uri->query != NULL) {
+	    if (len + 1 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '?';
+	    p = uri->query;
+	    while (*p != 0) {
+		if (len + 1 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+	    }
+	}
+    }
+    if (uri->fragment != NULL) {
+	if (len + 3 >= max) {
+            temp = realloc2n(ret, &max);
+            if (temp == NULL) goto mem_error;
+            ret = temp;
+	}
+	ret[len++] = '#';
+	p = uri->fragment;
+	while (*p != 0) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    if ((IS_UNRESERVED(*(p))) || (IS_RESERVED(*(p))))
+		ret[len++] = *p++;
+	    else {
+		int val = *(unsigned char *)p++;
+		int hi = val / 0x10, lo = val % 0x10;
+		ret[len++] = '%';
+		ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+	    }
+	}
+    }
+    if (len >= max) {
+        temp = realloc2n(ret, &max);
+        if (temp == NULL) goto mem_error;
+        ret = temp;
+    }
+    ret[len] = 0;
+    return(ret);
+
+mem_error:
+    g_free(ret);
+    return(NULL);
+}
+
+/**
+ * uri_clean:
+ * @uri:  pointer to an URI
+ *
+ * Make sure the URI struct is free of content
+ */
+static void
+uri_clean(URI *uri) {
+    if (uri == NULL) return;
+
+    if (uri->scheme != NULL) g_free(uri->scheme);
+    uri->scheme = NULL;
+    if (uri->server != NULL) g_free(uri->server);
+    uri->server = NULL;
+    if (uri->user != NULL) g_free(uri->user);
+    uri->user = NULL;
+    if (uri->path != NULL) g_free(uri->path);
+    uri->path = NULL;
+    if (uri->fragment != NULL) g_free(uri->fragment);
+    uri->fragment = NULL;
+    if (uri->opaque != NULL) g_free(uri->opaque);
+    uri->opaque = NULL;
+    if (uri->authority != NULL) g_free(uri->authority);
+    uri->authority = NULL;
+    if (uri->query != NULL) g_free(uri->query);
+    uri->query = NULL;
+}
+
+/**
+ * uri_free:
+ * @uri:  pointer to an URI
+ *
+ * Free up the URI struct
+ */
+void
+uri_free(URI *uri) {
+    uri_clean(uri);
+    g_free(uri);
+}
+
+/************************************************************************
+ *									*
+ *			Helper functions				*
+ *									*
+ ************************************************************************/
+
+/**
+ * normalize_uri_path:
+ * @path:  pointer to the path string
+ *
+ * Applies the 5 normalization steps to a path string--that is, RFC 2396
+ * Section 5.2, steps 6.c through 6.g.
+ *
+ * Normalization occurs directly on the string, no new allocation is done
+ *
+ * Returns 0 or an error code
+ */
+static int
+normalize_uri_path(char *path) {
+    char *cur, *out;
+
+    if (path == NULL)
+	return(-1);
+
+    /* Skip all initial "/" chars.  We want to get to the beginning of the
+     * first non-empty segment.
+     */
+    cur = path;
+    while (cur[0] == '/')
+      ++cur;
+    if (cur[0] == '\0')
+      return(0);
+
+    /* Keep everything we've seen so far.  */
+    out = cur;
+
+    /*
+     * Analyze each segment in sequence for cases (c) and (d).
+     */
+    while (cur[0] != '\0') {
+	/*
+	 * c) All occurrences of "./", where "." is a complete path segment,
+	 *    are removed from the buffer string.
+	 */
+	if ((cur[0] == '.') && (cur[1] == '/')) {
+	    cur += 2;
+	    /* '//' normalization should be done at this point too */
+	    while (cur[0] == '/')
+		cur++;
+	    continue;
+	}
+
+	/*
+	 * d) If the buffer string ends with "." as a complete path segment,
+	 *    that "." is removed.
+	 */
+	if ((cur[0] == '.') && (cur[1] == '\0'))
+	    break;
+
+	/* Otherwise keep the segment.  */
+	while (cur[0] != '/') {
+            if (cur[0] == '\0')
+              goto done_cd;
+	    (out++)[0] = (cur++)[0];
+	}
+	/* nomalize // */
+	while ((cur[0] == '/') && (cur[1] == '/'))
+	    cur++;
+
+        (out++)[0] = (cur++)[0];
+    }
+ done_cd:
+    out[0] = '\0';
+
+    /* Reset to the beginning of the first segment for the next sequence.  */
+    cur = path;
+    while (cur[0] == '/')
+      ++cur;
+    if (cur[0] == '\0')
+	return(0);
+
+    /*
+     * Analyze each segment in sequence for cases (e) and (f).
+     *
+     * e) All occurrences of "<segment>/../", where <segment> is a
+     *    complete path segment not equal to "..", are removed from the
+     *    buffer string.  Removal of these path segments is performed
+     *    iteratively, removing the leftmost matching pattern on each
+     *    iteration, until no matching pattern remains.
+     *
+     * f) If the buffer string ends with "<segment>/..", where <segment>
+     *    is a complete path segment not equal to "..", that
+     *    "<segment>/.." is removed.
+     *
+     * To satisfy the "iterative" clause in (e), we need to collapse the
+     * string every time we find something that needs to be removed.  Thus,
+     * we don't need to keep two pointers into the string: we only need a
+     * "current position" pointer.
+     */
+    while (1) {
+        char *segp, *tmp;
+
+        /* At the beginning of each iteration of this loop, "cur" points to
+         * the first character of the segment we want to examine.
+         */
+
+        /* Find the end of the current segment.  */
+        segp = cur;
+        while ((segp[0] != '/') && (segp[0] != '\0'))
+          ++segp;
+
+        /* If this is the last segment, we're done (we need at least two
+         * segments to meet the criteria for the (e) and (f) cases).
+         */
+        if (segp[0] == '\0')
+          break;
+
+        /* If the first segment is "..", or if the next segment _isn't_ "..",
+         * keep this segment and try the next one.
+         */
+        ++segp;
+        if (((cur[0] == '.') && (cur[1] == '.') && (segp == cur+3))
+            || ((segp[0] != '.') || (segp[1] != '.')
+                || ((segp[2] != '/') && (segp[2] != '\0')))) {
+          cur = segp;
+          continue;
+        }
+
+        /* If we get here, remove this segment and the next one and back up
+         * to the previous segment (if there is one), to implement the
+         * "iteratively" clause.  It's pretty much impossible to back up
+         * while maintaining two pointers into the buffer, so just compact
+         * the whole buffer now.
+         */
+
+        /* If this is the end of the buffer, we're done.  */
+        if (segp[2] == '\0') {
+          cur[0] = '\0';
+          break;
+        }
+        /* Valgrind complained, strcpy(cur, segp + 3); */
+        /* string will overlap, do not use strcpy */
+        tmp = cur;
+        segp += 3;
+        while ((*tmp++ = *segp++) != 0)
+          ;
+
+        /* If there are no previous segments, then keep going from here.  */
+        segp = cur;
+        while ((segp > path) && ((--segp)[0] == '/'))
+          ;
+        if (segp == path)
+          continue;
+
+        /* "segp" is pointing to the end of a previous segment; find it's
+         * start.  We need to back up to the previous segment and start
+         * over with that to handle things like "foo/bar/../..".  If we
+         * don't do this, then on the first pass we'll remove the "bar/..",
+         * but be pointing at the second ".." so we won't realize we can also
+         * remove the "foo/..".
+         */
+        cur = segp;
+        while ((cur > path) && (cur[-1] != '/'))
+          --cur;
+    }
+    out[0] = '\0';
+
+    /*
+     * g) If the resulting buffer string still begins with one or more
+     *    complete path segments of "..", then the reference is
+     *    considered to be in error. Implementations may handle this
+     *    error by retaining these components in the resolved path (i.e.,
+     *    treating them as part of the final URI), by removing them from
+     *    the resolved path (i.e., discarding relative levels above the
+     *    root), or by avoiding traversal of the reference.
+     *
+     * We discard them from the final path.
+     */
+    if (path[0] == '/') {
+      cur = path;
+      while ((cur[0] == '/') && (cur[1] == '.') && (cur[2] == '.')
+             && ((cur[3] == '/') || (cur[3] == '\0')))
+	cur += 3;
+
+      if (cur != path) {
+	out = path;
+	while (cur[0] != '\0')
+          (out++)[0] = (cur++)[0];
+	out[0] = 0;
+      }
+    }
+
+    return(0);
+}
+
+static int is_hex(char c) {
+    if (((c >= '0') && (c <= '9')) ||
+        ((c >= 'a') && (c <= 'f')) ||
+        ((c >= 'A') && (c <= 'F')))
+	return(1);
+    return(0);
+}
+
+
+/**
+ * uri_string_unescape:
+ * @str:  the string to unescape
+ * @len:   the length in bytes to unescape (or <= 0 to indicate full string)
+ * @target:  optional destination buffer
+ *
+ * Unescaping routine, but does not check that the string is an URI. The
+ * output is a direct unsigned char translation of %XX values (no encoding)
+ * Note that the length of the result can only be smaller or same size as
+ * the input string.
+ *
+ * Returns a copy of the string, but unescaped, will return NULL only in case
+ * of error
+ */
+char *
+uri_string_unescape(const char *str, int len, char *target) {
+    char *ret, *out;
+    const char *in;
+
+    if (str == NULL)
+	return(NULL);
+    if (len <= 0) len = strlen(str);
+    if (len < 0) return(NULL);
+
+    if (target == NULL) {
+	ret = g_malloc(len + 1);
+    } else
+	ret = target;
+    in = str;
+    out = ret;
+    while(len > 0) {
+	if ((len > 2) && (*in == '%') && (is_hex(in[1])) && (is_hex(in[2]))) {
+	    in++;
+	    if ((*in >= '0') && (*in <= '9'))
+	        *out = (*in - '0');
+	    else if ((*in >= 'a') && (*in <= 'f'))
+	        *out = (*in - 'a') + 10;
+	    else if ((*in >= 'A') && (*in <= 'F'))
+	        *out = (*in - 'A') + 10;
+	    in++;
+	    if ((*in >= '0') && (*in <= '9'))
+	        *out = *out * 16 + (*in - '0');
+	    else if ((*in >= 'a') && (*in <= 'f'))
+	        *out = *out * 16 + (*in - 'a') + 10;
+	    else if ((*in >= 'A') && (*in <= 'F'))
+	        *out = *out * 16 + (*in - 'A') + 10;
+	    in++;
+	    len -= 3;
+	    out++;
+	} else {
+	    *out++ = *in++;
+	    len--;
+	}
+    }
+    *out = 0;
+    return(ret);
+}
+
+/**
+ * uri_string_escape:
+ * @str:  string to escape
+ * @list: exception list string of chars not to escape
+ *
+ * This routine escapes a string to hex, ignoring reserved characters (a-z)
+ * and the characters in the exception list.
+ *
+ * Returns a new escaped string or NULL in case of error.
+ */
+char *
+uri_string_escape(const char *str, const char *list) {
+    char *ret, ch;
+    char *temp;
+    const char *in;
+    int len, out;
+
+    if (str == NULL)
+	return(NULL);
+    if (str[0] == 0)
+	return(g_strdup(str));
+    len = strlen(str);
+    if (!(len > 0)) return(NULL);
+
+    len += 20;
+    ret = g_malloc(len);
+    in = str;
+    out = 0;
+    while(*in != 0) {
+	if (len - out <= 3) {
+            temp = realloc2n(ret, &len);
+	    ret = temp;
+	}
+
+	ch = *in;
+
+	if ((ch != '@') && (!IS_UNRESERVED(ch)) && (!strchr(list, ch))) {
+	    unsigned char val;
+	    ret[out++] = '%';
+	    val = ch >> 4;
+	    if (val <= 9)
+		ret[out++] = '0' + val;
+	    else
+		ret[out++] = 'A' + val - 0xA;
+	    val = ch & 0xF;
+	    if (val <= 9)
+		ret[out++] = '0' + val;
+	    else
+		ret[out++] = 'A' + val - 0xA;
+	    in++;
+	} else {
+	    ret[out++] = *in++;
+	}
+
+    }
+    ret[out] = 0;
+    return(ret);
+}
+
+/************************************************************************
+ *									*
+ *			Public functions				*
+ *									*
+ ************************************************************************/
+
+/**
+ * uri_resolve:
+ * @URI:  the URI instance found in the document
+ * @base:  the base value
+ *
+ * Computes he final URI of the reference done by checking that
+ * the given URI is valid, and building the final URI using the
+ * base URI. This is processed according to section 5.2 of the
+ * RFC 2396
+ *
+ * 5.2. Resolving Relative References to Absolute Form
+ *
+ * Returns a new URI string (to be freed by the caller) or NULL in case
+ *         of error.
+ */
+char *
+uri_resolve(const char *uri, const char *base) {
+    char *val = NULL;
+    int ret, len, indx, cur, out;
+    URI *ref = NULL;
+    URI *bas = NULL;
+    URI *res = NULL;
+
+    /*
+     * 1) The URI reference is parsed into the potential four components and
+     *    fragment identifier, as described in Section 4.3.
+     *
+     *    NOTE that a completely empty URI is treated by modern browsers
+     *    as a reference to "." rather than as a synonym for the current
+     *    URI.  Should we do that here?
+     */
+    if (uri == NULL)
+	ret = -1;
+    else {
+	if (*uri) {
+	    ref = uri_new();
+	    if (ref == NULL)
+		goto done;
+	    ret = uri_parse_into(ref, uri);
+	}
+	else
+	    ret = 0;
+    }
+    if (ret != 0)
+	goto done;
+    if ((ref != NULL) && (ref->scheme != NULL)) {
+	/*
+	 * The URI is absolute don't modify.
+	 */
+	val = g_strdup(uri);
+	goto done;
+    }
+    if (base == NULL)
+	ret = -1;
+    else {
+	bas = uri_new();
+	if (bas == NULL)
+	    goto done;
+	ret = uri_parse_into(bas, base);
+    }
+    if (ret != 0) {
+	if (ref)
+	    val = uri_to_string(ref);
+	goto done;
+    }
+    if (ref == NULL) {
+	/*
+	 * the base fragment must be ignored
+	 */
+	if (bas->fragment != NULL) {
+	    g_free(bas->fragment);
+	    bas->fragment = NULL;
+	}
+	val = uri_to_string(bas);
+	goto done;
+    }
+
+    /*
+     * 2) If the path component is empty and the scheme, authority, and
+     *    query components are undefined, then it is a reference to the
+     *    current document and we are done.  Otherwise, the reference URI's
+     *    query and fragment components are defined as found (or not found)
+     *    within the URI reference and not inherited from the base URI.
+     *
+     *    NOTE that in modern browsers, the parsing differs from the above
+     *    in the following aspect:  the query component is allowed to be
+     *    defined while still treating this as a reference to the current
+     *    document.
+     */
+    res = uri_new();
+    if (res == NULL)
+	goto done;
+    if ((ref->scheme == NULL) && (ref->path == NULL) &&
+	((ref->authority == NULL) && (ref->server == NULL))) {
+	if (bas->scheme != NULL)
+	    res->scheme = g_strdup(bas->scheme);
+	if (bas->authority != NULL)
+	    res->authority = g_strdup(bas->authority);
+	else if (bas->server != NULL) {
+	    res->server = g_strdup(bas->server);
+	    if (bas->user != NULL)
+		res->user = g_strdup(bas->user);
+	    res->port = bas->port;
+	}
+	if (bas->path != NULL)
+	    res->path = g_strdup(bas->path);
+	if (ref->query != NULL)
+	    res->query = g_strdup (ref->query);
+	else if (bas->query != NULL)
+	    res->query = g_strdup(bas->query);
+	if (ref->fragment != NULL)
+	    res->fragment = g_strdup(ref->fragment);
+	goto step_7;
+    }
+
+    /*
+     * 3) If the scheme component is defined, indicating that the reference
+     *    starts with a scheme name, then the reference is interpreted as an
+     *    absolute URI and we are done.  Otherwise, the reference URI's
+     *    scheme is inherited from the base URI's scheme component.
+     */
+    if (ref->scheme != NULL) {
+	val = uri_to_string(ref);
+	goto done;
+    }
+    if (bas->scheme != NULL)
+	res->scheme = g_strdup(bas->scheme);
+
+    if (ref->query != NULL)
+	res->query = g_strdup(ref->query);
+    if (ref->fragment != NULL)
+	res->fragment = g_strdup(ref->fragment);
+
+    /*
+     * 4) If the authority component is defined, then the reference is a
+     *    network-path and we skip to step 7.  Otherwise, the reference
+     *    URI's authority is inherited from the base URI's authority
+     *    component, which will also be undefined if the URI scheme does not
+     *    use an authority component.
+     */
+    if ((ref->authority != NULL) || (ref->server != NULL)) {
+	if (ref->authority != NULL)
+	    res->authority = g_strdup(ref->authority);
+	else {
+	    res->server = g_strdup(ref->server);
+	    if (ref->user != NULL)
+		res->user = g_strdup(ref->user);
+            res->port = ref->port;
+	}
+	if (ref->path != NULL)
+	    res->path = g_strdup(ref->path);
+	goto step_7;
+    }
+    if (bas->authority != NULL)
+	res->authority = g_strdup(bas->authority);
+    else if (bas->server != NULL) {
+	res->server = g_strdup(bas->server);
+	if (bas->user != NULL)
+	    res->user = g_strdup(bas->user);
+	res->port = bas->port;
+    }
+
+    /*
+     * 5) If the path component begins with a slash character ("/"), then
+     *    the reference is an absolute-path and we skip to step 7.
+     */
+    if ((ref->path != NULL) && (ref->path[0] == '/')) {
+	res->path = g_strdup(ref->path);
+	goto step_7;
+    }
+
+
+    /*
+     * 6) If this step is reached, then we are resolving a relative-path
+     *    reference.  The relative path needs to be merged with the base
+     *    URI's path.  Although there are many ways to do this, we will
+     *    describe a simple method using a separate string buffer.
+     *
+     * Allocate a buffer large enough for the result string.
+     */
+    len = 2; /* extra / and 0 */
+    if (ref->path != NULL)
+	len += strlen(ref->path);
+    if (bas->path != NULL)
+	len += strlen(bas->path);
+    res->path = g_malloc(len);
+    res->path[0] = 0;
+
+    /*
+     * a) All but the last segment of the base URI's path component is
+     *    copied to the buffer.  In other words, any characters after the
+     *    last (right-most) slash character, if any, are excluded.
+     */
+    cur = 0;
+    out = 0;
+    if (bas->path != NULL) {
+	while (bas->path[cur] != 0) {
+	    while ((bas->path[cur] != 0) && (bas->path[cur] != '/'))
+		cur++;
+	    if (bas->path[cur] == 0)
+		break;
+
+	    cur++;
+	    while (out < cur) {
+		res->path[out] = bas->path[out];
+		out++;
+	    }
+	}
+    }
+    res->path[out] = 0;
+
+    /*
+     * b) The reference's path component is appended to the buffer
+     *    string.
+     */
+    if (ref->path != NULL && ref->path[0] != 0) {
+	indx = 0;
+	/*
+	 * Ensure the path includes a '/'
+	 */
+	if ((out == 0) && (bas->server != NULL))
+	    res->path[out++] = '/';
+	while (ref->path[indx] != 0) {
+	    res->path[out++] = ref->path[indx++];
+	}
+    }
+    res->path[out] = 0;
+
+    /*
+     * Steps c) to h) are really path normalization steps
+     */
+    normalize_uri_path(res->path);
+
+step_7:
+
+    /*
+     * 7) The resulting URI components, including any inherited from the
+     *    base URI, are recombined to give the absolute form of the URI
+     *    reference.
+     */
+    val = uri_to_string(res);
+
+done:
+    if (ref != NULL)
+	uri_free(ref);
+    if (bas != NULL)
+	uri_free(bas);
+    if (res != NULL)
+	uri_free(res);
+    return(val);
+}
+
+/**
+ * uri_resolve_relative:
+ * @URI:  the URI reference under consideration
+ * @base:  the base value
+ *
+ * Expresses the URI of the reference in terms relative to the
+ * base.  Some examples of this operation include:
+ *     base = "http://site1.com/docs/book1.html"
+ *        URI input                        URI returned
+ *     docs/pic1.gif                    pic1.gif
+ *     docs/img/pic1.gif                img/pic1.gif
+ *     img/pic1.gif                     ../img/pic1.gif
+ *     http://site1.com/docs/pic1.gif   pic1.gif
+ *     http://site2.com/docs/pic1.gif   http://site2.com/docs/pic1.gif
+ *
+ *     base = "docs/book1.html"
+ *        URI input                        URI returned
+ *     docs/pic1.gif                    pic1.gif
+ *     docs/img/pic1.gif                img/pic1.gif
+ *     img/pic1.gif                     ../img/pic1.gif
+ *     http://site1.com/docs/pic1.gif   http://site1.com/docs/pic1.gif
+ *
+ *
+ * Note: if the URI reference is really weird or complicated, it may be
+ *       worthwhile to first convert it into a "nice" one by calling
+ *       uri_resolve (using 'base') before calling this routine,
+ *       since this routine (for reasonable efficiency) assumes URI has
+ *       already been through some validation.
+ *
+ * Returns a new URI string (to be freed by the caller) or NULL in case
+ * error.
+ */
+char *
+uri_resolve_relative (const char *uri, const char * base)
+{
+    char *val = NULL;
+    int ret;
+    int ix;
+    int pos = 0;
+    int nbslash = 0;
+    int len;
+    URI *ref = NULL;
+    URI *bas = NULL;
+    char *bptr, *uptr, *vptr;
+    int remove_path = 0;
+
+    if ((uri == NULL) || (*uri == 0))
+	return NULL;
+
+    /*
+     * First parse URI into a standard form
+     */
+    ref = uri_new ();
+    if (ref == NULL)
+	return NULL;
+    /* If URI not already in "relative" form */
+    if (uri[0] != '.') {
+	ret = uri_parse_into (ref, uri);
+	if (ret != 0)
+	    goto done;		/* Error in URI, return NULL */
+    } else
+	ref->path = g_strdup(uri);
+
+    /*
+     * Next parse base into the same standard form
+     */
+    if ((base == NULL) || (*base == 0)) {
+	val = g_strdup (uri);
+	goto done;
+    }
+    bas = uri_new ();
+    if (bas == NULL)
+	goto done;
+    if (base[0] != '.') {
+	ret = uri_parse_into (bas, base);
+	if (ret != 0)
+	    goto done;		/* Error in base, return NULL */
+    } else
+	bas->path = g_strdup(base);
+
+    /*
+     * If the scheme / server on the URI differs from the base,
+     * just return the URI
+     */
+    if ((ref->scheme != NULL) &&
+	((bas->scheme == NULL) ||
+	 (strcmp (bas->scheme, ref->scheme)) ||
+	 (strcmp (bas->server, ref->server)))) {
+	val = g_strdup (uri);
+	goto done;
+    }
+    if (!strcmp(bas->path, ref->path)) {
+	val = g_strdup("");
+	goto done;
+    }
+    if (bas->path == NULL) {
+	val = g_strdup(ref->path);
+	goto done;
+    }
+    if (ref->path == NULL) {
+        ref->path = (char *) "/";
+	remove_path = 1;
+    }
+
+    /*
+     * At this point (at last!) we can compare the two paths
+     *
+     * First we take care of the special case where either of the
+     * two path components may be missing (bug 316224)
+     */
+    if (bas->path == NULL) {
+	if (ref->path != NULL) {
+	    uptr = ref->path;
+	    if (*uptr == '/')
+		uptr++;
+	    /* exception characters from uri_to_string */
+	    val = uri_string_escape(uptr, "/;&=+$,");
+	}
+	goto done;
+    }
+    bptr = bas->path;
+    if (ref->path == NULL) {
+	for (ix = 0; bptr[ix] != 0; ix++) {
+	    if (bptr[ix] == '/')
+		nbslash++;
+	}
+	uptr = NULL;
+	len = 1;	/* this is for a string terminator only */
+    } else {
+    /*
+     * Next we compare the two strings and find where they first differ
+     */
+	if ((ref->path[pos] == '.') && (ref->path[pos+1] == '/'))
+            pos += 2;
+	if ((*bptr == '.') && (bptr[1] == '/'))
+            bptr += 2;
+	else if ((*bptr == '/') && (ref->path[pos] != '/'))
+	    bptr++;
+	while ((bptr[pos] == ref->path[pos]) && (bptr[pos] != 0))
+	    pos++;
+
+	if (bptr[pos] == ref->path[pos]) {
+	    val = g_strdup("");
+	    goto done;		/* (I can't imagine why anyone would do this) */
+	}
+
+	/*
+	 * In URI, "back up" to the last '/' encountered.  This will be the
+	 * beginning of the "unique" suffix of URI
+	 */
+	ix = pos;
+	if ((ref->path[ix] == '/') && (ix > 0))
+	    ix--;
+	else if ((ref->path[ix] == 0) && (ix > 1) && (ref->path[ix - 1] == '/'))
+	    ix -= 2;
+	for (; ix > 0; ix--) {
+	    if (ref->path[ix] == '/')
+		break;
+	}
+	if (ix == 0) {
+	    uptr = ref->path;
+	} else {
+	    ix++;
+	    uptr = &ref->path[ix];
+	}
+
+	/*
+	 * In base, count the number of '/' from the differing point
+	 */
+	if (bptr[pos] != ref->path[pos]) {/* check for trivial URI == base */
+	    for (; bptr[ix] != 0; ix++) {
+		if (bptr[ix] == '/')
+		    nbslash++;
+	    }
+	}
+	len = strlen (uptr) + 1;
+    }
+
+    if (nbslash == 0) {
+	if (uptr != NULL)
+	    /* exception characters from uri_to_string */
+	    val = uri_string_escape(uptr, "/;&=+$,");
+	goto done;
+    }
+
+    /*
+     * Allocate just enough space for the returned string -
+     * length of the remainder of the URI, plus enough space
+     * for the "../" groups, plus one for the terminator
+     */
+    val = g_malloc (len + 3 * nbslash);
+    vptr = val;
+    /*
+     * Put in as many "../" as needed
+     */
+    for (; nbslash>0; nbslash--) {
+	*vptr++ = '.';
+	*vptr++ = '.';
+	*vptr++ = '/';
+    }
+    /*
+     * Finish up with the end of the URI
+     */
+    if (uptr != NULL) {
+        if ((vptr > val) && (len > 0) &&
+	    (uptr[0] == '/') && (vptr[-1] == '/')) {
+	    memcpy (vptr, uptr + 1, len - 1);
+	    vptr[len - 2] = 0;
+	} else {
+	    memcpy (vptr, uptr, len);
+	    vptr[len - 1] = 0;
+	}
+    } else {
+	vptr[len - 1] = 0;
+    }
+
+    /* escape the freshly-built path */
+    vptr = val;
+	/* exception characters from uri_to_string */
+    val = uri_string_escape(vptr, "/;&=+$,");
+    g_free(vptr);
+
+done:
+    /*
+     * Free the working variables
+     */
+    if (remove_path != 0)
+        ref->path = NULL;
+    if (ref != NULL)
+	uri_free (ref);
+    if (bas != NULL)
+	uri_free (bas);
+
+    return val;
+}
+
+/*
+ * Utility functions to help parse and assemble query strings.
+ */
+
+struct QueryParams *
+query_params_new (int init_alloc)
+{
+    struct QueryParams *ps;
+
+    if (init_alloc <= 0) init_alloc = 1;
+
+    ps = g_new(QueryParams, 1);
+    ps->n = 0;
+    ps->alloc = init_alloc;
+    ps->p = g_new(QueryParam, ps->alloc);
+
+    return ps;
+}
+
+/* Ensure there is space to store at least one more parameter
+ * at the end of the set.
+ */
+static int
+query_params_append (struct QueryParams *ps,
+               const char *name, const char *value)
+{
+    if (ps->n >= ps->alloc) {
+        ps->p = g_renew(QueryParam, ps->p, ps->alloc * 2);
+        ps->alloc *= 2;
+    }
+
+    ps->p[ps->n].name = g_strdup(name);
+    ps->p[ps->n].value = value ? g_strdup(value) : NULL;
+    ps->p[ps->n].ignore = 0;
+    ps->n++;
+
+    return 0;
+}
+
+void
+query_params_free (struct QueryParams *ps)
+{
+    int i;
+
+    for (i = 0; i < ps->n; ++i) {
+        g_free (ps->p[i].name);
+        g_free (ps->p[i].value);
+    }
+    g_free (ps->p);
+    g_free (ps);
+}
+
+struct QueryParams *
+query_params_parse (const char *query)
+{
+    struct QueryParams *ps;
+    const char *end, *eq;
+
+    ps = query_params_new (0);
+    if (!query || query[0] == '\0') return ps;
+
+    while (*query) {
+        char *name = NULL, *value = NULL;
+
+        /* Find the next separator, or end of the string. */
+        end = strchr (query, '&');
+        if (!end)
+            end = strchr (query, ';');
+        if (!end)
+            end = query + strlen (query);
+
+        /* Find the first '=' character between here and end. */
+        eq = strchr (query, '=');
+        if (eq && eq >= end) eq = NULL;
+
+        /* Empty section (eg. "&&"). */
+        if (end == query)
+            goto next;
+
+        /* If there is no '=' character, then we have just "name"
+         * and consistent with CGI.pm we assume value is "".
+         */
+        else if (!eq) {
+            name = uri_string_unescape (query, end - query, NULL);
+            value = NULL;
+        }
+        /* Or if we have "name=" here (works around annoying
+         * problem when calling uri_string_unescape with len = 0).
+         */
+        else if (eq+1 == end) {
+            name = uri_string_unescape (query, eq - query, NULL);
+            value = g_new0(char, 1);
+        }
+        /* If the '=' character is at the beginning then we have
+         * "=value" and consistent with CGI.pm we _ignore_ this.
+         */
+        else if (query == eq)
+            goto next;
+
+        /* Otherwise it's "name=value". */
+        else {
+            name = uri_string_unescape (query, eq - query, NULL);
+            value = uri_string_unescape (eq+1, end - (eq+1), NULL);
+        }
+
+        /* Append to the parameter set. */
+        query_params_append (ps, name, value);
+        g_free(name);
+        g_free(value);
+
+    next:
+        query = end;
+        if (*query) query ++; /* skip '&' separator */
+    }
+
+    return ps;
+}
diff --git a/vl.c b/vl.c
index 7c577fa5447..7aab73b736f 100644
--- a/vl.c
+++ b/vl.c
@@ -28,7 +28,7 @@
 #include <errno.h>
 #include <sys/time.h>
 #include <zlib.h>
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
@@ -65,7 +65,7 @@
 #endif
 
 #ifdef CONFIG_SECCOMP
-#include "qemu-seccomp.h"
+#include "sysemu/seccomp.h"
 #endif
 
 #ifdef __sun__
@@ -126,48 +126,49 @@ int main(int argc, char **argv)
 #include "hw/xen.h"
 #include "hw/qdev.h"
 #include "hw/loader.h"
-#include "bt-host.h"
-#include "net.h"
+#include "bt/bt.h"
+#include "net/net.h"
 #include "net/slirp.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
-#include "gdbstub.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "cache-utils.h"
-#include "blockdev.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
+#include "qemu/timer.h"
+#include "char/char.h"
+#include "qemu/cache-utils.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "block-migration.h"
-#include "dma.h"
+#include "migration/block.h"
+#include "sysemu/dma.h"
 #include "audio/audio.h"
-#include "migration.h"
-#include "kvm.h"
-#include "qjson.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "migration/migration.h"
+#include "sysemu/kvm.h"
+#include "qapi/qmp/qjson.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "qemu-options.h"
 #include "qmp-commands.h"
-#include "main-loop.h"
+#include "qemu/main-loop.h"
 #ifdef CONFIG_VIRTFS
 #include "fsdev/qemu-fsdev.h"
 #endif
-#include "qtest.h"
+#include "sysemu/qtest.h"
 
-#include "disas.h"
+#include "disas/disas.h"
 
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #include "slirp/libslirp.h"
 
 #include "trace.h"
 #include "trace/control.h"
-#include "qemu-queue.h"
-#include "cpus.h"
-#include "arch_init.h"
-#include "osdep.h"
+#include "qemu/queue.h"
+#include "sysemu/cpus.h"
+#include "sysemu/arch_init.h"
+#include "qemu/osdep.h"
 
 #include "ui/qemu-spice.h"
+#include "qapi/string-input-visitor.h"
 
 //#define DEBUG_NET
 //#define DEBUG_SLIRP
@@ -180,7 +181,7 @@ static const char *data_dir;
 const char *bios_name = NULL;
 enum vga_retrace_method vga_retrace_method = VGA_RETRACE_DUMB;
 DisplayType display_type = DT_DEFAULT;
-int display_remote = 0;
+static int display_remote;
 const char* keyboard_layout = NULL;
 ram_addr_t ram_size;
 const char *mem_path = NULL;
@@ -203,7 +204,6 @@ CharDriverState *serial_hds[MAX_SERIAL_PORTS];
 CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 CharDriverState *virtcon_hds[MAX_VIRTIO_CONSOLES];
 int win2k_install_hack = 0;
-int usb_enabled = 0;
 int singlestep = 0;
 int smp_cpus = 1;
 int max_cpus = 0;
@@ -215,7 +215,7 @@ const char *vnc_display;
 int acpi_enabled = 1;
 int no_hpet = 0;
 int fd_bootchk = 1;
-int no_reboot = 0;
+static int no_reboot;
 int no_shutdown = 0;
 int cursor_hide = 1;
 int graphic_rotate = 0;
@@ -231,7 +231,7 @@ unsigned int nb_prom_envs = 0;
 const char *prom_envs[MAX_PROM_ENVS];
 int boot_menu;
 uint8_t *boot_splash_filedata;
-int boot_splash_filedata_size;
+size_t boot_splash_filedata_size;
 uint8_t qemu_extra_params_fw[2];
 
 typedef struct FWBootEntry FWBootEntry;
@@ -243,7 +243,8 @@ struct FWBootEntry {
     char *suffix;
 };
 
-QTAILQ_HEAD(, FWBootEntry) fw_boot_order = QTAILQ_HEAD_INITIALIZER(fw_boot_order);
+static QTAILQ_HEAD(, FWBootEntry) fw_boot_order =
+    QTAILQ_HEAD_INITIALIZER(fw_boot_order);
 
 int nb_numa_nodes;
 uint64_t node_mem[MAX_NODES];
@@ -298,6 +299,195 @@ static struct {
     { .driver = "qxl-vga",              .flag = &default_vga       },
 };
 
+static QemuOptsList qemu_rtc_opts = {
+    .name = "rtc",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_rtc_opts.head),
+    .desc = {
+        {
+            .name = "base",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "clock",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "driftfix",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_sandbox_opts = {
+    .name = "sandbox",
+    .implied_opt_name = "enable",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_sandbox_opts.head),
+    .desc = {
+        {
+            .name = "enable",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_trace_opts = {
+    .name = "trace",
+    .implied_opt_name = "trace",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
+    .desc = {
+        {
+            .name = "events",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_option_rom_opts = {
+    .name = "option-rom",
+    .implied_opt_name = "romfile",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_option_rom_opts.head),
+    .desc = {
+        {
+            .name = "bootindex",
+            .type = QEMU_OPT_NUMBER,
+        }, {
+            .name = "romfile",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_machine_opts = {
+    .name = "machine",
+    .implied_opt_name = "type",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_machine_opts.head),
+    .desc = {
+        {
+            .name = "type",
+            .type = QEMU_OPT_STRING,
+            .help = "emulated machine"
+        }, {
+            .name = "accel",
+            .type = QEMU_OPT_STRING,
+            .help = "accelerator list",
+        }, {
+            .name = "kernel_irqchip",
+            .type = QEMU_OPT_BOOL,
+            .help = "use KVM in-kernel irqchip",
+        }, {
+            .name = "kvm_shadow_mem",
+            .type = QEMU_OPT_SIZE,
+            .help = "KVM shadow MMU size",
+        }, {
+            .name = "kernel",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel image file",
+        }, {
+            .name = "initrd",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux initial ramdisk file",
+        }, {
+            .name = "append",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel command line",
+        }, {
+            .name = "dtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel device tree file",
+        }, {
+            .name = "dumpdtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Dump current dtb to a file and quit",
+        }, {
+            .name = "phandle_start",
+            .type = QEMU_OPT_STRING,
+            .help = "The first phandle ID we may generate dynamically",
+        }, {
+            .name = "dt_compatible",
+            .type = QEMU_OPT_STRING,
+            .help = "Overrides the \"compatible\" property of the dt root node",
+        }, {
+            .name = "dump-guest-core",
+            .type = QEMU_OPT_BOOL,
+            .help = "Include guest memory in  a core dump",
+        }, {
+            .name = "mem-merge",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable memory merge support",
+        },{
+            .name = "usb",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable usb",
+        },
+        { /* End of list */ }
+    },
+};
+
+static QemuOptsList qemu_boot_opts = {
+    .name = "boot-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_boot_opts.head),
+    .desc = {
+        /* the three names below are not used now */
+        {
+            .name = "order",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "once",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "menu",
+            .type = QEMU_OPT_STRING,
+        /* following are really used */
+        }, {
+            .name = "splash",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "splash-time",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "reboot-timeout",
+            .type = QEMU_OPT_STRING,
+        },
+        { /*End of list */ }
+    },
+};
+
+static QemuOptsList qemu_add_fd_opts = {
+    .name = "add-fd",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_add_fd_opts.head),
+    .desc = {
+        {
+            .name = "fd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "file descriptor of which a duplicate is added to fd set",
+        },{
+            .name = "set",
+            .type = QEMU_OPT_NUMBER,
+            .help = "ID of the fd set to add fd to",
+        },{
+            .name = "opaque",
+            .type = QEMU_OPT_STRING,
+            .help = "free-form string used to describe fd",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
 const char *qemu_get_vm_name(void)
 {
     return qemu_name;
@@ -341,7 +531,7 @@ static const RunStateTransition runstate_transitions_def[] = {
     { RUN_STATE_DEBUG, RUN_STATE_RUNNING },
 
     { RUN_STATE_INMIGRATE, RUN_STATE_RUNNING },
-    { RUN_STATE_INMIGRATE, RUN_STATE_PRELAUNCH },
+    { RUN_STATE_INMIGRATE, RUN_STATE_PAUSED },
 
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
@@ -397,7 +587,7 @@ bool runstate_check(RunState state)
     return current_run_state == state;
 }
 
-void runstate_init(void)
+static void runstate_init(void)
 {
     const RunStateTransition *p;
 
@@ -447,21 +637,18 @@ StatusInfo *qmp_query_status(Error **errp)
 void qemu_get_timedate(struct tm *tm, int offset)
 {
     time_t ti;
-    struct tm *ret;
 
     time(&ti);
     ti += offset;
     if (rtc_date_offset == -1) {
         if (rtc_utc)
-            ret = gmtime(&ti);
+            gmtime_r(&ti, tm);
         else
-            ret = localtime(&ti);
+            localtime_r(&ti, tm);
     } else {
         ti -= rtc_date_offset;
-        ret = gmtime(&ti);
+        gmtime_r(&ti, tm);
     }
-
-    memcpy(tm, ret, sizeof(struct tm));
 }
 
 int qemu_timedate_diff(struct tm *tm)
@@ -790,6 +977,89 @@ static int parse_sandbox(QemuOpts *opts, void *opaque)
     return 0;
 }
 
+/*********QEMU USB setting******/
+bool usb_enabled(bool default_usb)
+{
+    QemuOpts *mach_opts;
+    mach_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (mach_opts) {
+        return qemu_opt_get_bool(mach_opts, "usb", default_usb);
+    }
+    return default_usb;
+}
+
+#ifndef _WIN32
+static int parse_add_fd(QemuOpts *opts, void *opaque)
+{
+    int fd, dupfd, flags;
+    int64_t fdset_id;
+    const char *fd_opaque = NULL;
+
+    fd = qemu_opt_get_number(opts, "fd", -1);
+    fdset_id = qemu_opt_get_number(opts, "set", -1);
+    fd_opaque = qemu_opt_get(opts, "opaque");
+
+    if (fd < 0) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd option is required and must be non-negative");
+        return -1;
+    }
+
+    if (fd <= STDERR_FILENO) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd cannot be a standard I/O stream");
+        return -1;
+    }
+
+    /*
+     * All fds inherited across exec() necessarily have FD_CLOEXEC
+     * clear, while qemu sets FD_CLOEXEC on all other fds used internally.
+     */
+    flags = fcntl(fd, F_GETFD);
+    if (flags == -1 || (flags & FD_CLOEXEC)) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd is not valid or already in use");
+        return -1;
+    }
+
+    if (fdset_id < 0) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "set option is required and must be non-negative");
+        return -1;
+    }
+
+#ifdef F_DUPFD_CLOEXEC
+    dupfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    dupfd = dup(fd);
+    if (dupfd != -1) {
+        qemu_set_cloexec(dupfd);
+    }
+#endif
+    if (dupfd == -1) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "Error duplicating fd: %s", strerror(errno));
+        return -1;
+    }
+
+    /* add the duplicate fd, and optionally the opaque string, to the fd set */
+    monitor_fdset_add_fd(dupfd, true, fdset_id, fd_opaque ? true : false,
+                         fd_opaque, NULL);
+
+    return 0;
+}
+
+static int cleanup_add_fd(QemuOpts *opts, void *opaque)
+{
+    int fd;
+
+    fd = qemu_opt_get_number(opts, "fd", -1);
+    close(fd);
+
+    return 0;
+}
+#endif
+
 /***********************************************************/
 /* QEMU Block devices */
 
@@ -802,9 +1072,9 @@ static int parse_sandbox(QemuOpts *opts, void *opaque)
 
 static int drive_init_func(QemuOpts *opts, void *opaque)
 {
-    int *use_scsi = opaque;
+    BlockInterfaceType *block_default_type = opaque;
 
-    return drive_init(opts, *use_scsi) == NULL;
+    return drive_init(opts, *block_default_type) == NULL;
 }
 
 static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
@@ -815,16 +1085,11 @@ static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
     return 0;
 }
 
-static void default_drive(int enable, int snapshot, int use_scsi,
-                          BlockInterfaceType type, int index,
-                          const char *optstr)
+static void default_drive(int enable, int snapshot, BlockInterfaceType type,
+                          int index, const char *optstr)
 {
     QemuOpts *opts;
 
-    if (type == IF_DEFAULT) {
-        type = use_scsi ? IF_SCSI : IF_IDE;
-    }
-
     if (!enable || drive_get_by_index(type, index)) {
         return;
     }
@@ -833,7 +1098,7 @@ static void default_drive(int enable, int snapshot, int use_scsi,
     if (snapshot) {
         drive_enable_snapshot(opts, NULL);
     }
-    if (!drive_init(opts, use_scsi)) {
+    if (!drive_init(opts, type)) {
         exit(1);
     }
 }
@@ -933,15 +1198,15 @@ void add_boot_device_path(int32_t bootindex, DeviceState *dev,
  * memory pointed by "size" is assigned total length of the array in bytes
  *
  */
-char *get_boot_devices_list(uint32_t *size)
+char *get_boot_devices_list(size_t *size)
 {
     FWBootEntry *i;
-    uint32_t total = 0;
+    size_t total = 0;
     char *list = NULL;
 
     QTAILQ_FOREACH(i, &fw_boot_order, link) {
         char *devpath = NULL, *bootpath;
-        int len;
+        size_t len;
 
         if (i->dev) {
             devpath = qdev_get_fw_dev_path(i->dev);
@@ -1023,7 +1288,6 @@ static void numa_add(const char *optarg)
         }
         nb_numa_nodes++;
     }
-    return;
 }
 
 static void smp_parse(const char *optarg)
@@ -1078,8 +1342,9 @@ static int usb_device_add(const char *devname)
     const char *p;
     USBDevice *dev = NULL;
 
-    if (!usb_enabled)
+    if (!usb_enabled(false)) {
         return -1;
+    }
 
     /* drivers with .usbdevice_name entry in USBDeviceInfo */
     dev = usbdevice_create(devname);
@@ -1115,8 +1380,9 @@ static int usb_device_del(const char *devname)
     if (strstart(devname, "host:", &p))
         return usb_host_device_close(p);
 
-    if (!usb_enabled)
+    if (!usb_enabled(false)) {
         return -1;
+    }
 
     p = strchr(devname, '.');
     if (!p)
@@ -1183,7 +1449,7 @@ void pcmcia_socket_unregister(PCMCIASocket *socket)
         }
 }
 
-void pcmcia_info(Monitor *mon)
+void pcmcia_info(Monitor *mon, const QDict *qdict)
 {
     struct pcmcia_socket_entry_s *iter;
 
@@ -1276,19 +1542,51 @@ static void gui_update(void *opaque)
 {
     uint64_t interval = GUI_REFRESH_INTERVAL;
     DisplayState *ds = opaque;
-    DisplayChangeListener *dcl = ds->listeners;
+    DisplayChangeListener *dcl;
 
     dpy_refresh(ds);
 
-    while (dcl != NULL) {
+    QLIST_FOREACH(dcl, &ds->listeners, next) {
         if (dcl->gui_timer_interval &&
             dcl->gui_timer_interval < interval)
             interval = dcl->gui_timer_interval;
-        dcl = dcl->next;
     }
     qemu_mod_timer(ds->gui_timer, interval + qemu_get_clock_ms(rt_clock));
 }
 
+void gui_setup_refresh(DisplayState *ds)
+{
+    DisplayChangeListener *dcl;
+    bool need_timer = false;
+    bool have_gfx = false;
+    bool have_text = false;
+
+    QLIST_FOREACH(dcl, &ds->listeners, next) {
+        if (dcl->dpy_refresh != NULL) {
+            need_timer = true;
+        }
+        if (dcl->dpy_gfx_update != NULL) {
+            have_gfx = true;
+        }
+        if (dcl->dpy_text_update != NULL) {
+            have_text = true;
+        }
+    }
+
+    if (need_timer && ds->gui_timer == NULL) {
+        ds->gui_timer = qemu_new_timer_ms(rt_clock, gui_update, ds);
+        qemu_mod_timer(ds->gui_timer, qemu_get_clock_ms(rt_clock));
+    }
+    if (!need_timer && ds->gui_timer != NULL) {
+        qemu_del_timer(ds->gui_timer);
+        qemu_free_timer(ds->gui_timer);
+        ds->gui_timer = NULL;
+    }
+
+    ds->have_gfx = have_gfx;
+    ds->have_text = have_text;
+}
+
 struct vm_change_state_entry {
     VMChangeStateHandler *cb;
     void *opaque;
@@ -1355,6 +1653,8 @@ static int powerdown_requested;
 static int debug_requested;
 static int suspend_requested;
 static int wakeup_requested;
+static NotifierList powerdown_notifiers =
+    NOTIFIER_LIST_INITIALIZER(powerdown_notifiers);
 static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
@@ -1372,14 +1672,14 @@ int qemu_reset_requested_get(void)
     return reset_requested;
 }
 
-int qemu_shutdown_requested(void)
+static int qemu_shutdown_requested(void)
 {
     int r = shutdown_requested;
     shutdown_requested = 0;
     return r;
 }
 
-void qemu_kill_report(void)
+static void qemu_kill_report(void)
 {
     if (!qtest_enabled() && shutdown_signal != -1) {
         fprintf(stderr, "qemu: terminating on signal %d", shutdown_signal);
@@ -1395,7 +1695,7 @@ void qemu_kill_report(void)
     }
 }
 
-int qemu_reset_requested(void)
+static int qemu_reset_requested(void)
 {
     int r = reset_requested;
     reset_requested = 0;
@@ -1416,7 +1716,7 @@ static int qemu_wakeup_requested(void)
     return r;
 }
 
-int qemu_powerdown_requested(void)
+static int qemu_powerdown_requested(void)
 {
     int r = powerdown_requested;
     powerdown_requested = 0;
@@ -1563,12 +1863,23 @@ void qemu_system_shutdown_request(void)
     qemu_notify_event();
 }
 
+static void qemu_system_powerdown(void)
+{
+    monitor_protocol_event(QEVENT_POWERDOWN, NULL);
+    notifier_list_notify(&powerdown_notifiers, NULL);
+}
+
 void qemu_system_powerdown_request(void)
 {
     powerdown_requested = 1;
     qemu_notify_event();
 }
 
+void qemu_register_powerdown_notifier(Notifier *notifier)
+{
+    notifier_list_add(&powerdown_notifiers, notifier);
+}
+
 void qemu_system_debug_request(void)
 {
     debug_requested = 1;
@@ -1581,8 +1892,6 @@ void qemu_system_vmstop_request(RunState state)
     qemu_notify_event();
 }
 
-qemu_irq qemu_system_powerdown;
-
 static bool main_loop_should_exit(void)
 {
     RunState r;
@@ -1619,8 +1928,7 @@ static bool main_loop_should_exit(void)
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-        monitor_protocol_event(QEVENT_POWERDOWN, NULL);
-        qemu_irq_raise(qemu_system_powerdown);
+        qemu_system_powerdown();
     }
     if (qemu_vmstop_requested(&r)) {
         vm_stop(r);
@@ -1690,17 +1998,23 @@ static const QEMUOption qemu_options[] = {
 
 static bool vga_available(void)
 {
-    return qdev_exists("VGA") || qdev_exists("isa-vga");
+    return object_class_by_name("VGA") || object_class_by_name("isa-vga");
 }
 
 static bool cirrus_vga_available(void)
 {
-    return qdev_exists("cirrus-vga") || qdev_exists("isa-cirrus-vga");
+    return object_class_by_name("cirrus-vga")
+           || object_class_by_name("isa-cirrus-vga");
 }
 
 static bool vmware_vga_available(void)
 {
-    return qdev_exists("vmware-svga");
+    return object_class_by_name("vmware-svga");
+}
+
+static bool qxl_vga_available(void)
+{
+    return object_class_by_name("qxl-vga");
 }
 
 static void select_vgahw (const char *p)
@@ -1732,7 +2046,12 @@ static void select_vgahw (const char *p)
     } else if (strstart(p, "xenfb", &opts)) {
         vga_interface_type = VGA_XENFB;
     } else if (strstart(p, "qxl", &opts)) {
-        vga_interface_type = VGA_QXL;
+        if (qxl_vga_available()) {
+            vga_interface_type = VGA_QXL;
+        } else {
+            fprintf(stderr, "Error: QXL VGA not available\n");
+            exit(0);
+        }
     } else if (!strstart(p, "none", &opts)) {
     invalid_vga:
         fprintf(stderr, "Unknown vga type: %s\n", p);
@@ -1863,7 +2182,7 @@ static int balloon_parse(const char *arg)
                 return  -1;
         } else {
             /* create empty opts */
-            opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+            opts = qemu_opts_create_nofail(qemu_find_opts("device"));
         }
         qemu_opt_set(opts, "driver", "virtio-balloon");
         return 0;
@@ -1919,11 +2238,14 @@ static int device_init_func(QemuOpts *opts, void *opaque)
 
 static int chardev_init_func(QemuOpts *opts, void *opaque)
 {
-    CharDriverState *chr;
+    Error *local_err = NULL;
 
-    chr = qemu_chr_new_from_opts(opts, NULL);
-    if (!chr)
+    qemu_chr_new_from_opts(opts, NULL, &local_err);
+    if (error_is_set(&local_err)) {
+        fprintf(stderr, "%s\n", error_get_pretty(local_err));
+        error_free(local_err);
         return -1;
+    }
     return 0;
 }
 
@@ -2023,7 +2345,9 @@ struct device_config {
     Location loc;
     QTAILQ_ENTRY(device_config) next;
 };
-QTAILQ_HEAD(, device_config) device_configs = QTAILQ_HEAD_INITIALIZER(device_configs);
+
+static QTAILQ_HEAD(, device_config) device_configs =
+    QTAILQ_HEAD_INITIALIZER(device_configs);
 
 static void add_device_config(int type, const char *cmdline)
 {
@@ -2111,14 +2435,14 @@ static int virtcon_parse(const char *devname)
         exit(1);
     }
 
-    bus_opts = qemu_opts_create(device, NULL, 0, NULL);
+    bus_opts = qemu_opts_create_nofail(device);
     if (arch_type == QEMU_ARCH_S390X) {
         qemu_opt_set(bus_opts, "driver", "virtio-serial-s390");
     } else {
         qemu_opt_set(bus_opts, "driver", "virtio-serial-pci");
     } 
 
-    dev_opts = qemu_opts_create(device, NULL, 0, NULL);
+    dev_opts = qemu_opts_create_nofail(device);
     qemu_opt_set(dev_opts, "driver", "virtconsole");
 
     snprintf(label, sizeof(label), "virtcon%d", index);
@@ -2337,9 +2661,51 @@ static void free_and_trace(gpointer mem)
     free(mem);
 }
 
-int qemu_init_main_loop(void)
+static int object_set_property(const char *name, const char *value, void *opaque)
 {
-    return main_loop_init();
+    Object *obj = OBJECT(opaque);
+    StringInputVisitor *siv;
+    Error *local_err = NULL;
+
+    if (strcmp(name, "qom-type") == 0 || strcmp(name, "id") == 0) {
+        return 0;
+    }
+
+    siv = string_input_visitor_new(value);
+    object_property_set(obj, string_input_get_visitor(siv), name, &local_err);
+    string_input_visitor_cleanup(siv);
+
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int object_create(QemuOpts *opts, void *opaque)
+{
+    const char *type = qemu_opt_get(opts, "qom-type");
+    const char *id = qemu_opts_id(opts);
+    Object *obj;
+
+    g_assert(type != NULL);
+
+    if (id == NULL) {
+        qerror_report(QERR_MISSING_PARAMETER, "id");
+        return -1;
+    }
+
+    obj = object_new(type);
+    if (qemu_opt_foreach(opts, object_set_property, obj, 1) < 0) {
+        return -1;
+    }
+
+    object_property_add_child(container_get(object_get_root(), "/objects"),
+                              id, obj, NULL);
+
+    return 0;
 }
 
 int main(int argc, char **argv, char **envp)
@@ -2349,9 +2715,8 @@ int main(int argc, char **argv, char **envp)
     const char *icount_option = NULL;
     const char *initrd_filename;
     const char *kernel_filename, *kernel_cmdline;
-    char boot_devices[33] = "cad"; /* default to HD->floppy->CD-ROM */
+    char boot_devices[33] = "";
     DisplayState *ds;
-    DisplayChangeListener *dcl;
     int cyls, heads, secs, translation;
     QemuOpts *hda_opts = NULL, *opts, *machine_opts;
     QemuOptsList *olist;
@@ -2393,6 +2758,22 @@ int main(int argc, char **argv, char **envp)
 
     module_call_init(MODULE_INIT_QOM);
 
+    qemu_add_opts(&qemu_drive_opts);
+    qemu_add_opts(&qemu_chardev_opts);
+    qemu_add_opts(&qemu_device_opts);
+    qemu_add_opts(&qemu_netdev_opts);
+    qemu_add_opts(&qemu_net_opts);
+    qemu_add_opts(&qemu_rtc_opts);
+    qemu_add_opts(&qemu_global_opts);
+    qemu_add_opts(&qemu_mon_opts);
+    qemu_add_opts(&qemu_trace_opts);
+    qemu_add_opts(&qemu_option_rom_opts);
+    qemu_add_opts(&qemu_machine_opts);
+    qemu_add_opts(&qemu_boot_opts);
+    qemu_add_opts(&qemu_sandbox_opts);
+    qemu_add_opts(&qemu_add_fd_opts);
+    qemu_add_opts(&qemu_object_opts);
+
     runstate_init();
 
     init_clocks();
@@ -2470,6 +2851,11 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_M:
                 machine = machine_parse(optarg);
                 break;
+            case QEMU_OPTION_no_kvm_irqchip: {
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "kernel_irqchip=off", 0);
+                break;
+            }
             case QEMU_OPTION_cpu:
                 /* hw initialization will check this */
                 cpu_model = optarg;
@@ -2622,7 +3008,8 @@ int main(int argc, char **argv, char **envp)
                 {
                     static const char * const params[] = {
                         "order", "once", "menu",
-                        "splash", "splash-time", NULL
+                        "splash", "splash-time",
+                        "reboot-timeout", NULL
                     };
                     char buf[sizeof(boot_devices)];
                     char *standard_boot_devices;
@@ -2923,8 +3310,7 @@ int main(int argc, char **argv, char **envp)
 
                 qemu_opt_set_bool(fsdev, "readonly",
                                 qemu_opt_get_bool(opts, "readonly", 0));
-                device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
-                                          NULL);
+                device = qemu_opts_create_nofail(qemu_find_opts("device"));
                 qemu_opt_set(device, "driver", "virtio-9p-pci");
                 qemu_opt_set(device, "fsdev",
                              qemu_opt_get(opts, "mount_tag"));
@@ -2944,8 +3330,7 @@ int main(int argc, char **argv, char **envp)
                 }
                 qemu_opt_set(fsdev, "fsdriver", "synth");
 
-                device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
-                                          NULL);
+                device = qemu_opts_create_nofail(qemu_find_opts("device"));
                 qemu_opt_set(device, "driver", "virtio-9p-pci");
                 qemu_opt_set(device, "fsdev", "v_synth");
                 qemu_opt_set(device, "mount_tag", "v_synth");
@@ -3061,11 +3446,37 @@ int main(int argc, char **argv, char **envp)
                     machine = machine_parse(optarg);
                 }
                 break;
+             case QEMU_OPTION_no_kvm:
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "accel=tcg", 0);
+                break;
+            case QEMU_OPTION_no_kvm_pit: {
+                fprintf(stderr, "Warning: KVM PIT can no longer be disabled "
+                                "separately.\n");
+                break;
+            }
+            case QEMU_OPTION_no_kvm_pit_reinjection: {
+                static GlobalProperty kvm_pit_lost_tick_policy[] = {
+                    {
+                        .driver   = "kvm-pit",
+                        .property = "lost_tick_policy",
+                        .value    = "discard",
+                    },
+                    { /* end of list */ }
+                };
+
+                fprintf(stderr, "Warning: option deprecated, use "
+                        "lost_tick_policy property of kvm-pit instead.\n");
+                qdev_prop_register_global_list(kvm_pit_lost_tick_policy);
+                break;
+            }
             case QEMU_OPTION_usb:
-                usb_enabled = 1;
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "usb=on", 0);
                 break;
             case QEMU_OPTION_usbdevice:
-                usb_enabled = 1;
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "usb=on", 0);
                 add_device_config(DEV_USB, optarg);
                 break;
             case QEMU_OPTION_device:
@@ -3144,6 +3555,10 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_semihosting:
                 semihosting_enabled = 1;
                 break;
+            case QEMU_OPTION_tdf:
+                fprintf(stderr, "Warning: user space PIT time drift fix "
+                                "is no longer supported.\n");
+                break;
             case QEMU_OPTION_name:
                 qemu_name = g_strdup(optarg);
 		 {
@@ -3288,6 +3703,21 @@ int main(int argc, char **argv, char **envp)
                     exit(0);
                 }
                 break;
+            case QEMU_OPTION_add_fd:
+#ifndef _WIN32
+                opts = qemu_opts_parse(qemu_find_opts("add-fd"), optarg, 0);
+                if (!opts) {
+                    exit(0);
+                }
+#else
+                error_report("File descriptor passing is disabled on this "
+                             "platform");
+                exit(1);
+#endif
+                break;
+            case QEMU_OPTION_object:
+                opts = qemu_opts_parse(qemu_find_opts("object"), optarg, 1);
+                break;
             default:
                 os_parse_cmd_args(popt->index, optarg);
             }
@@ -3295,10 +3725,25 @@ int main(int argc, char **argv, char **envp)
     }
     loc_set_none();
 
+    if (qemu_init_main_loop()) {
+        fprintf(stderr, "qemu_init_main_loop failed\n");
+        exit(1);
+    }
+
     if (qemu_opts_foreach(qemu_find_opts("sandbox"), parse_sandbox, NULL, 0)) {
         exit(1);
     }
 
+#ifndef _WIN32
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"), parse_add_fd, NULL, 1)) {
+        exit(1);
+    }
+
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"), cleanup_add_fd, NULL, 1)) {
+        exit(1);
+    }
+#endif
+
     if (machine == NULL) {
         fprintf(stderr, "No machine found.\n");
         exit(1);
@@ -3308,6 +3753,11 @@ int main(int argc, char **argv, char **envp)
         qemu_set_version(machine->hw_version);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          object_create, NULL, 0) != 0) {
+        exit(1);
+    }
+
     /* Init CPU def lists, based on config
      * - Must be called after all the qemu_read_config_file() calls
      * - Must be called before list_cpus()
@@ -3392,6 +3842,30 @@ int main(int argc, char **argv, char **envp)
         default_sdcard = 0;
     }
 
+    if (is_daemonized()) {
+        /* According to documentation and historically, -nographic redirects
+         * serial port, parallel port and monitor to stdio, which does not work
+         * with -daemonize.  We can redirect these to null instead, but since
+         * -nographic is legacy, let's just error out.
+         * We disallow -nographic only if all other ports are not redirected
+         * explicitly, to not break existing legacy setups which uses
+         * -nographic _and_ redirects all ports explicitly - this is valid
+         * usage, -nographic is just a no-op in this case.
+         */
+        if (display_type == DT_NOGRAPHIC
+            && (default_parallel || default_serial
+                || default_monitor || default_virtcon)) {
+            fprintf(stderr, "-nographic can not be used with -daemonize\n");
+            exit(1);
+        }
+#ifdef CONFIG_CURSES
+        if (display_type == DT_CURSES) {
+            fprintf(stderr, "curses display can not be used with -daemonize\n");
+            exit(1);
+        }
+#endif
+    }
+
     if (display_type == DT_NOGRAPHIC) {
         if (default_parallel)
             add_device_config(DEV_PARALLEL, "null");
@@ -3447,12 +3921,6 @@ int main(int argc, char **argv, char **envp)
 
     configure_accelerator();
 
-    qemu_init_cpu_loop();
-    if (qemu_init_main_loop()) {
-        fprintf(stderr, "qemu_init_main_loop failed\n");
-        exit(1);
-    }
-
     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
     if (machine_opts) {
         kernel_filename = qemu_opt_get(machine_opts, "kernel");
@@ -3485,10 +3953,8 @@ int main(int argc, char **argv, char **envp)
 
     os_set_line_buffering();
 
-    if (init_timer_alarm() < 0) {
-        fprintf(stderr, "could not initialize alarm timer\n");
-        exit(1);
-    }
+    qemu_init_cpu_loop();
+    qemu_mutex_lock_iothread();
 
 #ifdef CONFIG_SPICE
     /* spice needs the timers to be initialized by this point */
@@ -3501,6 +3967,9 @@ int main(int argc, char **argv, char **envp)
     }
     configure_icount(icount_option);
 
+    /* clean up network at qemu process termination */
+    atexit(&net_cleanup);
+
     if (net_init_clients() < 0) {
         exit(1);
     }
@@ -3526,15 +3995,15 @@ int main(int argc, char **argv, char **envp)
     /* open the virtual block devices */
     if (snapshot)
         qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot, NULL, 0);
-    if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func, &machine->use_scsi, 1) != 0)
+    if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func,
+                          &machine->block_default_type, 1) != 0) {
         exit(1);
+    }
 
-    default_drive(default_cdrom, snapshot, machine->use_scsi,
-                  IF_DEFAULT, 2, CDROM_OPTS);
-    default_drive(default_floppy, snapshot, machine->use_scsi,
-                  IF_FLOPPY, 0, FD_OPTS);
-    default_drive(default_sdcard, snapshot, machine->use_scsi,
-                  IF_SD, 0, SD_OPTS);
+    default_drive(default_cdrom, snapshot, machine->block_default_type, 2,
+                  CDROM_OPTS);
+    default_drive(default_floppy, snapshot, IF_FLOPPY, 0, FD_OPTS);
+    default_drive(default_sdcard, snapshot, IF_SD, 0, SD_OPTS);
 
     register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
 
@@ -3595,8 +4064,12 @@ int main(int argc, char **argv, char **envp)
         exit(1);
 
     /* If no default VGA is requested, the default is "none".  */
-    if (default_vga && cirrus_vga_available()) {
-        vga_model = "cirrus";
+    if (default_vga) {
+        if (cirrus_vga_available()) {
+            vga_model = "cirrus";
+        } else if (vga_available()) {
+            vga_model = "std";
+        }
     }
     select_vgahw(vga_model);
 
@@ -3613,8 +4086,15 @@ int main(int argc, char **argv, char **envp)
 
     qdev_machine_init();
 
-    machine->init(ram_size, boot_devices,
-                  kernel_filename, kernel_cmdline, initrd_filename, cpu_model);
+    QEMUMachineInitArgs args = { .ram_size = ram_size,
+                                 .boot_device = (boot_devices[0] == '\0') ?
+                                                machine->boot_order :
+                                                boot_devices,
+                                 .kernel_filename = kernel_filename,
+                                 .kernel_cmdline = kernel_cmdline,
+                                 .initrd_filename = initrd_filename,
+                                 .cpu_model = cpu_model };
+    machine->init(&args);
 
     cpu_synchronize_all_post_init();
 
@@ -3623,7 +4103,7 @@ int main(int argc, char **argv, char **envp)
     current_machine = machine;
 
     /* init USB devices */
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         if (foreach_device_config(DEV_USB, usb_parse) < 0)
             exit(1);
     }
@@ -3679,10 +4159,13 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_VNC
     /* init remote displays */
     if (vnc_display) {
+        Error *local_err = NULL;
         vnc_display_init(ds);
-        if (vnc_display_open(ds, vnc_display) < 0) {
-            fprintf(stderr, "Failed to start VNC server on `%s'\n",
-                    vnc_display);
+        vnc_display_open(ds, vnc_display, &local_err);
+        if (local_err != NULL) {
+            fprintf(stderr, "Failed to start VNC server on `%s': %s\n",
+                    vnc_display, error_get_pretty(local_err));
+            error_free(local_err);
             exit(1);
         }
 
@@ -3698,16 +4181,6 @@ int main(int argc, char **argv, char **envp)
 #endif
 
     /* display setup */
-    dpy_resize(ds);
-    dcl = ds->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_refresh != NULL) {
-            ds->gui_timer = qemu_new_timer_ms(rt_clock, gui_update, ds);
-            qemu_mod_timer(ds->gui_timer, qemu_get_clock_ms(rt_clock));
-            break;
-        }
-        dcl = dcl->next;
-    }
     text_consoles_set_display(ds);
 
     if (foreach_device_config(DEV_GDB, gdbserver_start) < 0) {
@@ -3734,16 +4207,12 @@ int main(int argc, char **argv, char **envp)
     }
 
     if (incoming) {
-        Error *errp = NULL;
-        int ret = qemu_start_incoming_migration(incoming, &errp);
-        if (ret < 0) {
-            if (error_is_set(&errp)) {
-                fprintf(stderr, "Migrate: %s\n", error_get_pretty(errp));
-                error_free(errp);
-            }
-            fprintf(stderr, "Migration failed. Exit code %s(%d), exiting.\n",
-                    incoming, ret);
-            exit(ret);
+        Error *local_err = NULL;
+        qemu_start_incoming_migration(incoming, &local_err);
+        if (local_err) {
+            fprintf(stderr, "-incoming %s: %s\n", incoming, error_get_pretty(local_err));
+            error_free(local_err);
+            exit(1);
         }
     } else if (autostart) {
         vm_start();
@@ -3755,7 +4224,6 @@ int main(int argc, char **argv, char **envp)
     main_loop();
     bdrv_close_all();
     pause_all_vcpus();
-    net_cleanup();
     res_free();
 
     return 0;
diff --git a/xen-all.c b/xen-all.c
index f76b051eee2..110f958a535 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -10,15 +10,17 @@
 
 #include <sys/mman.h>
 
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/pc.h"
 #include "hw/xen_common.h"
 #include "hw/xen_backend.h"
+#include "qmp-commands.h"
 
-#include "range.h"
-#include "xen-mapcache.h"
+#include "char/char.h"
+#include "qemu/range.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include <xen/hvm/ioreq.h>
 #include <xen/hvm/params.h>
@@ -36,6 +38,7 @@
 
 static MemoryRegion ram_memory, ram_640k, ram_lo, ram_hi;
 static MemoryRegion *framebuffer;
+static bool xen_in_migration;
 
 /* Compatibility with older version */
 #if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
@@ -66,10 +69,10 @@ static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
 #define BUFFER_IO_MAX_DELAY  100
 
 typedef struct XenPhysmap {
-    target_phys_addr_t start_addr;
+    hwaddr start_addr;
     ram_addr_t size;
     char *name;
-    target_phys_addr_t phys_offset;
+    hwaddr phys_offset;
 
     QLIST_ENTRY(XenPhysmap) list;
 } XenPhysmap;
@@ -90,7 +93,7 @@ typedef struct XenIOState {
     struct xs_handle *xenstore;
     MemoryListener memory_listener;
     QLIST_HEAD(, XenPhysmap) physmap;
-    target_phys_addr_t free_phys_offset;
+    hwaddr free_phys_offset;
     const XenPhysmap *log_for_dirtybit;
 
     Notifier exit;
@@ -229,7 +232,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
 }
 
 static XenPhysmap *get_physmapping(XenIOState *state,
-                                   target_phys_addr_t start_addr, ram_addr_t size)
+                                   hwaddr start_addr, ram_addr_t size)
 {
     XenPhysmap *physmap = NULL;
 
@@ -243,10 +246,10 @@ static XenPhysmap *get_physmapping(XenIOState *state,
     return NULL;
 }
 
-static target_phys_addr_t xen_phys_offset_to_gaddr(target_phys_addr_t start_addr,
+static hwaddr xen_phys_offset_to_gaddr(hwaddr start_addr,
                                                    ram_addr_t size, void *opaque)
 {
-    target_phys_addr_t addr = start_addr & TARGET_PAGE_MASK;
+    hwaddr addr = start_addr & TARGET_PAGE_MASK;
     XenIOState *xen_io_state = opaque;
     XenPhysmap *physmap = NULL;
 
@@ -261,16 +264,16 @@ static target_phys_addr_t xen_phys_offset_to_gaddr(target_phys_addr_t start_addr
 
 #if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 340
 static int xen_add_to_physmap(XenIOState *state,
-                              target_phys_addr_t start_addr,
+                              hwaddr start_addr,
                               ram_addr_t size,
                               MemoryRegion *mr,
-                              target_phys_addr_t offset_within_region)
+                              hwaddr offset_within_region)
 {
     unsigned long i = 0;
     int rc = 0;
     XenPhysmap *physmap = NULL;
-    target_phys_addr_t pfn, start_gpfn;
-    target_phys_addr_t phys_offset = memory_region_get_ram_addr(mr);
+    hwaddr pfn, start_gpfn;
+    hwaddr phys_offset = memory_region_get_ram_addr(mr);
     char path[80], value[17];
 
     if (get_physmapping(state, start_addr, size)) {
@@ -290,7 +293,8 @@ static int xen_add_to_physmap(XenIOState *state,
     return -1;
 
 go_physmap:
-    DPRINTF("mapping vram to %llx - %llx\n", start_addr, start_addr + size);
+    DPRINTF("mapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+            start_addr, start_addr + size);
 
     pfn = phys_offset >> TARGET_PAGE_BITS;
     start_gpfn = start_addr >> TARGET_PAGE_BITS;
@@ -347,13 +351,13 @@ go_physmap:
 }
 
 static int xen_remove_from_physmap(XenIOState *state,
-                                   target_phys_addr_t start_addr,
+                                   hwaddr start_addr,
                                    ram_addr_t size)
 {
     unsigned long i = 0;
     int rc = 0;
     XenPhysmap *physmap = NULL;
-    target_phys_addr_t phys_offset = 0;
+    hwaddr phys_offset = 0;
 
     physmap = get_physmapping(state, start_addr, size);
     if (physmap == NULL) {
@@ -363,8 +367,8 @@ static int xen_remove_from_physmap(XenIOState *state,
     phys_offset = physmap->phys_offset;
     size = physmap->size;
 
-    DPRINTF("unmapping vram to %llx - %llx, from %llx\n",
-            phys_offset, phys_offset + size, start_addr);
+    DPRINTF("unmapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx", from ",
+            "%"HWADDR_PRIx"\n", phys_offset, phys_offset + size, start_addr);
 
     size >>= TARGET_PAGE_BITS;
     start_addr >>= TARGET_PAGE_BITS;
@@ -392,16 +396,16 @@ static int xen_remove_from_physmap(XenIOState *state,
 
 #else
 static int xen_add_to_physmap(XenIOState *state,
-                              target_phys_addr_t start_addr,
+                              hwaddr start_addr,
                               ram_addr_t size,
                               MemoryRegion *mr,
-                              target_phys_addr_t offset_within_region)
+                              hwaddr offset_within_region)
 {
     return -ENOSYS;
 }
 
 static int xen_remove_from_physmap(XenIOState *state,
-                                   target_phys_addr_t start_addr,
+                                   hwaddr start_addr,
                                    ram_addr_t size)
 {
     return -ENOSYS;
@@ -413,7 +417,7 @@ static void xen_set_memory(struct MemoryListener *listener,
                            bool add)
 {
     XenIOState *state = container_of(listener, XenIOState, memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     bool log_dirty = memory_region_is_logging(section->mr);
     hvmmem_type_t mem_type;
@@ -452,14 +456,6 @@ static void xen_set_memory(struct MemoryListener *listener,
     }
 }
 
-static void xen_begin(MemoryListener *listener)
-{
-}
-
-static void xen_commit(MemoryListener *listener)
-{
-}
-
 static void xen_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -472,16 +468,11 @@ static void xen_region_del(MemoryListener *listener,
     xen_set_memory(listener, section, false);
 }
 
-static void xen_region_nop(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
 static void xen_sync_dirty_bitmap(XenIOState *state,
-                                  target_phys_addr_t start_addr,
+                                  hwaddr start_addr,
                                   ram_addr_t size)
 {
-    target_phys_addr_t npages = size >> TARGET_PAGE_BITS;
+    hwaddr npages = size >> TARGET_PAGE_BITS;
     const int width = sizeof(unsigned long) * 8;
     unsigned long bitmap[(npages + width - 1) / width];
     int rc, i, j;
@@ -505,7 +496,8 @@ static void xen_sync_dirty_bitmap(XenIOState *state,
                                  bitmap);
     if (rc < 0) {
         if (rc != -ENODATA) {
-            fprintf(stderr, "xen: track_dirty_vram failed (0x" TARGET_FMT_plx
+            memory_region_set_dirty(framebuffer, 0, size);
+            DPRINTF("xen: track_dirty_vram failed (0x" TARGET_FMT_plx
                     ", 0x" TARGET_FMT_plx "): %s\n",
                     start_addr, start_addr + size, strerror(-rc));
         }
@@ -552,42 +544,36 @@ static void xen_log_sync(MemoryListener *listener, MemoryRegionSection *section)
 
 static void xen_log_global_start(MemoryListener *listener)
 {
+    if (xen_enabled()) {
+        xen_in_migration = true;
+    }
 }
 
 static void xen_log_global_stop(MemoryListener *listener)
 {
-}
-
-static void xen_eventfd_add(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-}
-
-static void xen_eventfd_del(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
+    xen_in_migration = false;
 }
 
 static MemoryListener xen_memory_listener = {
-    .begin = xen_begin,
-    .commit = xen_commit,
     .region_add = xen_region_add,
     .region_del = xen_region_del,
-    .region_nop = xen_region_nop,
     .log_start = xen_log_start,
     .log_stop = xen_log_stop,
     .log_sync = xen_log_sync,
     .log_global_start = xen_log_global_start,
     .log_global_stop = xen_log_global_stop,
-    .eventfd_add = xen_eventfd_add,
-    .eventfd_del = xen_eventfd_del,
     .priority = 10,
 };
 
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+    if (enable) {
+        memory_global_dirty_log_start();
+    } else {
+        memory_global_dirty_log_stop();
+    }
+}
+
 /* VCPU Operations, MMIO, IO ring ... */
 
 static void xen_reset_vcpu(void *opaque)
@@ -599,9 +585,7 @@ static void xen_reset_vcpu(void *opaque)
 
 void xen_vcpu_init(void)
 {
-    CPUArchState *first_cpu;
-
-    if ((first_cpu = qemu_get_cpu(0))) {
+    if (first_cpu != NULL) {
         qemu_register_reset(xen_reset_vcpu, first_cpu);
         xen_reset_vcpu(first_cpu);
     }
@@ -698,11 +682,45 @@ static void do_outp(pio_addr_t addr,
     }
 }
 
-static void cpu_ioreq_pio(ioreq_t *req)
+/*
+ * Helper functions which read/write an object from/to physical guest
+ * memory, as part of the implementation of an ioreq.
+ *
+ * Equivalent to
+ *   cpu_physical_memory_rw(addr + (req->df ? -1 : +1) * req->size * i,
+ *                          val, req->size, 0/1)
+ * except without the integer overflow problems.
+ */
+static void rw_phys_req_item(hwaddr addr,
+                             ioreq_t *req, uint32_t i, void *val, int rw)
+{
+    /* Do everything unsigned so overflow just results in a truncated result
+     * and accesses to undesired parts of guest memory, which is up
+     * to the guest */
+    hwaddr offset = (hwaddr)req->size * i;
+    if (req->df) {
+        addr -= offset;
+    } else {
+        addr += offset;
+    }
+    cpu_physical_memory_rw(addr, val, req->size, rw);
+}
+
+static inline void read_phys_req_item(hwaddr addr,
+                                      ioreq_t *req, uint32_t i, void *val)
+{
+    rw_phys_req_item(addr, req, i, val, 0);
+}
+static inline void write_phys_req_item(hwaddr addr,
+                                       ioreq_t *req, uint32_t i, void *val)
 {
-    int i, sign;
+    rw_phys_req_item(addr, req, i, val, 1);
+}
 
-    sign = req->df ? -1 : 1;
+
+static void cpu_ioreq_pio(ioreq_t *req)
+{
+    uint32_t i;
 
     if (req->dir == IOREQ_READ) {
         if (!req->data_is_ptr) {
@@ -712,9 +730,7 @@ static void cpu_ioreq_pio(ioreq_t *req)
 
             for (i = 0; i < req->count; i++) {
                 tmp = do_inp(req->addr, req->size);
-                cpu_physical_memory_write(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &tmp, req->size);
+                write_phys_req_item(req->data, req, i, &tmp);
             }
         }
     } else if (req->dir == IOREQ_WRITE) {
@@ -724,9 +740,7 @@ static void cpu_ioreq_pio(ioreq_t *req)
             for (i = 0; i < req->count; i++) {
                 uint32_t tmp = 0;
 
-                cpu_physical_memory_read(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->data, req, i, &tmp);
                 do_outp(req->addr, req->size, tmp);
             }
         }
@@ -735,22 +749,16 @@ static void cpu_ioreq_pio(ioreq_t *req)
 
 static void cpu_ioreq_move(ioreq_t *req)
 {
-    int i, sign;
-
-    sign = req->df ? -1 : 1;
+    uint32_t i;
 
     if (!req->data_is_ptr) {
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &req->data, req->size);
+                read_phys_req_item(req->addr, req, i, &req->data);
             }
         } else if (req->dir == IOREQ_WRITE) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_write(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &req->data, req->size);
+                write_phys_req_item(req->addr, req, i, &req->data);
             }
         }
     } else {
@@ -758,21 +766,13 @@ static void cpu_ioreq_move(ioreq_t *req)
 
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
-                cpu_physical_memory_write(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->addr, req, i, &tmp);
+                write_phys_req_item(req->data, req, i, &tmp);
             }
         } else if (req->dir == IOREQ_WRITE) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
-                cpu_physical_memory_write(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->data, req, i, &tmp);
+                write_phys_req_item(req->addr, req, i, &tmp);
             }
         }
     }
@@ -1076,7 +1076,6 @@ static void xen_read_physmap(XenIOState *state)
         QLIST_INSERT_HEAD(&state->physmap, physmap, list);
     }
     free(entries);
-    return;
 }
 
 int xen_hvm_init(void)
@@ -1158,7 +1157,7 @@ int xen_hvm_init(void)
 
     state->memory_listener = xen_memory_listener;
     QLIST_INIT(&state->physmap);
-    memory_listener_register(&state->memory_listener, get_system_memory());
+    memory_listener_register(&state->memory_listener, &address_space_memory);
     state->log_for_dirtybit = NULL;
 
     /* Initialize backend core & drivers */
@@ -1213,3 +1212,24 @@ void xen_shutdown_fatal_error(const char *fmt, ...)
     /* destroy the domain */
     qemu_system_shutdown_request();
 }
+
+void xen_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+    if (unlikely(xen_in_migration)) {
+        int rc;
+        ram_addr_t start_pfn, nb_pages;
+
+        if (length == 0) {
+            length = TARGET_PAGE_SIZE;
+        }
+        start_pfn = start >> TARGET_PAGE_BITS;
+        nb_pages = ((start + length + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS)
+            - start_pfn;
+        rc = xc_hvm_modified_memory(xen_xc, xen_domid, start_pfn, nb_pages);
+        if (rc) {
+            fprintf(stderr,
+                    "%s failed for "RAM_ADDR_FMT" ("RAM_ADDR_FMT"): %i, %s\n",
+                    __func__, start, nb_pages, rc, strerror(-rc));
+        }
+    }
+}
diff --git a/xen-mapcache.c b/xen-mapcache.c
index 9cd6db3d7b0..dc6d1fadb75 100644
--- a/xen-mapcache.c
+++ b/xen-mapcache.c
@@ -13,13 +13,13 @@
 #include <sys/resource.h>
 
 #include "hw/xen_backend.h"
-#include "blockdev.h"
-#include "bitmap.h"
+#include "sysemu/blockdev.h"
+#include "qemu/bitmap.h"
 
 #include <xen/hvm/params.h>
 #include <sys/mman.h>
 
-#include "xen-mapcache.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
 
@@ -53,18 +53,18 @@
 #define mapcache_unlock() ((void)0)
 
 typedef struct MapCacheEntry {
-    target_phys_addr_t paddr_index;
+    hwaddr paddr_index;
     uint8_t *vaddr_base;
     unsigned long *valid_mapping;
     uint8_t lock;
-    target_phys_addr_t size;
+    hwaddr size;
     struct MapCacheEntry *next;
 } MapCacheEntry;
 
 typedef struct MapCacheRev {
     uint8_t *vaddr_req;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     QTAILQ_ENTRY(MapCacheRev) next;
 } MapCacheRev;
 
@@ -74,7 +74,7 @@ typedef struct MapCache {
     QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
 
     /* For most cases (>99.9%), the page address is the same. */
-    target_phys_addr_t last_address_index;
+    hwaddr last_address_index;
     uint8_t *last_address_vaddr;
     unsigned long max_mcache_size;
     unsigned int mcache_bucket_shift;
@@ -142,14 +142,14 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque)
 }
 
 static void xen_remap_bucket(MapCacheEntry *entry,
-                             target_phys_addr_t size,
-                             target_phys_addr_t address_index)
+                             hwaddr size,
+                             hwaddr address_index)
 {
     uint8_t *vaddr_base;
     xen_pfn_t *pfns;
     int *err;
     unsigned int i;
-    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
+    hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
     trace_xen_remap_bucket(address_index);
 
@@ -195,13 +195,13 @@ static void xen_remap_bucket(MapCacheEntry *entry,
     g_free(err);
 }
 
-uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
+uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
                        uint8_t lock)
 {
     MapCacheEntry *entry, *pentry = NULL;
-    target_phys_addr_t address_index;
-    target_phys_addr_t address_offset;
-    target_phys_addr_t __size = size;
+    hwaddr address_index;
+    hwaddr address_offset;
+    hwaddr __size = size;
     bool translated = false;
 
 tryagain:
@@ -278,8 +278,8 @@ ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
 {
     MapCacheEntry *entry = NULL;
     MapCacheRev *reventry;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     int found = 0;
 
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
@@ -316,8 +316,8 @@ void xen_invalidate_map_cache_entry(uint8_t *buffer)
 {
     MapCacheEntry *entry = NULL, *pentry = NULL;
     MapCacheRev *reventry;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     int found = 0;
 
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
diff --git a/xen-stub.c b/xen-stub.c
index 8ff2b79ac6f..1ee841137e2 100644
--- a/xen-stub.c
+++ b/xen-stub.c
@@ -10,7 +10,8 @@
 
 #include "qemu-common.h"
 #include "hw/xen.h"
-#include "memory.h"
+#include "exec/memory.h"
+#include "qmp-commands.h"
 
 void xenstore_store_pv_console_info(int i, CharDriverState *chr)
 {
@@ -54,3 +55,11 @@ int xen_init(void)
 void xen_register_framebuffer(MemoryRegion *mr)
 {
 }
+
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+}
+
+void xen_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+}